From a25eb03290439a9a74a9443f1af59562d84c4b70 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Tue, 7 Aug 2018 12:16:45 -0400 Subject: [PATCH 01/29] Add BackupStorageLocation API type This commit only provides the data model for further work. It does not implement any logic around locations, nor does it remove anything from the Config API type. Closes #736 Closes #732 Signed-off-by: Nolan Brubaker --- examples/common/00-prereqs.yaml | 15 +++ pkg/apis/ark/v1/backup.go | 7 +- pkg/apis/ark/v1/backup_storage_location.go | 93 ++++++++++++++ pkg/apis/ark/v1/register.go | 19 +-- pkg/apis/ark/v1/zz_generated.deepcopy.go | 142 +++++++++++++++++++++ 5 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 pkg/apis/ark/v1/backup_storage_location.go diff --git a/examples/common/00-prereqs.yaml b/examples/common/00-prereqs.yaml index 53660427e..c9c096415 100644 --- a/examples/common/00-prereqs.yaml +++ b/examples/common/00-prereqs.yaml @@ -147,6 +147,21 @@ spec: plural: resticrepositories kind: ResticRepository +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: backupstoragelocations.ark.heptio.com + labels: + component: ark +spec: + group: ark.heptio.com + version: v1 + scope: Namespaced + names: + plural: backupstoragelocations + kind: BackupStorageLocation + --- apiVersion: v1 kind: Namespace diff --git a/pkg/apis/ark/v1/backup.go b/pkg/apis/ark/v1/backup.go index 041d3b5b1..8aef922ca 100644 --- a/pkg/apis/ark/v1/backup.go +++ b/pkg/apis/ark/v1/backup.go @@ -16,7 +16,9 @@ limitations under the License. package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // BackupSpec defines the specification for an Ark backup. type BackupSpec struct { @@ -56,6 +58,9 @@ type BackupSpec struct { // Hooks represent custom behaviors that should be executed at different phases of the backup. Hooks BackupHooks `json:"hooks"` + + // StorageLocation is a string containing the name of a BackupStorageLocation where the backup should be stored. + StorageLocation string `json:"storageLocation"` } // BackupHooks contains custom behaviors that should be executed at different phases of the backup. diff --git a/pkg/apis/ark/v1/backup_storage_location.go b/pkg/apis/ark/v1/backup_storage_location.go new file mode 100644 index 000000000..adafec1b7 --- /dev/null +++ b/pkg/apis/ark/v1/backup_storage_location.go @@ -0,0 +1,93 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// BackupStorageLocation is a location where Ark stores backup objects. +type BackupStorageLocation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec BackupStorageLocationSpec `json:"spec"` + Status BackupStorageLocationStatus `json:"status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// BackupStorageLocationList is a list of BackupStorageLocations. +type BackupStorageLocationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + Items []BackupStorageLocation `json:"items"` +} + +// StorageType represents the type of storage that a backup location uses. +// ObjectStorage must be non-nil, since it is currently the only supported StorageType. +type StorageType struct { + ObjectStorage *ObjectStorageLocation `json:"objectStorage,omitempty"` +} + +// ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. +type ObjectStorageLocation struct { + // Bucket is the bucket to use for object storage. + Bucket string `json:"bucket"` + + // Prefix is the path inside a bucket to use for Ark storage. Optional. + Prefix string `json:"prefix"` +} + +// BackupStorageLocationSpec defines the specification for an Ark BackupStorageLocation. +type BackupStorageLocationSpec struct { + // Provider is the provider of the backup storage. + Provider string `json:"provider"` + + // Config is for provider-specific configuration fields. + Config map[string]string `json:"config"` + + StorageType `json:",inline"` +} + +// BackupStorageLocationPhase is the lifecyle phase of an Ark BackupStorageLocation. +type BackupStorageLocationPhase string + +const ( + // BackupStorageLocationPhaseAvailable means the location is available to read and write from. + BackupStorageLocationPhaseAvailable BackupStorageLocationPhase = "Available" + + // BackupStorageLocationPhaseUnavailable means the location is unavailable to read and write from. + BackupStorageLocationPhaseUnavailable BackupStorageLocationPhase = "Unavailable" +) + +// BackupStorageLocationAccessMode represents the permissions for a BackupStorageLocation. +type BackupStorageLocationAccessMode string + +const ( + // BackupStorageLocationAccessModeReadOnly represents read-only access to a BackupStorageLocation. + BackupStorageLocationAccessModeReadOnly BackupStorageLocationAccessMode = "ReadOnly" + + // BackupStorageLocationAccessModeReadWrite represents read and write access to a BackupStorageLocation. + BackupStorageLocationAccessModeReadWrite BackupStorageLocationAccessMode = "ReadWrite" +) + +// BackupStorageLocationStatus describes the current status of an Ark BackupStorageLocation. +type BackupStorageLocationStatus struct { + Phase BackupStorageLocationPhase `json:"phase,omitempty"` + AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` +} diff --git a/pkg/apis/ark/v1/register.go b/pkg/apis/ark/v1/register.go index 1e9913c9e..5d70dcc76 100644 --- a/pkg/apis/ark/v1/register.go +++ b/pkg/apis/ark/v1/register.go @@ -59,15 +59,16 @@ func newTypeInfo(pluralName string, itemType, itemListType runtime.Object) typeI // API group, keyed on Kind. func CustomResources() map[string]typeInfo { return map[string]typeInfo{ - "Backup": newTypeInfo("backups", &Backup{}, &BackupList{}), - "Restore": newTypeInfo("restores", &Restore{}, &RestoreList{}), - "Schedule": newTypeInfo("schedules", &Schedule{}, &ScheduleList{}), - "Config": newTypeInfo("configs", &Config{}, &ConfigList{}), - "DownloadRequest": newTypeInfo("downloadrequests", &DownloadRequest{}, &DownloadRequestList{}), - "DeleteBackupRequest": newTypeInfo("deletebackuprequests", &DeleteBackupRequest{}, &DeleteBackupRequestList{}), - "PodVolumeBackup": newTypeInfo("podvolumebackups", &PodVolumeBackup{}, &PodVolumeBackupList{}), - "PodVolumeRestore": newTypeInfo("podvolumerestores", &PodVolumeRestore{}, &PodVolumeRestoreList{}), - "ResticRepository": newTypeInfo("resticrepositories", &ResticRepository{}, &ResticRepositoryList{}), + "Backup": newTypeInfo("backups", &Backup{}, &BackupList{}), + "Restore": newTypeInfo("restores", &Restore{}, &RestoreList{}), + "Schedule": newTypeInfo("schedules", &Schedule{}, &ScheduleList{}), + "Config": newTypeInfo("configs", &Config{}, &ConfigList{}), + "DownloadRequest": newTypeInfo("downloadrequests", &DownloadRequest{}, &DownloadRequestList{}), + "DeleteBackupRequest": newTypeInfo("deletebackuprequests", &DeleteBackupRequest{}, &DeleteBackupRequestList{}), + "PodVolumeBackup": newTypeInfo("podvolumebackups", &PodVolumeBackup{}, &PodVolumeBackupList{}), + "PodVolumeRestore": newTypeInfo("podvolumerestores", &PodVolumeRestore{}, &PodVolumeRestoreList{}), + "ResticRepository": newTypeInfo("resticrepositories", &ResticRepository{}, &ResticRepositoryList{}), + "BackupStorageLocation": newTypeInfo("backupstoragelocations", &BackupStorageLocation{}, &BackupStorageLocationList{}), } } diff --git a/pkg/apis/ark/v1/zz_generated.deepcopy.go b/pkg/apis/ark/v1/zz_generated.deepcopy.go index 7915be376..0d55e70b6 100644 --- a/pkg/apis/ark/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ark/v1/zz_generated.deepcopy.go @@ -301,6 +301,107 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStorageLocation) DeepCopyInto(out *BackupStorageLocation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocation. +func (in *BackupStorageLocation) DeepCopy() *BackupStorageLocation { + if in == nil { + return nil + } + out := new(BackupStorageLocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BackupStorageLocation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStorageLocationList) DeepCopyInto(out *BackupStorageLocationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BackupStorageLocation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationList. +func (in *BackupStorageLocationList) DeepCopy() *BackupStorageLocationList { + if in == nil { + return nil + } + out := new(BackupStorageLocationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BackupStorageLocationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStorageLocationSpec) DeepCopyInto(out *BackupStorageLocationSpec) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.StorageType.DeepCopyInto(&out.StorageType) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationSpec. +func (in *BackupStorageLocationSpec) DeepCopy() *BackupStorageLocationSpec { + if in == nil { + return nil + } + out := new(BackupStorageLocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStorageLocationStatus) DeepCopyInto(out *BackupStorageLocationStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageLocationStatus. +func (in *BackupStorageLocationStatus) DeepCopy() *BackupStorageLocationStatus { + if in == nil { + return nil + } + out := new(BackupStorageLocationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProviderConfig) DeepCopyInto(out *CloudProviderConfig) { *out = *in @@ -624,6 +725,22 @@ func (in *ExecHook) DeepCopy() *ExecHook { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStorageLocation) DeepCopyInto(out *ObjectStorageLocation) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageLocation. +func (in *ObjectStorageLocation) DeepCopy() *ObjectStorageLocation { + if in == nil { + return nil + } + out := new(ObjectStorageLocation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStorageProviderConfig) DeepCopyInto(out *ObjectStorageProviderConfig) { *out = *in @@ -1221,6 +1338,31 @@ func (in *ScheduleStatus) DeepCopy() *ScheduleStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageType) DeepCopyInto(out *StorageType) { + *out = *in + if in.ObjectStorage != nil { + in, out := &in.ObjectStorage, &out.ObjectStorage + if *in == nil { + *out = nil + } else { + *out = new(ObjectStorageLocation) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageType. +func (in *StorageType) DeepCopy() *StorageType { + if in == nil { + return nil + } + out := new(StorageType) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeBackupInfo) DeepCopyInto(out *VolumeBackupInfo) { *out = *in From 345c3c39b10d9580b9720d030b5d90010e41d819 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 9 Aug 2018 12:19:49 -0400 Subject: [PATCH 02/29] Generate clients for BackupStorageLocation Signed-off-by: Nolan Brubaker --- pkg/apis/ark/v1/backup_storage_location.go | 1 + .../versioned/typed/ark/v1/ark_client.go | 5 + .../typed/ark/v1/backupstoragelocation.go | 174 ++++++++++++++++++ .../typed/ark/v1/fake/fake_ark_client.go | 4 + .../ark/v1/fake/fake_backupstoragelocation.go | 140 ++++++++++++++ .../typed/ark/v1/generated_expansion.go | 2 + .../ark/v1/backupstoragelocation.go | 89 +++++++++ .../externalversions/ark/v1/interface.go | 7 + .../informers/externalversions/generic.go | 2 + .../listers/ark/v1/backupstoragelocation.go | 94 ++++++++++ .../listers/ark/v1/expansion_generated.go | 8 + 11 files changed, 526 insertions(+) create mode 100644 pkg/generated/clientset/versioned/typed/ark/v1/backupstoragelocation.go create mode 100644 pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_backupstoragelocation.go create mode 100644 pkg/generated/informers/externalversions/ark/v1/backupstoragelocation.go create mode 100644 pkg/generated/listers/ark/v1/backupstoragelocation.go diff --git a/pkg/apis/ark/v1/backup_storage_location.go b/pkg/apis/ark/v1/backup_storage_location.go index adafec1b7..1277449ce 100644 --- a/pkg/apis/ark/v1/backup_storage_location.go +++ b/pkg/apis/ark/v1/backup_storage_location.go @@ -18,6 +18,7 @@ package v1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // BackupStorageLocation is a location where Ark stores backup objects. diff --git a/pkg/generated/clientset/versioned/typed/ark/v1/ark_client.go b/pkg/generated/clientset/versioned/typed/ark/v1/ark_client.go index 184634c47..8abe24f1e 100644 --- a/pkg/generated/clientset/versioned/typed/ark/v1/ark_client.go +++ b/pkg/generated/clientset/versioned/typed/ark/v1/ark_client.go @@ -28,6 +28,7 @@ import ( type ArkV1Interface interface { RESTClient() rest.Interface BackupsGetter + BackupStorageLocationsGetter ConfigsGetter DeleteBackupRequestsGetter DownloadRequestsGetter @@ -47,6 +48,10 @@ func (c *ArkV1Client) Backups(namespace string) BackupInterface { return newBackups(c, namespace) } +func (c *ArkV1Client) BackupStorageLocations(namespace string) BackupStorageLocationInterface { + return newBackupStorageLocations(c, namespace) +} + func (c *ArkV1Client) Configs(namespace string) ConfigInterface { return newConfigs(c, namespace) } diff --git a/pkg/generated/clientset/versioned/typed/ark/v1/backupstoragelocation.go b/pkg/generated/clientset/versioned/typed/ark/v1/backupstoragelocation.go new file mode 100644 index 000000000..1ca79113f --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/ark/v1/backupstoragelocation.go @@ -0,0 +1,174 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/heptio/ark/pkg/apis/ark/v1" + scheme "github.com/heptio/ark/pkg/generated/clientset/versioned/scheme" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// BackupStorageLocationsGetter has a method to return a BackupStorageLocationInterface. +// A group's client should implement this interface. +type BackupStorageLocationsGetter interface { + BackupStorageLocations(namespace string) BackupStorageLocationInterface +} + +// BackupStorageLocationInterface has methods to work with BackupStorageLocation resources. +type BackupStorageLocationInterface interface { + Create(*v1.BackupStorageLocation) (*v1.BackupStorageLocation, error) + Update(*v1.BackupStorageLocation) (*v1.BackupStorageLocation, error) + UpdateStatus(*v1.BackupStorageLocation) (*v1.BackupStorageLocation, error) + Delete(name string, options *meta_v1.DeleteOptions) error + DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error + Get(name string, options meta_v1.GetOptions) (*v1.BackupStorageLocation, error) + List(opts meta_v1.ListOptions) (*v1.BackupStorageLocationList, error) + Watch(opts meta_v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.BackupStorageLocation, err error) + BackupStorageLocationExpansion +} + +// backupStorageLocations implements BackupStorageLocationInterface +type backupStorageLocations struct { + client rest.Interface + ns string +} + +// newBackupStorageLocations returns a BackupStorageLocations +func newBackupStorageLocations(c *ArkV1Client, namespace string) *backupStorageLocations { + return &backupStorageLocations{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the backupStorageLocation, and returns the corresponding backupStorageLocation object, and an error if there is any. +func (c *backupStorageLocations) Get(name string, options meta_v1.GetOptions) (result *v1.BackupStorageLocation, err error) { + result = &v1.BackupStorageLocation{} + err = c.client.Get(). + Namespace(c.ns). + Resource("backupstoragelocations"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of BackupStorageLocations that match those selectors. +func (c *backupStorageLocations) List(opts meta_v1.ListOptions) (result *v1.BackupStorageLocationList, err error) { + result = &v1.BackupStorageLocationList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("backupstoragelocations"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested backupStorageLocations. +func (c *backupStorageLocations) Watch(opts meta_v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("backupstoragelocations"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a backupStorageLocation and creates it. Returns the server's representation of the backupStorageLocation, and an error, if there is any. +func (c *backupStorageLocations) Create(backupStorageLocation *v1.BackupStorageLocation) (result *v1.BackupStorageLocation, err error) { + result = &v1.BackupStorageLocation{} + err = c.client.Post(). + Namespace(c.ns). + Resource("backupstoragelocations"). + Body(backupStorageLocation). + Do(). + Into(result) + return +} + +// Update takes the representation of a backupStorageLocation and updates it. Returns the server's representation of the backupStorageLocation, and an error, if there is any. +func (c *backupStorageLocations) Update(backupStorageLocation *v1.BackupStorageLocation) (result *v1.BackupStorageLocation, err error) { + result = &v1.BackupStorageLocation{} + err = c.client.Put(). + Namespace(c.ns). + Resource("backupstoragelocations"). + Name(backupStorageLocation.Name). + Body(backupStorageLocation). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *backupStorageLocations) UpdateStatus(backupStorageLocation *v1.BackupStorageLocation) (result *v1.BackupStorageLocation, err error) { + result = &v1.BackupStorageLocation{} + err = c.client.Put(). + Namespace(c.ns). + Resource("backupstoragelocations"). + Name(backupStorageLocation.Name). + SubResource("status"). + Body(backupStorageLocation). + Do(). + Into(result) + return +} + +// Delete takes name of the backupStorageLocation and deletes it. Returns an error if one occurs. +func (c *backupStorageLocations) Delete(name string, options *meta_v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("backupstoragelocations"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *backupStorageLocations) DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("backupstoragelocations"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched backupStorageLocation. +func (c *backupStorageLocations) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.BackupStorageLocation, err error) { + result = &v1.BackupStorageLocation{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("backupstoragelocations"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_ark_client.go b/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_ark_client.go index 25f8f0a2e..656fc0e57 100644 --- a/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_ark_client.go +++ b/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_ark_client.go @@ -32,6 +32,10 @@ func (c *FakeArkV1) Backups(namespace string) v1.BackupInterface { return &FakeBackups{c, namespace} } +func (c *FakeArkV1) BackupStorageLocations(namespace string) v1.BackupStorageLocationInterface { + return &FakeBackupStorageLocations{c, namespace} +} + func (c *FakeArkV1) Configs(namespace string) v1.ConfigInterface { return &FakeConfigs{c, namespace} } diff --git a/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_backupstoragelocation.go b/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_backupstoragelocation.go new file mode 100644 index 000000000..cfda30347 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/ark/v1/fake/fake_backupstoragelocation.go @@ -0,0 +1,140 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + ark_v1 "github.com/heptio/ark/pkg/apis/ark/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeBackupStorageLocations implements BackupStorageLocationInterface +type FakeBackupStorageLocations struct { + Fake *FakeArkV1 + ns string +} + +var backupstoragelocationsResource = schema.GroupVersionResource{Group: "ark.heptio.com", Version: "v1", Resource: "backupstoragelocations"} + +var backupstoragelocationsKind = schema.GroupVersionKind{Group: "ark.heptio.com", Version: "v1", Kind: "BackupStorageLocation"} + +// Get takes name of the backupStorageLocation, and returns the corresponding backupStorageLocation object, and an error if there is any. +func (c *FakeBackupStorageLocations) Get(name string, options v1.GetOptions) (result *ark_v1.BackupStorageLocation, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(backupstoragelocationsResource, c.ns, name), &ark_v1.BackupStorageLocation{}) + + if obj == nil { + return nil, err + } + return obj.(*ark_v1.BackupStorageLocation), err +} + +// List takes label and field selectors, and returns the list of BackupStorageLocations that match those selectors. +func (c *FakeBackupStorageLocations) List(opts v1.ListOptions) (result *ark_v1.BackupStorageLocationList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(backupstoragelocationsResource, backupstoragelocationsKind, c.ns, opts), &ark_v1.BackupStorageLocationList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &ark_v1.BackupStorageLocationList{ListMeta: obj.(*ark_v1.BackupStorageLocationList).ListMeta} + for _, item := range obj.(*ark_v1.BackupStorageLocationList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested backupStorageLocations. +func (c *FakeBackupStorageLocations) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(backupstoragelocationsResource, c.ns, opts)) + +} + +// Create takes the representation of a backupStorageLocation and creates it. Returns the server's representation of the backupStorageLocation, and an error, if there is any. +func (c *FakeBackupStorageLocations) Create(backupStorageLocation *ark_v1.BackupStorageLocation) (result *ark_v1.BackupStorageLocation, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(backupstoragelocationsResource, c.ns, backupStorageLocation), &ark_v1.BackupStorageLocation{}) + + if obj == nil { + return nil, err + } + return obj.(*ark_v1.BackupStorageLocation), err +} + +// Update takes the representation of a backupStorageLocation and updates it. Returns the server's representation of the backupStorageLocation, and an error, if there is any. +func (c *FakeBackupStorageLocations) Update(backupStorageLocation *ark_v1.BackupStorageLocation) (result *ark_v1.BackupStorageLocation, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(backupstoragelocationsResource, c.ns, backupStorageLocation), &ark_v1.BackupStorageLocation{}) + + if obj == nil { + return nil, err + } + return obj.(*ark_v1.BackupStorageLocation), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeBackupStorageLocations) UpdateStatus(backupStorageLocation *ark_v1.BackupStorageLocation) (*ark_v1.BackupStorageLocation, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(backupstoragelocationsResource, "status", c.ns, backupStorageLocation), &ark_v1.BackupStorageLocation{}) + + if obj == nil { + return nil, err + } + return obj.(*ark_v1.BackupStorageLocation), err +} + +// Delete takes name of the backupStorageLocation and deletes it. Returns an error if one occurs. +func (c *FakeBackupStorageLocations) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(backupstoragelocationsResource, c.ns, name), &ark_v1.BackupStorageLocation{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeBackupStorageLocations) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(backupstoragelocationsResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &ark_v1.BackupStorageLocationList{}) + return err +} + +// Patch applies the patch and returns the patched backupStorageLocation. +func (c *FakeBackupStorageLocations) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *ark_v1.BackupStorageLocation, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(backupstoragelocationsResource, c.ns, name, data, subresources...), &ark_v1.BackupStorageLocation{}) + + if obj == nil { + return nil, err + } + return obj.(*ark_v1.BackupStorageLocation), err +} diff --git a/pkg/generated/clientset/versioned/typed/ark/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/ark/v1/generated_expansion.go index e09577a39..4181aa73a 100644 --- a/pkg/generated/clientset/versioned/typed/ark/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/ark/v1/generated_expansion.go @@ -20,6 +20,8 @@ package v1 type BackupExpansion interface{} +type BackupStorageLocationExpansion interface{} + type ConfigExpansion interface{} type DeleteBackupRequestExpansion interface{} diff --git a/pkg/generated/informers/externalversions/ark/v1/backupstoragelocation.go b/pkg/generated/informers/externalversions/ark/v1/backupstoragelocation.go new file mode 100644 index 000000000..f4281cab0 --- /dev/null +++ b/pkg/generated/informers/externalversions/ark/v1/backupstoragelocation.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + time "time" + + ark_v1 "github.com/heptio/ark/pkg/apis/ark/v1" + versioned "github.com/heptio/ark/pkg/generated/clientset/versioned" + internalinterfaces "github.com/heptio/ark/pkg/generated/informers/externalversions/internalinterfaces" + v1 "github.com/heptio/ark/pkg/generated/listers/ark/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BackupStorageLocationInformer provides access to a shared informer and lister for +// BackupStorageLocations. +type BackupStorageLocationInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.BackupStorageLocationLister +} + +type backupStorageLocationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBackupStorageLocationInformer constructs a new informer for BackupStorageLocation type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBackupStorageLocationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBackupStorageLocationInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredBackupStorageLocationInformer constructs a new informer for BackupStorageLocation type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBackupStorageLocationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ArkV1().BackupStorageLocations(namespace).List(options) + }, + WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ArkV1().BackupStorageLocations(namespace).Watch(options) + }, + }, + &ark_v1.BackupStorageLocation{}, + resyncPeriod, + indexers, + ) +} + +func (f *backupStorageLocationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBackupStorageLocationInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *backupStorageLocationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&ark_v1.BackupStorageLocation{}, f.defaultInformer) +} + +func (f *backupStorageLocationInformer) Lister() v1.BackupStorageLocationLister { + return v1.NewBackupStorageLocationLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/ark/v1/interface.go b/pkg/generated/informers/externalversions/ark/v1/interface.go index b197b2a66..3b253965c 100644 --- a/pkg/generated/informers/externalversions/ark/v1/interface.go +++ b/pkg/generated/informers/externalversions/ark/v1/interface.go @@ -26,6 +26,8 @@ import ( type Interface interface { // Backups returns a BackupInformer. Backups() BackupInformer + // BackupStorageLocations returns a BackupStorageLocationInformer. + BackupStorageLocations() BackupStorageLocationInformer // Configs returns a ConfigInformer. Configs() ConfigInformer // DeleteBackupRequests returns a DeleteBackupRequestInformer. @@ -60,6 +62,11 @@ func (v *version) Backups() BackupInformer { return &backupInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// BackupStorageLocations returns a BackupStorageLocationInformer. +func (v *version) BackupStorageLocations() BackupStorageLocationInformer { + return &backupStorageLocationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // Configs returns a ConfigInformer. func (v *version) Configs() ConfigInformer { return &configInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 6e1c22334..bea512916 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -55,6 +55,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource // Group=ark.heptio.com, Version=v1 case v1.SchemeGroupVersion.WithResource("backups"): return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().Backups().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("backupstoragelocations"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().BackupStorageLocations().Informer()}, nil case v1.SchemeGroupVersion.WithResource("configs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().Configs().Informer()}, nil case v1.SchemeGroupVersion.WithResource("deletebackuprequests"): diff --git a/pkg/generated/listers/ark/v1/backupstoragelocation.go b/pkg/generated/listers/ark/v1/backupstoragelocation.go new file mode 100644 index 000000000..2bfdc8cac --- /dev/null +++ b/pkg/generated/listers/ark/v1/backupstoragelocation.go @@ -0,0 +1,94 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/heptio/ark/pkg/apis/ark/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// BackupStorageLocationLister helps list BackupStorageLocations. +type BackupStorageLocationLister interface { + // List lists all BackupStorageLocations in the indexer. + List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) + // BackupStorageLocations returns an object that can list and get BackupStorageLocations. + BackupStorageLocations(namespace string) BackupStorageLocationNamespaceLister + BackupStorageLocationListerExpansion +} + +// backupStorageLocationLister implements the BackupStorageLocationLister interface. +type backupStorageLocationLister struct { + indexer cache.Indexer +} + +// NewBackupStorageLocationLister returns a new BackupStorageLocationLister. +func NewBackupStorageLocationLister(indexer cache.Indexer) BackupStorageLocationLister { + return &backupStorageLocationLister{indexer: indexer} +} + +// List lists all BackupStorageLocations in the indexer. +func (s *backupStorageLocationLister) List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.BackupStorageLocation)) + }) + return ret, err +} + +// BackupStorageLocations returns an object that can list and get BackupStorageLocations. +func (s *backupStorageLocationLister) BackupStorageLocations(namespace string) BackupStorageLocationNamespaceLister { + return backupStorageLocationNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// BackupStorageLocationNamespaceLister helps list and get BackupStorageLocations. +type BackupStorageLocationNamespaceLister interface { + // List lists all BackupStorageLocations in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) + // Get retrieves the BackupStorageLocation from the indexer for a given namespace and name. + Get(name string) (*v1.BackupStorageLocation, error) + BackupStorageLocationNamespaceListerExpansion +} + +// backupStorageLocationNamespaceLister implements the BackupStorageLocationNamespaceLister +// interface. +type backupStorageLocationNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all BackupStorageLocations in the indexer for a given namespace. +func (s backupStorageLocationNamespaceLister) List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1.BackupStorageLocation)) + }) + return ret, err +} + +// Get retrieves the BackupStorageLocation from the indexer for a given namespace and name. +func (s backupStorageLocationNamespaceLister) Get(name string) (*v1.BackupStorageLocation, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("backupstoragelocation"), name) + } + return obj.(*v1.BackupStorageLocation), nil +} diff --git a/pkg/generated/listers/ark/v1/expansion_generated.go b/pkg/generated/listers/ark/v1/expansion_generated.go index 9b4a1d5aa..2a1d153e7 100644 --- a/pkg/generated/listers/ark/v1/expansion_generated.go +++ b/pkg/generated/listers/ark/v1/expansion_generated.go @@ -26,6 +26,14 @@ type BackupListerExpansion interface{} // BackupNamespaceLister. type BackupNamespaceListerExpansion interface{} +// BackupStorageLocationListerExpansion allows custom methods to be added to +// BackupStorageLocationLister. +type BackupStorageLocationListerExpansion interface{} + +// BackupStorageLocationNamespaceListerExpansion allows custom methods to be added to +// BackupStorageLocationNamespaceLister. +type BackupStorageLocationNamespaceListerExpansion interface{} + // ConfigListerExpansion allows custom methods to be added to // ConfigLister. type ConfigListerExpansion interface{} From 56f1617049ca42c096cb239de82fdf0393395072 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 9 Aug 2018 12:40:13 -0400 Subject: [PATCH 03/29] Correct metadata for BackupStorageLocationList Signed-off-by: Nolan Brubaker --- pkg/apis/ark/v1/backup_storage_location.go | 6 +++--- pkg/apis/ark/v1/zz_generated.deepcopy.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/apis/ark/v1/backup_storage_location.go b/pkg/apis/ark/v1/backup_storage_location.go index 1277449ce..57eb4792f 100644 --- a/pkg/apis/ark/v1/backup_storage_location.go +++ b/pkg/apis/ark/v1/backup_storage_location.go @@ -34,9 +34,9 @@ type BackupStorageLocation struct { // BackupStorageLocationList is a list of BackupStorageLocations. type BackupStorageLocationList struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata"` - Items []BackupStorageLocation `json:"items"` + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []BackupStorageLocation `json:"items"` } // StorageType represents the type of storage that a backup location uses. diff --git a/pkg/apis/ark/v1/zz_generated.deepcopy.go b/pkg/apis/ark/v1/zz_generated.deepcopy.go index 0d55e70b6..3403f946c 100644 --- a/pkg/apis/ark/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ark/v1/zz_generated.deepcopy.go @@ -333,7 +333,7 @@ func (in *BackupStorageLocation) DeepCopyObject() runtime.Object { func (in *BackupStorageLocationList) DeepCopyInto(out *BackupStorageLocationList) { *out = *in out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.ListMeta = in.ListMeta if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]BackupStorageLocation, len(*in)) From 2a34772ed5ebf3592be89c900c20290a0f76c6ac Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 9 Aug 2018 12:41:31 -0400 Subject: [PATCH 04/29] Add --storage-location argument to create commands Closes #738 Signed-off-by: Nolan Brubaker --- docs/cli-reference/ark_backup_create.md | 1 + docs/cli-reference/ark_create_backup.md | 1 + docs/cli-reference/ark_create_schedule.md | 1 + docs/cli-reference/ark_schedule_create.md | 1 + pkg/cmd/cli/backup/create.go | 33 ++++++++++++++++------- pkg/cmd/cli/schedule/create.go | 13 ++++----- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/cli-reference/ark_backup_create.md b/docs/cli-reference/ark_backup_create.md index 2e5467683..f6a8569ef 100644 --- a/docs/cli-reference/ark_backup_create.md +++ b/docs/cli-reference/ark_backup_create.md @@ -26,6 +26,7 @@ ark backup create NAME [flags] -l, --selector labelSelector only back up resources matching this label selector (default ) --show-labels show labels in the last column --snapshot-volumes optionalBool[=true] take snapshots of PersistentVolumes as part of the backup + --storage-location string location in which to store the backup --ttl duration how long before the backup can be garbage collected (default 720h0m0s) -w, --wait wait for the operation to complete ``` diff --git a/docs/cli-reference/ark_create_backup.md b/docs/cli-reference/ark_create_backup.md index f3fd16f2c..8d45c9692 100644 --- a/docs/cli-reference/ark_create_backup.md +++ b/docs/cli-reference/ark_create_backup.md @@ -26,6 +26,7 @@ ark create backup NAME [flags] -l, --selector labelSelector only back up resources matching this label selector (default ) --show-labels show labels in the last column --snapshot-volumes optionalBool[=true] take snapshots of PersistentVolumes as part of the backup + --storage-location string location in which to store the backup --ttl duration how long before the backup can be garbage collected (default 720h0m0s) -w, --wait wait for the operation to complete ``` diff --git a/docs/cli-reference/ark_create_schedule.md b/docs/cli-reference/ark_create_schedule.md index 4ddb7b575..809d67870 100644 --- a/docs/cli-reference/ark_create_schedule.md +++ b/docs/cli-reference/ark_create_schedule.md @@ -41,6 +41,7 @@ ark create schedule NAME --schedule="0 */6 * * *" -l, --selector labelSelector only back up resources matching this label selector (default ) --show-labels show labels in the last column --snapshot-volumes optionalBool[=true] take snapshots of PersistentVolumes as part of the backup + --storage-location string location in which to store the backup --ttl duration how long before the backup can be garbage collected (default 720h0m0s) ``` diff --git a/docs/cli-reference/ark_schedule_create.md b/docs/cli-reference/ark_schedule_create.md index c60c91a0d..c445f080e 100644 --- a/docs/cli-reference/ark_schedule_create.md +++ b/docs/cli-reference/ark_schedule_create.md @@ -41,6 +41,7 @@ ark create schedule NAME --schedule="0 */6 * * *" -l, --selector labelSelector only back up resources matching this label selector (default ) --show-labels show labels in the last column --snapshot-volumes optionalBool[=true] take snapshots of PersistentVolumes as part of the backup + --storage-location string location in which to store the backup --ttl duration how long before the backup can be garbage collected (default 720h0m0s) ``` diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index d899ffacd..837a87db7 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -32,6 +32,7 @@ import ( "github.com/heptio/ark/pkg/cmd" "github.com/heptio/ark/pkg/cmd/util/flag" "github.com/heptio/ark/pkg/cmd/util/output" + arkclient "github.com/heptio/ark/pkg/generated/clientset/versioned" ) func NewCreateCommand(f client.Factory, use string) *cobra.Command { @@ -42,8 +43,8 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { Short: "Create a backup", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { - cmd.CheckError(o.Complete(args)) - cmd.CheckError(o.Validate(c, args)) + cmd.CheckError(o.Complete(args, f)) + cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } @@ -68,6 +69,9 @@ type CreateOptions struct { Selector flag.LabelSelector IncludeClusterResources flag.OptionalBool Wait bool + StorageLocation string + + client arkclient.Interface } func NewCreateOptions() *CreateOptions { @@ -87,6 +91,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.Var(&o.IncludeResources, "include-resources", "resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)") flags.Var(&o.ExcludeResources, "exclude-resources", "resources to exclude from the backup, formatted as resource.group, such as storageclasses.storage.k8s.io") flags.Var(&o.Labels, "labels", "labels to apply to the backup") + flags.StringVar(&o.StorageLocation, "storage-location", "", "location in which to store the backup") flags.VarP(&o.Selector, "selector", "l", "only back up resources matching this label selector") f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "take snapshots of PersistentVolumes as part of the backup") // this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true" @@ -103,24 +108,31 @@ func (o *CreateOptions) BindWait(flags *pflag.FlagSet) { flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "wait for the operation to complete") } -func (o *CreateOptions) Validate(c *cobra.Command, args []string) error { +func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if err := output.ValidateFlags(c); err != nil { return err } + if o.StorageLocation != "" { + if _, err := o.client.ArkV1().BackupStorageLocations(f.Namespace()).Get(o.StorageLocation, metav1.GetOptions{}); err != nil { + return err + } + } + return nil } -func (o *CreateOptions) Complete(args []string) error { +func (o *CreateOptions) Complete(args []string, f client.Factory) error { o.Name = args[0] + client, err := f.Client() + if err != nil { + return err + } + o.client = client return nil } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { - arkClient, err := f.Client() - if err != nil { - return err - } backup := &api.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -137,6 +149,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { SnapshotVolumes: o.SnapshotVolumes.Value, TTL: metav1.Duration{Duration: o.TTL}, IncludeClusterResources: o.IncludeClusterResources.Value, + StorageLocation: o.StorageLocation, }, } @@ -152,7 +165,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { updates = make(chan *api.Backup) - backupInformer = v1.NewBackupInformer(arkClient, f.Namespace(), 0, nil) + backupInformer = v1.NewBackupInformer(o.client, f.Namespace(), 0, nil) backupInformer.AddEventHandler( cache.FilteringResourceEventHandler{ @@ -184,7 +197,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { go backupInformer.Run(stop) } - _, err = arkClient.ArkV1().Backups(backup.Namespace).Create(backup) + _, err := o.client.ArkV1().Backups(backup.Namespace).Create(backup) if err != nil { return err } diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index a49c86be9..079414615 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -51,8 +51,8 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { Example: `ark create schedule NAME --schedule="0 */6 * * *"`, Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { - cmd.CheckError(o.Complete(args)) - cmd.CheckError(o.Validate(c, args)) + cmd.CheckError(o.Complete(args, f)) + cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } @@ -82,16 +82,16 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Schedule, "schedule", o.Schedule, "a cron expression specifying a recurring schedule for this backup to run") } -func (o *CreateOptions) Validate(c *cobra.Command, args []string) error { +func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { if len(o.Schedule) == 0 { return errors.New("--schedule is required") } - return o.BackupOptions.Validate(c, args) + return o.BackupOptions.Validate(c, args, f) } -func (o *CreateOptions) Complete(args []string) error { - return o.BackupOptions.Complete(args) +func (o *CreateOptions) Complete(args []string, f client.Factory) error { + return o.BackupOptions.Complete(args, f) } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { @@ -114,6 +114,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { LabelSelector: o.BackupOptions.Selector.LabelSelector, SnapshotVolumes: o.BackupOptions.SnapshotVolumes.Value, TTL: metav1.Duration{Duration: o.BackupOptions.TTL}, + StorageLocation: o.BackupOptions.StorageLocation, }, Schedule: o.Schedule, }, From adbcd3703bc4f6419c7620119a5bfc55bc93ec58 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 8 Aug 2018 17:36:50 -0700 Subject: [PATCH 05/29] add --default-backup-storage-location flag to server cmd Signed-off-by: Steve Kriss --- docs/cli-reference/ark_server.md | 1 + pkg/cmd/server/server.go | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/cli-reference/ark_server.md b/docs/cli-reference/ark_server.md index ab3d2dff3..623251f11 100644 --- a/docs/cli-reference/ark_server.md +++ b/docs/cli-reference/ark_server.md @@ -15,6 +15,7 @@ ark server [flags] ``` --backup-sync-period duration how often to ensure all Ark backups in object storage exist as Backup API objects in the cluster (default 1h0m0s) + --default-backup-storage-location string name of the default backup storage location (default "default") -h, --help help for server --log-level the level at which to log. Valid values are debug, info, warning, error, fatal, panic. (default info) --metrics-address string the address to expose prometheus metrics (default ":8085") diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 265efac40..8f95f84c0 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -92,6 +92,7 @@ func NewCommand() *cobra.Command { podVolumeOperationTimeout: defaultPodVolumeOperationTimeout, restoreResourcePriorities: defaultRestorePriorities, } + defaultBackupLocation = "default" ) var command = &cobra.Command{ @@ -126,7 +127,7 @@ func NewCommand() *cobra.Command { } namespace := getServerNamespace(namespaceFlag) - s, err := newServer(namespace, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()), config, logger) + s, err := newServer(namespace, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()), config, defaultBackupLocation, logger) cmd.CheckError(err) cmd.CheckError(s.run()) @@ -140,6 +141,7 @@ func NewCommand() *cobra.Command { command.Flags().DurationVar(&config.podVolumeOperationTimeout, "restic-timeout", config.podVolumeOperationTimeout, "how long backups/restores of pod volumes should be allowed to run before timing out") command.Flags().BoolVar(&config.restoreOnly, "restore-only", config.restoreOnly, "run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled") command.Flags().StringSliceVar(&config.restoreResourcePriorities, "restore-resource-priorities", config.restoreResourcePriorities, "desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources") + command.Flags().StringVar(&defaultBackupLocation, "default-backup-storage-location", defaultBackupLocation, "name of the default backup storage location") return command } @@ -179,9 +181,10 @@ type server struct { resticManager restic.RepositoryManager metrics *metrics.ServerMetrics config serverConfig + defaultBackupLocation string } -func newServer(namespace, baseName string, config serverConfig, logger *logrus.Logger) (*server, error) { +func newServer(namespace, baseName string, config serverConfig, defaultBackupLocation string, logger *logrus.Logger) (*server, error) { clientConfig, err := client.Config("", "", baseName) if err != nil { return nil, err @@ -222,13 +225,14 @@ func newServer(namespace, baseName string, config serverConfig, logger *logrus.L discoveryClient: arkClient.Discovery(), dynamicClient: dynamicClient, sharedInformerFactory: informers.NewFilteredSharedInformerFactory(arkClient, 0, namespace, nil), - ctx: ctx, - cancelFunc: cancelFunc, - logger: logger, - logLevel: logger.Level, - pluginRegistry: pluginRegistry, - pluginManager: pluginManager, - config: config, + ctx: ctx, + cancelFunc: cancelFunc, + logger: logger, + logLevel: logger.Level, + pluginRegistry: pluginRegistry, + pluginManager: pluginManager, + config: config, + defaultBackupLocation: defaultBackupLocation, } return s, nil From 06b5af449f043d76f01876607010402beb4354c3 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 9 Aug 2018 15:08:50 -0700 Subject: [PATCH 06/29] add create and get CLI commands for backup locations Signed-off-by: Steve Kriss --- docs/cli-reference/ark.md | 1 + docs/cli-reference/ark_backup-location.md | 35 +++++ .../ark_backup-location_create.md | 45 ++++++ docs/cli-reference/ark_backup-location_get.md | 41 ++++++ docs/cli-reference/ark_create.md | 1 + .../ark_create_backup-location.md | 45 ++++++ docs/cli-reference/ark_get.md | 1 + .../cli-reference/ark_get_backup-locations.md | 41 ++++++ pkg/cmd/ark/ark.go | 2 + pkg/cmd/cli/backuplocation/backup_location.go | 38 +++++ pkg/cmd/cli/backuplocation/create.go | 134 ++++++++++++++++++ pkg/cmd/cli/backuplocation/get.go | 66 +++++++++ pkg/cmd/cli/create/create.go | 2 + pkg/cmd/cli/get/get.go | 5 + .../output/backup_storage_location_printer.go | 71 ++++++++++ pkg/cmd/util/output/output.go | 2 + 16 files changed, 530 insertions(+) create mode 100644 docs/cli-reference/ark_backup-location.md create mode 100644 docs/cli-reference/ark_backup-location_create.md create mode 100644 docs/cli-reference/ark_backup-location_get.md create mode 100644 docs/cli-reference/ark_create_backup-location.md create mode 100644 docs/cli-reference/ark_get_backup-locations.md create mode 100644 pkg/cmd/cli/backuplocation/backup_location.go create mode 100644 pkg/cmd/cli/backuplocation/create.go create mode 100644 pkg/cmd/cli/backuplocation/get.go create mode 100644 pkg/cmd/util/output/backup_storage_location_printer.go diff --git a/docs/cli-reference/ark.md b/docs/cli-reference/ark.md index f6a77e6a2..eee6a7949 100644 --- a/docs/cli-reference/ark.md +++ b/docs/cli-reference/ark.md @@ -31,6 +31,7 @@ operations can also be performed as 'ark backup get' and 'ark schedule create'. ### SEE ALSO * [ark backup](ark_backup.md) - Work with backups +* [ark backup-location](ark_backup-location.md) - Work with backup storage locations * [ark bug](ark_bug.md) - Report an Ark bug * [ark client](ark_client.md) - Ark client related commands * [ark completion](ark_completion.md) - Output shell completion code for the specified shell (bash or zsh) diff --git a/docs/cli-reference/ark_backup-location.md b/docs/cli-reference/ark_backup-location.md new file mode 100644 index 000000000..f371f3b2d --- /dev/null +++ b/docs/cli-reference/ark_backup-location.md @@ -0,0 +1,35 @@ +## ark backup-location + +Work with backup storage locations + +### Synopsis + + +Work with backup storage locations + +### Options + +``` + -h, --help help for backup-location +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --kubecontext string The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -n, --namespace string The namespace in which Ark should operate (default "heptio-ark") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark](ark.md) - Back up and restore Kubernetes cluster resources. +* [ark backup-location create](ark_backup-location_create.md) - Create a backup storage location +* [ark backup-location get](ark_backup-location_get.md) - Get backup storage locations + diff --git a/docs/cli-reference/ark_backup-location_create.md b/docs/cli-reference/ark_backup-location_create.md new file mode 100644 index 000000000..126c9dfd2 --- /dev/null +++ b/docs/cli-reference/ark_backup-location_create.md @@ -0,0 +1,45 @@ +## ark backup-location create + +Create a backup storage location + +### Synopsis + + +Create a backup storage location + +``` +ark backup-location create NAME [flags] +``` + +### Options + +``` + --bucket string name of the object storage bucket where backups should be stored + --config mapStringString configuration key-value pairs + -h, --help help for create + --label-columns stringArray a comma-separated list of labels to be displayed as columns + --labels mapStringString labels to apply to the backup storage location + -o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. + --prefix string prefix under which all Ark data should be stored within the bucket. Optional. + --provider string name of the backup storage provider (e.g. aws, azure, gcp) + --show-labels show labels in the last column +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --kubecontext string The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -n, --namespace string The namespace in which Ark should operate (default "heptio-ark") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark backup-location](ark_backup-location.md) - Work with backup storage locations + diff --git a/docs/cli-reference/ark_backup-location_get.md b/docs/cli-reference/ark_backup-location_get.md new file mode 100644 index 000000000..5f878f62f --- /dev/null +++ b/docs/cli-reference/ark_backup-location_get.md @@ -0,0 +1,41 @@ +## ark backup-location get + +Get backup storage locations + +### Synopsis + + +Get backup storage locations + +``` +ark backup-location get [flags] +``` + +### Options + +``` + -h, --help help for get + --label-columns stringArray a comma-separated list of labels to be displayed as columns + -o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. (default "table") + -l, --selector string only show items matching this label selector + --show-labels show labels in the last column +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --kubecontext string The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -n, --namespace string The namespace in which Ark should operate (default "heptio-ark") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark backup-location](ark_backup-location.md) - Work with backup storage locations + diff --git a/docs/cli-reference/ark_create.md b/docs/cli-reference/ark_create.md index fc7934f14..2676f1041 100644 --- a/docs/cli-reference/ark_create.md +++ b/docs/cli-reference/ark_create.md @@ -31,6 +31,7 @@ Create ark resources ### SEE ALSO * [ark](ark.md) - Back up and restore Kubernetes cluster resources. * [ark create backup](ark_create_backup.md) - Create a backup +* [ark create backup-location](ark_create_backup-location.md) - Create a backup storage location * [ark create restore](ark_create_restore.md) - Create a restore * [ark create schedule](ark_create_schedule.md) - Create a schedule diff --git a/docs/cli-reference/ark_create_backup-location.md b/docs/cli-reference/ark_create_backup-location.md new file mode 100644 index 000000000..96cb1ec46 --- /dev/null +++ b/docs/cli-reference/ark_create_backup-location.md @@ -0,0 +1,45 @@ +## ark create backup-location + +Create a backup storage location + +### Synopsis + + +Create a backup storage location + +``` +ark create backup-location NAME [flags] +``` + +### Options + +``` + --bucket string name of the object storage bucket where backups should be stored + --config mapStringString configuration key-value pairs + -h, --help help for backup-location + --label-columns stringArray a comma-separated list of labels to be displayed as columns + --labels mapStringString labels to apply to the backup storage location + -o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. + --prefix string prefix under which all Ark data should be stored within the bucket. Optional. + --provider string name of the backup storage provider (e.g. aws, azure, gcp) + --show-labels show labels in the last column +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --kubecontext string The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -n, --namespace string The namespace in which Ark should operate (default "heptio-ark") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark create](ark_create.md) - Create ark resources + diff --git a/docs/cli-reference/ark_get.md b/docs/cli-reference/ark_get.md index 4a91e8a66..e0a542ff0 100644 --- a/docs/cli-reference/ark_get.md +++ b/docs/cli-reference/ark_get.md @@ -30,6 +30,7 @@ Get ark resources ### SEE ALSO * [ark](ark.md) - Back up and restore Kubernetes cluster resources. +* [ark get backup-locations](ark_get_backup-locations.md) - Get backup storage locations * [ark get backups](ark_get_backups.md) - Get backups * [ark get restores](ark_get_restores.md) - Get restores * [ark get schedules](ark_get_schedules.md) - Get schedules diff --git a/docs/cli-reference/ark_get_backup-locations.md b/docs/cli-reference/ark_get_backup-locations.md new file mode 100644 index 000000000..4aa3a5061 --- /dev/null +++ b/docs/cli-reference/ark_get_backup-locations.md @@ -0,0 +1,41 @@ +## ark get backup-locations + +Get backup storage locations + +### Synopsis + + +Get backup storage locations + +``` +ark get backup-locations [flags] +``` + +### Options + +``` + -h, --help help for backup-locations + --label-columns stringArray a comma-separated list of labels to be displayed as columns + -o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. (default "table") + -l, --selector string only show items matching this label selector + --show-labels show labels in the last column +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --kubecontext string The context to use to talk to the Kubernetes apiserver. If unset defaults to whatever your current-context is (kubectl config current-context) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -n, --namespace string The namespace in which Ark should operate (default "heptio-ark") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark get](ark_get.md) - Get ark resources + diff --git a/pkg/cmd/ark/ark.go b/pkg/cmd/ark/ark.go index f3c324484..ddf4532db 100644 --- a/pkg/cmd/ark/ark.go +++ b/pkg/cmd/ark/ark.go @@ -23,6 +23,7 @@ import ( "github.com/heptio/ark/pkg/client" "github.com/heptio/ark/pkg/cmd/cli/backup" + "github.com/heptio/ark/pkg/cmd/cli/backuplocation" "github.com/heptio/ark/pkg/cmd/cli/bug" cliclient "github.com/heptio/ark/pkg/cmd/cli/client" "github.com/heptio/ark/pkg/cmd/cli/completion" @@ -71,6 +72,7 @@ operations can also be performed as 'ark backup get' and 'ark schedule create'.` completion.NewCommand(), restic.NewCommand(f), bug.NewCommand(), + backuplocation.NewCommand(f), ) // add the glog flags diff --git a/pkg/cmd/cli/backuplocation/backup_location.go b/pkg/cmd/cli/backuplocation/backup_location.go new file mode 100644 index 000000000..8c0fd8b91 --- /dev/null +++ b/pkg/cmd/cli/backuplocation/backup_location.go @@ -0,0 +1,38 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuplocation + +import ( + "github.com/spf13/cobra" + + "github.com/heptio/ark/pkg/client" +) + +func NewCommand(f client.Factory) *cobra.Command { + c := &cobra.Command{ + Use: "backup-location", + Short: "Work with backup storage locations", + Long: "Work with backup storage locations", + } + + c.AddCommand( + NewCreateCommand(f, "create"), + NewGetCommand(f, "get"), + ) + + return c +} diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go new file mode 100644 index 000000000..d67a193b2 --- /dev/null +++ b/pkg/cmd/cli/backuplocation/create.go @@ -0,0 +1,134 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuplocation + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/client" + "github.com/heptio/ark/pkg/cmd" + "github.com/heptio/ark/pkg/cmd/util/flag" + "github.com/heptio/ark/pkg/cmd/util/output" +) + +func NewCreateCommand(f client.Factory, use string) *cobra.Command { + o := NewCreateOptions() + + c := &cobra.Command{ + Use: use + " NAME", + Short: "Create a backup storage location", + Args: cobra.ExactArgs(1), + Run: func(c *cobra.Command, args []string) { + cmd.CheckError(o.Complete(args, f)) + cmd.CheckError(o.Validate(c, args, f)) + cmd.CheckError(o.Run(c, f)) + }, + } + + o.BindFlags(c.Flags()) + output.BindFlags(c.Flags()) + output.ClearOutputFlagDefault(c) + + return c +} + +type CreateOptions struct { + Name string + Provider string + Bucket string + Prefix string + Config flag.Map + Labels flag.Map +} + +func NewCreateOptions() *CreateOptions { + return &CreateOptions{ + Config: flag.NewMap(), + } +} + +func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { + flags.StringVar(&o.Provider, "provider", o.Provider, "name of the backup storage provider (e.g. aws, azure, gcp)") + flags.StringVar(&o.Bucket, "bucket", o.Bucket, "name of the object storage bucket where backups should be stored") + flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Ark data should be stored within the bucket. Optional.") + flags.Var(&o.Config, "config", "configuration key-value pairs") + flags.Var(&o.Labels, "labels", "labels to apply to the backup storage location") +} + +func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error { + if err := output.ValidateFlags(c); err != nil { + return err + } + + if o.Provider == "" { + return errors.New("--provider is required") + } + + if o.Bucket == "" { + return errors.New("--bucket is required") + } + + return nil +} + +func (o *CreateOptions) Complete(args []string, f client.Factory) error { + o.Name = args[0] + return nil +} + +func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { + backupStorageLocation := &api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace(), + Name: o.Name, + Labels: o.Labels.Data(), + }, + Spec: api.BackupStorageLocationSpec{ + Provider: o.Provider, + StorageType: api.StorageType{ + ObjectStorage: &api.ObjectStorageLocation{ + Bucket: o.Bucket, + Prefix: o.Prefix, + }, + }, + Config: o.Config.Data(), + }, + } + + if printed, err := output.PrintWithFormat(c, backupStorageLocation); printed || err != nil { + return err + } + + client, err := f.Client() + if err != nil { + return err + } + + if _, err := client.ArkV1().BackupStorageLocations(backupStorageLocation.Namespace).Create(backupStorageLocation); err != nil { + return errors.WithStack(err) + } + + fmt.Printf("Backup storage location %q configured successfully.\n", backupStorageLocation.Name) + return nil +} diff --git a/pkg/cmd/cli/backuplocation/get.go b/pkg/cmd/cli/backuplocation/get.go new file mode 100644 index 000000000..5573eea43 --- /dev/null +++ b/pkg/cmd/cli/backuplocation/get.go @@ -0,0 +1,66 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuplocation + +import ( + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/client" + "github.com/heptio/ark/pkg/cmd" + "github.com/heptio/ark/pkg/cmd/util/output" +) + +func NewGetCommand(f client.Factory, use string) *cobra.Command { + var listOptions metav1.ListOptions + + c := &cobra.Command{ + Use: use, + Short: "Get backup storage locations", + Run: func(c *cobra.Command, args []string) { + err := output.ValidateFlags(c) + cmd.CheckError(err) + + arkClient, err := f.Client() + cmd.CheckError(err) + + var locations *api.BackupStorageLocationList + if len(args) > 0 { + locations = new(api.BackupStorageLocationList) + for _, name := range args { + location, err := arkClient.Ark().BackupStorageLocations(f.Namespace()).Get(name, metav1.GetOptions{}) + cmd.CheckError(err) + locations.Items = append(locations.Items, *location) + } + } else { + locations, err = arkClient.ArkV1().BackupStorageLocations(f.Namespace()).List(listOptions) + cmd.CheckError(err) + } + + _, err = output.PrintWithFormat(c, locations) + cmd.CheckError(err) + }, + } + + c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "only show items matching this label selector") + + output.BindFlags(c.Flags()) + + return c +} diff --git a/pkg/cmd/cli/create/create.go b/pkg/cmd/cli/create/create.go index c22d2494d..83881dbdf 100644 --- a/pkg/cmd/cli/create/create.go +++ b/pkg/cmd/cli/create/create.go @@ -21,6 +21,7 @@ import ( "github.com/heptio/ark/pkg/client" "github.com/heptio/ark/pkg/cmd/cli/backup" + "github.com/heptio/ark/pkg/cmd/cli/backuplocation" "github.com/heptio/ark/pkg/cmd/cli/restore" "github.com/heptio/ark/pkg/cmd/cli/schedule" ) @@ -36,6 +37,7 @@ func NewCommand(f client.Factory) *cobra.Command { backup.NewCreateCommand(f, "backup"), schedule.NewCreateCommand(f, "schedule"), restore.NewCreateCommand(f, "restore"), + backuplocation.NewCreateCommand(f, "backup-location"), ) return c diff --git a/pkg/cmd/cli/get/get.go b/pkg/cmd/cli/get/get.go index ac565b72f..27b3eee55 100644 --- a/pkg/cmd/cli/get/get.go +++ b/pkg/cmd/cli/get/get.go @@ -21,6 +21,7 @@ import ( "github.com/heptio/ark/pkg/client" "github.com/heptio/ark/pkg/cmd/cli/backup" + "github.com/heptio/ark/pkg/cmd/cli/backuplocation" "github.com/heptio/ark/pkg/cmd/cli/restore" "github.com/heptio/ark/pkg/cmd/cli/schedule" ) @@ -41,10 +42,14 @@ func NewCommand(f client.Factory) *cobra.Command { restoreCommand := restore.NewGetCommand(f, "restores") restoreCommand.Aliases = []string{"restore"} + backupLocationCommand := backuplocation.NewGetCommand(f, "backup-locations") + backupLocationCommand.Aliases = []string{"backup-location"} + c.AddCommand( backupCommand, scheduleCommand, restoreCommand, + backupLocationCommand, ) return c diff --git a/pkg/cmd/util/output/backup_storage_location_printer.go b/pkg/cmd/util/output/backup_storage_location_printer.go new file mode 100644 index 000000000..7fa0d773e --- /dev/null +++ b/pkg/cmd/util/output/backup_storage_location_printer.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package output + +import ( + "fmt" + "io" + + "k8s.io/kubernetes/pkg/printers" + + "github.com/heptio/ark/pkg/apis/ark/v1" +) + +var ( + backupStorageLocationColumns = []string{"NAME", "PROVIDER", "BUCKET/PREFIX"} +) + +func printBackupStorageLocationList(list *v1.BackupStorageLocationList, w io.Writer, options printers.PrintOptions) error { + for i := range list.Items { + if err := printBackupStorageLocation(&list.Items[i], w, options); err != nil { + return err + } + } + return nil +} + +func printBackupStorageLocation(location *v1.BackupStorageLocation, w io.Writer, options printers.PrintOptions) error { + name := printers.FormatResourceName(options.Kind, location.Name, options.WithKind) + + if options.WithNamespace { + if _, err := fmt.Fprintf(w, "%s\t", location.Namespace); err != nil { + return err + } + } + + bucketAndPrefix := location.Spec.ObjectStorage.Bucket + if location.Spec.ObjectStorage.Prefix != "" { + bucketAndPrefix += "/" + location.Spec.ObjectStorage.Prefix + } + + if _, err := fmt.Fprintf( + w, + "%s\t%s\t%s", + name, + location.Spec.Provider, + bucketAndPrefix, + ); err != nil { + return err + } + + if _, err := fmt.Fprint(w, printers.AppendLabels(location.Labels, options.ColumnLabels)); err != nil { + return err + } + + _, err := fmt.Fprint(w, printers.AppendAllLabels(options.ShowLabels, location.Labels)) + return err +} diff --git a/pkg/cmd/util/output/output.go b/pkg/cmd/util/output/output.go index de8d2433d..02c651054 100644 --- a/pkg/cmd/util/output/output.go +++ b/pkg/cmd/util/output/output.go @@ -143,6 +143,8 @@ func printTable(cmd *cobra.Command, obj runtime.Object) (bool, error) { printer.Handler(scheduleColumns, nil, printScheduleList) printer.Handler(resticRepoColumns, nil, printResticRepo) printer.Handler(resticRepoColumns, nil, printResticRepoList) + printer.Handler(backupStorageLocationColumns, nil, printBackupStorageLocation) + printer.Handler(backupStorageLocationColumns, nil, printBackupStorageLocationList) err = printer.PrintObj(obj, os.Stdout) if err != nil { From c6f488f75fb7435c2bd1ba49a43a70adf7379aa4 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Aug 2018 18:41:59 -0400 Subject: [PATCH 07/29] Use backup location in the backup controller Fixes #739 Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 4 +- pkg/controller/backup_controller.go | 109 +++++++++++++++-------- pkg/controller/backup_controller_test.go | 85 ++++++++++++++++-- pkg/util/test/test_backup.go | 5 ++ 4 files changed, 153 insertions(+), 50 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 8f95f84c0..d95aed7da 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -635,13 +635,13 @@ func (s *server) runControllers(config *api.Config) error { s.sharedInformerFactory.Ark().V1().Backups(), s.arkClient.ArkV1(), backupper, - config.BackupStorageProvider.CloudProviderConfig, - config.BackupStorageProvider.Bucket, s.blockStore != nil, s.logger, s.logLevel, s.pluginRegistry, backupTracker, + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), + s.defaultBackupLocation, s.metrics, ) wg.Add(1) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 36613191b..88f62fbe7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -57,21 +57,22 @@ import ( const backupVersion = 1 type backupController struct { - backupper backup.Backupper - objectStoreConfig api.CloudProviderConfig - bucket string - pvProviderExists bool - lister listers.BackupLister - listerSynced cache.InformerSynced - client arkv1client.BackupsGetter - syncHandler func(backupName string) error - queue workqueue.RateLimitingInterface - clock clock.Clock - logger logrus.FieldLogger - logLevel logrus.Level - pluginRegistry plugin.Registry - backupTracker BackupTracker - metrics *metrics.ServerMetrics + backupper backup.Backupper + pvProviderExists bool + lister listers.BackupLister + listerSynced cache.InformerSynced + client arkv1client.BackupsGetter + syncHandler func(backupName string) error + queue workqueue.RateLimitingInterface + clock clock.Clock + logger logrus.FieldLogger + logLevel logrus.Level + pluginRegistry plugin.Registry + backupTracker BackupTracker + backupLocationLister listers.BackupStorageLocationLister + backupLocationListerSynced cache.InformerSynced + defaultBackupLocation string + metrics *metrics.ServerMetrics newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager } @@ -80,30 +81,31 @@ func NewBackupController( backupInformer informers.BackupInformer, client arkv1client.BackupsGetter, backupper backup.Backupper, - objectStoreConfig api.CloudProviderConfig, - bucket string, pvProviderExists bool, logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry, backupTracker BackupTracker, + backupLocationInformer informers.BackupStorageLocationInformer, + defaultBackupLocation string, metrics *metrics.ServerMetrics, ) Interface { c := &backupController{ - backupper: backupper, - objectStoreConfig: objectStoreConfig, - bucket: bucket, - pvProviderExists: pvProviderExists, - lister: backupInformer.Lister(), - listerSynced: backupInformer.Informer().HasSynced, - client: client, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "backup"), - clock: &clock.RealClock{}, - logger: logger, - logLevel: logLevel, - pluginRegistry: pluginRegistry, - backupTracker: backupTracker, - metrics: metrics, + backupper: backupper, + pvProviderExists: pvProviderExists, + lister: backupInformer.Lister(), + listerSynced: backupInformer.Informer().HasSynced, + client: client, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "backup"), + clock: &clock.RealClock{}, + logger: logger, + logLevel: logLevel, + pluginRegistry: pluginRegistry, + backupTracker: backupTracker, + backupLocationLister: backupLocationInformer.Lister(), + backupLocationListerSynced: backupLocationInformer.Informer().HasSynced, + defaultBackupLocation: defaultBackupLocation, + metrics: metrics, newPluginManager: func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { return plugin.NewManager(logger, logLevel, pluginRegistry) @@ -165,7 +167,7 @@ func (controller *backupController) Run(ctx context.Context, numWorkers int) err defer controller.logger.Info("Shutting down BackupController") controller.logger.Info("Waiting for caches to sync") - if !cache.WaitForCacheSync(ctx.Done(), controller.listerSynced) { + if !cache.WaitForCacheSync(ctx.Done(), controller.listerSynced, controller.backupLocationListerSynced) { return errors.New("timed out waiting for caches to sync") } controller.logger.Info("Caches are synced") @@ -259,8 +261,9 @@ func (controller *backupController) processBackup(key string) error { backup.Status.Expiration = metav1.NewTime(controller.clock.Now().Add(backup.Spec.TTL.Duration)) } + var backupLocation *api.BackupStorageLocation // validation - if backup.Status.ValidationErrors = controller.getValidationErrors(backup); len(backup.Status.ValidationErrors) > 0 { + if backupLocation, backup.Status.ValidationErrors = controller.getLocationAndValidate(backup, controller.defaultBackupLocation); len(backup.Status.ValidationErrors) > 0 { backup.Status.Phase = api.BackupPhaseFailedValidation } else { backup.Status.Phase = api.BackupPhaseInProgress @@ -287,7 +290,7 @@ func (controller *backupController) processBackup(key string) error { backupScheduleName := backup.GetLabels()["ark-schedule"] controller.metrics.RegisterBackupAttempt(backupScheduleName) - if err := controller.runBackup(backup, controller.bucket); err != nil { + if err := controller.runBackup(backup, backupLocation); err != nil { logContext.WithError(err).Error("backup failed") backup.Status.Phase = api.BackupPhaseFailed controller.metrics.RegisterBackupFailed(backupScheduleName) @@ -327,7 +330,7 @@ func patchBackup(original, updated *api.Backup, client arkv1client.BackupsGetter return res, nil } -func (controller *backupController) getValidationErrors(itm *api.Backup) []string { +func (controller *backupController) getLocationAndValidate(itm *api.Backup, defaultBackupLocation string) (*api.BackupStorageLocation, []string) { var validationErrors []string for _, err := range collections.ValidateIncludesExcludes(itm.Spec.IncludedResources, itm.Spec.ExcludedResources) { @@ -342,10 +345,20 @@ func (controller *backupController) getValidationErrors(itm *api.Backup) []strin validationErrors = append(validationErrors, "Server is not configured for PV snapshots") } - return validationErrors + if itm.Spec.StorageLocation == "" { + itm.Spec.StorageLocation = defaultBackupLocation + } + + var backupLocation *api.BackupStorageLocation + backupLocation, err := controller.backupLocationLister.BackupStorageLocations(itm.Namespace).Get(itm.Spec.StorageLocation) + if err != nil { + validationErrors = append(validationErrors, fmt.Sprintf("Error getting backup storage location: %v", err)) + } + + return backupLocation, validationErrors } -func (controller *backupController) runBackup(backup *api.Backup, bucket string) error { +func (controller *backupController) runBackup(backup *api.Backup, backupLocation *api.BackupStorageLocation) error { log := controller.logger.WithField("backup", kubeutil.NamespaceAndName(backup)) log.Info("Starting backup") backup.Status.StartTimestamp.Time = controller.clock.Now() @@ -382,7 +395,7 @@ func (controller *backupController) runBackup(backup *api.Backup, bucket string) return err } - objectStore, err := getObjectStore(controller.objectStoreConfig, pluginManager) + objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager) if err != nil { return err } @@ -424,7 +437,7 @@ func (controller *backupController) runBackup(backup *api.Backup, bucket string) controller.logger.WithError(err).Error("error closing gzippedLogFile") } - if err := cloudprovider.UploadBackup(log, objectStore, bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil { + if err := cloudprovider.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil { errs = append(errs, err) } @@ -458,6 +471,24 @@ func getObjectStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) return objectStore, nil } +// TODO(nrb): Consolidate with other implementations +func getObjectStoreForLocation(location *api.BackupStorageLocation, manager plugin.Manager) (cloudprovider.ObjectStore, error) { + if location.Spec.Provider == "" { + return nil, errors.New("backup storage location provider name must not be empty") + } + + objectStore, err := manager.GetObjectStore(location.Spec.Provider) + if err != nil { + return nil, err + } + + if err := objectStore.Init(location.Spec.Config); err != nil { + return nil, err + } + + return objectStore, nil +} + func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { if err := file.Close(); err != nil { log.WithError(err).WithField("file", file.Name()).Error("error closing file") diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 27a000f1f..49d9c6876 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -152,6 +152,24 @@ func TestProcessBackup(t *testing.T) { allowSnapshots: true, expectBackup: true, }, + { + name: "Backup without a location will have it set to the default", + key: "heptio-ark/backup1", + backup: arktest.NewTestBackup().WithName("backup1").WithPhase(v1.BackupPhaseNew), + expectBackup: true, + }, + { + name: "Backup with a location completes", + key: "heptio-ark/backup1", + backup: arktest.NewTestBackup().WithName("backup1").WithPhase(v1.BackupPhaseNew).WithStorageLocation("loc1"), + expectBackup: true, + }, + { + name: "Backup with non-existent location will fail validation", + key: "heptio-ark/backup1", + backup: arktest.NewTestBackup().WithName("backup1").WithPhase(v1.BackupPhaseNew).WithStorageLocation("loc2"), + expectBackup: false, + }, } for _, test := range tests { @@ -174,13 +192,13 @@ func TestProcessBackup(t *testing.T) { sharedInformers.Ark().V1().Backups(), client.ArkV1(), backupper, - v1.CloudProviderConfig{Name: "myCloud"}, - "bucket", test.allowSnapshots, logger, logrus.InfoLevel, pluginRegistry, NewBackupTracker(), + sharedInformers.Ark().V1().BackupStorageLocations(), + "default", metrics.NewServerMetrics(), ).(*backupController) @@ -224,6 +242,37 @@ func TestProcessBackup(t *testing.T) { mock.Anything, // actions ).Return(nil) + defaultLocation := &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "default", + }, + Spec: v1.BackupStorageLocationSpec{ + Provider: "myCloud", + StorageType: v1.StorageType{ + ObjectStorage: &v1.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + } + loc1 := &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "loc1", + }, + Spec: v1.BackupStorageLocationSpec{ + Provider: "myCloud", + StorageType: v1.StorageType{ + ObjectStorage: &v1.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + } + require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(defaultLocation)) + require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(loc1)) + pluginManager.On("GetBackupItemActions").Return(nil, nil) // Ensure we have a CompletionTimestamp when uploading. @@ -312,9 +361,13 @@ func TestProcessBackup(t *testing.T) { StartTimestamp metav1.Time `json:"startTimestamp"` CompletionTimestamp metav1.Time `json:"completionTimestamp"` } + type SpecPatch struct { + StorageLocation string `json:"storageLocation"` + } type Patch struct { Status StatusPatch `json:"status"` + Spec SpecPatch `json:"spec,omitempty"` } decode := func(decoder *json.Decoder) (interface{}, error) { @@ -324,13 +377,27 @@ func TestProcessBackup(t *testing.T) { return *actual, err } - // validate Patch call 1 (setting version, expiration, and phase) - expected := Patch{ - Status: StatusPatch{ - Version: 1, - Phase: v1.BackupPhaseInProgress, - Expiration: expiration, - }, + // validate Patch call 1 (setting version, expiration, phase, and storage location) + var expected Patch + if test.backup.Spec.StorageLocation == "" { + expected = Patch{ + Status: StatusPatch{ + Version: 1, + Phase: v1.BackupPhaseInProgress, + Expiration: expiration, + }, + Spec: SpecPatch{ + StorageLocation: "default", + }, + } + } else { + expected = Patch{ + Status: StatusPatch{ + Version: 1, + Phase: v1.BackupPhaseInProgress, + Expiration: expiration, + }, + } } arktest.ValidatePatch(t, actions[0], expected, decode) diff --git a/pkg/util/test/test_backup.go b/pkg/util/test/test_backup.go index d7f877e51..041dd9e84 100644 --- a/pkg/util/test/test_backup.go +++ b/pkg/util/test/test_backup.go @@ -135,3 +135,8 @@ func (b *TestBackup) WithStartTimestamp(startTime time.Time) *TestBackup { b.Status.StartTimestamp = metav1.Time{Time: startTime} return b } + +func (b *TestBackup) WithStorageLocation(location string) *TestBackup { + b.Spec.StorageLocation = location + return b +} From bab08ed1a61905b12c69e5c20e599f902ee8c7e1 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 15 Aug 2018 16:27:27 -0700 Subject: [PATCH 08/29] backup deletion controller: use backup location for object store Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 4 +- pkg/controller/backup_deletion_controller.go | 61 +++++++++++++------ .../backup_deletion_controller_test.go | 61 +++++++++++++++---- 3 files changed, 93 insertions(+), 33 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index d95aed7da..eed216aa4 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -682,13 +682,13 @@ func (s *server) runControllers(config *api.Config) error { s.arkClient.ArkV1(), // deleteBackupRequestClient s.arkClient.ArkV1(), // backupClient s.blockStore, - s.objectStore, - config.BackupStorageProvider.Bucket, s.sharedInformerFactory.Ark().V1().Restores(), s.arkClient.ArkV1(), // restoreClient backupTracker, s.resticManager, s.sharedInformerFactory.Ark().V1().PodVolumeBackups(), + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), + s.pluginRegistry, ) wg.Add(1) go func() { diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 209f9fec3..3e0e6a9fb 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -28,6 +28,7 @@ import ( arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/restic" "github.com/heptio/ark/pkg/util/kube" "github.com/pkg/errors" @@ -50,17 +51,17 @@ type backupDeletionController struct { deleteBackupRequestLister listers.DeleteBackupRequestLister backupClient arkv1client.BackupsGetter blockStore cloudprovider.BlockStore - objectStore cloudprovider.ObjectStore - bucket string restoreLister listers.RestoreLister restoreClient arkv1client.RestoresGetter backupTracker BackupTracker resticMgr restic.RepositoryManager podvolumeBackupLister listers.PodVolumeBackupLister - - deleteBackupDir cloudprovider.DeleteBackupDirFunc - processRequestFunc func(*v1.DeleteBackupRequest) error - clock clock.Clock + backupLocationLister listers.BackupStorageLocationLister + pluginRegistry plugin.Registry + deleteBackupDir cloudprovider.DeleteBackupDirFunc + processRequestFunc func(*v1.DeleteBackupRequest) error + clock clock.Clock + newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager } // NewBackupDeletionController creates a new backup deletion controller. @@ -70,13 +71,13 @@ func NewBackupDeletionController( deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter, backupClient arkv1client.BackupsGetter, blockStore cloudprovider.BlockStore, - objectStore cloudprovider.ObjectStore, - bucket string, restoreInformer informers.RestoreInformer, restoreClient arkv1client.RestoresGetter, backupTracker BackupTracker, resticMgr restic.RepositoryManager, podvolumeBackupInformer informers.PodVolumeBackupInformer, + backupLocationInformer informers.BackupStorageLocationInformer, + pluginRegistry plugin.Registry, ) Interface { c := &backupDeletionController{ genericController: newGenericController("backup-deletion", logger), @@ -84,16 +85,20 @@ func NewBackupDeletionController( deleteBackupRequestLister: deleteBackupRequestInformer.Lister(), backupClient: backupClient, blockStore: blockStore, - objectStore: objectStore, - bucket: bucket, restoreLister: restoreInformer.Lister(), restoreClient: restoreClient, backupTracker: backupTracker, resticMgr: resticMgr, + podvolumeBackupLister: podvolumeBackupInformer.Lister(), + backupLocationLister: backupLocationInformer.Lister(), + pluginRegistry: pluginRegistry, - podvolumeBackupLister: podvolumeBackupInformer.Lister(), - deleteBackupDir: cloudprovider.DeleteBackupDir, - clock: &clock.RealClock{}, + // use variables to refer to these functions so they can be + // replaced with fakes for testing. + deleteBackupDir: cloudprovider.DeleteBackupDir, + newPluginManager: plugin.NewManager, + + clock: &clock.RealClock{}, } c.syncHandler = c.processQueueItem @@ -102,6 +107,7 @@ func NewBackupDeletionController( deleteBackupRequestInformer.Informer().HasSynced, restoreInformer.Informer().HasSynced, podvolumeBackupInformer.Informer().HasSynced, + backupLocationInformer.Informer().HasSynced, ) c.processRequestFunc = c.processRequest @@ -240,7 +246,6 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e var errs []string - // Try to delete snapshots log.Info("Removing PV snapshots") for _, volumeBackup := range backup.Status.VolumeBackups { log.WithField("snapshotID", volumeBackup.SnapshotID).Info("Removing snapshot associated with backup") @@ -249,7 +254,6 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e } } - // Try to delete restic snapshots log.Info("Removing restic snapshots") if deleteErrs := c.deleteResticSnapshots(backup); len(deleteErrs) > 0 { for _, err := range deleteErrs { @@ -257,13 +261,11 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e } } - // Try to delete backup from backup storage log.Info("Removing backup from backup storage") - if err := c.deleteBackupDir(log, c.objectStore, c.bucket, backup.Name); err != nil { - errs = append(errs, errors.Wrap(err, "error deleting backup from backup storage").Error()) + if err := c.deleteBackupFromStorage(backup, log); err != nil { + errs = append(errs, err.Error()) } - // Try to delete restores log.Info("Removing restores") if restores, err := c.restoreLister.Restores(backup.Namespace).List(labels.Everything()); err != nil { log.WithError(errors.WithStack(err)).Error("Error listing restore API objects") @@ -312,6 +314,27 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e return nil } +func (c *backupDeletionController) deleteBackupFromStorage(backup *v1.Backup, log *logrus.Entry) error { + pluginManager := c.newPluginManager(log, log.Level, c.pluginRegistry) + defer pluginManager.CleanupClients() + + backupLocation, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation) + if err != nil { + return errors.WithStack(err) + } + + objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager) + if err != nil { + return err + } + + if err := c.deleteBackupDir(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name); err != nil { + return errors.Wrap(err, "error deleting backup from backup storage") + } + + return nil +} + func (c *backupDeletionController) deleteExistingDeletionRequests(req *v1.DeleteBackupRequest, log logrus.FieldLogger) []error { log.Info("Removing existing deletion requests for backup") selector := labels.SelectorFromSet(labels.Set(map[string]string{ diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index dcbe5c736..307f18d81 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -26,10 +26,13 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" "github.com/heptio/ark/pkg/generated/clientset/versioned/fake" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" + "github.com/heptio/ark/pkg/plugin" + pluginmocks "github.com/heptio/ark/pkg/plugin/mocks" arktest "github.com/heptio/ark/pkg/util/test" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,13 +52,13 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient nil, // blockStore - nil, // backupService - "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), + sharedInformers.Ark().V1().BackupStorageLocations(), + nil, // pluginRegistry ).(*backupDeletionController) // Error splitting key @@ -109,37 +112,51 @@ type backupDeletionControllerTestData struct { client *fake.Clientset sharedInformers informers.SharedInformerFactory blockStore *arktest.FakeBlockStore + objectStore *arktest.ObjectStore controller *backupDeletionController req *v1.DeleteBackupRequest } func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletionControllerTestData { - client := fake.NewSimpleClientset(objects...) - sharedInformers := informers.NewSharedInformerFactory(client, 0) - blockStore := &arktest.FakeBlockStore{SnapshotsTaken: sets.NewString()} - req := pkgbackup.NewDeleteBackupRequest("foo", "uid") + var ( + client = fake.NewSimpleClientset(objects...) + sharedInformers = informers.NewSharedInformerFactory(client, 0) + blockStore = &arktest.FakeBlockStore{SnapshotsTaken: sets.NewString()} + pluginManager = &pluginmocks.Manager{} + objectStore = &arktest.ObjectStore{} + req = pkgbackup.NewDeleteBackupRequest("foo", "uid") + ) data := &backupDeletionControllerTestData{ client: client, sharedInformers: sharedInformers, blockStore: blockStore, + objectStore: objectStore, controller: NewBackupDeletionController( arktest.NewLogger(), sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient blockStore, - nil, // objectStore - "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), + sharedInformers.Ark().V1().BackupStorageLocations(), + nil, // pluginRegistry ).(*backupDeletionController), req: req, } + + data.controller.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { + return pluginManager + } + + pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) + pluginManager.On("CleanupClients").Return(nil) + req.Namespace = "heptio-ark" req.Name = "foo-abcde" @@ -347,6 +364,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { t.Run("full delete, no errors", func(t *testing.T) { backup := arktest.NewTestBackup().WithName("foo").WithSnapshot("pv-1", "snap-1").Backup backup.UID = "uid" + backup.Spec.StorageLocation = "primary" restore1 := arktest.NewTestRestore("heptio-ark", "restore-1", v1.RestorePhaseCompleted).WithBackup("foo").Restore restore2 := arktest.NewTestRestore("heptio-ark", "restore-2", v1.RestorePhaseCompleted).WithBackup("foo").Restore @@ -358,6 +376,24 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore2) td.sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore3) + location := &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: backup.Spec.StorageLocation, + }, + Spec: v1.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: v1.StorageType{ + ObjectStorage: &v1.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + } + require.NoError(t, td.sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location)) + + td.objectStore.On("Init", mock.Anything).Return(nil) + // Clear out req labels to make sure the controller adds them td.req.Labels = make(map[string]string) @@ -374,8 +410,9 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { return true, backup, nil }) - td.controller.deleteBackupDir = func(_ logrus.FieldLogger, _ cloudprovider.ObjectStore, bucket, backupName string) error { - require.Equal(t, "bucket", bucket) + td.controller.deleteBackupDir = func(_ logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error { + require.NotNil(t, objectStore) + require.Equal(t, location.Spec.ObjectStorage.Bucket, bucket) require.Equal(t, td.req.Spec.BackupName, backupName) return nil } @@ -561,13 +598,13 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient nil, // blockStore - nil, // backupService - "bucket", sharedInformers.Ark().V1().Restores(), client.ArkV1(), // restoreClient NewBackupTracker(), nil, sharedInformers.Ark().V1().PodVolumeBackups(), + sharedInformers.Ark().V1().BackupStorageLocations(), + nil, // pluginRegistry ).(*backupDeletionController) fakeClock := &clock.FakeClock{} From 8f5346150c7c94e875b53cf659cc54715f113b45 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 15 Aug 2018 16:42:38 -0700 Subject: [PATCH 09/29] download request controller: use backup location for object store Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 5 +- pkg/controller/download_request_controller.go | 177 ++++++------------ .../download_request_controller_test.go | 51 ++++- 3 files changed, 108 insertions(+), 125 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index eed216aa4..2df81996a 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -737,8 +737,9 @@ func (s *server) runControllers(config *api.Config) error { s.arkClient.ArkV1(), s.sharedInformerFactory.Ark().V1().DownloadRequests(), s.sharedInformerFactory.Ark().V1().Restores(), - s.objectStore, - config.BackupStorageProvider.Bucket, + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), + s.sharedInformerFactory.Ark().V1().Backups(), + s.pluginRegistry, s.logger, ) wg.Add(1) diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index a497421cc..cee333162 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -17,9 +17,7 @@ limitations under the License. package controller import ( - "context" "encoding/json" - "sync" "time" jsonpatch "github.com/evanphx/json-patch" @@ -31,32 +29,29 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" "github.com/heptio/ark/pkg/apis/ark/v1" "github.com/heptio/ark/pkg/cloudprovider" arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/util/kube" ) type downloadRequestController struct { - downloadRequestClient arkv1client.DownloadRequestsGetter - downloadRequestLister listers.DownloadRequestLister - downloadRequestListerSynced cache.InformerSynced - restoreLister listers.RestoreLister - restoreListerSynced cache.InformerSynced - objectStore cloudprovider.ObjectStore - bucket string - syncHandler func(key string) error - queue workqueue.RateLimitingInterface - clock clock.Clock - logger logrus.FieldLogger + *genericController - createSignedURL cloudprovider.CreateSignedURLFunc + downloadRequestClient arkv1client.DownloadRequestsGetter + downloadRequestLister listers.DownloadRequestLister + restoreLister listers.RestoreLister + clock clock.Clock + createSignedURL cloudprovider.CreateSignedURLFunc + backupLocationLister listers.BackupStorageLocationLister + backupLister listers.BackupLister + pluginRegistry plugin.Registry + newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager } // NewDownloadRequestController creates a new DownloadRequestController. @@ -64,26 +59,35 @@ func NewDownloadRequestController( downloadRequestClient arkv1client.DownloadRequestsGetter, downloadRequestInformer informers.DownloadRequestInformer, restoreInformer informers.RestoreInformer, - objectStore cloudprovider.ObjectStore, - bucket string, + backupLocationInformer informers.BackupStorageLocationInformer, + backupInformer informers.BackupInformer, + pluginRegistry plugin.Registry, logger logrus.FieldLogger, ) Interface { c := &downloadRequestController{ - downloadRequestClient: downloadRequestClient, - downloadRequestLister: downloadRequestInformer.Lister(), - downloadRequestListerSynced: downloadRequestInformer.Informer().HasSynced, - restoreLister: restoreInformer.Lister(), - restoreListerSynced: restoreInformer.Informer().HasSynced, - objectStore: objectStore, - bucket: bucket, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "downloadrequest"), - clock: &clock.RealClock{}, - logger: logger, + genericController: newGenericController("downloadrequest", logger), + downloadRequestClient: downloadRequestClient, + downloadRequestLister: downloadRequestInformer.Lister(), + restoreLister: restoreInformer.Lister(), + backupLocationLister: backupLocationInformer.Lister(), + backupLister: backupInformer.Lister(), - createSignedURL: cloudprovider.CreateSignedURL, + // use variables to refer to these functions so they can be + // replaced with fakes for testing. + createSignedURL: cloudprovider.CreateSignedURL, + newPluginManager: plugin.NewManager, + + clock: &clock.RealClock{}, } c.syncHandler = c.processDownloadRequest + c.cacheSyncWaiters = append( + c.cacheSyncWaiters, + downloadRequestInformer.Informer().HasSynced, + restoreInformer.Informer().HasSynced, + backupLocationInformer.Informer().HasSynced, + backupInformer.Informer().HasSynced, + ) downloadRequestInformer.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -104,102 +108,21 @@ func NewDownloadRequestController( return c } -// Run is a blocking function that runs the specified number of worker goroutines -// to process items in the work queue. It will return when it receives on the -// ctx.Done() channel. -func (c *downloadRequestController) Run(ctx context.Context, numWorkers int) error { - var wg sync.WaitGroup - - defer func() { - c.logger.Info("Waiting for workers to finish their work") - - c.queue.ShutDown() - - // We have to wait here in the deferred function instead of at the bottom of the function body - // because we have to shut down the queue in order for the workers to shut down gracefully, and - // we want to shut down the queue via defer and not at the end of the body. - wg.Wait() - - c.logger.Info("All workers have finished") - }() - - c.logger.Info("Starting DownloadRequestController") - defer c.logger.Info("Shutting down DownloadRequestController") - - c.logger.Info("Waiting for caches to sync") - if !cache.WaitForCacheSync(ctx.Done(), c.downloadRequestListerSynced, c.restoreListerSynced) { - return errors.New("timed out waiting for caches to sync") - } - c.logger.Info("Caches are synced") - - wg.Add(numWorkers) - for i := 0; i < numWorkers; i++ { - go func() { - wait.Until(c.runWorker, time.Second, ctx.Done()) - wg.Done() - }() - } - - wg.Add(1) - go func() { - wait.Until(c.resync, time.Minute, ctx.Done()) - wg.Done() - }() - - <-ctx.Done() - - return nil -} - -// runWorker runs a worker until the controller's queue indicates it's time to shut down. -func (c *downloadRequestController) runWorker() { - // continually take items off the queue (waits if it's - // empty) until we get a shutdown signal from the queue - for c.processNextWorkItem() { - } -} - -// processNextWorkItem processes a single item from the queue. -func (c *downloadRequestController) processNextWorkItem() bool { - key, quit := c.queue.Get() - if quit { - return false - } - // always call done on this item, since if it fails we'll add - // it back with rate-limiting below - defer c.queue.Done(key) - - err := c.syncHandler(key.(string)) - if err == nil { - // If you had no error, tell the queue to stop tracking history for your key. This will reset - // things like failure counts for per-item rate limiting. - c.queue.Forget(key) - return true - } - - c.logger.WithError(err).WithField("key", key).Error("Error in syncHandler, re-adding item to queue") - - // we had an error processing the item so add it back - // into the queue for re-processing with rate-limiting - c.queue.AddRateLimited(key) - - return true -} - // processDownloadRequest is the default per-item sync handler. It generates a pre-signed URL for // a new DownloadRequest or deletes the DownloadRequest if it has expired. func (c *downloadRequestController) processDownloadRequest(key string) error { - logContext := c.logger.WithField("key", key) + log := c.logger.WithField("key", key) - logContext.Debug("Running processDownloadRequest") + log.Debug("Running processDownloadRequest") ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - return errors.Wrap(err, "error splitting queue key") + log.WithError(err).Error("error splitting queue key") + return nil } downloadRequest, err := c.downloadRequestLister.DownloadRequests(ns).Get(name) if apierrors.IsNotFound(err) { - logContext.Debug("Unable to find DownloadRequest") + log.Debug("Unable to find DownloadRequest") return nil } if err != nil { @@ -208,7 +131,7 @@ func (c *downloadRequestController) processDownloadRequest(key string) error { switch downloadRequest.Status.Phase { case "", v1.DownloadRequestPhaseNew: - return c.generatePreSignedURL(downloadRequest) + return c.generatePreSignedURL(downloadRequest, log) case v1.DownloadRequestPhaseProcessed: return c.deleteIfExpired(downloadRequest) } @@ -220,7 +143,7 @@ const signedURLTTL = 10 * time.Minute // generatePreSignedURL generates a pre-signed URL for downloadRequest, changes the phase to // Processed, and persists the changes to storage. -func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest) error { +func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest, log *logrus.Entry) error { update := downloadRequest.DeepCopy() var ( @@ -240,7 +163,25 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow directory = downloadRequest.Spec.Target.Name } - update.Status.DownloadURL, err = c.createSignedURL(c.objectStore, downloadRequest.Spec.Target, c.bucket, directory, signedURLTTL) + backup, err := c.backupLister.Backups(downloadRequest.Namespace).Get(directory) + if err != nil { + return errors.WithStack(err) + } + + backupLocation, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation) + if err != nil { + return errors.WithStack(err) + } + + pluginManager := c.newPluginManager(log, log.Level, c.pluginRegistry) + defer pluginManager.CleanupClients() + + objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager) + if err != nil { + return errors.WithStack(err) + } + + update.Status.DownloadURL, err = c.createSignedURL(objectStore, downloadRequest.Spec.Target, backupLocation.Spec.ObjectStorage.Bucket, directory, signedURLTTL) if err != nil { return err } diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 25505e36a..350f6cc19 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -21,7 +21,9 @@ import ( "testing" "time" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +33,8 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" "github.com/heptio/ark/pkg/generated/clientset/versioned/fake" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" + "github.com/heptio/ark/pkg/plugin" + pluginmocks "github.com/heptio/ark/pkg/plugin/mocks" arktest "github.com/heptio/ark/pkg/util/test" ) @@ -52,9 +56,8 @@ func TestProcessDownloadRequest(t *testing.T) { key: "", }, { - name: "bad key format", - key: "a/b/c", - expectedError: `error splitting queue key: unexpected key format: "a/b/c"`, + name: "bad key format", + key: "a/b/c", }, { name: "backup log request with phase '' gets a url", @@ -109,17 +112,29 @@ func TestProcessDownloadRequest(t *testing.T) { restoresInformer = sharedInformers.Ark().V1().Restores() logger = arktest.NewLogger() clockTime, _ = time.Parse("Mon Jan 2 15:04:05 2006", "Mon Jan 2 15:04:05 2006") + pluginManager = &pluginmocks.Manager{} + objectStore = &arktest.ObjectStore{} ) c := NewDownloadRequestController( client.ArkV1(), downloadRequestsInformer, restoresInformer, - nil, // objectStore - "bucket", + sharedInformers.Ark().V1().BackupStorageLocations(), + sharedInformers.Ark().V1().Backups(), + nil, // pluginRegistry logger, ).(*downloadRequestController) + c.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { + return pluginManager + } + + pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) + pluginManager.On("CleanupClients").Return(nil) + + objectStore.On("Init", mock.Anything).Return(nil) + c.clock = clock.NewFakeClock(clockTime) var downloadRequest *v1.DownloadRequest @@ -145,6 +160,32 @@ func TestProcessDownloadRequest(t *testing.T) { restoresInformer.Informer().GetStore().Add(tc.restore) } + if tc.expectedDir != "" { + backup := &v1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.expectedDir, + Namespace: v1.DefaultNamespace, + }, + } + require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup)) + + location := &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: backup.Spec.StorageLocation, + Namespace: backup.Namespace, + }, + Spec: v1.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: v1.StorageType{ + ObjectStorage: &v1.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + } + require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location)) + } + c.createSignedURL = func(objectStore cloudprovider.ObjectStore, target v1.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) { require.Equal(t, expectedTarget, target) require.Equal(t, "bucket", bucket) From 7007f198e1b803691e04b97ad4980ce9b08ba0d6 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 16 Aug 2018 16:52:52 -0700 Subject: [PATCH 10/29] refactor download request controller test and add test cases Signed-off-by: Steve Kriss --- .../download_request_controller_test.go | 407 ++++++++++-------- 1 file changed, 239 insertions(+), 168 deletions(-) diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 350f6cc19..29508a261 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "encoding/json" "testing" "time" @@ -26,223 +25,295 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" "github.com/heptio/ark/pkg/apis/ark/v1" - "github.com/heptio/ark/pkg/cloudprovider" "github.com/heptio/ark/pkg/generated/clientset/versioned/fake" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" "github.com/heptio/ark/pkg/plugin" pluginmocks "github.com/heptio/ark/pkg/plugin/mocks" + kubeutil "github.com/heptio/ark/pkg/util/kube" arktest "github.com/heptio/ark/pkg/util/test" ) +type downloadRequestTestHarness struct { + client *fake.Clientset + informerFactory informers.SharedInformerFactory + pluginManager *pluginmocks.Manager + objectStore *arktest.ObjectStore + + controller *downloadRequestController +} + +func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { + var ( + client = fake.NewSimpleClientset() + informerFactory = informers.NewSharedInformerFactory(client, 0) + pluginManager = new(pluginmocks.Manager) + objectStore = new(arktest.ObjectStore) + controller = NewDownloadRequestController( + client.ArkV1(), + informerFactory.Ark().V1().DownloadRequests(), + informerFactory.Ark().V1().Restores(), + informerFactory.Ark().V1().BackupStorageLocations(), + informerFactory.Ark().V1().Backups(), + nil, + arktest.NewLogger(), + ).(*downloadRequestController) + ) + + clockTime, err := time.Parse(time.RFC1123, time.RFC1123) + require.NoError(t, err) + + controller.clock = clock.NewFakeClock(clockTime) + + controller.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { + return pluginManager + } + + pluginManager.On("CleanupClients").Return() + objectStore.On("Init", mock.Anything).Return(nil) + + return &downloadRequestTestHarness{ + client: client, + informerFactory: informerFactory, + pluginManager: pluginManager, + objectStore: objectStore, + controller: controller, + } +} + +func newDownloadRequest(phase v1.DownloadRequestPhase, targetKind v1.DownloadTargetKind, targetName string) *v1.DownloadRequest { + return &v1.DownloadRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-download-request", + Namespace: v1.DefaultNamespace, + }, + Spec: v1.DownloadRequestSpec{ + Target: v1.DownloadTarget{ + Kind: targetKind, + Name: targetName, + }, + }, + Status: v1.DownloadRequestStatus{ + Phase: phase, + }, + } +} + +func newBackupLocation(name, provider, bucket string) *v1.BackupStorageLocation { + return &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: v1.DefaultNamespace, + }, + Spec: v1.BackupStorageLocationSpec{ + Provider: provider, + StorageType: v1.StorageType{ + ObjectStorage: &v1.ObjectStorageLocation{ + Bucket: bucket, + }, + }, + }, + } +} + func TestProcessDownloadRequest(t *testing.T) { tests := []struct { - name string - key string - phase v1.DownloadRequestPhase - targetKind v1.DownloadTargetKind - targetName string - restore *v1.Restore - expectedError string - expectedDir string - expectedPhase v1.DownloadRequestPhase - expectedURL string + name string + key string + downloadRequest *v1.DownloadRequest + backup *v1.Backup + restore *v1.Restore + backupLocation *v1.BackupStorageLocation + expired bool + expectedErr string + expectedRequestedObject string }{ { - name: "empty key", + name: "empty key returns without error", key: "", }, { - name: "bad key format", + name: "bad key format returns without error", key: "a/b/c", }, { - name: "backup log request with phase '' gets a url", - key: "heptio-ark/dr1", - phase: "", - targetKind: v1.DownloadTargetKindBackupLog, - targetName: "backup1", - expectedDir: "backup1", - expectedPhase: v1.DownloadRequestPhaseProcessed, - expectedURL: "signedURL", + name: "no download request for key returns without error", + key: "nonexistent/key", }, { - name: "backup log request with phase 'New' gets a url", - key: "heptio-ark/dr1", - phase: v1.DownloadRequestPhaseNew, - targetKind: v1.DownloadTargetKindBackupLog, - targetName: "backup1", - expectedDir: "backup1", - expectedPhase: v1.DownloadRequestPhaseProcessed, - expectedURL: "signedURL", + name: "backup contents request for nonexistent backup returns an error", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), + backup: arktest.NewTestBackup().WithName("non-matching-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedErr: "backup.ark.heptio.com \"a-backup\" not found", }, { - name: "restore log request with phase '' gets a url", - key: "heptio-ark/dr1", - phase: "", - targetKind: v1.DownloadTargetKindRestoreLog, - targetName: "backup1-20170912150214", - restore: arktest.NewTestRestore(v1.DefaultNamespace, "backup1-20170912150214", v1.RestorePhaseCompleted).WithBackup("backup1").Restore, - expectedDir: "backup1", - expectedPhase: v1.DownloadRequestPhaseProcessed, - expectedURL: "signedURL", + name: "restore log request for nonexistent restore returns an error", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: arktest.NewTestRestore(v1.DefaultNamespace, "non-matching-restore", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore, + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedErr: "error getting Restore: restore.ark.heptio.com \"a-backup-20170912150214\" not found", }, { - name: "restore log request with phase New gets a url", - key: "heptio-ark/dr1", - phase: v1.DownloadRequestPhaseNew, - targetKind: v1.DownloadTargetKindRestoreLog, - targetName: "backup1-20170912150214", - restore: arktest.NewTestRestore(v1.DefaultNamespace, "backup1-20170912150214", v1.RestorePhaseCompleted).WithBackup("backup1").Restore, - expectedDir: "backup1", - expectedPhase: v1.DownloadRequestPhaseProcessed, - expectedURL: "signedURL", + name: "backup contents request for backup with nonexistent location returns an error", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("non-matching-location", "a-provider", "a-bucket"), + expectedErr: "backupstoragelocation.ark.heptio.com \"a-location\" not found", + }, + { + name: "backup contents request with phase '' gets a url", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/a-backup.tar.gz", + }, + { + name: "backup contents request with phase 'New' gets a url", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupContents, "a-backup"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/a-backup.tar.gz", + }, + { + name: "backup log request with phase '' gets a url", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupLog, "a-backup"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/a-backup-logs.gz", + }, + { + name: "backup log request with phase 'New' gets a url", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupLog, "a-backup"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/a-backup-logs.gz", + }, + { + name: "restore log request with phase '' gets a url", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore, + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-logs.gz", + }, + { + name: "restore log request with phase 'New' gets a url", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore, + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-logs.gz", + }, + { + name: "restore results request with phase '' gets a url", + downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), + restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore, + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-results.gz", + }, + { + name: "restore results request with phase 'New' gets a url", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), + restore: arktest.NewTestRestore(v1.DefaultNamespace, "a-backup-20170912150214", v1.RestorePhaseCompleted).WithBackup("a-backup").Restore, + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), + expectedRequestedObject: "a-backup/restore-a-backup-20170912150214-results.gz", + }, + { + name: "request with phase 'Processed' is not deleted if not expired", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseProcessed, v1.DownloadTargetKindBackupLog, "a-backup-20170912150214"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + }, + { + name: "request with phase 'Processed' is deleted if expired", + downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseProcessed, v1.DownloadTargetKindBackupLog, "a-backup-20170912150214"), + backup: arktest.NewTestBackup().WithName("a-backup").WithStorageLocation("a-location").Backup, + expired: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - var ( - client = fake.NewSimpleClientset() - sharedInformers = informers.NewSharedInformerFactory(client, 0) - downloadRequestsInformer = sharedInformers.Ark().V1().DownloadRequests() - restoresInformer = sharedInformers.Ark().V1().Restores() - logger = arktest.NewLogger() - clockTime, _ = time.Parse("Mon Jan 2 15:04:05 2006", "Mon Jan 2 15:04:05 2006") - pluginManager = &pluginmocks.Manager{} - objectStore = &arktest.ObjectStore{} - ) + harness := newDownloadRequestTestHarness(t) - c := NewDownloadRequestController( - client.ArkV1(), - downloadRequestsInformer, - restoresInformer, - sharedInformers.Ark().V1().BackupStorageLocations(), - sharedInformers.Ark().V1().Backups(), - nil, // pluginRegistry - logger, - ).(*downloadRequestController) + // set up test case data - c.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { - return pluginManager - } - - pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) - pluginManager.On("CleanupClients").Return(nil) - - objectStore.On("Init", mock.Anything).Return(nil) - - c.clock = clock.NewFakeClock(clockTime) - - var downloadRequest *v1.DownloadRequest - - if tc.expectedPhase == v1.DownloadRequestPhaseProcessed { - expectedTarget := v1.DownloadTarget{ - Kind: tc.targetKind, - Name: tc.targetName, - } - - downloadRequest = &v1.DownloadRequest{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: v1.DefaultNamespace, - Name: "dr1", - }, - Spec: v1.DownloadRequestSpec{ - Target: expectedTarget, - }, - } - downloadRequestsInformer.Informer().GetStore().Add(downloadRequest) - - if tc.restore != nil { - restoresInformer.Informer().GetStore().Add(tc.restore) - } - - if tc.expectedDir != "" { - backup := &v1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: tc.expectedDir, - Namespace: v1.DefaultNamespace, - }, - } - require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup)) - - location := &v1.BackupStorageLocation{ - ObjectMeta: metav1.ObjectMeta{ - Name: backup.Spec.StorageLocation, - Namespace: backup.Namespace, - }, - Spec: v1.BackupStorageLocationSpec{ - Provider: "objStoreProvider", - StorageType: v1.StorageType{ - ObjectStorage: &v1.ObjectStorageLocation{ - Bucket: "bucket", - }, - }, - }, - } - require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location)) - } - - c.createSignedURL = func(objectStore cloudprovider.ObjectStore, target v1.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) { - require.Equal(t, expectedTarget, target) - require.Equal(t, "bucket", bucket) - require.Equal(t, tc.expectedDir, directory) - require.Equal(t, 10*time.Minute, ttl) - return "signedURL", nil + // Set .status.expiration properly for processed requests. Since "expired" is relative to the controller's + // clock time, it's easier to do this here than as part of the test case definitions. + if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == v1.DownloadRequestPhaseProcessed { + if tc.expired { + tc.downloadRequest.Status.Expiration.Time = harness.controller.clock.Now().Add(-1 * time.Minute) + } else { + tc.downloadRequest.Status.Expiration.Time = harness.controller.clock.Now().Add(time.Minute) } } - // method under test - err := c.processDownloadRequest(tc.key) + if tc.downloadRequest != nil { + require.NoError(t, harness.informerFactory.Ark().V1().DownloadRequests().Informer().GetStore().Add(tc.downloadRequest)) - if tc.expectedError != "" { - assert.EqualError(t, err, tc.expectedError) - return + _, err := harness.client.ArkV1().DownloadRequests(tc.downloadRequest.Namespace).Create(tc.downloadRequest) + require.NoError(t, err) } - require.NoError(t, err) - - actions := client.Actions() - - // if we don't expect a phase update, this means - // we don't expect any actions to take place - if tc.expectedPhase == "" { - require.Equal(t, 0, len(actions)) - return + if tc.restore != nil { + require.NoError(t, harness.informerFactory.Ark().V1().Restores().Informer().GetStore().Add(tc.restore)) } - // otherwise, we should get exactly 1 patch - require.Equal(t, 1, len(actions)) - - type PatchStatus struct { - DownloadURL string `json:"downloadURL"` - Phase v1.DownloadRequestPhase `json:"phase"` - Expiration time.Time `json:"expiration"` + if tc.backup != nil { + require.NoError(t, harness.informerFactory.Ark().V1().Backups().Informer().GetStore().Add(tc.backup)) } - type Patch struct { - Status PatchStatus `json:"status"` + if tc.backupLocation != nil { + require.NoError(t, harness.informerFactory.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(tc.backupLocation)) + + harness.pluginManager.On("GetObjectStore", tc.backupLocation.Spec.Provider).Return(harness.objectStore, nil) } - decode := func(decoder *json.Decoder) (interface{}, error) { - actual := new(Patch) - err := decoder.Decode(actual) - - return *actual, err + if tc.expectedRequestedObject != "" { + harness.objectStore.On("CreateSignedURL", tc.backupLocation.Spec.ObjectStorage.Bucket, tc.expectedRequestedObject, mock.Anything).Return("a-url", nil) } - expected := Patch{ - Status: PatchStatus{ - DownloadURL: tc.expectedURL, - Phase: tc.expectedPhase, - Expiration: clockTime.Add(signedURLTTL), - }, + // exercise method under test + key := tc.key + if key == "" && tc.downloadRequest != nil { + key = kubeutil.NamespaceAndName(tc.downloadRequest) } - arktest.ValidatePatch(t, actions[0], expected, decode) + err := harness.controller.processDownloadRequest(key) + + // verify results + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + } else { + assert.Nil(t, err) + } + + if tc.expectedRequestedObject != "" { + output, err := harness.client.ArkV1().DownloadRequests(tc.downloadRequest.Namespace).Get(tc.downloadRequest.Name, metav1.GetOptions{}) + require.NoError(t, err) + + assert.Equal(t, string(v1.DownloadRequestPhaseProcessed), string(output.Status.Phase)) + assert.Equal(t, "a-url", output.Status.DownloadURL) + assert.True(t, arktest.TimesAreEqual(harness.controller.clock.Now().Add(signedURLTTL), output.Status.Expiration.Time), "expiration does not match") + } + + if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == v1.DownloadRequestPhaseProcessed { + res, err := harness.client.ArkV1().DownloadRequests(tc.downloadRequest.Namespace).Get(tc.downloadRequest.Name, metav1.GetOptions{}) + + if tc.expired { + assert.True(t, apierrors.IsNotFound(err)) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.downloadRequest, res) + } + } }) } } From 74043ab428199f5595187afb2df873c9f4cb9faa Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 16 Aug 2018 17:12:31 -0700 Subject: [PATCH 11/29] download request controller: fix bug in determining expiration Signed-off-by: Steve Kriss --- pkg/controller/download_request_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index cee333162..46ed38391 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -197,7 +197,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow func (c *downloadRequestController) deleteIfExpired(downloadRequest *v1.DownloadRequest) error { logContext := c.logger.WithField("key", kube.NamespaceAndName(downloadRequest)) logContext.Info("checking for expiration of DownloadRequest") - if downloadRequest.Status.Expiration.Time.Before(c.clock.Now()) { + if downloadRequest.Status.Expiration.Time.After(c.clock.Now()) { logContext.Debug("DownloadRequest has not expired") return nil } From 3234124afe27ed9e48f0707f2e766f354423700a Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 17 Aug 2018 11:20:44 -0700 Subject: [PATCH 12/29] backup deletion: fix setting of log level in plugin manager Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 1 + pkg/controller/backup_deletion_controller.go | 15 ++++++++------- pkg/controller/backup_deletion_controller_test.go | 7 ++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 2df81996a..1528e1227 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -678,6 +678,7 @@ func (s *server) runControllers(config *api.Config) error { backupDeletionController := controller.NewBackupDeletionController( s.logger, + s.logLevel, s.sharedInformerFactory.Ark().V1().DeleteBackupRequests(), s.arkClient.ArkV1(), // deleteBackupRequestClient s.arkClient.ArkV1(), // backupClient diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 3e0e6a9fb..b4b33c6b3 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -57,16 +57,16 @@ type backupDeletionController struct { resticMgr restic.RepositoryManager podvolumeBackupLister listers.PodVolumeBackupLister backupLocationLister listers.BackupStorageLocationLister - pluginRegistry plugin.Registry deleteBackupDir cloudprovider.DeleteBackupDirFunc processRequestFunc func(*v1.DeleteBackupRequest) error clock clock.Clock - newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager + newPluginManager func(logrus.FieldLogger) plugin.Manager } // NewBackupDeletionController creates a new backup deletion controller. func NewBackupDeletionController( logger logrus.FieldLogger, + logLevel logrus.Level, deleteBackupRequestInformer informers.DeleteBackupRequestInformer, deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter, backupClient arkv1client.BackupsGetter, @@ -91,12 +91,13 @@ func NewBackupDeletionController( resticMgr: resticMgr, podvolumeBackupLister: podvolumeBackupInformer.Lister(), backupLocationLister: backupLocationInformer.Lister(), - pluginRegistry: pluginRegistry, // use variables to refer to these functions so they can be // replaced with fakes for testing. - deleteBackupDir: cloudprovider.DeleteBackupDir, - newPluginManager: plugin.NewManager, + deleteBackupDir: cloudprovider.DeleteBackupDir, + newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { + return plugin.NewManager(logger, logLevel, pluginRegistry) + }, clock: &clock.RealClock{}, } @@ -314,8 +315,8 @@ func (c *backupDeletionController) processRequest(req *v1.DeleteBackupRequest) e return nil } -func (c *backupDeletionController) deleteBackupFromStorage(backup *v1.Backup, log *logrus.Entry) error { - pluginManager := c.newPluginManager(log, log.Level, c.pluginRegistry) +func (c *backupDeletionController) deleteBackupFromStorage(backup *v1.Backup, log logrus.FieldLogger) error { + pluginManager := c.newPluginManager(log) defer pluginManager.CleanupClients() backupLocation, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation) diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index 307f18d81..5bd39e890 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -48,6 +48,7 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { controller := NewBackupDeletionController( arktest.NewLogger(), + logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient @@ -134,6 +135,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio objectStore: objectStore, controller: NewBackupDeletionController( arktest.NewLogger(), + logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient @@ -150,9 +152,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio req: req, } - data.controller.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { - return pluginManager - } + data.controller.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) pluginManager.On("CleanupClients").Return(nil) @@ -594,6 +594,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { controller := NewBackupDeletionController( arktest.NewLogger(), + logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient From cf7c8587f04378ff5373d75b08a7807598d7ec11 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 17 Aug 2018 11:24:21 -0700 Subject: [PATCH 13/29] download request: fix setting of log level for plugin manager Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 1 + pkg/controller/download_request_controller.go | 14 ++++++++------ pkg/controller/download_request_controller_test.go | 5 ++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 1528e1227..84621d933 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -742,6 +742,7 @@ func (s *server) runControllers(config *api.Config) error { s.sharedInformerFactory.Ark().V1().Backups(), s.pluginRegistry, s.logger, + s.logLevel, ) wg.Add(1) go func() { diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 46ed38391..d06dbb6dc 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -50,8 +50,7 @@ type downloadRequestController struct { createSignedURL cloudprovider.CreateSignedURLFunc backupLocationLister listers.BackupStorageLocationLister backupLister listers.BackupLister - pluginRegistry plugin.Registry - newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager + newPluginManager func(logrus.FieldLogger) plugin.Manager } // NewDownloadRequestController creates a new DownloadRequestController. @@ -63,6 +62,7 @@ func NewDownloadRequestController( backupInformer informers.BackupInformer, pluginRegistry plugin.Registry, logger logrus.FieldLogger, + logLevel logrus.Level, ) Interface { c := &downloadRequestController{ genericController: newGenericController("downloadrequest", logger), @@ -74,8 +74,10 @@ func NewDownloadRequestController( // use variables to refer to these functions so they can be // replaced with fakes for testing. - createSignedURL: cloudprovider.CreateSignedURL, - newPluginManager: plugin.NewManager, + createSignedURL: cloudprovider.CreateSignedURL, + newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { + return plugin.NewManager(logger, logLevel, pluginRegistry) + }, clock: &clock.RealClock{}, } @@ -143,7 +145,7 @@ const signedURLTTL = 10 * time.Minute // generatePreSignedURL generates a pre-signed URL for downloadRequest, changes the phase to // Processed, and persists the changes to storage. -func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest, log *logrus.Entry) error { +func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest, log logrus.FieldLogger) error { update := downloadRequest.DeepCopy() var ( @@ -173,7 +175,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow return errors.WithStack(err) } - pluginManager := c.newPluginManager(log, log.Level, c.pluginRegistry) + pluginManager := c.newPluginManager(log) defer pluginManager.CleanupClients() objectStore, err := getObjectStoreForLocation(backupLocation, pluginManager) diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 29508a261..645fccbba 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -61,6 +61,7 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { informerFactory.Ark().V1().Backups(), nil, arktest.NewLogger(), + logrus.InfoLevel, ).(*downloadRequestController) ) @@ -69,9 +70,7 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { controller.clock = clock.NewFakeClock(clockTime) - controller.newPluginManager = func(_ logrus.FieldLogger, _ logrus.Level, _ plugin.Registry) plugin.Manager { - return pluginManager - } + controller.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } pluginManager.On("CleanupClients").Return() objectStore.On("Init", mock.Anything).Return(nil) From 833a6307a92fc2eaa8c3e1868e25f9a2910df05f Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 20 Aug 2018 17:17:13 -0400 Subject: [PATCH 14/29] Add storage location to backup get/describe Fixes #775 Also conforms ark imports to https://github.com/heptio/ark/issues/494 Signed-off-by: Nolan Brubaker --- pkg/cmd/util/output/backup_describer.go | 43 +++++++++++++------------ pkg/cmd/util/output/backup_printer.go | 16 +++++---- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 6516a3bc8..b34e49d5d 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -21,19 +21,19 @@ import ( "sort" "strings" - "github.com/heptio/ark/pkg/apis/ark/v1" + arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // DescribeBackup describes a backup in human-readable format. -func DescribeBackup(backup *v1.Backup, deleteRequests []v1.DeleteBackupRequest, podVolumeBackups []v1.PodVolumeBackup, volumeDetails bool) string { +func DescribeBackup(backup *arkv1api.Backup, deleteRequests []arkv1api.DeleteBackupRequest, podVolumeBackups []arkv1api.PodVolumeBackup, volumeDetails bool) string { return Describe(func(d *Describer) { d.DescribeMetadata(backup.ObjectMeta) d.Println() phase := backup.Status.Phase if phase == "" { - phase = v1.BackupPhaseNew + phase = arkv1api.BackupPhaseNew } d.Printf("Phase:\t%s\n", phase) @@ -56,7 +56,7 @@ func DescribeBackup(backup *v1.Backup, deleteRequests []v1.DeleteBackupRequest, } // DescribeBackupSpec describes a backup spec in human-readable format. -func DescribeBackupSpec(d *Describer, spec v1.BackupSpec) { +func DescribeBackupSpec(d *Describer, spec arkv1api.BackupSpec) { // TODO make a helper for this and use it in all the describers. d.Printf("Namespaces:\n") var s string @@ -97,6 +97,9 @@ func DescribeBackupSpec(d *Describer, spec v1.BackupSpec) { } d.Printf("Label selector:\t%s\n", s) + d.Println() + d.Printf("Storage Location:\t%s\n", spec.StorageLocation) + d.Println() d.Printf("Snapshot PVs:\t%s\n", BoolPointerString(spec.SnapshotVolumes, "false", "true", "auto")) @@ -164,7 +167,7 @@ func DescribeBackupSpec(d *Describer, spec v1.BackupSpec) { } // DescribeBackupStatus describes a backup status in human-readable format. -func DescribeBackupStatus(d *Describer, status v1.BackupStatus) { +func DescribeBackupStatus(d *Describer, status arkv1api.BackupStatus) { d.Printf("Backup Format Version:\t%d\n", status.Version) d.Println() @@ -213,7 +216,7 @@ func DescribeBackupStatus(d *Describer, status v1.BackupStatus) { } // DescribeDeleteBackupRequests describes delete backup requests in human-readable format. -func DescribeDeleteBackupRequests(d *Describer, requests []v1.DeleteBackupRequest) { +func DescribeDeleteBackupRequests(d *Describer, requests []arkv1api.DeleteBackupRequest) { d.Printf("Deletion Attempts") if count := failedDeletionCount(requests); count > 0 { d.Printf(" (%d failed)", count) @@ -238,10 +241,10 @@ func DescribeDeleteBackupRequests(d *Describer, requests []v1.DeleteBackupReques } } -func failedDeletionCount(requests []v1.DeleteBackupRequest) int { +func failedDeletionCount(requests []arkv1api.DeleteBackupRequest) int { var count int for _, req := range requests { - if req.Status.Phase == v1.DeleteBackupRequestPhaseProcessed && len(req.Status.Errors) > 0 { + if req.Status.Phase == arkv1api.DeleteBackupRequestPhaseProcessed && len(req.Status.Errors) > 0 { count++ } } @@ -249,7 +252,7 @@ func failedDeletionCount(requests []v1.DeleteBackupRequest) int { } // DescribePodVolumeBackups describes pod volume backups in human-readable format. -func DescribePodVolumeBackups(d *Describer, backups []v1.PodVolumeBackup, details bool) { +func DescribePodVolumeBackups(d *Describer, backups []arkv1api.PodVolumeBackup, details bool) { if details { d.Printf("Restic Backups:\n") } else { @@ -261,10 +264,10 @@ func DescribePodVolumeBackups(d *Describer, backups []v1.PodVolumeBackup, detail // go through phases in a specific order for _, phase := range []string{ - string(v1.PodVolumeBackupPhaseCompleted), - string(v1.PodVolumeBackupPhaseFailed), + string(arkv1api.PodVolumeBackupPhaseCompleted), + string(arkv1api.PodVolumeBackupPhaseFailed), "In Progress", - string(v1.PodVolumeBackupPhaseNew), + string(arkv1api.PodVolumeBackupPhaseNew), } { if len(backupsByPhase[phase]) == 0 { continue @@ -293,15 +296,15 @@ func DescribePodVolumeBackups(d *Describer, backups []v1.PodVolumeBackup, detail } } -func groupByPhase(backups []v1.PodVolumeBackup) map[string][]v1.PodVolumeBackup { - backupsByPhase := make(map[string][]v1.PodVolumeBackup) +func groupByPhase(backups []arkv1api.PodVolumeBackup) map[string][]arkv1api.PodVolumeBackup { + backupsByPhase := make(map[string][]arkv1api.PodVolumeBackup) - phaseToGroup := map[v1.PodVolumeBackupPhase]string{ - v1.PodVolumeBackupPhaseCompleted: string(v1.PodVolumeBackupPhaseCompleted), - v1.PodVolumeBackupPhaseFailed: string(v1.PodVolumeBackupPhaseFailed), - v1.PodVolumeBackupPhaseInProgress: "In Progress", - v1.PodVolumeBackupPhaseNew: string(v1.PodVolumeBackupPhaseNew), - "": string(v1.PodVolumeBackupPhaseNew), + phaseToGroup := map[arkv1api.PodVolumeBackupPhase]string{ + arkv1api.PodVolumeBackupPhaseCompleted: string(arkv1api.PodVolumeBackupPhaseCompleted), + arkv1api.PodVolumeBackupPhaseFailed: string(arkv1api.PodVolumeBackupPhaseFailed), + arkv1api.PodVolumeBackupPhaseInProgress: "In Progress", + arkv1api.PodVolumeBackupPhaseNew: string(arkv1api.PodVolumeBackupPhaseNew), + "": string(arkv1api.PodVolumeBackupPhaseNew), } for _, backup := range backups { diff --git a/pkg/cmd/util/output/backup_printer.go b/pkg/cmd/util/output/backup_printer.go index 17153d223..2db0363e7 100644 --- a/pkg/cmd/util/output/backup_printer.go +++ b/pkg/cmd/util/output/backup_printer.go @@ -27,14 +27,14 @@ import ( "k8s.io/apimachinery/pkg/util/duration" "k8s.io/kubernetes/pkg/printers" - "github.com/heptio/ark/pkg/apis/ark/v1" + arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" ) var ( - backupColumns = []string{"NAME", "STATUS", "CREATED", "EXPIRES", "SELECTOR"} + backupColumns = []string{"NAME", "STATUS", "CREATED", "EXPIRES", "STORAGE LOCATION", "SELECTOR"} ) -func printBackupList(list *v1.BackupList, w io.Writer, options printers.PrintOptions) error { +func printBackupList(list *arkv1api.BackupList, w io.Writer, options printers.PrintOptions) error { sortBackupsByPrefixAndTimestamp(list) for i := range list.Items { @@ -45,7 +45,7 @@ func printBackupList(list *v1.BackupList, w io.Writer, options printers.PrintOpt return nil } -func sortBackupsByPrefixAndTimestamp(list *v1.BackupList) { +func sortBackupsByPrefixAndTimestamp(list *arkv1api.BackupList) { // sort by default alphabetically, but if backups stem from a common schedule // (detected by the presence of a 14-digit timestamp suffix), then within that // group, sort by newest to oldest (i.e. prefix ASC, suffix DESC) @@ -70,7 +70,7 @@ func sortBackupsByPrefixAndTimestamp(list *v1.BackupList) { }) } -func printBackup(backup *v1.Backup, w io.Writer, options printers.PrintOptions) error { +func printBackup(backup *arkv1api.Backup, w io.Writer, options printers.PrintOptions) error { name := printers.FormatResourceName(options.Kind, backup.Name, options.WithKind) if options.WithNamespace { @@ -86,13 +86,15 @@ func printBackup(backup *v1.Backup, w io.Writer, options printers.PrintOptions) status := backup.Status.Phase if status == "" { - status = v1.BackupPhaseNew + status = arkv1api.BackupPhaseNew } if backup.DeletionTimestamp != nil && !backup.DeletionTimestamp.Time.IsZero() { status = "Deleting" } - if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s", name, status, backup.CreationTimestamp.Time, humanReadableTimeFromNow(expiration), metav1.FormatLabelSelector(backup.Spec.LabelSelector)); err != nil { + location := backup.Spec.StorageLocation + + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s", name, status, backup.CreationTimestamp.Time, humanReadableTimeFromNow(expiration), location, metav1.FormatLabelSelector(backup.Spec.LabelSelector)); err != nil { return err } From 20f89fbcefe34f1e07a9219e74129f6d215af5fc Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 9 Aug 2018 13:28:57 -0700 Subject: [PATCH 15/29] use the default backup storage location for restic Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 19 +++-- .../restic_repository_controller.go | 8 +- pkg/restic/common.go | 1 + pkg/restic/config.go | 17 +++-- pkg/restic/config_test.go | 73 +++++++++++-------- 5 files changed, 69 insertions(+), 49 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 84621d933..f3c6c1ea9 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -271,6 +271,11 @@ func (s *server) run() error { s.watchConfig(originalConfig) + backupStorageLocation, err := s.arkClient.ArkV1().BackupStorageLocations(s.namespace).Get(s.defaultBackupLocation, metav1.GetOptions{}) + if err != nil { + return errors.WithStack(err) + } + objectStore, err := getObjectStore(config.BackupStorageProvider.CloudProviderConfig, s.pluginManager) if err != nil { return err @@ -288,13 +293,13 @@ func (s *server) run() error { s.blockStore = blockStore } - if config.BackupStorageProvider.ResticLocation != "" { - if err := s.initRestic(config.BackupStorageProvider); err != nil { + if backupStorageLocation.Spec.Config[restic.ResticLocationConfigKey] != "" { + if err := s.initRestic(backupStorageLocation.Spec.Provider); err != nil { return err } } - if err := s.runControllers(config); err != nil { + if err := s.runControllers(config, backupStorageLocation); err != nil { return err } @@ -525,7 +530,7 @@ func durationMin(a, b time.Duration) time.Duration { return b } -func (s *server) initRestic(config api.ObjectStorageProviderConfig) error { +func (s *server) initRestic(providerName string) error { // warn if restic daemonset does not exist if _, err := s.kubeClient.AppsV1().DaemonSets(s.namespace).Get(restic.DaemonSet, metav1.GetOptions{}); apierrors.IsNotFound(err) { s.logger.Warn("Ark restic daemonset not found; restic backups/restores will not work until it's created") @@ -539,7 +544,7 @@ func (s *server) initRestic(config api.ObjectStorageProviderConfig) error { } // set the env vars that restic uses for creds purposes - if config.Name == string(restic.AzureBackend) { + if providerName == string(restic.AzureBackend) { os.Setenv("AZURE_ACCOUNT_NAME", os.Getenv("AZURE_STORAGE_ACCOUNT_ID")) os.Setenv("AZURE_ACCOUNT_KEY", os.Getenv("AZURE_STORAGE_KEY")) } @@ -578,7 +583,7 @@ func (s *server) initRestic(config api.ObjectStorageProviderConfig) error { return nil } -func (s *server) runControllers(config *api.Config) error { +func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.BackupStorageLocation) error { s.logger.Info("Starting controllers") ctx := s.ctx @@ -755,7 +760,7 @@ func (s *server) runControllers(config *api.Config) error { s.logger, s.sharedInformerFactory.Ark().V1().ResticRepositories(), s.arkClient.ArkV1(), - config.BackupStorageProvider, + defaultBackupLocation, s.resticManager, ) wg.Add(1) diff --git a/pkg/controller/restic_repository_controller.go b/pkg/controller/restic_repository_controller.go index 783baa915..8c232b55f 100644 --- a/pkg/controller/restic_repository_controller.go +++ b/pkg/controller/restic_repository_controller.go @@ -44,7 +44,7 @@ type resticRepositoryController struct { resticRepositoryClient arkv1client.ResticRepositoriesGetter resticRepositoryLister listers.ResticRepositoryLister - objectStorageConfig arkv1api.ObjectStorageProviderConfig + storageLocation *arkv1api.BackupStorageLocation repositoryManager restic.RepositoryManager clock clock.Clock @@ -55,14 +55,14 @@ func NewResticRepositoryController( logger logrus.FieldLogger, resticRepositoryInformer informers.ResticRepositoryInformer, resticRepositoryClient arkv1client.ResticRepositoriesGetter, - objectStorageConfig arkv1api.ObjectStorageProviderConfig, + storageLocation *arkv1api.BackupStorageLocation, repositoryManager restic.RepositoryManager, ) Interface { c := &resticRepositoryController{ genericController: newGenericController("restic-repository", logger), resticRepositoryClient: resticRepositoryClient, resticRepositoryLister: resticRepositoryInformer.Lister(), - objectStorageConfig: objectStorageConfig, + storageLocation: storageLocation, repositoryManager: repositoryManager, clock: &clock.RealClock{}, } @@ -139,7 +139,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo // defaulting - if the patch fails, return an error so the item is returned to the queue if err := c.patchResticRepository(req, func(r *v1.ResticRepository) { - r.Spec.ResticIdentifier = restic.GetRepoIdentifier(c.objectStorageConfig, r.Name) + r.Spec.ResticIdentifier = restic.GetRepoIdentifier(c.storageLocation, r.Name) if r.Spec.MaintenanceFrequency.Duration <= 0 { r.Spec.MaintenanceFrequency = metav1.Duration{Duration: restic.DefaultMaintenanceFrequency} diff --git a/pkg/restic/common.go b/pkg/restic/common.go index cbaa136af..3b8797430 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -36,6 +36,7 @@ const ( DaemonSet = "restic" InitContainer = "restic-wait" DefaultMaintenanceFrequency = 24 * time.Hour + ResticLocationConfigKey = "restic-location" podAnnotationPrefix = "snapshot.ark.heptio.com/" volumesToBackupAnnotation = "backup.ark.heptio.com/backup-volumes" diff --git a/pkg/restic/config.go b/pkg/restic/config.go index 7a9dc9f11..1e70d7d91 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -38,9 +38,10 @@ var getAWSBucketRegion = aws.GetBucketRegion // getRepoPrefix returns the prefix of the value of the --repo flag for // restic commands, i.e. everything except the "/". -func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { +func getRepoPrefix(location *arkv1api.BackupStorageLocation) string { var ( - parts = strings.SplitN(config.ResticLocation, "/", 2) + resticLocation = location.Spec.Config[ResticLocationConfigKey] + parts = strings.SplitN(resticLocation, "/", 2) bucket, path, prefix string ) @@ -51,13 +52,13 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { path = parts[1] } - switch BackendType(config.Name) { + switch BackendType(location.Spec.Provider) { case AWSBackend: var url string switch { // non-AWS, S3-compatible object store - case config.Config["s3Url"] != "": - url = config.Config["s3Url"] + case location.Spec.Config["s3Url"] != "": + url = location.Spec.Config["s3Url"] default: region, err := getAWSBucketRegion(bucket) if err != nil { @@ -68,7 +69,7 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { url = fmt.Sprintf("s3-%s.amazonaws.com", region) } - return fmt.Sprintf("s3:%s/%s", url, config.ResticLocation) + return fmt.Sprintf("s3:%s/%s", url, resticLocation) case AzureBackend: prefix = "azure" case GCPBackend: @@ -80,8 +81,8 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { // GetRepoIdentifier returns the string to be used as the value of the --repo flag in // restic commands for the given repository. -func GetRepoIdentifier(config arkv1api.ObjectStorageProviderConfig, name string) string { - prefix := getRepoPrefix(config) +func GetRepoIdentifier(location *arkv1api.BackupStorageLocation, name string) string { + prefix := getRepoPrefix(location) return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name) } diff --git a/pkg/restic/config_test.go b/pkg/restic/config_test.go index 9130b717c..982c1dca1 100644 --- a/pkg/restic/config_test.go +++ b/pkg/restic/config_test.go @@ -30,47 +30,60 @@ func TestGetRepoIdentifier(t *testing.T) { getAWSBucketRegion = func(string) (string, error) { return "", errors.New("no region found") } - config := arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"}, - ResticLocation: "bucket/prefix", + + backupLocation := &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "aws", + Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + }, } - assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1")) + assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) // stub implementation of getAWSBucketRegion getAWSBucketRegion = func(string) (string, error) { return "us-west-2", nil } - config = arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"}, - ResticLocation: "bucket", - } - assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/repo-1", GetRepoIdentifier(config, "repo-1")) - - config = arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"}, - ResticLocation: "bucket/prefix", - } - assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1")) - - config = arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{ - Name: "aws", - Config: map[string]string{"s3Url": "alternate-url"}, + backupLocation = &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "aws", + Config: map[string]string{ResticLocationConfigKey: "bucket"}, }, - ResticLocation: "bucket/prefix", } - assert.Equal(t, "s3:alternate-url/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1")) + assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) - config = arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "azure"}, - ResticLocation: "bucket/prefix", + backupLocation = &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "aws", + Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + }, } - assert.Equal(t, "azure:bucket:/prefix/repo-1", GetRepoIdentifier(config, "repo-1")) + assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) - config = arkv1api.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "gcp"}, - ResticLocation: "bucket-2/prefix-2", + backupLocation = &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "aws", + Config: map[string]string{ + ResticLocationConfigKey: "bucket/prefix", + "s3Url": "alternate-url", + }, + }, } - assert.Equal(t, "gs:bucket-2:/prefix-2/repo-2", GetRepoIdentifier(config, "repo-2")) + assert.Equal(t, "s3:alternate-url/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + + backupLocation = &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "azure", + Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + }, + } + assert.Equal(t, "azure:bucket:/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + + backupLocation = &arkv1api.BackupStorageLocation{ + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "gcp", + Config: map[string]string{ResticLocationConfigKey: "bucket-2/prefix-2"}, + }, + } + assert.Equal(t, "gs:bucket-2:/prefix-2/repo-2", GetRepoIdentifier(backupLocation, "repo-2")) } From 2750aa71b9c7d75577027c2d5b71cbaa07191c0c Mon Sep 17 00:00:00 2001 From: Carlisia Date: Tue, 14 Aug 2018 07:28:11 -0700 Subject: [PATCH 16/29] Use backup storage location during restore Closes #740 Signed-off-by: Carlisia --- pkg/cmd/server/server.go | 4 +- pkg/controller/restore_controller.go | 272 +++++++++++------- pkg/controller/restore_controller_test.go | 130 ++++++--- pkg/util/test/test_backup_storage_location.go | 68 +++++ 4 files changed, 326 insertions(+), 148 deletions(-) create mode 100644 pkg/util/test/test_backup_storage_location.go diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index f3c6c1ea9..32c20b6eb 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -723,13 +723,13 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.arkClient.ArkV1(), s.arkClient.ArkV1(), restorer, - config.BackupStorageProvider.CloudProviderConfig, - config.BackupStorageProvider.Bucket, s.sharedInformerFactory.Ark().V1().Backups(), + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.blockStore != nil, s.logger, s.logLevel, s.pluginRegistry, + s.defaultBackupLocation, s.metrics, ) diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 7746b378e..4fe2842b0 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -71,23 +71,24 @@ var nonRestorableResources = []string{ } type restoreController struct { - namespace string - restoreClient arkv1client.RestoresGetter - backupClient arkv1client.BackupsGetter - restorer restore.Restorer - objectStoreConfig api.CloudProviderConfig - bucket string - pvProviderExists bool - backupLister listers.BackupLister - backupListerSynced cache.InformerSynced - restoreLister listers.RestoreLister - restoreListerSynced cache.InformerSynced - syncHandler func(restoreName string) error - queue workqueue.RateLimitingInterface - logger logrus.FieldLogger - logLevel logrus.Level - pluginRegistry plugin.Registry - metrics *metrics.ServerMetrics + namespace string + restoreClient arkv1client.RestoresGetter + backupClient arkv1client.BackupsGetter + restorer restore.Restorer + pvProviderExists bool + backupLister listers.BackupLister + backupListerSynced cache.InformerSynced + restoreLister listers.RestoreLister + restoreListerSynced cache.InformerSynced + backupLocationLister listers.BackupStorageLocationLister + backupLocationListerSynced cache.InformerSynced + syncHandler func(restoreName string) error + queue workqueue.RateLimitingInterface + logger logrus.FieldLogger + logLevel logrus.Level + pluginRegistry plugin.Registry + defaultBackupLocation string + metrics *metrics.ServerMetrics getBackup cloudprovider.GetBackupFunc downloadBackup cloudprovider.DownloadBackupFunc @@ -102,33 +103,34 @@ func NewRestoreController( restoreClient arkv1client.RestoresGetter, backupClient arkv1client.BackupsGetter, restorer restore.Restorer, - objectStoreConfig api.CloudProviderConfig, - bucket string, backupInformer informers.BackupInformer, + backupLocationInformer informers.BackupStorageLocationInformer, pvProviderExists bool, logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry, + defaultBackupLocation string, metrics *metrics.ServerMetrics, ) Interface { c := &restoreController{ - namespace: namespace, - restoreClient: restoreClient, - backupClient: backupClient, - restorer: restorer, - objectStoreConfig: objectStoreConfig, - bucket: bucket, - pvProviderExists: pvProviderExists, - backupLister: backupInformer.Lister(), - backupListerSynced: backupInformer.Informer().HasSynced, - restoreLister: restoreInformer.Lister(), - restoreListerSynced: restoreInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"), - logger: logger, - logLevel: logLevel, - pluginRegistry: pluginRegistry, - metrics: metrics, + namespace: namespace, + restoreClient: restoreClient, + backupClient: backupClient, + restorer: restorer, + pvProviderExists: pvProviderExists, + backupLister: backupInformer.Lister(), + backupListerSynced: backupInformer.Informer().HasSynced, + restoreLister: restoreInformer.Lister(), + restoreListerSynced: restoreInformer.Informer().HasSynced, + backupLocationLister: backupLocationInformer.Lister(), + backupLocationListerSynced: backupLocationInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"), + logger: logger, + logLevel: logLevel, + pluginRegistry: pluginRegistry, + defaultBackupLocation: defaultBackupLocation, + metrics: metrics, getBackup: cloudprovider.GetBackup, downloadBackup: cloudprovider.DownloadBackup, @@ -193,7 +195,7 @@ func (c *restoreController) Run(ctx context.Context, numWorkers int) error { defer c.logger.Info("Shutting down RestoreController") c.logger.Info("Waiting for caches to sync") - if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced) { + if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced, c.backupLocationListerSynced) { return errors.New("timed out waiting for caches to sync") } c.logger.Info("Caches are synced") @@ -283,28 +285,25 @@ func (c *restoreController) processRestore(key string) error { pluginManager := c.newPluginManager(logContext, logContext.Level, c.pluginRegistry) defer pluginManager.CleanupClients() - objectStore, err := getObjectStore(c.objectStoreConfig, pluginManager) - if err != nil { - return errors.Wrap(err, "error initializing object store") - } - actions, err := pluginManager.GetRestoreItemActions() if err != nil { return errors.Wrap(err, "error initializing restore item actions") } - // complete & validate restore - if restore.Status.ValidationErrors = c.completeAndValidate(objectStore, restore); len(restore.Status.ValidationErrors) > 0 { - restore.Status.Phase = api.RestorePhaseFailedValidation - } else { - restore.Status.Phase = api.RestorePhaseInProgress - } - + // validate the restore and fetch the backup + info := c.validateAndComplete(restore, pluginManager) backupScheduleName := restore.Spec.ScheduleName // Register attempts after validation so we don't have to fetch the backup multiple times c.metrics.RegisterRestoreAttempt(backupScheduleName) - // update status + if len(restore.Status.ValidationErrors) > 0 { + restore.Status.Phase = api.RestorePhaseFailedValidation + c.metrics.RegisterRestoreValidationFailed(backupScheduleName) + } else { + restore.Status.Phase = api.RestorePhaseInProgress + } + + // patch to update status and persist to API updatedRestore, err := patchRestore(original, restore, c.restoreClient) if err != nil { return errors.Wrapf(err, "error updating Restore phase to %s", restore.Status.Phase) @@ -314,15 +313,16 @@ func (c *restoreController) processRestore(key string) error { restore = updatedRestore.DeepCopy() if restore.Status.Phase == api.RestorePhaseFailedValidation { - c.metrics.RegisterRestoreValidationFailed(backupScheduleName) return nil } + logContext.Debug("Running restore") + // execution & upload of restore restoreWarnings, restoreErrors, restoreFailure := c.runRestore( restore, actions, - objectStore, + info, ) restore.Status.Warnings = len(restoreWarnings.Ark) + len(restoreWarnings.Cluster) @@ -355,7 +355,13 @@ func (c *restoreController) processRestore(key string) error { return nil } -func (c *restoreController) completeAndValidate(objectStore cloudprovider.ObjectStore, restore *api.Restore) []string { +type backupInfo struct { + bucketName string + backup *api.Backup + objectStore cloudprovider.ObjectStore +} + +func (c *restoreController) validateAndComplete(restore *api.Restore, pluginManager plugin.Manager) backupInfo { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { @@ -363,34 +369,34 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object restore.Spec.ExcludedResources = append(restore.Spec.ExcludedResources, nonrestorable) } } - var validationErrors []string // validate that included resources don't contain any non-restorable resources includedResources := sets.NewString(restore.Spec.IncludedResources...) for _, nonRestorableResource := range nonRestorableResources { if includedResources.Has(nonRestorableResource) { - validationErrors = append(validationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource)) } } // validate included/excluded resources for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedResources, restore.Spec.ExcludedResources) { - validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) } // validate included/excluded namespaces for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedNamespaces, restore.Spec.ExcludedNamespaces) { - validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } // validate that PV provider exists if we're restoring PVs if boolptr.IsSetToTrue(restore.Spec.RestorePVs) && !c.pvProviderExists { - validationErrors = append(validationErrors, "Server is not configured for PV snapshot restores") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Server is not configured for PV snapshot restores") } // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { - return append(validationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") + return backupInfo{} } // if ScheduleName is specified, fill in BackupName with the most recent successful backup from @@ -402,33 +408,33 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object backups, err := c.backupLister.Backups(c.namespace).List(selector) if err != nil { - return append(validationErrors, "Unable to list backups for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") + return backupInfo{} } if len(backups) == 0 { - return append(validationErrors, "No backups found for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") } if backup := mostRecentCompletedBackup(backups); backup != nil { restore.Spec.BackupName = backup.Name } else { - return append(validationErrors, "No completed backups found for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") + return backupInfo{} } } - var ( - backup *api.Backup - err error - ) - if backup, err = c.fetchBackup(objectStore, restore.Spec.BackupName); err != nil { - return append(validationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) + info, err := c.fetchBackupInfo(restore.Spec.BackupName, pluginManager) + if err != nil { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) + return backupInfo{} } // Fill in the ScheduleName so it's easier to consume for metrics. if restore.Spec.ScheduleName == "" { - restore.Spec.ScheduleName = backup.GetLabels()["ark-schedule"] + restore.Spec.ScheduleName = info.backup.GetLabels()["ark-schedule"] } - return validationErrors + return info } // backupXorScheduleProvided returns true if exactly one of BackupName and @@ -462,43 +468,110 @@ func mostRecentCompletedBackup(backups []*api.Backup) *api.Backup { return nil } -func (c *restoreController) fetchBackup(objectStore cloudprovider.ObjectStore, name string) (*api.Backup, error) { - backup, err := c.backupLister.Backups(c.namespace).Get(name) - if err == nil { - return backup, nil - } - - if !apierrors.IsNotFound(err) { - return nil, errors.WithStack(err) - } - - logContext := c.logger.WithField("backupName", name) - - logContext.Debug("Backup not found in backupLister, checking object storage directly") - backup, err = c.getBackup(objectStore, c.bucket, name) +// fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't +// find it, it tries to retrieve it from one of the backup storage locations. +func (c *restoreController) fetchBackupInfo(backupName string, pluginManager plugin.Manager) (backupInfo, error) { + var info backupInfo + var err error + info.backup, err = c.backupLister.Backups(c.namespace).Get(backupName) if err != nil { - return nil, err + if !apierrors.IsNotFound(err) { + return backupInfo{}, errors.WithStack(err) + } + + logContext := c.logger.WithField("backupName", backupName) + logContext.Debug("Backup not found in backupLister, checking each backup location directly, starting with default...") + return c.fetchFromBackupStorage(backupName, pluginManager) + } + + location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(info.backup.Spec.StorageLocation) + if err != nil { + return backupInfo{}, errors.WithStack(err) + } + + info.objectStore, err = getObjectStoreForLocation(location, pluginManager) + if err != nil { + return backupInfo{}, errors.Wrap(err, "error initializing object store") + } + info.bucketName = location.Spec.ObjectStorage.Bucket + + return info, nil +} + +// fetchFromBackupStorage checks each backup storage location, starting with the default, +// looking for a backup that matches the given backup name. +func (c *restoreController) fetchFromBackupStorage(backupName string, pluginManager plugin.Manager) (backupInfo, error) { + locations, err := c.backupLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything()) + if err != nil { + return backupInfo{}, errors.WithStack(err) + } + + orderedLocations := orderedBackupLocations(locations, c.defaultBackupLocation) + + logContext := c.logger.WithField("backupName", backupName) + for _, location := range orderedLocations { + info, err := c.backupInfoForLocation(location, backupName, pluginManager) + if err != nil { + logContext.WithField("locationName", location.Name).WithError(err).Error("Unable to fetch backup from object storage location") + continue + } + return info, nil + } + + return backupInfo{}, errors.New("not able to fetch from backup storage") +} + +func orderedBackupLocations(locations []*api.BackupStorageLocation, defaultLocationName string) []*api.BackupStorageLocation { + var result []*api.BackupStorageLocation + + for i := range locations { + if locations[i].Name == defaultLocationName { + // put the default location first + result = append(result, locations[i]) + // append everything before the default + result = append(result, locations[:i]...) + // append everything after the default + result = append(result, locations[i+1:]...) + + return result + } + } + + return locations +} + +func (c *restoreController) backupInfoForLocation(location *api.BackupStorageLocation, backupName string, pluginManager plugin.Manager) (backupInfo, error) { + objectStore, err := getObjectStoreForLocation(location, pluginManager) + if err != nil { + return backupInfo{}, err + } + + backup, err := c.getBackup(objectStore, location.Spec.ObjectStorage.Bucket, backupName) + if err != nil { + return backupInfo{}, err } // ResourceVersion needs to be cleared in order to create the object in the API backup.ResourceVersion = "" - // Clear out the namespace too, just in case + // Clear out the namespace, in case the backup was made in a different cluster, with a different namespace backup.Namespace = "" - created, createErr := c.backupClient.Backups(c.namespace).Create(backup) - if createErr != nil { - logContext.WithError(errors.WithStack(createErr)).Error("Unable to create API object for Backup") - } else { - backup = created + backupCreated, err := c.backupClient.Backups(c.namespace).Create(backup) + if err != nil { + return backupInfo{}, errors.WithStack(err) } - return backup, nil + return backupInfo{ + bucketName: location.Spec.ObjectStorage.Bucket, + backup: backupCreated, + objectStore: objectStore, + }, nil } func (c *restoreController) runRestore( restore *api.Restore, actions []restore.ItemAction, - objectStore cloudprovider.ObjectStore, + info backupInfo, ) (restoreWarnings, restoreErrors api.RestoreResult, restoreFailure error) { logFile, err := ioutil.TempFile("", "") if err != nil { @@ -530,14 +603,7 @@ func (c *restoreController) runRestore( "backup": restore.Spec.BackupName, }) - backup, err := c.fetchBackup(objectStore, restore.Spec.BackupName) - if err != nil { - logContext.WithError(err).Error("Error getting backup") - restoreErrors.Ark = append(restoreErrors.Ark, err.Error()) - return - } - - backupFile, err := downloadToTempFile(objectStore, c.bucket, restore.Spec.BackupName, c.downloadBackup, c.logger) + backupFile, err := downloadToTempFile(info.objectStore, info.bucketName, restore.Spec.BackupName, c.downloadBackup, c.logger) if err != nil { logContext.WithError(err).Error("Error downloading backup") restoreErrors.Ark = append(restoreErrors.Ark, err.Error()) @@ -558,7 +624,7 @@ func (c *restoreController) runRestore( // Any return statement above this line means a total restore failure // Some failures after this line *may* be a total restore failure logContext.Info("starting restore") - restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, backup, backupFile, actions) + restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, info.backup, backupFile, actions) logContext.Info("restore completed") // Try to upload the log file. This is best-effort. If we fail, we'll add to the ark errors. @@ -571,7 +637,7 @@ func (c *restoreController) runRestore( return } - if err := c.uploadRestoreLog(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, logFile); err != nil { + if err := c.uploadRestoreLog(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, logFile); err != nil { restoreErrors.Ark = append(restoreErrors.Ark, fmt.Sprintf("error uploading log file to object storage: %v", err)) } @@ -592,7 +658,7 @@ func (c *restoreController) runRestore( logContext.WithError(errors.WithStack(err)).Error("Error resetting results file offset to 0") return } - if err := c.uploadRestoreResults(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, resultsFile); err != nil { + if err := c.uploadRestoreResults(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, resultsFile); err != nil { logContext.WithError(errors.WithStack(err)).Error("Error uploading results files to object storage") } diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index a618e2ee8..b67d423e2 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -19,17 +19,16 @@ package controller import ( "bytes" "encoding/json" - "errors" "io" "io/ioutil" "testing" "time" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" core "k8s.io/client-go/testing" @@ -47,10 +46,11 @@ import ( arktest "github.com/heptio/ark/pkg/util/test" ) -func TestFetchBackup(t *testing.T) { +func TestFetchBackupInfo(t *testing.T) { tests := []struct { name string backupName string + informerLocations []*api.BackupStorageLocation informerBackups []*api.Backup backupServiceBackup *api.Backup backupServiceError error @@ -58,16 +58,19 @@ func TestFetchBackup(t *testing.T) { expectedErr bool }{ { - name: "lister has backup", - backupName: "backup-1", - informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").Backup}, - expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup, + name: "lister has backup", + backupName: "backup-1", + informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation}, + informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup}, + expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, { - name: "backupSvc has backup", + name: "lister does not have a backup, but backupSvc does", backupName: "backup-1", - backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").Backup, - expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup, + backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, + informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation}, + informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup}, + expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, { name: "no backup", @@ -84,26 +87,43 @@ func TestFetchBackup(t *testing.T) { restorer = &fakeRestorer{} sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = arktest.NewLogger() + pluginManager = &pluginmocks.Manager{} + objectStore = &arktest.ObjectStore{} ) + defer restorer.AssertExpectations(t) + defer objectStore.AssertExpectations(t) + c := NewRestoreController( api.DefaultNamespace, sharedInformers.Ark().V1().Restores(), client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, logger, logrus.InfoLevel, nil, //pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) + c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { + return pluginManager + } - for _, itm := range test.informerBackups { - sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm) + if test.backupServiceError == nil { + pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil) + objectStore.On("Init", mock.Anything).Return(nil) + + for _, itm := range test.informerLocations { + sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(itm) + } + + for _, itm := range test.informerBackups { + sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm) + } } if test.backupServiceBackup != nil || test.backupServiceError != nil { @@ -114,10 +134,10 @@ func TestFetchBackup(t *testing.T) { } } - backup, err := c.fetchBackup(nil, test.backupName) + info, err := c.fetchBackupInfo(test.backupName, pluginManager) if assert.Equal(t, test.expectedErr, err != nil) { - assert.Equal(t, test.expectedRes, backup) + assert.Equal(t, test.expectedRes, info.backup) } }) } @@ -175,13 +195,13 @@ func TestProcessRestoreSkips(t *testing.T) { client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, // pvProviderExists logger, logrus.InfoLevel, nil, // pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { @@ -197,10 +217,12 @@ func TestProcessRestoreSkips(t *testing.T) { }) } } + func TestProcessRestore(t *testing.T) { tests := []struct { name string restoreKey string + location *api.BackupStorageLocation restore *api.Restore backup *api.Backup restorerError error @@ -217,16 +239,18 @@ func TestProcessRestore(t *testing.T) { }{ { name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1"}, }, { name: "restore with resource in both includedResources and excludedResources fails validation", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource"}, @@ -246,11 +270,13 @@ func TestProcessRestore(t *testing.T) { expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { - name: "valid restore with schedule name gets executed", - restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore, + name: "valid restore with schedule name gets executed", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, + restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore, backup: arktest. NewTestBackup(). WithName("backup-1"). + WithStorageLocation("default"). WithLabel("ark-schedule", "sched-1"). WithPhase(api.BackupPhaseCompleted). Backup, @@ -263,13 +289,14 @@ func TestProcessRestore(t *testing.T) { restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Restore, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), - expectedValidationErrors: []string{"Error retrieving backup: no backup here"}, + expectedValidationErrors: []string{"Error retrieving backup: not able to fetch from backup storage"}, backupServiceGetBackupError: errors.New("no backup here"), }, { name: "restorer throwing an error causes the restore to fail", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, restorerError: errors.New("blarg"), expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), @@ -278,16 +305,18 @@ func TestProcessRestore(t *testing.T) { }, { name: "valid restore gets executed", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Restore, }, { name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, allowRestoreSnapshots: true, expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), @@ -295,16 +324,18 @@ func TestProcessRestore(t *testing.T) { }, { name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Server is not configured for PV snapshot restores"}, }, { name: "restoration of nodes is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -314,8 +345,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of events is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -325,8 +357,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of events.events.k8s.io is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -336,8 +369,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of backups.ark.heptio.com is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.ark.heptio.com", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -347,8 +381,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of restores.ark.heptio.com is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.ark.heptio.com", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -358,11 +393,12 @@ func TestProcessRestore(t *testing.T) { }, { name: "backup download error results in failed restore", - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, + restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, expectedPhase: string(api.RestorePhaseInProgress), expectedFinalPhase: string(api.RestorePhaseFailed), backupServiceDownloadBackupError: errors.New("Couldn't download backup"), - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, } @@ -377,6 +413,7 @@ func TestProcessRestore(t *testing.T) { objectStore = &arktest.ObjectStore{} ) defer restorer.AssertExpectations(t) + defer objectStore.AssertExpectations(t) c := NewRestoreController( @@ -385,23 +422,29 @@ func TestProcessRestore(t *testing.T) { client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), test.allowRestoreSnapshots, logger, logrus.InfoLevel, nil, // pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { return pluginManager } - if test.restore != nil { + if test.location != nil { + sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(test.location) + } + if test.backup != nil { + sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup) pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil) objectStore.On("Init", mock.Anything).Return(nil) + } + if test.restore != nil { sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(test.restore) // this is necessary so the Patch() call returns the appropriate object @@ -590,11 +633,12 @@ func TestProcessRestore(t *testing.T) { } } -func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { +func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { var ( client = fake.NewSimpleClientset() sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = arktest.NewLogger() + pluginManager = &pluginmocks.Manager{} ) c := NewRestoreController( @@ -603,13 +647,13 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { client.ArkV1(), client.ArkV1(), nil, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, logger, logrus.DebugLevel, nil, + "default", nil, ).(*restoreController) @@ -632,7 +676,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs := c.completeAndValidate(nil, restore) + errs := c.validateAndComplete(restore, pluginManager) assert.Equal(t, []string{"No backups found for schedule"}, errs) assert.Empty(t, restore.Spec.BackupName) @@ -645,7 +689,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs = c.completeAndValidate(nil, restore) + errs = c.validateAndComplete(restore, pluginManager) assert.Equal(t, []string{"No completed backups found for schedule"}, errs) assert.Empty(t, restore.Spec.BackupName) @@ -669,7 +713,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs = c.completeAndValidate(nil, restore) + errs = c.validateAndComplete(restore, pluginManager) assert.Nil(t, errs) assert.Equal(t, "bar", restore.Spec.BackupName) } diff --git a/pkg/util/test/test_backup_storage_location.go b/pkg/util/test/test_backup_storage_location.go new file mode 100644 index 000000000..d18d7c428 --- /dev/null +++ b/pkg/util/test/test_backup_storage_location.go @@ -0,0 +1,68 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/heptio/ark/pkg/apis/ark/v1" +) + +type TestBackupStorageLocation struct { + *v1.BackupStorageLocation +} + +func NewTestBackupStorageLocation() *TestBackupStorageLocation { + return &TestBackupStorageLocation{ + BackupStorageLocation: &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.DefaultNamespace, + }, + }, + } +} + +func (b *TestBackupStorageLocation) WithNamespace(namespace string) *TestBackupStorageLocation { + b.Namespace = namespace + return b +} + +func (b *TestBackupStorageLocation) WithName(name string) *TestBackupStorageLocation { + b.Name = name + return b +} + +func (b *TestBackupStorageLocation) WithLabel(key, value string) *TestBackupStorageLocation { + if b.Labels == nil { + b.Labels = make(map[string]string) + } + b.Labels[key] = value + return b +} + +func (b *TestBackupStorageLocation) WithProvider(name string) *TestBackupStorageLocation { + b.Spec.Provider = name + return b +} + +func (b *TestBackupStorageLocation) WithObjectStorage(bucketName string) *TestBackupStorageLocation { + if b.Spec.StorageType.ObjectStorage == nil { + b.Spec.StorageType.ObjectStorage = &v1.ObjectStorageLocation{} + } + b.Spec.ObjectStorage.Bucket = bucketName + return b +} From 0e94fa37f97de8e6df5c7238503cd5d9d7481d66 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Tue, 21 Aug 2018 16:52:49 -0700 Subject: [PATCH 17/29] update sync controller for backup locations Signed-off-by: Steve Kriss --- pkg/apis/ark/v1/labels_annotations.go | 4 + pkg/cloudprovider/backup_cache.go | 97 ----- pkg/cloudprovider/backup_cache_test.go | 170 -------- pkg/cloudprovider/backup_service.go | 16 - pkg/cmd/server/server.go | 20 +- pkg/controller/backup_controller.go | 6 + pkg/controller/backup_controller_test.go | 21 +- pkg/controller/backup_sync_controller.go | 190 +++++---- pkg/controller/backup_sync_controller_test.go | 393 +++++++++++------- 9 files changed, 382 insertions(+), 535 deletions(-) delete mode 100644 pkg/cloudprovider/backup_cache.go delete mode 100644 pkg/cloudprovider/backup_cache_test.go diff --git a/pkg/apis/ark/v1/labels_annotations.go b/pkg/apis/ark/v1/labels_annotations.go index 00b016a02..300344c5f 100644 --- a/pkg/apis/ark/v1/labels_annotations.go +++ b/pkg/apis/ark/v1/labels_annotations.go @@ -36,4 +36,8 @@ const ( // a backup/restore-specific timeout value for pod volume operations (i.e. // restic backups/restores). PodVolumeOperationTimeoutAnnotation = "ark.heptio.com/pod-volume-timeout" + + // StorageLocationLabel is the label key used to identify the storage + // location of a backup. + StorageLocationLabel = "ark.heptio.com/storage-location" ) diff --git a/pkg/cloudprovider/backup_cache.go b/pkg/cloudprovider/backup_cache.go deleted file mode 100644 index 06abc99fb..000000000 --- a/pkg/cloudprovider/backup_cache.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2017 the Heptio Ark contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cloudprovider - -import ( - "context" - "sync" - "time" - - "github.com/sirupsen/logrus" - - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/heptio/ark/pkg/apis/ark/v1" -) - -// backupCacheBucket holds the backups and error from a ListBackups call. -type backupCacheBucket struct { - backups []*v1.Backup - error error -} - -// backupCache caches ListBackups calls, refreshing them periodically. -type backupCache struct { - delegate BackupLister - lock sync.RWMutex - // This doesn't really need to be a map right now, but if we ever move to supporting multiple - // buckets, this will be ready for it. - buckets map[string]*backupCacheBucket - logger logrus.FieldLogger -} - -var _ BackupLister = &backupCache{} - -// NewBackupCache returns a new backup cache that refreshes from delegate every resyncPeriod. -func NewBackupCache(ctx context.Context, delegate BackupLister, resyncPeriod time.Duration, logger logrus.FieldLogger) BackupLister { - c := &backupCache{ - delegate: delegate, - buckets: make(map[string]*backupCacheBucket), - logger: logger, - } - - // Start the goroutine to refresh all buckets every resyncPeriod. This stops when ctx.Done() is - // available. - go wait.Until(c.refresh, resyncPeriod, ctx.Done()) - - return c -} - -// refresh refreshes all the buckets currently in the cache by doing a live lookup via c.delegate. -func (c *backupCache) refresh() { - c.lock.Lock() - defer c.lock.Unlock() - - c.logger.Debug("refreshing all cached backup lists from object storage") - - for bucketName, bucket := range c.buckets { - c.logger.WithField("bucket", bucketName).Debug("Refreshing bucket") - bucket.backups, bucket.error = c.delegate.ListBackups(bucketName) - } -} - -func (c *backupCache) ListBackups(bucketName string) ([]*v1.Backup, error) { - c.lock.RLock() - bucket, found := c.buckets[bucketName] - c.lock.RUnlock() - - logContext := c.logger.WithField("bucket", bucketName) - - if found { - logContext.Debug("Returning cached backup list") - return bucket.backups, bucket.error - } - - logContext.Debug("Bucket is not in cache - doing a live lookup") - - backups, err := c.delegate.ListBackups(bucketName) - c.lock.Lock() - c.buckets[bucketName] = &backupCacheBucket{backups: backups, error: err} - c.lock.Unlock() - - return backups, err -} diff --git a/pkg/cloudprovider/backup_cache_test.go b/pkg/cloudprovider/backup_cache_test.go deleted file mode 100644 index 4eec9481b..000000000 --- a/pkg/cloudprovider/backup_cache_test.go +++ /dev/null @@ -1,170 +0,0 @@ -/* -Copyright 2017 the Heptio Ark contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cloudprovider - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/heptio/ark/pkg/apis/ark/v1" - cloudprovidermocks "github.com/heptio/ark/pkg/cloudprovider/mocks" - "github.com/heptio/ark/pkg/util/test" -) - -func TestNewBackupCache(t *testing.T) { - var ( - delegate = &cloudprovidermocks.BackupLister{} - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) - logger = test.NewLogger() - ) - defer cancel() - - c := NewBackupCache(ctx, delegate, 100*time.Millisecond, logger) - - // nothing in cache, live lookup - bucket1 := []*v1.Backup{ - test.NewTestBackup().WithName("backup1").Backup, - test.NewTestBackup().WithName("backup2").Backup, - } - delegate.On("ListBackups", "bucket1").Return(bucket1, nil).Once() - - // should be updated via refresh - updatedBucket1 := []*v1.Backup{ - test.NewTestBackup().WithName("backup2").Backup, - } - delegate.On("ListBackups", "bucket1").Return(updatedBucket1, nil) - - // nothing in cache, live lookup - bucket2 := []*v1.Backup{ - test.NewTestBackup().WithName("backup5").Backup, - test.NewTestBackup().WithName("backup6").Backup, - } - delegate.On("ListBackups", "bucket2").Return(bucket2, nil).Once() - - // should be updated via refresh - updatedBucket2 := []*v1.Backup{ - test.NewTestBackup().WithName("backup7").Backup, - } - delegate.On("ListBackups", "bucket2").Return(updatedBucket2, nil) - - backups, err := c.ListBackups("bucket1") - assert.Equal(t, bucket1, backups) - assert.NoError(t, err) - - backups, err = c.ListBackups("bucket2") - assert.Equal(t, bucket2, backups) - assert.NoError(t, err) - - var done1, done2 bool - for { - select { - case <-ctx.Done(): - t.Fatal("timed out") - default: - if done1 && done2 { - return - } - } - - backups, err = c.ListBackups("bucket1") - if len(backups) == 1 { - if assert.Equal(t, updatedBucket1[0], backups[0]) { - done1 = true - } - } - - backups, err = c.ListBackups("bucket2") - if len(backups) == 1 { - if assert.Equal(t, updatedBucket2[0], backups[0]) { - done2 = true - } - } - time.Sleep(100 * time.Millisecond) - } -} - -func TestBackupCacheRefresh(t *testing.T) { - var ( - delegate = &cloudprovidermocks.BackupLister{} - logger = test.NewLogger() - ) - - c := &backupCache{ - delegate: delegate, - buckets: map[string]*backupCacheBucket{ - "bucket1": {}, - "bucket2": {}, - }, - logger: logger, - } - - bucket1 := []*v1.Backup{ - test.NewTestBackup().WithName("backup1").Backup, - test.NewTestBackup().WithName("backup2").Backup, - } - delegate.On("ListBackups", "bucket1").Return(bucket1, nil) - - delegate.On("ListBackups", "bucket2").Return(nil, errors.New("bad")) - - c.refresh() - - assert.Equal(t, bucket1, c.buckets["bucket1"].backups) - assert.NoError(t, c.buckets["bucket1"].error) - - assert.Empty(t, c.buckets["bucket2"].backups) - assert.EqualError(t, c.buckets["bucket2"].error, "bad") -} - -func TestBackupCacheGetAllBackupsUsesCacheIfPresent(t *testing.T) { - var ( - delegate = &cloudprovidermocks.BackupLister{} - logger = test.NewLogger() - bucket1 = []*v1.Backup{ - test.NewTestBackup().WithName("backup1").Backup, - test.NewTestBackup().WithName("backup2").Backup, - } - ) - - c := &backupCache{ - delegate: delegate, - buckets: map[string]*backupCacheBucket{ - "bucket1": { - backups: bucket1, - }, - }, - logger: logger, - } - - bucket2 := []*v1.Backup{ - test.NewTestBackup().WithName("backup3").Backup, - test.NewTestBackup().WithName("backup4").Backup, - } - - delegate.On("ListBackups", "bucket2").Return(bucket2, nil) - - backups, err := c.ListBackups("bucket1") - assert.Equal(t, bucket1, backups) - assert.NoError(t, err) - - backups, err = c.ListBackups("bucket2") - assert.Equal(t, bucket2, backups) - assert.NoError(t, err) -} diff --git a/pkg/cloudprovider/backup_service.go b/pkg/cloudprovider/backup_service.go index 49aaebefc..e17859af6 100644 --- a/pkg/cloudprovider/backup_service.go +++ b/pkg/cloudprovider/backup_service.go @@ -147,22 +147,6 @@ func DownloadBackup(objectStore ObjectStore, bucket, backupName string) (io.Read return objectStore.GetObject(bucket, getBackupContentsKey(backupName, backupName)) } -type liveBackupLister struct { - logger logrus.FieldLogger - objectStore ObjectStore -} - -func NewLiveBackupLister(logger logrus.FieldLogger, objectStore ObjectStore) BackupLister { - return &liveBackupLister{ - logger: logger, - objectStore: objectStore, - } -} - -func (l *liveBackupLister) ListBackups(bucket string) ([]*api.Backup, error) { - return ListBackups(l.logger, l.objectStore, bucket) -} - func ListBackups(logger logrus.FieldLogger, objectStore ObjectStore, bucket string) ([]*api.Backup, error) { prefixes, err := objectStore.ListCommonPrefixes(bucket, "/") if err != nil { diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 32c20b6eb..ec8f4d3cb 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -589,12 +589,6 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B ctx := s.ctx var wg sync.WaitGroup - cloudBackupCacheResyncPeriod := durationMin(controller.GCSyncPeriod, s.config.backupSyncPeriod) - s.logger.Infof("Caching cloud backups every %s", cloudBackupCacheResyncPeriod) - - liveBackupLister := cloudprovider.NewLiveBackupLister(s.logger, s.objectStore) - cachedBackupLister := cloudprovider.NewBackupCache(ctx, liveBackupLister, cloudBackupCacheResyncPeriod, s.logger) - go func() { metricsMux := http.NewServeMux() metricsMux.Handle("/metrics", promhttp.Handler()) @@ -608,12 +602,13 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B backupSyncController := controller.NewBackupSyncController( s.arkClient.ArkV1(), - cachedBackupLister, - config.BackupStorageProvider.Bucket, + s.sharedInformerFactory.Ark().V1().Backups(), + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.config.backupSyncPeriod, s.namespace, - s.sharedInformerFactory.Ark().V1().Backups(), + s.pluginRegistry, s.logger, + s.logLevel, ) wg.Add(1) go func() { @@ -775,7 +770,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B // SHARED INFORMERS HAVE TO BE STARTED AFTER ALL CONTROLLERS go s.sharedInformerFactory.Start(ctx.Done()) - // Remove this sometime after v0.8.0 + // TODO(1.0): remove cache.WaitForCacheSync(ctx.Done(), s.sharedInformerFactory.Ark().V1().Backups().Informer().HasSynced) s.removeDeprecatedGCFinalizer() @@ -789,9 +784,10 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B return nil } -const gcFinalizer = "gc.ark.heptio.com" - +// TODO(1.0): remove func (s *server) removeDeprecatedGCFinalizer() { + const gcFinalizer = "gc.ark.heptio.com" + backups, err := s.sharedInformerFactory.Ark().V1().Backups().Lister().List(labels.Everything()) if err != nil { s.logger.WithError(errors.WithStack(err)).Error("error listing backups from cache - unable to remove old finalizers") diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 88f62fbe7..364331734 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -349,6 +349,12 @@ func (controller *backupController) getLocationAndValidate(itm *api.Backup, defa itm.Spec.StorageLocation = defaultBackupLocation } + // add the storage location as a label for easy filtering later. + if itm.Labels == nil { + itm.Labels = make(map[string]string) + } + itm.Labels[api.StorageLocationLabel] = itm.Spec.StorageLocation + var backupLocation *api.BackupStorageLocation backupLocation, err := controller.backupLocationLister.BackupStorageLocations(itm.Namespace).Get(itm.Spec.StorageLocation) if err != nil { diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 49d9c6876..9a41d6d29 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -25,13 +25,12 @@ import ( "testing" "time" - "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" core "k8s.io/client-go/testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -364,10 +363,14 @@ func TestProcessBackup(t *testing.T) { type SpecPatch struct { StorageLocation string `json:"storageLocation"` } + type ObjectMetaPatch struct { + Labels map[string]string `json:"labels"` + } type Patch struct { - Status StatusPatch `json:"status"` - Spec SpecPatch `json:"spec,omitempty"` + Status StatusPatch `json:"status"` + Spec SpecPatch `json:"spec,omitempty"` + ObjectMeta ObjectMetaPatch `json:"metadata,omitempty"` } decode := func(decoder *json.Decoder) (interface{}, error) { @@ -389,6 +392,11 @@ func TestProcessBackup(t *testing.T) { Spec: SpecPatch{ StorageLocation: "default", }, + ObjectMeta: ObjectMetaPatch{ + Labels: map[string]string{ + v1.StorageLocationLabel: "default", + }, + }, } } else { expected = Patch{ @@ -397,6 +405,11 @@ func TestProcessBackup(t *testing.T) { Phase: v1.BackupPhaseInProgress, Expiration: expiration, }, + ObjectMeta: ObjectMetaPatch{ + Labels: map[string]string{ + v1.StorageLocationLabel: test.backup.Spec.StorageLocation, + }, + }, } } diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index 5b3a482fb..0706108fc 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "context" "time" "github.com/pkg/errors" @@ -27,133 +26,172 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" - api "github.com/heptio/ark/pkg/apis/ark/v1" + arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" "github.com/heptio/ark/pkg/cloudprovider" arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1" listers "github.com/heptio/ark/pkg/generated/listers/ark/v1" + "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/util/kube" "github.com/heptio/ark/pkg/util/stringslice" ) type backupSyncController struct { - client arkv1client.BackupsGetter - cloudBackupLister cloudprovider.BackupLister - bucket string - syncPeriod time.Duration - namespace string - backupLister listers.BackupLister - backupInformerSynced cache.InformerSynced - logger logrus.FieldLogger + *genericController + + client arkv1client.BackupsGetter + backupLister listers.BackupLister + backupStorageLocationLister listers.BackupStorageLocationLister + namespace string + newPluginManager func(logrus.FieldLogger) plugin.Manager + listCloudBackups func(logrus.FieldLogger, cloudprovider.ObjectStore, string) ([]*arkv1api.Backup, error) } func NewBackupSyncController( client arkv1client.BackupsGetter, - cloudBackupLister cloudprovider.BackupLister, - bucket string, + backupInformer informers.BackupInformer, + backupStorageLocationInformer informers.BackupStorageLocationInformer, syncPeriod time.Duration, namespace string, - backupInformer informers.BackupInformer, + pluginRegistry plugin.Registry, logger logrus.FieldLogger, + logLevel logrus.Level, ) Interface { if syncPeriod < time.Minute { logger.Infof("Provided backup sync period %v is too short. Setting to 1 minute", syncPeriod) syncPeriod = time.Minute } - return &backupSyncController{ - client: client, - cloudBackupLister: cloudBackupLister, - bucket: bucket, - syncPeriod: syncPeriod, - namespace: namespace, - backupLister: backupInformer.Lister(), - backupInformerSynced: backupInformer.Informer().HasSynced, - logger: logger, - } -} -// Run is a blocking function that continually runs the object storage -> Ark API -// sync process according to the controller's syncPeriod. It will return when it -// receives on the ctx.Done() channel. -func (c *backupSyncController) Run(ctx context.Context, workers int) error { - c.logger.Info("Running backup sync controller") - c.logger.Info("Waiting for caches to sync") - if !cache.WaitForCacheSync(ctx.Done(), c.backupInformerSynced) { - return errors.New("timed out waiting for caches to sync") + c := &backupSyncController{ + genericController: newGenericController("backup-sync", logger), + client: client, + namespace: namespace, + backupLister: backupInformer.Lister(), + backupStorageLocationLister: backupStorageLocationInformer.Lister(), + + newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { + return plugin.NewManager(logger, logLevel, pluginRegistry) + }, + listCloudBackups: cloudprovider.ListBackups, } - c.logger.Info("Caches are synced") - wait.Until(c.run, c.syncPeriod, ctx.Done()) - return nil + + c.resyncFunc = c.run + c.resyncPeriod = syncPeriod + c.cacheSyncWaiters = []cache.InformerSynced{ + backupInformer.Informer().HasSynced, + backupStorageLocationInformer.Informer().HasSynced, + } + + return c } const gcFinalizer = "gc.ark.heptio.com" func (c *backupSyncController) run() { - c.logger.Info("Syncing backups from object storage") - backups, err := c.cloudBackupLister.ListBackups(c.bucket) + c.logger.Info("Syncing backups from backup storage into cluster") + + locations, err := c.backupStorageLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything()) if err != nil { - c.logger.WithError(err).Error("error listing backups") + c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister") return } - c.logger.WithField("backupCount", len(backups)).Info("Got backups from object storage") - cloudBackupNames := sets.NewString() - for _, cloudBackup := range backups { - logContext := c.logger.WithField("backup", kube.NamespaceAndName(cloudBackup)) - logContext.Info("Syncing backup") + pluginManager := c.newPluginManager(c.logger) - cloudBackupNames.Insert(cloudBackup.Name) + for _, location := range locations { + log := c.logger.WithField("backupLocation", location.Name) + log.Info("Syncing backups from backup location") - // If we're syncing backups made by pre-0.8.0 versions, the server removes all finalizers - // faster than the sync finishes. Just process them as we find them. - cloudBackup.Finalizers = stringslice.Except(cloudBackup.Finalizers, gcFinalizer) - - cloudBackup.Namespace = c.namespace - cloudBackup.ResourceVersion = "" - - // Backup only if backup does not exist in Kubernetes or if we are not able to get the backup for any reason. - _, err := c.client.Backups(cloudBackup.Namespace).Get(cloudBackup.Name, metav1.GetOptions{}) + objectStore, err := getObjectStoreForLocation(location, pluginManager) if err != nil { + log.WithError(err).Error("Error getting object store for location") + continue + } + + backupsInBackupStore, err := c.listCloudBackups(log, objectStore, location.Spec.ObjectStorage.Bucket) + if err != nil { + log.WithError(err).Error("Error listing backups in object store") + continue + } + + log.WithField("backupCount", len(backupsInBackupStore)).Info("Got backups from object store") + + cloudBackupNames := sets.NewString() + for _, cloudBackup := range backupsInBackupStore { + log = log.WithField("backup", kube.NamespaceAndName(cloudBackup)) + log.Debug("Checking cloud backup to see if it needs to be synced into the cluster") + + cloudBackupNames.Insert(cloudBackup.Name) + + // use the controller's namespace when getting the backup because that's where we + // are syncing backups to, regardless of the namespace of the cloud backup. + _, err := c.client.Backups(c.namespace).Get(cloudBackup.Name, metav1.GetOptions{}) + if err == nil { + log.Debug("Backup already exists in cluster") + continue + } if !kuberrs.IsNotFound(err) { - logContext.WithError(errors.WithStack(err)).Error("Error getting backup from client, proceeding with backup sync") + log.WithError(errors.WithStack(err)).Error("Error getting backup from client, proceeding with sync into cluster") } - if _, err := c.client.Backups(cloudBackup.Namespace).Create(cloudBackup); err != nil && !kuberrs.IsAlreadyExists(err) { - logContext.WithError(errors.WithStack(err)).Error("Error syncing backup from object storage") + // remove the pre-v0.8.0 gcFinalizer if it exists + // TODO(1.0): remove this + cloudBackup.Finalizers = stringslice.Except(cloudBackup.Finalizers, gcFinalizer) + cloudBackup.Namespace = c.namespace + cloudBackup.ResourceVersion = "" + + // update the StorageLocation field and label since the name of the location + // may be different in this cluster than in the cluster that created the + // backup. + cloudBackup.Spec.StorageLocation = location.Name + if cloudBackup.Labels == nil { + cloudBackup.Labels = make(map[string]string) + } + cloudBackup.Labels[arkv1api.StorageLocationLabel] = cloudBackup.Spec.StorageLocation + + _, err = c.client.Backups(cloudBackup.Namespace).Create(cloudBackup) + switch { + case err != nil && kuberrs.IsAlreadyExists(err): + log.Debug("Backup already exists in cluster") + case err != nil && !kuberrs.IsAlreadyExists(err): + log.WithError(errors.WithStack(err)).Error("Error syncing backup into cluster") + default: + log.Debug("Synced backup into cluster") } } - } - c.deleteUnused(cloudBackupNames) - return + c.deleteOrphanedBackups(location.Name, cloudBackupNames, log) + } } -// deleteUnused deletes backup objects from Kubernetes if they are complete -// and there is no corresponding backup in the object storage. -func (c *backupSyncController) deleteUnused(cloudBackupNames sets.String) { - // Backups objects in Kubernetes - backups, err := c.backupLister.Backups(c.namespace).List(labels.Everything()) +// deleteOrphanedBackups deletes backup objects from Kubernetes that have the specified location +// and a phase of Completed, but no corresponding backup in object storage. +func (c *backupSyncController) deleteOrphanedBackups(locationName string, cloudBackupNames sets.String, log logrus.FieldLogger) { + locationSelector := labels.Set(map[string]string{ + arkv1api.StorageLocationLabel: locationName, + }).AsSelector() + + backups, err := c.backupLister.Backups(c.namespace).List(locationSelector) if err != nil { - c.logger.WithError(errors.WithStack(err)).Error("Error listing backup from Kubernetes") + log.WithError(errors.WithStack(err)).Error("Error listing backups from cluster") + return } if len(backups) == 0 { return } - // For each completed backup object in Kubernetes, delete it if it - // does not have a corresponding backup in object storage for _, backup := range backups { - if backup.Status.Phase == api.BackupPhaseCompleted && !cloudBackupNames.Has(backup.Name) { - if err := c.client.Backups(backup.Namespace).Delete(backup.Name, nil); err != nil { - c.logger.WithError(errors.WithStack(err)).Error("Error deleting unused backup from Kubernetes") - } else { - c.logger.Debugf("Deleted backup: %s", backup.Name) - } + log = log.WithField("backup", backup.Name) + if backup.Status.Phase != arkv1api.BackupPhaseCompleted || cloudBackupNames.Has(backup.Name) { + continue + } + + if err := c.client.Backups(backup.Namespace).Delete(backup.Name, nil); err != nil { + log.WithError(errors.WithStack(err)).Error("Error deleting orphaned backup from cluster") + } else { + log.Debug("Deleted orphaned backup from cluster") } } - - return } diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 71df0cf8f..a91a45c0b 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -20,281 +20,354 @@ import ( "testing" "time" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/pkg/errors" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" core "k8s.io/client-go/testing" - "github.com/heptio/ark/pkg/apis/ark/v1" - cloudprovidermocks "github.com/heptio/ark/pkg/cloudprovider/mocks" + arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/cloudprovider" "github.com/heptio/ark/pkg/generated/clientset/versioned/fake" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" + "github.com/heptio/ark/pkg/plugin" + pluginmocks "github.com/heptio/ark/pkg/plugin/mocks" "github.com/heptio/ark/pkg/util/stringslice" arktest "github.com/heptio/ark/pkg/util/test" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) +func defaultLocationsList(namespace string) []*arkv1api.BackupStorageLocation { + return []*arkv1api.BackupStorageLocation{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "location-1", + }, + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket-1", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "location-2", + }, + Spec: arkv1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket-2", + }, + }, + }, + }, + } +} + func TestBackupSyncControllerRun(t *testing.T) { tests := []struct { - name string - listBackupsError error - cloudBackups []*v1.Backup - namespace string - existingBackups sets.String + name string + namespace string + locations []*arkv1api.BackupStorageLocation + cloudBackups map[string][]*arkv1api.Backup + existingBackups []*arkv1api.Backup }{ { name: "no cloud backups", }, { - name: "backup lister returns error on ListBackups", - listBackupsError: errors.New("listBackups"), - }, - { - name: "normal case", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "normal case", namespace: "ns-1", + locations: defaultLocationsList("ns-1"), + cloudBackups: map[string][]*arkv1api.Backup{ + "bucket-1": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, + }, + "bucket-2": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, + }, + }, }, { - name: "Finalizer gets removed on sync", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers(gcFinalizer).Backup, - }, + name: "gcFinalizer (only) gets removed on sync", namespace: "ns-1", + locations: defaultLocationsList("ns-1"), + cloudBackups: map[string][]*arkv1api.Backup{ + "bucket-1": { + arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers("a-finalizer", gcFinalizer, "some-other-finalizer").Backup, + }, + }, }, { - name: "Only target finalizer is removed", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithFinalizers(gcFinalizer, "blah").Backup, - }, - namespace: "ns-1", - }, - { - name: "backups get created in Ark server's namespace", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-2").WithName("backup-2").Backup, - }, + name: "all synced backups get created in Ark server's namespace", namespace: "heptio-ark", + locations: defaultLocationsList("heptio-ark"), + cloudBackups: map[string][]*arkv1api.Backup{ + "bucket-1": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, + }, + "bucket-2": { + arktest.NewTestBackup().WithNamespace("ns-2").WithName("backup-3").Backup, + arktest.NewTestBackup().WithNamespace("heptio-ark").WithName("backup-4").Backup, + }, + }, }, { - name: "normal case with backups that already exist in Kubernetes", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, + name: "new backups get synced when some cloud backups already exist in the cluster", + namespace: "ns-1", + locations: defaultLocationsList("ns-1"), + cloudBackups: map[string][]*arkv1api.Backup{ + "bucket-1": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, + }, + "bucket-2": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-4").Backup, + }, + }, + existingBackups: []*arkv1api.Backup{ + // add a label to each existing backup so we can differentiate it from the cloud + // backup during verification + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel("i-exist", "true").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel("i-exist", "true").Backup, + }, + }, + { + name: "backup storage location names and labels get updated", + namespace: "ns-1", + locations: defaultLocationsList("ns-1"), + cloudBackups: map[string][]*arkv1api.Backup{ + "bucket-1": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithStorageLocation("foo").WithLabel(arkv1api.StorageLocationLabel, "foo").Backup, + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, + }, + "bucket-2": { + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithStorageLocation("bar").WithLabel(arkv1api.StorageLocationLabel, "bar").Backup, + }, }, - existingBackups: sets.NewString("backup-2", "backup-3"), - namespace: "ns-1", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( - backupLister = &cloudprovidermocks.BackupLister{} client = fake.NewSimpleClientset() sharedInformers = informers.NewSharedInformerFactory(client, 0) - logger = arktest.NewLogger() + pluginManager = &pluginmocks.Manager{} + objectStore = &arktest.ObjectStore{} ) c := NewBackupSyncController( client.ArkV1(), - backupLister, - "bucket", + sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), time.Duration(0), test.namespace, - sharedInformers.Ark().V1().Backups(), - logger, + nil, // pluginRegistry + arktest.NewLogger(), + logrus.DebugLevel, ).(*backupSyncController) - backupLister.On("ListBackups", "bucket").Return(test.cloudBackups, test.listBackupsError) + c.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } + pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) + pluginManager.On("CleanupClients").Return(nil) + objectStore.On("Init", mock.Anything).Return(nil) - expectedActions := make([]core.Action, 0) + for _, location := range test.locations { + require.NoError(t, sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(location)) + } - client.PrependReactor("get", "backups", func(action core.Action) (bool, runtime.Object, error) { - getAction := action.(core.GetAction) - if test.existingBackups.Has(getAction.GetName()) { - return true, nil, nil + c.listCloudBackups = func(_ logrus.FieldLogger, _ cloudprovider.ObjectStore, bucket string) ([]*arkv1api.Backup, error) { + backups, ok := test.cloudBackups[bucket] + if !ok { + return nil, errors.New("bucket not found") } - // We return nil in place of the found backup object because - // we exclusively check for the error and don't use the object - // returned by the Get / Backups call. - return true, nil, apierrors.NewNotFound(v1.SchemeGroupVersion.WithResource("backups").GroupResource(), getAction.GetName()) - }) + + return backups, nil + } + + for _, existingBackup := range test.existingBackups { + require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(existingBackup)) + + _, err := client.ArkV1().Backups(test.namespace).Create(existingBackup) + require.NoError(t, err) + } + client.ClearActions() c.run() - // we only expect creates for items within the target bucket - for _, cloudBackup := range test.cloudBackups { - // Verify that the run function stripped the GC finalizer - assert.False(t, stringslice.Has(cloudBackup.Finalizers, gcFinalizer)) - assert.Equal(t, test.namespace, cloudBackup.Namespace) + for bucket, backups := range test.cloudBackups { + for _, cloudBackup := range backups { + obj, err := client.ArkV1().Backups(test.namespace).Get(cloudBackup.Name, metav1.GetOptions{}) + require.NoError(t, err) - actionGet := core.NewGetAction( - v1.SchemeGroupVersion.WithResource("backups"), - test.namespace, - cloudBackup.Name, - ) - expectedActions = append(expectedActions, actionGet) + // did this cloud backup already exist in the cluster? + var existing *arkv1api.Backup + for _, obj := range test.existingBackups { + if obj.Name == cloudBackup.Name { + existing = obj + break + } + } - if test.existingBackups.Has(cloudBackup.Name) { - continue + if existing != nil { + // if this cloud backup already exists in the cluster, make sure that what we get from the + // client is the existing backup, not the cloud one. + assert.Equal(t, existing, obj) + } else { + // verify that the GC finalizer is removed + assert.Equal(t, stringslice.Except(cloudBackup.Finalizers, gcFinalizer), obj.Finalizers) + + // verify that the storage location field and label are set properly + for _, location := range test.locations { + if location.Spec.ObjectStorage.Bucket == bucket { + assert.Equal(t, location.Name, obj.Spec.StorageLocation) + assert.Equal(t, location.Name, obj.Labels[arkv1api.StorageLocationLabel]) + break + } + } + } } - actionCreate := core.NewCreateAction( - v1.SchemeGroupVersion.WithResource("backups"), - test.namespace, - cloudBackup, - ) - expectedActions = append(expectedActions, actionCreate) } - - assert.Equal(t, expectedActions, client.Actions()) }) } } -func TestDeleteUnused(t *testing.T) { +func TestDeleteOrphanedBackups(t *testing.T) { tests := []struct { name string - cloudBackups []*v1.Backup + cloudBackups sets.String k8sBackups []*arktest.TestBackup namespace string expectedDeletes sets.String }{ { - name: "no overlapping backups", - namespace: "ns-1", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "no overlapping backups", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), k8sBackups: []*arktest.TestBackup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupB").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithPhase(v1.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupB").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), }, expectedDeletes: sets.NewString("backupA", "backupB", "backupC"), }, { - name: "some overlapping backups", - namespace: "ns-1", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "some overlapping backups", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), k8sBackups: []*arktest.TestBackup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupC").WithPhase(v1.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-C").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), }, - expectedDeletes: sets.NewString("backupC"), + expectedDeletes: sets.NewString("backup-C"), }, { - name: "all overlapping backups", - namespace: "ns-1", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "all overlapping backups", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), k8sBackups: []*arktest.TestBackup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithPhase(v1.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), }, expectedDeletes: sets.NewString(), }, { - name: "no overlapping backups but including backups that are not complete", - namespace: "ns-1", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "no overlapping backups but including backups that are not complete", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), k8sBackups: []*arktest.TestBackup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithPhase(v1.BackupPhaseCompleted), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("Deleting").WithPhase(v1.BackupPhaseDeleting), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("Failed").WithPhase(v1.BackupPhaseFailed), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("FailedValidation").WithPhase(v1.BackupPhaseFailedValidation), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("InProgress").WithPhase(v1.BackupPhaseInProgress), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("New").WithPhase(v1.BackupPhaseNew), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backupA").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("Deleting").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseDeleting), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("Failed").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailed), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("FailedValidation").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailedValidation), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("InProgress").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseInProgress), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("New").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseNew), }, expectedDeletes: sets.NewString("backupA"), }, { - name: "all overlapping backups and all backups that are not complete", - namespace: "ns-1", - cloudBackups: []*v1.Backup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup, - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").Backup, - }, + name: "all overlapping backups and all backups that are not complete", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), k8sBackups: []*arktest.TestBackup{ - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithPhase(v1.BackupPhaseFailed), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithPhase(v1.BackupPhaseFailedValidation), - arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithPhase(v1.BackupPhaseInProgress), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailed), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseFailedValidation), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-3").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseInProgress), }, expectedDeletes: sets.NewString(), }, + { + name: "no completed backups in other locations are deleted", + namespace: "ns-1", + cloudBackups: sets.NewString("backup-1", "backup-2", "backup-3"), + k8sBackups: []*arktest.TestBackup{ + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-1").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-2").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-C").WithLabel(arkv1api.StorageLocationLabel, "default").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-4").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-5").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted), + arktest.NewTestBackup().WithNamespace("ns-1").WithName("backup-6").WithLabel(arkv1api.StorageLocationLabel, "alternate").WithPhase(arkv1api.BackupPhaseCompleted), + }, + expectedDeletes: sets.NewString("backup-C"), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( - backupLister = &cloudprovidermocks.BackupLister{} client = fake.NewSimpleClientset() sharedInformers = informers.NewSharedInformerFactory(client, 0) - logger = arktest.NewLogger() ) c := NewBackupSyncController( client.ArkV1(), - backupLister, - "bucket", + sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), time.Duration(0), test.namespace, - sharedInformers.Ark().V1().Backups(), - logger, + nil, // pluginRegistry + arktest.NewLogger(), + logrus.InfoLevel, ).(*backupSyncController) expectedDeleteActions := make([]core.Action, 0) - // setup: insert backups into Kubernetes for _, backup := range test.k8sBackups { + // add test backup to informer + require.NoError(t, sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup.Backup), "Error adding backup to informer") + + // add test backup to client + _, err := client.Ark().Backups(test.namespace).Create(backup.Backup) + require.NoError(t, err, "Error adding backup to clientset") + + // if we expect this backup to be deleted, set up the expected DeleteAction if test.expectedDeletes.Has(backup.Name) { actionDelete := core.NewDeleteAction( - v1.SchemeGroupVersion.WithResource("backups"), + arkv1api.SchemeGroupVersion.WithResource("backups"), test.namespace, backup.Name, ) expectedDeleteActions = append(expectedDeleteActions, actionDelete) } - - // add test backup to informer: - err := sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(backup.Backup) - assert.NoError(t, err, "Error adding backup to informer") - - // add test backup to kubernetes: - _, err = client.Ark().Backups(test.namespace).Create(backup.Backup) - assert.NoError(t, err, "Error deleting from clientset") } - // get names of client backups - testBackupNames := sets.NewString() - for _, cloudBackup := range test.cloudBackups { - testBackupNames.Insert(cloudBackup.Name) - } - - c.deleteUnused(testBackupNames) + c.deleteOrphanedBackups("default", test.cloudBackups, arktest.NewLogger()) numBackups, err := numBackups(t, client, c.namespace) assert.NoError(t, err) From bd4d97b9e42d5e2bd237b704b53c10c7ec1a0f8d Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 22 Aug 2018 19:51:10 -0700 Subject: [PATCH 18/29] move server's defaultBackupLocation into config struct Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ec8f4d3cb..d50990318 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -76,10 +76,10 @@ const ( ) type serverConfig struct { - pluginDir, metricsAddress string - backupSyncPeriod, podVolumeOperationTimeout time.Duration - restoreResourcePriorities []string - restoreOnly bool + pluginDir, metricsAddress, defaultBackupLocation string + backupSyncPeriod, podVolumeOperationTimeout time.Duration + restoreResourcePriorities []string + restoreOnly bool } func NewCommand() *cobra.Command { @@ -88,11 +88,11 @@ func NewCommand() *cobra.Command { config = serverConfig{ pluginDir: "/plugins", metricsAddress: defaultMetricsAddress, + defaultBackupLocation: "default", backupSyncPeriod: defaultBackupSyncPeriod, podVolumeOperationTimeout: defaultPodVolumeOperationTimeout, restoreResourcePriorities: defaultRestorePriorities, } - defaultBackupLocation = "default" ) var command = &cobra.Command{ @@ -127,7 +127,7 @@ func NewCommand() *cobra.Command { } namespace := getServerNamespace(namespaceFlag) - s, err := newServer(namespace, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()), config, defaultBackupLocation, logger) + s, err := newServer(namespace, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()), config, logger) cmd.CheckError(err) cmd.CheckError(s.run()) @@ -141,7 +141,7 @@ func NewCommand() *cobra.Command { command.Flags().DurationVar(&config.podVolumeOperationTimeout, "restic-timeout", config.podVolumeOperationTimeout, "how long backups/restores of pod volumes should be allowed to run before timing out") command.Flags().BoolVar(&config.restoreOnly, "restore-only", config.restoreOnly, "run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled") command.Flags().StringSliceVar(&config.restoreResourcePriorities, "restore-resource-priorities", config.restoreResourcePriorities, "desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources") - command.Flags().StringVar(&defaultBackupLocation, "default-backup-storage-location", defaultBackupLocation, "name of the default backup storage location") + command.Flags().StringVar(&config.defaultBackupLocation, "default-backup-storage-location", config.defaultBackupLocation, "name of the default backup storage location") return command } @@ -181,10 +181,9 @@ type server struct { resticManager restic.RepositoryManager metrics *metrics.ServerMetrics config serverConfig - defaultBackupLocation string } -func newServer(namespace, baseName string, config serverConfig, defaultBackupLocation string, logger *logrus.Logger) (*server, error) { +func newServer(namespace, baseName string, config serverConfig, logger *logrus.Logger) (*server, error) { clientConfig, err := client.Config("", "", baseName) if err != nil { return nil, err @@ -225,14 +224,13 @@ func newServer(namespace, baseName string, config serverConfig, defaultBackupLoc discoveryClient: arkClient.Discovery(), dynamicClient: dynamicClient, sharedInformerFactory: informers.NewFilteredSharedInformerFactory(arkClient, 0, namespace, nil), - ctx: ctx, - cancelFunc: cancelFunc, - logger: logger, - logLevel: logger.Level, - pluginRegistry: pluginRegistry, - pluginManager: pluginManager, - config: config, - defaultBackupLocation: defaultBackupLocation, + ctx: ctx, + cancelFunc: cancelFunc, + logger: logger, + logLevel: logger.Level, + pluginRegistry: pluginRegistry, + pluginManager: pluginManager, + config: config, } return s, nil @@ -271,7 +269,7 @@ func (s *server) run() error { s.watchConfig(originalConfig) - backupStorageLocation, err := s.arkClient.ArkV1().BackupStorageLocations(s.namespace).Get(s.defaultBackupLocation, metav1.GetOptions{}) + backupStorageLocation, err := s.arkClient.ArkV1().BackupStorageLocations(s.namespace).Get(s.config.defaultBackupLocation, metav1.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -641,7 +639,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.pluginRegistry, backupTracker, s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), - s.defaultBackupLocation, + s.config.defaultBackupLocation, s.metrics, ) wg.Add(1) @@ -724,7 +722,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.logger, s.logLevel, s.pluginRegistry, - s.defaultBackupLocation, + s.config.defaultBackupLocation, s.metrics, ) From 6f7bfe545de88db228c5b1ff4413073cadc998db Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 22 Aug 2018 20:06:55 -0700 Subject: [PATCH 19/29] remove Config CRD's BackupStorageProvider & other obsolete code Signed-off-by: Steve Kriss --- pkg/apis/ark/v1/config.go | 23 --------- pkg/apis/ark/v1/zz_generated.deepcopy.go | 18 ------- pkg/cmd/server/server.go | 33 ------------ pkg/controller/backup_controller.go | 28 ++++------- pkg/install/config.go | 64 ------------------------ 5 files changed, 10 insertions(+), 156 deletions(-) diff --git a/pkg/apis/ark/v1/config.go b/pkg/apis/ark/v1/config.go index 7f1c1d36f..90115fa48 100644 --- a/pkg/apis/ark/v1/config.go +++ b/pkg/apis/ark/v1/config.go @@ -40,11 +40,6 @@ type Config struct { // PersistentVolumeProvider is the configuration information for the cloud where // the cluster is running and has PersistentVolumes to snapshot or restore. Optional. PersistentVolumeProvider *CloudProviderConfig `json:"persistentVolumeProvider"` - - // BackupStorageProvider is the configuration information for the cloud where - // Ark backups are stored in object storage. This may be a different cloud than - // where the cluster is running. - BackupStorageProvider ObjectStorageProviderConfig `json:"backupStorageProvider"` } // CloudProviderConfig is configuration information about how to connect @@ -54,21 +49,3 @@ type CloudProviderConfig struct { Config map[string]string `json:"config"` } - -// ObjectStorageProviderConfig is configuration information for connecting to -// a particular bucket in object storage to access Ark backups. -type ObjectStorageProviderConfig struct { - // CloudProviderConfig is the configuration information for the cloud where - // Ark backups are stored in object storage. - CloudProviderConfig `json:",inline"` - - // Bucket is the name of the bucket in object storage where Ark backups - // are stored. - Bucket string `json:"bucket"` - - // ResticLocation is the bucket and optional prefix in object storage where - // Ark stores restic backups of pod volumes, specified either as "bucket" or - // "bucket/prefix". This bucket must be different than the `Bucket` field. - // Optional. - ResticLocation string `json:"resticLocation"` -} diff --git a/pkg/apis/ark/v1/zz_generated.deepcopy.go b/pkg/apis/ark/v1/zz_generated.deepcopy.go index 3403f946c..f6e0e4433 100644 --- a/pkg/apis/ark/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ark/v1/zz_generated.deepcopy.go @@ -439,7 +439,6 @@ func (in *Config) DeepCopyInto(out *Config) { (*in).DeepCopyInto(*out) } } - in.BackupStorageProvider.DeepCopyInto(&out.BackupStorageProvider) return } @@ -741,23 +740,6 @@ func (in *ObjectStorageLocation) DeepCopy() *ObjectStorageLocation { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ObjectStorageProviderConfig) DeepCopyInto(out *ObjectStorageProviderConfig) { - *out = *in - in.CloudProviderConfig.DeepCopyInto(&out.CloudProviderConfig) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageProviderConfig. -func (in *ObjectStorageProviderConfig) DeepCopy() *ObjectStorageProviderConfig { - if in == nil { - return nil - } - out := new(ObjectStorageProviderConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeBackup) DeepCopyInto(out *PodVolumeBackup) { *out = *in diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index d50990318..b2a53d865 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -166,7 +166,6 @@ type server struct { kubeClientConfig *rest.Config kubeClient kubernetes.Interface arkClient clientset.Interface - objectStore cloudprovider.ObjectStore blockStore cloudprovider.BlockStore discoveryClient discovery.DiscoveryInterface discoveryHelper arkdiscovery.Helper @@ -274,12 +273,6 @@ func (s *server) run() error { return errors.WithStack(err) } - objectStore, err := getObjectStore(config.BackupStorageProvider.CloudProviderConfig, s.pluginManager) - if err != nil { - return err - } - s.objectStore = objectStore - if config.PersistentVolumeProvider == nil { s.logger.Info("PersistentVolumeProvider config not provided, volume snapshots and restores are disabled") } else { @@ -319,15 +312,6 @@ func (s *server) applyConfigDefaults(c *api.Config) { } else { s.logger.WithField("priorities", s.config.restoreResourcePriorities).Info("Using given resource priorities") } - - if c.BackupStorageProvider.Config == nil { - c.BackupStorageProvider.Config = make(map[string]string) - } - - // add the bucket name to the config map so that object stores can use - // it when initializing. The AWS object store uses this to determine the - // bucket's region when setting up its client. - c.BackupStorageProvider.Config["bucket"] = c.BackupStorageProvider.Bucket } // namespaceExists returns nil if namespace can be successfully @@ -487,23 +471,6 @@ func (s *server) watchConfig(config *api.Config) { }) } -func getObjectStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) (cloudprovider.ObjectStore, error) { - if cloudConfig.Name == "" { - return nil, errors.New("object storage provider name must not be empty") - } - - objectStore, err := manager.GetObjectStore(cloudConfig.Name) - if err != nil { - return nil, err - } - - if err := objectStore.Init(cloudConfig.Config); err != nil { - return nil, err - } - - return objectStore, nil -} - func getBlockStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) (cloudprovider.BlockStore, error) { if cloudConfig.Name == "" { return nil, errors.New("block storage provider name must not be empty") diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 364331734..18c487bd3 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -460,24 +460,6 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation } // TODO(ncdc): move this to a better location that isn't backup specific -func getObjectStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) (cloudprovider.ObjectStore, error) { - if cloudConfig.Name == "" { - return nil, errors.New("object storage provider name must not be empty") - } - - objectStore, err := manager.GetObjectStore(cloudConfig.Name) - if err != nil { - return nil, err - } - - if err := objectStore.Init(cloudConfig.Config); err != nil { - return nil, err - } - - return objectStore, nil -} - -// TODO(nrb): Consolidate with other implementations func getObjectStoreForLocation(location *api.BackupStorageLocation, manager plugin.Manager) (cloudprovider.ObjectStore, error) { if location.Spec.Provider == "" { return nil, errors.New("backup storage location provider name must not be empty") @@ -488,6 +470,16 @@ func getObjectStoreForLocation(location *api.BackupStorageLocation, manager plug return nil, err } + // add the bucket name to the config map so that object stores can use + // it when initializing. The AWS object store uses this to determine the + // bucket's region when setting up its client. + if location.Spec.ObjectStorage != nil { + if location.Spec.Config == nil { + location.Spec.Config = make(map[string]string) + } + location.Spec.Config["bucket"] = location.Spec.ObjectStorage.Bucket + } + if err := objectStore.Init(location.Spec.Config); err != nil { return nil, err } diff --git a/pkg/install/config.go b/pkg/install/config.go index f81956932..ed74b8bed 100644 --- a/pkg/install/config.go +++ b/pkg/install/config.go @@ -17,83 +17,19 @@ limitations under the License. package install import ( - "time" - arkv1 "github.com/heptio/ark/pkg/apis/ark/v1" ) -type arkConfigOption func(*arkConfig) - -type arkConfig struct { - backupSyncPeriod time.Duration - gcSyncPeriod time.Duration - podVolumeOperationTimeout time.Duration - restoreOnly bool - resticLocation string -} - -func WithBackupSyncPeriod(t time.Duration) arkConfigOption { - return func(c *arkConfig) { - c.backupSyncPeriod = t - } -} - -func WithGCSyncPeriod(t time.Duration) arkConfigOption { - return func(c *arkConfig) { - c.gcSyncPeriod = t - } -} - -func WithPodVolumeOperationTimeout(t time.Duration) arkConfigOption { - return func(c *arkConfig) { - c.podVolumeOperationTimeout = t - } -} - -func WithRestoreOnly() arkConfigOption { - return func(c *arkConfig) { - c.restoreOnly = true - } -} - -func WithResticLocation(location string) arkConfigOption { - return func(c *arkConfig) { - c.resticLocation = location - } -} - func Config( namespace string, pvCloudProviderName string, pvCloudProviderConfig map[string]string, - backupCloudProviderName string, - backupCloudProviderConfig map[string]string, - bucket string, - opts ...arkConfigOption, ) *arkv1.Config { - c := &arkConfig{ - backupSyncPeriod: 30 * time.Minute, - gcSyncPeriod: 30 * time.Minute, - podVolumeOperationTimeout: 60 * time.Minute, - } - - for _, opt := range opts { - opt(c) - } - return &arkv1.Config{ ObjectMeta: objectMeta(namespace, "default"), PersistentVolumeProvider: &arkv1.CloudProviderConfig{ Name: pvCloudProviderName, Config: pvCloudProviderConfig, }, - BackupStorageProvider: arkv1.ObjectStorageProviderConfig{ - CloudProviderConfig: arkv1.CloudProviderConfig{ - Name: backupCloudProviderName, - Config: backupCloudProviderConfig, - }, - Bucket: bucket, - ResticLocation: c.resticLocation, - }, } } From 7a1e6d16cc690e52f43892b3469d51282185887d Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 23 Aug 2018 10:14:25 -0700 Subject: [PATCH 20/29] generic controller: allow controllers with only a resync func Signed-off-by: Steve Kriss --- pkg/controller/generic_controller.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/controller/generic_controller.go b/pkg/controller/generic_controller.go index e18b3e243..11ec624ca 100644 --- a/pkg/controller/generic_controller.go +++ b/pkg/controller/generic_controller.go @@ -53,9 +53,9 @@ func newGenericController(name string, logger logrus.FieldLogger) *genericContro // to process items in the work queue. It will return when it receives on the // ctx.Done() channel. func (c *genericController) Run(ctx context.Context, numWorkers int) error { - if c.syncHandler == nil { + if c.syncHandler == nil && c.resyncFunc == nil { // programmer error - panic("syncHandler is required") + panic("at least one of syncHandler or resyncFunc is required") } var wg sync.WaitGroup @@ -83,12 +83,14 @@ func (c *genericController) Run(ctx context.Context, numWorkers int) error { } c.logger.Info("Caches are synced") - wg.Add(numWorkers) - for i := 0; i < numWorkers; i++ { - go func() { - wait.Until(c.runWorker, time.Second, ctx.Done()) - wg.Done() - }() + if c.syncHandler != nil { + wg.Add(numWorkers) + for i := 0; i < numWorkers; i++ { + go func() { + wait.Until(c.runWorker, time.Second, ctx.Done()) + wg.Done() + }() + } } if c.resyncFunc != nil { From 133dc185ca5eeadd04b40edb0575f84b80b10434 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 24 Aug 2018 11:07:01 -0700 Subject: [PATCH 21/29] backup sync: process the default location first Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 1 + pkg/controller/backup_sync_controller.go | 5 +++++ pkg/controller/backup_sync_controller_test.go | 2 ++ pkg/controller/restore_controller.go | 2 ++ 4 files changed, 10 insertions(+) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index b2a53d865..60d6c96b8 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -571,6 +571,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.config.backupSyncPeriod, s.namespace, + s.config.defaultBackupLocation, s.pluginRegistry, s.logger, s.logLevel, diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index 0706108fc..a19d7aea0 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -45,6 +45,7 @@ type backupSyncController struct { backupLister listers.BackupLister backupStorageLocationLister listers.BackupStorageLocationLister namespace string + defaultBackupLocation string newPluginManager func(logrus.FieldLogger) plugin.Manager listCloudBackups func(logrus.FieldLogger, cloudprovider.ObjectStore, string) ([]*arkv1api.Backup, error) } @@ -55,6 +56,7 @@ func NewBackupSyncController( backupStorageLocationInformer informers.BackupStorageLocationInformer, syncPeriod time.Duration, namespace string, + defaultBackupLocation string, pluginRegistry plugin.Registry, logger logrus.FieldLogger, logLevel logrus.Level, @@ -68,6 +70,7 @@ func NewBackupSyncController( genericController: newGenericController("backup-sync", logger), client: client, namespace: namespace, + defaultBackupLocation: defaultBackupLocation, backupLister: backupInformer.Lister(), backupStorageLocationLister: backupStorageLocationInformer.Lister(), @@ -97,6 +100,8 @@ func (c *backupSyncController) run() { c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister") return } + // sync the default location first, if it exists + locations = orderedBackupLocations(locations, c.defaultBackupLocation) pluginManager := c.newPluginManager(c.logger) diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index a91a45c0b..4badb0368 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -176,6 +176,7 @@ func TestBackupSyncControllerRun(t *testing.T) { sharedInformers.Ark().V1().BackupStorageLocations(), time.Duration(0), test.namespace, + "", nil, // pluginRegistry arktest.NewLogger(), logrus.DebugLevel, @@ -341,6 +342,7 @@ func TestDeleteOrphanedBackups(t *testing.T) { sharedInformers.Ark().V1().BackupStorageLocations(), time.Duration(0), test.namespace, + "", nil, // pluginRegistry arktest.NewLogger(), logrus.InfoLevel, diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 4fe2842b0..215c51405 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -521,6 +521,8 @@ func (c *restoreController) fetchFromBackupStorage(backupName string, pluginMana return backupInfo{}, errors.New("not able to fetch from backup storage") } +// orderedBackupLocations returns a new slice with the default backup location first (if it exists), +// followed by the rest of the locations in no particular order. func orderedBackupLocations(locations []*api.BackupStorageLocation, defaultLocationName string) []*api.BackupStorageLocation { var result []*api.BackupStorageLocation From 6445dbf1c7d6b70752030a089b73ed1795001901 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 23 Aug 2018 15:44:25 -0400 Subject: [PATCH 22/29] Update examples and docs for backup locations Signed-off-by: Nolan Brubaker --- docs/aws-config.md | 9 ++- docs/azure-config.md | 24 +++---- docs/backupstoragelocation-definition.md | 65 +++++++++++++++++++ docs/config-definition.md | 28 -------- docs/gcp-config.md | 6 +- docs/ibm-config.md | 6 +- examples/aws/00-ark-config.yaml | 11 ---- .../05-ark-backupstoragelocation.yaml} | 30 ++++----- .../azure/05-ark-backupstoragelocation.yaml | 30 +++++++++ examples/azure/10-ark-config.yaml | 11 +--- examples/gcp/00-ark-config.yaml | 12 +--- .../gcp/05-ark-backupstoragelocation.yaml | 30 +++++++++ .../ibm/05-ark-backupstoragelocation.yaml | 34 ++++++++++ 13 files changed, 201 insertions(+), 95 deletions(-) create mode 100644 docs/backupstoragelocation-definition.md rename examples/{ibm/00-ark-config.yaml => aws/05-ark-backupstoragelocation.yaml} (58%) create mode 100644 examples/azure/05-ark-backupstoragelocation.yaml create mode 100644 examples/gcp/05-ark-backupstoragelocation.yaml create mode 100644 examples/ibm/05-ark-backupstoragelocation.yaml diff --git a/docs/aws-config.md b/docs/aws-config.md index 35f85d033..add8b5a87 100644 --- a/docs/aws-config.md +++ b/docs/aws-config.md @@ -141,7 +141,11 @@ Specify the following values in the example files: * In `examples/aws/00-ark-config.yaml`: - * Replace `` and `` (for S3, region is optional and will be queried from the AWS S3 API if not provided). See the [Config definition][6] for details. + * Replace ``. See the [Config definition][6] for details. + +* In `examples/aws/05-ark-backupstoragelocation.yaml`: + + * Replace `` and `` (for S3 backup storage, region is optional and will be queried from the AWS S3 API if not provided). See the [BackupStorageLocation definition][21] for details. * (Optional) If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`: @@ -273,4 +277,5 @@ It can be set up for Ark by creating a role that will have required permissions, [5]: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-welcome.html [6]: config-definition.md#aws [14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html -[20]: faq.md \ No newline at end of file +[20]: faq.md +[21]: backupstoragelocation-definition.md#aws \ No newline at end of file diff --git a/docs/azure-config.md b/docs/azure-config.md index c92fedc8b..b02d29229 100644 --- a/docs/azure-config.md +++ b/docs/azure-config.md @@ -135,9 +135,13 @@ Now that you have your Azure credentials stored in a Secret, you need to replace * In file `examples/azure/10-ark-config.yaml`: - * Replace `` and ``. See the [Config definition][8] for details. + * Replace ``. See the [Config definition][8] for details. -Here is an example of a completed file. +* In file `examples/azure/05-ark-backupstoragelocation.yaml`: + + * Replace ``. See the [BackupStorageLocation definition][21] for details. + +Here is an example of a completed config file. ```yaml apiVersion: ark.heptio.com/v1 @@ -149,9 +153,6 @@ persistentVolumeProvider: name: azure config: apiTimeout: 15m -backupStorageProvider: - name: azure - bucket: ark backupSyncPeriod: 30m gcSyncPeriod: 30m scheduleSyncPeriod: 1m @@ -166,9 +167,10 @@ In the root of your Ark directory, run: kubectl apply -f examples/azure/ ``` - [0]: namespace.md - [8]: config-definition.md#azure - [17]: https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-application-objects - [18]: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli - [19]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#storage - [20]: faq.md +[0]: namespace.md +[8]: config-definition.md#azure +[17]: https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-application-objects +[18]: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli +[19]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#storage +[20]: faq.md +[21]: backupstoragelocation-definition.md#azure \ No newline at end of file diff --git a/docs/backupstoragelocation-definition.md b/docs/backupstoragelocation-definition.md new file mode 100644 index 000000000..f8529b0fc --- /dev/null +++ b/docs/backupstoragelocation-definition.md @@ -0,0 +1,65 @@ +# Ark Backup Storage Locations + +## Backup Storage Location + +Ark can store backups in a number of locations. These are represented in the cluster via the `BackupStorageLocation` CRD. + +Ark must have at least one `BackupStorageLocation`. By default, this is expected to be named `default`, however the name can be changed by specifying `--default-backup-storage-location` on `ark server`. Backups that do not explicitly specify a storage location will be saved to this `BackupStorageLocation`. + +> *NOTE*: `BackupStorageLocation` takes the place of the `Config.backupStorageProvider` key as of v0.10.0 + +A sample YAML `BackupStorageLocation` looks like the following: + +```yaml +apiVersion: ark.heptio.com/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: heptio-ark +spec: + provider: aws + objectStorage: + bucket: myBucket + config: + region: us-west-2 +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String (Ark natively supports `aws`, `gcp`, and `azure`. Other providers may be available via external plugins.)| Required Field | The name for whichever cloud provider will be used to actually store the backups. | +| `objectStorage` | ObjectStorageLocation | Specification of the object storage for the given provider. | +| `objectStorage/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | +| `objectStorage/prefix` | String | Optional Field | The directory inside a storage bucket where backups are to be uploaded. | +| `objectStorage/config` | map[string]string

(See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs or your provider's documentation.) | None (Optional) | Configuration keys/values to be passed to the cloud provider for backup storage. | + +#### AWS + +**(Or other S3-compatible storage)** + +##### objectStorage/config + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `region` | string | Empty | *Example*: "us-east-1"

See [AWS documentation][3] for the full list.

Queried from the AWS S3 API if not provided. | +| `s3ForcePathStyle` | bool | `false` | Set this to `true` if you are using a local storage service like Minio. | +| `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000

You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.| +| `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/``"

Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.| + +#### GCP + +No parameters required. + +#### Azure + +No parameters required. + +[0]: #aws +[1]: #gcp +[2]: #azure +[3]: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions \ No newline at end of file diff --git a/docs/config-definition.md b/docs/config-definition.md index ca689e906..4b4bf0942 100644 --- a/docs/config-definition.md +++ b/docs/config-definition.md @@ -31,11 +31,6 @@ persistentVolumeProvider: name: aws config: region: us-west-2 -backupStorageProvider: - name: aws - bucket: ark - config: - region: us-west-2 ``` ### Parameter Reference @@ -49,24 +44,9 @@ The configurable parameters are as follows: | `persistentVolumeProvider` | CloudProviderConfig | None (Optional) | The specification for whichever cloud provider the cluster is using for persistent volumes (to be snapshotted), if any.

If not specified, Backups and Restores requesting PV snapshots & restores, respectively, are considered invalid.

*NOTE*: For Azure, your Kubernetes cluster needs to be version 1.7.2+ in order to support PV snapshotting of its managed disks. | | `persistentVolumeProvider/name` | String

(Ark natively supports `aws`, `gcp`, and `azure`. Other providers may be available via external plugins.) | None (Optional) | The name of the cloud provider the cluster is using for persistent volumes, if any. | | `persistentVolumeProvider/config` | map[string]string

(See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs or your provider's documentation.) | None (Optional) | Configuration keys/values to be passed to the cloud provider for persistent volumes. | -| `backupStorageProvider` | CloudProviderConfig | Required Field | The specification for whichever cloud provider will be used to actually store the backups. | -| `backupStorageProvider/name` | String

(Ark natively supports `aws`, `gcp`, and `azure`. Other providers may be available via external plugins.) | Required Field | The name of the cloud provider that will be used to actually store the backups. | -| `backupStorageProvider/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | -| `backupStorageProvider/config` | map[string]string

(See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs or your provider's documentation.) | None (Optional) | Configuration keys/values to be passed to the cloud provider for backup storage. | #### AWS -**(Or other S3-compatible storage)** - -##### backupStorageProvider/config - -| Key | Type | Default | Meaning | -| --- | --- | --- | --- | -| `region` | string | Empty | *Example*: "us-east-1"

See [AWS documentation][3] for the full list.

Queried from the AWS S3 API if not provided. | -| `s3ForcePathStyle` | bool | `false` | Set this to `true` if you are using a local storage service like Minio. | -| `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000

You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.| -| `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/``"

Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.| - ##### persistentVolumeProvider/config (AWS Only) | Key | Type | Default | Meaning | @@ -75,20 +55,12 @@ The configurable parameters are as follows: #### GCP -#### backupStorageProvider/config - -No parameters required. - ##### persistentVolumeProvider/config No parameters required. #### Azure -##### backupStorageProvider/config - -No parameters required. - ##### persistentVolumeProvider/config | Key | Type | Default | Meaning | diff --git a/docs/gcp-config.md b/docs/gcp-config.md index 9561010fa..2943fbb56 100644 --- a/docs/gcp-config.md +++ b/docs/gcp-config.md @@ -112,9 +112,9 @@ _Note: If you use a custom namespace, replace `heptio-ark` with the name of the Specify the following values in the example files: -* In file `examples/gcp/00-ark-config.yaml`: +* In file `examples/gcp/05-ark-backupstoragelocation.yaml`: - * Replace ``. See the [Config definition][7] for details. + * Replace ``. See the [BackupStorageLocation definition][7] for details. * (Optional) If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`: @@ -130,7 +130,7 @@ In the root of your Ark directory, run: ``` [0]: namespace.md - [7]: config-definition.md#gcp + [7]: backupstoragelocation-definition.md#gcp [15]: https://cloud.google.com/compute/docs/access/service-accounts [16]: https://cloud.google.com/sdk/docs/ [20]: faq.md diff --git a/docs/ibm-config.md b/docs/ibm-config.md index 218727a3a..b27f2c13f 100644 --- a/docs/ibm-config.md +++ b/docs/ibm-config.md @@ -53,9 +53,9 @@ kubectl create secret generic cloud-credentials \ Specify the following values in the example files: -* In `examples/ibm/00-ark-config.yaml`: +* In `examples/ibm/05-ark-backupstoragelocation.yaml`: - * Replace ``, `` and ``. See the [Config definition][6] for details. + * Replace ``, `` and ``. See the [BackupStorageLocation definition][6] for details. @@ -78,5 +78,5 @@ In the root of your Ark directory, run: [3]: https://console.bluemix.net/docs/services/cloud-object-storage/iam/service-credentials.html#service-credentials [4]: https://www.ibm.com/support/knowledgecenter/SSBS6K_2.1.0/kc_welcome_containers.html [5]: https://console.bluemix.net/docs/containers/container_index.html#container_index - [6]: config-definition.md#aws + [6]: backupstoragelocation-definition.md#aws [14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html diff --git a/examples/aws/00-ark-config.yaml b/examples/aws/00-ark-config.yaml index 716c865fc..2068deec2 100644 --- a/examples/aws/00-ark-config.yaml +++ b/examples/aws/00-ark-config.yaml @@ -20,16 +20,5 @@ metadata: name: default persistentVolumeProvider: name: aws - config: - region: -backupStorageProvider: - name: aws - bucket: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # resticLocation: config: region: \ No newline at end of file diff --git a/examples/ibm/00-ark-config.yaml b/examples/aws/05-ark-backupstoragelocation.yaml similarity index 58% rename from examples/ibm/00-ark-config.yaml rename to examples/aws/05-ark-backupstoragelocation.yaml index b37b33e71..04789c478 100644 --- a/examples/ibm/00-ark-config.yaml +++ b/examples/aws/05-ark-backupstoragelocation.yaml @@ -14,21 +14,19 @@ --- apiVersion: ark.heptio.com/v1 -kind: Config +kind: BackupStorageLocation metadata: - namespace: heptio-ark name: default -backupStorageProvider: - name: aws - bucket: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # resticLocation: - config: - region: - s3ForcePathStyle: "true" - s3Url: ---- + namespace: heptio-ark +spec: + provider: aws + objectStorage: + bucket: + config: + region: + # Uncomment the below line to enable restic integration. + # The format for resticLocation is [/], + # e.g. "my-restic-bucket" or "my-restic-bucket/repos". + # This MUST be a different bucket than the main Ark bucket + # specified just above. + # restic-location: diff --git a/examples/azure/05-ark-backupstoragelocation.yaml b/examples/azure/05-ark-backupstoragelocation.yaml new file mode 100644 index 000000000..76c178206 --- /dev/null +++ b/examples/azure/05-ark-backupstoragelocation.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 the Heptio Ark contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: ark.heptio.com/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: heptio-ark +spec: + provider: azure + objectStorage: + bucket: + # Uncomment the below line to enable restic integration. + # The format for resticLocation is [/], + # e.g. "my-restic-bucket" or "my-restic-bucket/repos". + # This MUST be a different bucket than the main Ark bucket + # specified just above. + # restic-location: \ No newline at end of file diff --git a/examples/azure/10-ark-config.yaml b/examples/azure/10-ark-config.yaml index 4a2208ee8..9acb3d1c3 100644 --- a/examples/azure/10-ark-config.yaml +++ b/examples/azure/10-ark-config.yaml @@ -21,13 +21,4 @@ metadata: persistentVolumeProvider: name: azure config: - apiTimeout: -backupStorageProvider: - name: azure - bucket: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # resticLocation: \ No newline at end of file + apiTimeout: \ No newline at end of file diff --git a/examples/gcp/00-ark-config.yaml b/examples/gcp/00-ark-config.yaml index f686a3c19..9ebc5622d 100644 --- a/examples/gcp/00-ark-config.yaml +++ b/examples/gcp/00-ark-config.yaml @@ -19,14 +19,4 @@ metadata: namespace: heptio-ark name: default persistentVolumeProvider: - name: gcp -backupStorageProvider: - name: gcp - bucket: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # resticLocation: - + name: gcp \ No newline at end of file diff --git a/examples/gcp/05-ark-backupstoragelocation.yaml b/examples/gcp/05-ark-backupstoragelocation.yaml new file mode 100644 index 000000000..346aff838 --- /dev/null +++ b/examples/gcp/05-ark-backupstoragelocation.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 the Heptio Ark contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: ark.heptio.com/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: heptio-ark +spec: + provider: gcp + objectStorage: + bucket: + # Uncomment the below line to enable restic integration. + # The format for resticLocation is [/], + # e.g. "my-restic-bucket" or "my-restic-bucket/repos". + # This MUST be a different bucket than the main Ark bucket + # specified just above. + # restic-location: \ No newline at end of file diff --git a/examples/ibm/05-ark-backupstoragelocation.yaml b/examples/ibm/05-ark-backupstoragelocation.yaml new file mode 100644 index 000000000..2e65b45ad --- /dev/null +++ b/examples/ibm/05-ark-backupstoragelocation.yaml @@ -0,0 +1,34 @@ +# Copyright 2018 the Heptio Ark contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: ark.heptio.com/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: heptio-ark +spec: + provider: aws + objectStorage: + bucket: + config: + s3ForcePathStyle: "true" + s3Url: + region: + # Uncomment the below line to enable restic integration. + # The format for resticLocation is [/], + # e.g. "my-restic-bucket" or "my-restic-bucket/repos". + # This MUST be a different bucket than the main Ark bucket + # specified just above. + # restic-location: From 729d73398608ef7d0f882809bc7b540b54dfc98a Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Sat, 25 Aug 2018 12:53:56 -0700 Subject: [PATCH 23/29] controllers: take a newPluginManager func in constructors Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 17 +++++++++-------- pkg/controller/backup_controller.go | 14 ++++---------- pkg/controller/backup_controller_test.go | 6 +----- pkg/controller/backup_deletion_controller.go | 9 +++------ .../backup_deletion_controller_test.go | 11 +++-------- pkg/controller/backup_sync_controller.go | 9 ++++----- pkg/controller/backup_sync_controller_test.go | 8 +++----- pkg/controller/download_request_controller.go | 9 +++------ .../download_request_controller_test.go | 5 +---- pkg/controller/restore_controller.go | 15 ++++++--------- pkg/controller/restore_controller_test.go | 15 +++------------ 11 files changed, 40 insertions(+), 78 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 60d6c96b8..73c07bb7c 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -565,6 +565,10 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.metrics = metrics.NewServerMetrics() s.metrics.RegisterAllMetrics() + newPluginManager := func(logger logrus.FieldLogger) plugin.Manager { + return plugin.NewManager(logger, s.logLevel, s.pluginRegistry) + } + backupSyncController := controller.NewBackupSyncController( s.arkClient.ArkV1(), s.sharedInformerFactory.Ark().V1().Backups(), @@ -572,9 +576,8 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.config.backupSyncPeriod, s.namespace, s.config.defaultBackupLocation, - s.pluginRegistry, + newPluginManager, s.logger, - s.logLevel, ) wg.Add(1) go func() { @@ -604,7 +607,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.blockStore != nil, s.logger, s.logLevel, - s.pluginRegistry, + newPluginManager, backupTracker, s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.config.defaultBackupLocation, @@ -644,7 +647,6 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B backupDeletionController := controller.NewBackupDeletionController( s.logger, - s.logLevel, s.sharedInformerFactory.Ark().V1().DeleteBackupRequests(), s.arkClient.ArkV1(), // deleteBackupRequestClient s.arkClient.ArkV1(), // backupClient @@ -655,7 +657,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.resticManager, s.sharedInformerFactory.Ark().V1().PodVolumeBackups(), s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), - s.pluginRegistry, + newPluginManager, ) wg.Add(1) go func() { @@ -689,7 +691,7 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.blockStore != nil, s.logger, s.logLevel, - s.pluginRegistry, + newPluginManager, s.config.defaultBackupLocation, s.metrics, ) @@ -706,9 +708,8 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.sharedInformerFactory.Ark().V1().Restores(), s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.sharedInformerFactory.Ark().V1().Backups(), - s.pluginRegistry, + newPluginManager, s.logger, - s.logLevel, ) wg.Add(1) go func() { diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 18c487bd3..1ffd4a81a 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -67,14 +67,12 @@ type backupController struct { clock clock.Clock logger logrus.FieldLogger logLevel logrus.Level - pluginRegistry plugin.Registry + newPluginManager func(logrus.FieldLogger) plugin.Manager backupTracker BackupTracker backupLocationLister listers.BackupStorageLocationLister backupLocationListerSynced cache.InformerSynced defaultBackupLocation string metrics *metrics.ServerMetrics - - newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager } func NewBackupController( @@ -84,7 +82,7 @@ func NewBackupController( pvProviderExists bool, logger logrus.FieldLogger, logLevel logrus.Level, - pluginRegistry plugin.Registry, + newPluginManager func(logrus.FieldLogger) plugin.Manager, backupTracker BackupTracker, backupLocationInformer informers.BackupStorageLocationInformer, defaultBackupLocation string, @@ -100,16 +98,12 @@ func NewBackupController( clock: &clock.RealClock{}, logger: logger, logLevel: logLevel, - pluginRegistry: pluginRegistry, + newPluginManager: newPluginManager, backupTracker: backupTracker, backupLocationLister: backupLocationInformer.Lister(), backupLocationListerSynced: backupLocationInformer.Informer().HasSynced, defaultBackupLocation: defaultBackupLocation, metrics: metrics, - - newPluginManager: func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return plugin.NewManager(logger, logLevel, pluginRegistry) - }, } c.syncHandler = c.processBackup @@ -387,7 +381,7 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation log.Info("Starting backup") - pluginManager := controller.newPluginManager(log, log.Level, controller.pluginRegistry) + pluginManager := controller.newPluginManager(log) defer pluginManager.CleanupClients() backupFile, err := ioutil.TempFile("", "") diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 9a41d6d29..97c6fd63a 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -178,7 +178,6 @@ func TestProcessBackup(t *testing.T) { backupper = &fakeBackupper{} sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = logging.DefaultLogger(logrus.DebugLevel) - pluginRegistry = plugin.NewRegistry("/dir", logger, logrus.InfoLevel) clockTime, _ = time.Parse("Mon Jan 2 15:04:05 2006", "Mon Jan 2 15:04:05 2006") objectStore = &arktest.ObjectStore{} pluginManager = &pluginmocks.Manager{} @@ -194,7 +193,7 @@ func TestProcessBackup(t *testing.T) { test.allowSnapshots, logger, logrus.InfoLevel, - pluginRegistry, + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, NewBackupTracker(), sharedInformers.Ark().V1().BackupStorageLocations(), "default", @@ -202,9 +201,6 @@ func TestProcessBackup(t *testing.T) { ).(*backupController) c.clock = clock.NewFakeClock(clockTime) - c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return pluginManager - } var expiration, startTime time.Time diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index b4b33c6b3..3fd77d4c2 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -66,7 +66,6 @@ type backupDeletionController struct { // NewBackupDeletionController creates a new backup deletion controller. func NewBackupDeletionController( logger logrus.FieldLogger, - logLevel logrus.Level, deleteBackupRequestInformer informers.DeleteBackupRequestInformer, deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter, backupClient arkv1client.BackupsGetter, @@ -77,7 +76,7 @@ func NewBackupDeletionController( resticMgr restic.RepositoryManager, podvolumeBackupInformer informers.PodVolumeBackupInformer, backupLocationInformer informers.BackupStorageLocationInformer, - pluginRegistry plugin.Registry, + newPluginManager func(logrus.FieldLogger) plugin.Manager, ) Interface { c := &backupDeletionController{ genericController: newGenericController("backup-deletion", logger), @@ -94,10 +93,8 @@ func NewBackupDeletionController( // use variables to refer to these functions so they can be // replaced with fakes for testing. - deleteBackupDir: cloudprovider.DeleteBackupDir, - newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { - return plugin.NewManager(logger, logLevel, pluginRegistry) - }, + newPluginManager: newPluginManager, + deleteBackupDir: cloudprovider.DeleteBackupDir, clock: &clock.RealClock{}, } diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index 5bd39e890..1a1b6df77 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -48,7 +48,6 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { controller := NewBackupDeletionController( arktest.NewLogger(), - logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient @@ -59,7 +58,7 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), sharedInformers.Ark().V1().BackupStorageLocations(), - nil, // pluginRegistry + nil, // new plugin manager func ).(*backupDeletionController) // Error splitting key @@ -135,7 +134,6 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio objectStore: objectStore, controller: NewBackupDeletionController( arktest.NewLogger(), - logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient @@ -146,14 +144,12 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio nil, // restic repository manager sharedInformers.Ark().V1().PodVolumeBackups(), sharedInformers.Ark().V1().BackupStorageLocations(), - nil, // pluginRegistry + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, ).(*backupDeletionController), req: req, } - data.controller.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } - pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) pluginManager.On("CleanupClients").Return(nil) @@ -594,7 +590,6 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { controller := NewBackupDeletionController( arktest.NewLogger(), - logrus.InfoLevel, sharedInformers.Ark().V1().DeleteBackupRequests(), client.ArkV1(), // deleteBackupRequestClient client.ArkV1(), // backupClient @@ -605,7 +600,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { nil, sharedInformers.Ark().V1().PodVolumeBackups(), sharedInformers.Ark().V1().BackupStorageLocations(), - nil, // pluginRegistry + nil, // new plugin manager func ).(*backupDeletionController) fakeClock := &clock.FakeClock{} diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index a19d7aea0..38904051d 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -57,9 +57,8 @@ func NewBackupSyncController( syncPeriod time.Duration, namespace string, defaultBackupLocation string, - pluginRegistry plugin.Registry, + newPluginManager func(logrus.FieldLogger) plugin.Manager, logger logrus.FieldLogger, - logLevel logrus.Level, ) Interface { if syncPeriod < time.Minute { logger.Infof("Provided backup sync period %v is too short. Setting to 1 minute", syncPeriod) @@ -74,9 +73,9 @@ func NewBackupSyncController( backupLister: backupInformer.Lister(), backupStorageLocationLister: backupStorageLocationInformer.Lister(), - newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { - return plugin.NewManager(logger, logLevel, pluginRegistry) - }, + // use variables to refer to these functions so they can be + // replaced with fakes for testing. + newPluginManager: newPluginManager, listCloudBackups: cloudprovider.ListBackups, } diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 4badb0368..a69d31c4b 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -177,14 +177,13 @@ func TestBackupSyncControllerRun(t *testing.T) { time.Duration(0), test.namespace, "", - nil, // pluginRegistry + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, arktest.NewLogger(), - logrus.DebugLevel, ).(*backupSyncController) - c.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } pluginManager.On("GetObjectStore", "objStoreProvider").Return(objectStore, nil) pluginManager.On("CleanupClients").Return(nil) + objectStore.On("Init", mock.Anything).Return(nil) for _, location := range test.locations { @@ -343,9 +342,8 @@ func TestDeleteOrphanedBackups(t *testing.T) { time.Duration(0), test.namespace, "", - nil, // pluginRegistry + nil, // new plugin manager func arktest.NewLogger(), - logrus.InfoLevel, ).(*backupSyncController) expectedDeleteActions := make([]core.Action, 0) diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index d06dbb6dc..7a4c025f3 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -60,9 +60,8 @@ func NewDownloadRequestController( restoreInformer informers.RestoreInformer, backupLocationInformer informers.BackupStorageLocationInformer, backupInformer informers.BackupInformer, - pluginRegistry plugin.Registry, + newPluginManager func(logrus.FieldLogger) plugin.Manager, logger logrus.FieldLogger, - logLevel logrus.Level, ) Interface { c := &downloadRequestController{ genericController: newGenericController("downloadrequest", logger), @@ -74,10 +73,8 @@ func NewDownloadRequestController( // use variables to refer to these functions so they can be // replaced with fakes for testing. - createSignedURL: cloudprovider.CreateSignedURL, - newPluginManager: func(logger logrus.FieldLogger) plugin.Manager { - return plugin.NewManager(logger, logLevel, pluginRegistry) - }, + createSignedURL: cloudprovider.CreateSignedURL, + newPluginManager: newPluginManager, clock: &clock.RealClock{}, } diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 645fccbba..7098d9f11 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -59,9 +59,8 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { informerFactory.Ark().V1().Restores(), informerFactory.Ark().V1().BackupStorageLocations(), informerFactory.Ark().V1().Backups(), - nil, + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, arktest.NewLogger(), - logrus.InfoLevel, ).(*downloadRequestController) ) @@ -70,8 +69,6 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { controller.clock = clock.NewFakeClock(clockTime) - controller.newPluginManager = func(_ logrus.FieldLogger) plugin.Manager { return pluginManager } - pluginManager.On("CleanupClients").Return() objectStore.On("Init", mock.Anything).Return(nil) diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 215c51405..fead34d3a 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -86,7 +86,6 @@ type restoreController struct { queue workqueue.RateLimitingInterface logger logrus.FieldLogger logLevel logrus.Level - pluginRegistry plugin.Registry defaultBackupLocation string metrics *metrics.ServerMetrics @@ -94,7 +93,7 @@ type restoreController struct { downloadBackup cloudprovider.DownloadBackupFunc uploadRestoreLog cloudprovider.UploadRestoreLogFunc uploadRestoreResults cloudprovider.UploadRestoreResultsFunc - newPluginManager func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager + newPluginManager func(logger logrus.FieldLogger) plugin.Manager } func NewRestoreController( @@ -108,10 +107,9 @@ func NewRestoreController( pvProviderExists bool, logger logrus.FieldLogger, logLevel logrus.Level, - pluginRegistry plugin.Registry, + newPluginManager func(logrus.FieldLogger) plugin.Manager, defaultBackupLocation string, metrics *metrics.ServerMetrics, - ) Interface { c := &restoreController{ namespace: namespace, @@ -128,17 +126,16 @@ func NewRestoreController( queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"), logger: logger, logLevel: logLevel, - pluginRegistry: pluginRegistry, defaultBackupLocation: defaultBackupLocation, metrics: metrics, + // use variables to refer to these functions so they can be + // replaced with fakes for testing. + newPluginManager: newPluginManager, getBackup: cloudprovider.GetBackup, downloadBackup: cloudprovider.DownloadBackup, uploadRestoreLog: cloudprovider.UploadRestoreLog, uploadRestoreResults: cloudprovider.UploadRestoreResults, - newPluginManager: func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return plugin.NewManager(logger, logLevel, pluginRegistry) - }, } c.syncHandler = c.processRestore @@ -282,7 +279,7 @@ func (c *restoreController) processRestore(key string) error { // don't modify items in the cache restore = restore.DeepCopy() - pluginManager := c.newPluginManager(logContext, logContext.Level, c.pluginRegistry) + pluginManager := c.newPluginManager(logContext) defer pluginManager.CleanupClients() actions, err := pluginManager.GetRestoreItemActions() diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index b67d423e2..407a30739 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -105,13 +105,10 @@ func TestFetchBackupInfo(t *testing.T) { false, logger, logrus.InfoLevel, - nil, //pluginRegistry + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, "default", metrics.NewServerMetrics(), ).(*restoreController) - c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return pluginManager - } if test.backupServiceError == nil { pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil) @@ -200,13 +197,10 @@ func TestProcessRestoreSkips(t *testing.T) { false, // pvProviderExists logger, logrus.InfoLevel, - nil, // pluginRegistry + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, "default", metrics.NewServerMetrics(), ).(*restoreController) - c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return pluginManager - } if test.restore != nil { sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(test.restore) @@ -427,13 +421,10 @@ func TestProcessRestore(t *testing.T) { test.allowRestoreSnapshots, logger, logrus.InfoLevel, - nil, // pluginRegistry + func(logrus.FieldLogger) plugin.Manager { return pluginManager }, "default", metrics.NewServerMetrics(), ).(*restoreController) - c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { - return pluginManager - } if test.location != nil { sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(test.location) From b31e25bf6ee665ae832d5ef4620691805874cff8 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Sat, 25 Aug 2018 12:54:35 -0700 Subject: [PATCH 24/29] server: remove unused code, replace deprecated func Signed-off-by: Steve Kriss --- pkg/cmd/server/server.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 73c07bb7c..4666d21cc 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -30,6 +30,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -67,7 +68,6 @@ import ( "github.com/heptio/ark/pkg/util/kube" "github.com/heptio/ark/pkg/util/logging" "github.com/heptio/ark/pkg/util/stringslice" - "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( @@ -222,7 +222,7 @@ func newServer(namespace, baseName string, config serverConfig, logger *logrus.L arkClient: arkClient, discoveryClient: arkClient.Discovery(), dynamicClient: dynamicClient, - sharedInformerFactory: informers.NewFilteredSharedInformerFactory(arkClient, 0, namespace, nil), + sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(arkClient, 0, informers.WithNamespace(namespace)), ctx: ctx, cancelFunc: cancelFunc, logger: logger, @@ -488,13 +488,6 @@ func getBlockStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) return blockStore, nil } -func durationMin(a, b time.Duration) time.Duration { - if a < b { - return a - } - return b -} - func (s *server) initRestic(providerName string) error { // warn if restic daemonset does not exist if _, err := s.kubeClient.AppsV1().DaemonSets(s.namespace).Get(restic.DaemonSet, metav1.GetOptions{}); apierrors.IsNotFound(err) { From a440029c2fb0486ca8a58e5b1ba9ebfc51b8b61f Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 23 Aug 2018 16:12:18 -0700 Subject: [PATCH 25/29] bump Azure SDK version and include storage mgmt package Signed-off-by: Steve Kriss --- Gopkg.lock | 7 +- Gopkg.toml | 2 +- .../github.com/Azure/azure-sdk-for-go/NOTICE | 5 + .../azure-sdk-for-go/arm/disk/version.go | 4 +- .../arm/examples/helpers/helpers.go | 15 + .../mgmt/2017-10-01/storage/accounts.go | 952 ++++++++++++++++++ .../storage/mgmt/2017-10-01/storage/client.go | 51 + .../storage/mgmt/2017-10-01/storage/models.go | 605 +++++++++++ .../mgmt/2017-10-01/storage/operations.go | 98 ++ .../storage/mgmt/2017-10-01/storage/skus.go | 102 ++ .../storage/mgmt/2017-10-01/storage/usage.go | 102 ++ .../mgmt/2017-10-01/storage/version.go | 28 + .../azure-sdk-for-go/storage/appendblob.go | 27 +- .../azure-sdk-for-go/storage/authorization.go | 35 +- .../Azure/azure-sdk-for-go/storage/blob.go | 55 +- .../azure-sdk-for-go/storage/blobsasuri.go | 156 ++- .../storage/blobserviceclient.go | 97 +- .../azure-sdk-for-go/storage/blockblob.go | 20 +- .../Azure/azure-sdk-for-go/storage/client.go | 304 +++++- .../azure-sdk-for-go/storage/commonsasuri.go | 38 + .../azure-sdk-for-go/storage/container.go | 175 +++- .../azure-sdk-for-go/storage/copyblob.go | 14 + .../azure-sdk-for-go/storage/directory.go | 14 + .../Azure/azure-sdk-for-go/storage/entity.go | 14 + .../Azure/azure-sdk-for-go/storage/file.go | 16 +- .../storage/fileserviceclient.go | 14 + .../azure-sdk-for-go/storage/leaseblob.go | 14 + .../Azure/azure-sdk-for-go/storage/message.go | 14 + .../Azure/azure-sdk-for-go/storage/odata.go | 14 + .../azure-sdk-for-go/storage/pageblob.go | 23 +- .../Azure/azure-sdk-for-go/storage/queue.go | 14 + .../azure-sdk-for-go/storage/queuesasuri.go | 146 +++ .../storage/queueserviceclient.go | 14 + .../Azure/azure-sdk-for-go/storage/share.go | 14 + .../azure-sdk-for-go/storage/storagepolicy.go | 14 + .../storage/storageservice.go | 14 + .../Azure/azure-sdk-for-go/storage/table.go | 25 +- .../azure-sdk-for-go/storage/table_batch.go | 14 + .../storage/tableserviceclient.go | 14 + .../Azure/azure-sdk-for-go/storage/util.go | 44 +- .../Azure/azure-sdk-for-go/storage/version.go | 16 +- 41 files changed, 3199 insertions(+), 145 deletions(-) create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/NOTICE create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/accounts.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/client.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/models.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/operations.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/skus.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/usage.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/version.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/storage/commonsasuri.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/storage/queuesasuri.go diff --git a/Gopkg.lock b/Gopkg.lock index 42f68e609..88fe09b1e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -19,10 +19,11 @@ packages = [ "arm/disk", "arm/examples/helpers", + "services/storage/mgmt/2017-10-01/storage", "storage" ] - revision = "2d49bb8f2cee530cc16f1f1a9f0aae763dee257d" - version = "v10.2.1-beta" + revision = "2d1d76c9013c4feb6695a2346f0e66ea0ef77aa6" + version = "v11.3.0-beta" [[projects]] name = "github.com/Azure/go-autorest" @@ -806,6 +807,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "70b3cfc235408d89934ada479417194e2a82df523f459f7d9d3264538805ea98" + inputs-digest = "84d160fa2e769b80040762566acadbe7c23ee774124dfdf7a498c0e65cd8011a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c4657dcf6..67062a6ea 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -66,7 +66,7 @@ [[constraint]] name = "github.com/Azure/azure-sdk-for-go" - version = "~10.2.1-beta" + version = "~11.3.0-beta" [[constraint]] name = "cloud.google.com/go" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/NOTICE b/vendor/github.com/Azure/azure-sdk-for-go/NOTICE new file mode 100644 index 000000000..2d1d72608 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/NOTICE @@ -0,0 +1,5 @@ +Microsoft Azure-SDK-for-Go +Copyright 2014-2017 Microsoft + +This product includes software developed at +the Microsoft Corporation (https://www.microsoft.com). diff --git a/vendor/github.com/Azure/azure-sdk-for-go/arm/disk/version.go b/vendor/github.com/Azure/azure-sdk-for-go/arm/disk/version.go index 11c4a35ee..08d10df15 100755 --- a/vendor/github.com/Azure/azure-sdk-for-go/arm/disk/version.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/arm/disk/version.go @@ -19,10 +19,10 @@ package disk // UserAgent returns the UserAgent string to use when sending http.Requests. func UserAgent() string { - return "Azure-SDK-For-Go/v10.2.0-beta arm-disk/2016-04-30-preview" + return "Azure-SDK-For-Go/v11.3.0-beta arm-disk/2016-04-30-preview" } // Version returns the semantic version (see http://semver.org) of the client. func Version() string { - return "v10.2.0-beta" + return "v11.3.0-beta" } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go b/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go index f883215ef..de9d55166 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go @@ -1,8 +1,23 @@ package helpers +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/json" "fmt" + "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" ) diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/accounts.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/accounts.go new file mode 100644 index 000000000..1068281b1 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/accounts.go @@ -0,0 +1,952 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/validation" + "net/http" +) + +// AccountsClient is the the Azure Storage Management API. +type AccountsClient struct { + ManagementClient +} + +// NewAccountsClient creates an instance of the AccountsClient client. +func NewAccountsClient(subscriptionID string) AccountsClient { + return NewAccountsClientWithBaseURI(DefaultBaseURI, subscriptionID) +} + +// NewAccountsClientWithBaseURI creates an instance of the AccountsClient client. +func NewAccountsClientWithBaseURI(baseURI string, subscriptionID string) AccountsClient { + return AccountsClient{NewWithBaseURI(baseURI, subscriptionID)} +} + +// CheckNameAvailability checks that the storage account name is valid and is not already in use. +// +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. +func (client AccountsClient) CheckNameAvailability(accountName AccountCheckNameAvailabilityParameters) (result CheckNameAvailabilityResult, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName.Name", Name: validation.Null, Rule: true, Chain: nil}, + {Target: "accountName.Type", Name: validation.Null, Rule: true, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "CheckNameAvailability") + } + + req, err := client.CheckNameAvailabilityPreparer(accountName) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "CheckNameAvailability", nil, "Failure preparing request") + return + } + + resp, err := client.CheckNameAvailabilitySender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "CheckNameAvailability", resp, "Failure sending request") + return + } + + result, err = client.CheckNameAvailabilityResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "CheckNameAvailability", resp, "Failure responding to request") + } + + return +} + +// CheckNameAvailabilityPreparer prepares the CheckNameAvailability request. +func (client AccountsClient) CheckNameAvailabilityPreparer(accountName AccountCheckNameAvailabilityParameters) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPost(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/providers/Microsoft.Storage/checkNameAvailability", pathParameters), + autorest.WithJSON(accountName), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// CheckNameAvailabilitySender sends the CheckNameAvailability request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) CheckNameAvailabilitySender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// CheckNameAvailabilityResponder handles the response to the CheckNameAvailability request. The method always +// closes the http.Response Body. +func (client AccountsClient) CheckNameAvailabilityResponder(resp *http.Response) (result CheckNameAvailabilityResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// Create asynchronously creates a new storage account with the specified parameters. If an account is already created +// and a subsequent create request is issued with different properties, the account properties will be updated. If an +// account is already created and a subsequent create or update request is issued with the exact same set of +// properties, the request will succeed. This method may poll for completion. Polling can be canceled by passing the +// cancel channel argument. The channel will be used to cancel polling and any outstanding HTTP requests. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. parameters is the parameters to +// provide for the created account. +func (client AccountsClient) Create(resourceGroupName string, accountName string, parameters AccountCreateParameters, cancel <-chan struct{}) (<-chan Account, <-chan error) { + resultChan := make(chan Account, 1) + errChan := make(chan error, 1) + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}, + {TargetValue: parameters, + Constraints: []validation.Constraint{{Target: "parameters.Sku", Name: validation.Null, Rule: true, Chain: nil}, + {Target: "parameters.Location", Name: validation.Null, Rule: true, Chain: nil}, + {Target: "parameters.Identity", Name: validation.Null, Rule: false, + Chain: []validation.Constraint{{Target: "parameters.Identity.Type", Name: validation.Null, Rule: true, Chain: nil}}}, + {Target: "parameters.AccountPropertiesCreateParameters", Name: validation.Null, Rule: false, + Chain: []validation.Constraint{{Target: "parameters.AccountPropertiesCreateParameters.CustomDomain", Name: validation.Null, Rule: false, + Chain: []validation.Constraint{{Target: "parameters.AccountPropertiesCreateParameters.CustomDomain.Name", Name: validation.Null, Rule: true, Chain: nil}}}, + }}}}}); err != nil { + errChan <- validation.NewErrorWithValidationError(err, "storage.AccountsClient", "Create") + close(errChan) + close(resultChan) + return resultChan, errChan + } + + go func() { + var err error + var result Account + defer func() { + if err != nil { + errChan <- err + } + resultChan <- result + close(resultChan) + close(errChan) + }() + req, err := client.CreatePreparer(resourceGroupName, accountName, parameters, cancel) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Create", nil, "Failure preparing request") + return + } + + resp, err := client.CreateSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Create", resp, "Failure sending request") + return + } + + result, err = client.CreateResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Create", resp, "Failure responding to request") + } + }() + return resultChan, errChan +} + +// CreatePreparer prepares the Create request. +func (client AccountsClient) CreatePreparer(resourceGroupName string, accountName string, parameters AccountCreateParameters, cancel <-chan struct{}) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPut(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}", pathParameters), + autorest.WithJSON(parameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{Cancel: cancel}) +} + +// CreateSender sends the Create request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) CreateSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client), + azure.DoPollForAsynchronous(client.PollingDelay)) +} + +// CreateResponder handles the response to the Create request. The method always +// closes the http.Response Body. +func (client AccountsClient) CreateResponder(resp *http.Response) (result Account, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusAccepted), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// Delete deletes a storage account in Microsoft Azure. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. +func (client AccountsClient) Delete(resourceGroupName string, accountName string) (result autorest.Response, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "Delete") + } + + req, err := client.DeletePreparer(resourceGroupName, accountName) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Delete", nil, "Failure preparing request") + return + } + + resp, err := client.DeleteSender(req) + if err != nil { + result.Response = resp + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Delete", resp, "Failure sending request") + return + } + + result, err = client.DeleteResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Delete", resp, "Failure responding to request") + } + + return +} + +// DeletePreparer prepares the Delete request. +func (client AccountsClient) DeletePreparer(resourceGroupName string, accountName string) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsDelete(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// DeleteSender sends the Delete request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) DeleteSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// DeleteResponder handles the response to the Delete request. The method always +// closes the http.Response Body. +func (client AccountsClient) DeleteResponder(resp *http.Response) (result autorest.Response, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusNoContent), + autorest.ByClosing()) + result.Response = resp + return +} + +// GetProperties returns the properties for the specified storage account including but not limited to name, SKU name, +// location, and account status. The ListKeys operation should be used to retrieve storage keys. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. +func (client AccountsClient) GetProperties(resourceGroupName string, accountName string) (result Account, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "GetProperties") + } + + req, err := client.GetPropertiesPreparer(resourceGroupName, accountName) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "GetProperties", nil, "Failure preparing request") + return + } + + resp, err := client.GetPropertiesSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "GetProperties", resp, "Failure sending request") + return + } + + result, err = client.GetPropertiesResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "GetProperties", resp, "Failure responding to request") + } + + return +} + +// GetPropertiesPreparer prepares the GetProperties request. +func (client AccountsClient) GetPropertiesPreparer(resourceGroupName string, accountName string) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// GetPropertiesSender sends the GetProperties request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) GetPropertiesSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// GetPropertiesResponder handles the response to the GetProperties request. The method always +// closes the http.Response Body. +func (client AccountsClient) GetPropertiesResponder(resp *http.Response) (result Account, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// List lists all the storage accounts available under the subscription. Note that storage keys are not returned; use +// the ListKeys operation for this. +func (client AccountsClient) List() (result AccountListResult, err error) { + req, err := client.ListPreparer() + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "List", nil, "Failure preparing request") + return + } + + resp, err := client.ListSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "List", resp, "Failure sending request") + return + } + + result, err = client.ListResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "List", resp, "Failure responding to request") + } + + return +} + +// ListPreparer prepares the List request. +func (client AccountsClient) ListPreparer() (*http.Request, error) { + pathParameters := map[string]interface{}{ + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/providers/Microsoft.Storage/storageAccounts", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListSender sends the List request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) ListSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListResponder handles the response to the List request. The method always +// closes the http.Response Body. +func (client AccountsClient) ListResponder(resp *http.Response) (result AccountListResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// ListAccountSAS list SAS credentials of a storage account. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. parameters is the parameters to +// provide to list SAS credentials for the storage account. +func (client AccountsClient) ListAccountSAS(resourceGroupName string, accountName string, parameters AccountSasParameters) (result ListAccountSasResponse, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}, + {TargetValue: parameters, + Constraints: []validation.Constraint{{Target: "parameters.SharedAccessExpiryTime", Name: validation.Null, Rule: true, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "ListAccountSAS") + } + + req, err := client.ListAccountSASPreparer(resourceGroupName, accountName, parameters) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListAccountSAS", nil, "Failure preparing request") + return + } + + resp, err := client.ListAccountSASSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListAccountSAS", resp, "Failure sending request") + return + } + + result, err = client.ListAccountSASResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListAccountSAS", resp, "Failure responding to request") + } + + return +} + +// ListAccountSASPreparer prepares the ListAccountSAS request. +func (client AccountsClient) ListAccountSASPreparer(resourceGroupName string, accountName string, parameters AccountSasParameters) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPost(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}/ListAccountSas", pathParameters), + autorest.WithJSON(parameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListAccountSASSender sends the ListAccountSAS request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) ListAccountSASSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListAccountSASResponder handles the response to the ListAccountSAS request. The method always +// closes the http.Response Body. +func (client AccountsClient) ListAccountSASResponder(resp *http.Response) (result ListAccountSasResponse, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// ListByResourceGroup lists all the storage accounts available under the given resource group. Note that storage keys +// are not returned; use the ListKeys operation for this. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +func (client AccountsClient) ListByResourceGroup(resourceGroupName string) (result AccountListResult, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "ListByResourceGroup") + } + + req, err := client.ListByResourceGroupPreparer(resourceGroupName) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListByResourceGroup", nil, "Failure preparing request") + return + } + + resp, err := client.ListByResourceGroupSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListByResourceGroup", resp, "Failure sending request") + return + } + + result, err = client.ListByResourceGroupResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListByResourceGroup", resp, "Failure responding to request") + } + + return +} + +// ListByResourceGroupPreparer prepares the ListByResourceGroup request. +func (client AccountsClient) ListByResourceGroupPreparer(resourceGroupName string) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListByResourceGroupSender sends the ListByResourceGroup request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) ListByResourceGroupSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListByResourceGroupResponder handles the response to the ListByResourceGroup request. The method always +// closes the http.Response Body. +func (client AccountsClient) ListByResourceGroupResponder(resp *http.Response) (result AccountListResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// ListKeys lists the access keys for the specified storage account. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. +func (client AccountsClient) ListKeys(resourceGroupName string, accountName string) (result AccountListKeysResult, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "ListKeys") + } + + req, err := client.ListKeysPreparer(resourceGroupName, accountName) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListKeys", nil, "Failure preparing request") + return + } + + resp, err := client.ListKeysSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListKeys", resp, "Failure sending request") + return + } + + result, err = client.ListKeysResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListKeys", resp, "Failure responding to request") + } + + return +} + +// ListKeysPreparer prepares the ListKeys request. +func (client AccountsClient) ListKeysPreparer(resourceGroupName string, accountName string) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsPost(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}/listKeys", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListKeysSender sends the ListKeys request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) ListKeysSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListKeysResponder handles the response to the ListKeys request. The method always +// closes the http.Response Body. +func (client AccountsClient) ListKeysResponder(resp *http.Response) (result AccountListKeysResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// ListServiceSAS list service SAS credentials of a specific resource. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. parameters is the parameters to +// provide to list service SAS credentials. +func (client AccountsClient) ListServiceSAS(resourceGroupName string, accountName string, parameters ServiceSasParameters) (result ListServiceSasResponse, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}, + {TargetValue: parameters, + Constraints: []validation.Constraint{{Target: "parameters.CanonicalizedResource", Name: validation.Null, Rule: true, Chain: nil}, + {Target: "parameters.Identifier", Name: validation.Null, Rule: false, + Chain: []validation.Constraint{{Target: "parameters.Identifier", Name: validation.MaxLength, Rule: 64, Chain: nil}}}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "ListServiceSAS") + } + + req, err := client.ListServiceSASPreparer(resourceGroupName, accountName, parameters) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListServiceSAS", nil, "Failure preparing request") + return + } + + resp, err := client.ListServiceSASSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListServiceSAS", resp, "Failure sending request") + return + } + + result, err = client.ListServiceSASResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "ListServiceSAS", resp, "Failure responding to request") + } + + return +} + +// ListServiceSASPreparer prepares the ListServiceSAS request. +func (client AccountsClient) ListServiceSASPreparer(resourceGroupName string, accountName string, parameters ServiceSasParameters) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPost(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}/ListServiceSas", pathParameters), + autorest.WithJSON(parameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListServiceSASSender sends the ListServiceSAS request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) ListServiceSASSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListServiceSASResponder handles the response to the ListServiceSAS request. The method always +// closes the http.Response Body. +func (client AccountsClient) ListServiceSASResponder(resp *http.Response) (result ListServiceSasResponse, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// RegenerateKey regenerates one of the access keys for the specified storage account. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. regenerateKey is specifies name +// of the key which should be regenerated -- key1 or key2. +func (client AccountsClient) RegenerateKey(resourceGroupName string, accountName string, regenerateKey AccountRegenerateKeyParameters) (result AccountListKeysResult, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}, + {TargetValue: regenerateKey, + Constraints: []validation.Constraint{{Target: "regenerateKey.KeyName", Name: validation.Null, Rule: true, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "RegenerateKey") + } + + req, err := client.RegenerateKeyPreparer(resourceGroupName, accountName, regenerateKey) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "RegenerateKey", nil, "Failure preparing request") + return + } + + resp, err := client.RegenerateKeySender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "RegenerateKey", resp, "Failure sending request") + return + } + + result, err = client.RegenerateKeyResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "RegenerateKey", resp, "Failure responding to request") + } + + return +} + +// RegenerateKeyPreparer prepares the RegenerateKey request. +func (client AccountsClient) RegenerateKeyPreparer(resourceGroupName string, accountName string, regenerateKey AccountRegenerateKeyParameters) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPost(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}/regenerateKey", pathParameters), + autorest.WithJSON(regenerateKey), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// RegenerateKeySender sends the RegenerateKey request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) RegenerateKeySender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// RegenerateKeyResponder handles the response to the RegenerateKey request. The method always +// closes the http.Response Body. +func (client AccountsClient) RegenerateKeyResponder(resp *http.Response) (result AccountListKeysResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} + +// Update the update operation can be used to update the SKU, encryption, access tier, or tags for a storage account. +// It can also be used to map the account to a custom domain. Only one custom domain is supported per storage account; +// the replacement/change of custom domain is not supported. In order to replace an old custom domain, the old value +// must be cleared/unregistered before a new value can be set. The update of multiple properties is supported. This +// call does not change the storage keys for the account. If you want to change the storage account keys, use the +// regenerate keys operation. The location and name of the storage account cannot be changed after creation. +// +// resourceGroupName is the name of the resource group within the user's subscription. The name is case insensitive. +// accountName is the name of the storage account within the specified resource group. Storage account names must be +// between 3 and 24 characters in length and use numbers and lower-case letters only. parameters is the parameters to +// provide for the updated account. +func (client AccountsClient) Update(resourceGroupName string, accountName string, parameters AccountUpdateParameters) (result Account, err error) { + if err := validation.Validate([]validation.Validation{ + {TargetValue: resourceGroupName, + Constraints: []validation.Constraint{{Target: "resourceGroupName", Name: validation.MaxLength, Rule: 90, Chain: nil}, + {Target: "resourceGroupName", Name: validation.MinLength, Rule: 1, Chain: nil}, + {Target: "resourceGroupName", Name: validation.Pattern, Rule: `^[-\w\._\(\)]+$`, Chain: nil}}}, + {TargetValue: accountName, + Constraints: []validation.Constraint{{Target: "accountName", Name: validation.MaxLength, Rule: 24, Chain: nil}, + {Target: "accountName", Name: validation.MinLength, Rule: 3, Chain: nil}}}}); err != nil { + return result, validation.NewErrorWithValidationError(err, "storage.AccountsClient", "Update") + } + + req, err := client.UpdatePreparer(resourceGroupName, accountName, parameters) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Update", nil, "Failure preparing request") + return + } + + resp, err := client.UpdateSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Update", resp, "Failure sending request") + return + } + + result, err = client.UpdateResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.AccountsClient", "Update", resp, "Failure responding to request") + } + + return +} + +// UpdatePreparer prepares the Update request. +func (client AccountsClient) UpdatePreparer(resourceGroupName string, accountName string, parameters AccountUpdateParameters) (*http.Request, error) { + pathParameters := map[string]interface{}{ + "accountName": autorest.Encode("path", accountName), + "resourceGroupName": autorest.Encode("path", resourceGroupName), + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsJSON(), + autorest.AsPatch(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Storage/storageAccounts/{accountName}", pathParameters), + autorest.WithJSON(parameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// UpdateSender sends the Update request. The method will close the +// http.Response Body if it receives an error. +func (client AccountsClient) UpdateSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// UpdateResponder handles the response to the Update request. The method always +// closes the http.Response Body. +func (client AccountsClient) UpdateResponder(resp *http.Response) (result Account, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/client.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/client.go new file mode 100644 index 000000000..05d570127 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/client.go @@ -0,0 +1,51 @@ +// Package storage implements the Azure ARM Storage service API version 2017-10-01. +// +// The Azure Storage Management API. +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" +) + +const ( + // DefaultBaseURI is the default URI used for the service Storage + DefaultBaseURI = "https://management.azure.com" +) + +// ManagementClient is the base client for Storage. +type ManagementClient struct { + autorest.Client + BaseURI string + SubscriptionID string +} + +// New creates an instance of the ManagementClient client. +func New(subscriptionID string) ManagementClient { + return NewWithBaseURI(DefaultBaseURI, subscriptionID) +} + +// NewWithBaseURI creates an instance of the ManagementClient client. +func NewWithBaseURI(baseURI string, subscriptionID string) ManagementClient { + return ManagementClient{ + Client: autorest.NewClientWithUserAgent(UserAgent()), + BaseURI: baseURI, + SubscriptionID: subscriptionID, + } +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/models.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/models.go new file mode 100644 index 000000000..c4045794d --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/models.go @@ -0,0 +1,605 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/date" +) + +// AccessTier enumerates the values for access tier. +type AccessTier string + +const ( + // Cool specifies the cool state for access tier. + Cool AccessTier = "Cool" + // Hot specifies the hot state for access tier. + Hot AccessTier = "Hot" +) + +// AccountStatus enumerates the values for account status. +type AccountStatus string + +const ( + // Available specifies the available state for account status. + Available AccountStatus = "available" + // Unavailable specifies the unavailable state for account status. + Unavailable AccountStatus = "unavailable" +) + +// Action enumerates the values for action. +type Action string + +const ( + // Allow specifies the allow state for action. + Allow Action = "Allow" +) + +// Bypass enumerates the values for bypass. +type Bypass string + +const ( + // AzureServices specifies the azure services state for bypass. + AzureServices Bypass = "AzureServices" + // Logging specifies the logging state for bypass. + Logging Bypass = "Logging" + // Metrics specifies the metrics state for bypass. + Metrics Bypass = "Metrics" + // None specifies the none state for bypass. + None Bypass = "None" +) + +// DefaultAction enumerates the values for default action. +type DefaultAction string + +const ( + // DefaultActionAllow specifies the default action allow state for default action. + DefaultActionAllow DefaultAction = "Allow" + // DefaultActionDeny specifies the default action deny state for default action. + DefaultActionDeny DefaultAction = "Deny" +) + +// HTTPProtocol enumerates the values for http protocol. +type HTTPProtocol string + +const ( + // HTTPS specifies the https state for http protocol. + HTTPS HTTPProtocol = "https" + // Httpshttp specifies the httpshttp state for http protocol. + Httpshttp HTTPProtocol = "https,http" +) + +// KeyPermission enumerates the values for key permission. +type KeyPermission string + +const ( + // Full specifies the full state for key permission. + Full KeyPermission = "Full" + // Read specifies the read state for key permission. + Read KeyPermission = "Read" +) + +// KeySource enumerates the values for key source. +type KeySource string + +const ( + // MicrosoftKeyvault specifies the microsoft keyvault state for key source. + MicrosoftKeyvault KeySource = "Microsoft.Keyvault" + // MicrosoftStorage specifies the microsoft storage state for key source. + MicrosoftStorage KeySource = "Microsoft.Storage" +) + +// Kind enumerates the values for kind. +type Kind string + +const ( + // BlobStorage specifies the blob storage state for kind. + BlobStorage Kind = "BlobStorage" + // Storage specifies the storage state for kind. + Storage Kind = "Storage" + // StorageV2 specifies the storage v2 state for kind. + StorageV2 Kind = "StorageV2" +) + +// Permissions enumerates the values for permissions. +type Permissions string + +const ( + // A specifies the a state for permissions. + A Permissions = "a" + // C specifies the c state for permissions. + C Permissions = "c" + // D specifies the d state for permissions. + D Permissions = "d" + // L specifies the l state for permissions. + L Permissions = "l" + // P specifies the p state for permissions. + P Permissions = "p" + // R specifies the r state for permissions. + R Permissions = "r" + // U specifies the u state for permissions. + U Permissions = "u" + // W specifies the w state for permissions. + W Permissions = "w" +) + +// ProvisioningState enumerates the values for provisioning state. +type ProvisioningState string + +const ( + // Creating specifies the creating state for provisioning state. + Creating ProvisioningState = "Creating" + // ResolvingDNS specifies the resolving dns state for provisioning state. + ResolvingDNS ProvisioningState = "ResolvingDNS" + // Succeeded specifies the succeeded state for provisioning state. + Succeeded ProvisioningState = "Succeeded" +) + +// Reason enumerates the values for reason. +type Reason string + +const ( + // AccountNameInvalid specifies the account name invalid state for reason. + AccountNameInvalid Reason = "AccountNameInvalid" + // AlreadyExists specifies the already exists state for reason. + AlreadyExists Reason = "AlreadyExists" +) + +// ReasonCode enumerates the values for reason code. +type ReasonCode string + +const ( + // NotAvailableForSubscription specifies the not available for subscription state for reason code. + NotAvailableForSubscription ReasonCode = "NotAvailableForSubscription" + // QuotaID specifies the quota id state for reason code. + QuotaID ReasonCode = "QuotaId" +) + +// Services enumerates the values for services. +type Services string + +const ( + // B specifies the b state for services. + B Services = "b" + // F specifies the f state for services. + F Services = "f" + // Q specifies the q state for services. + Q Services = "q" + // T specifies the t state for services. + T Services = "t" +) + +// SignedResource enumerates the values for signed resource. +type SignedResource string + +const ( + // SignedResourceB specifies the signed resource b state for signed resource. + SignedResourceB SignedResource = "b" + // SignedResourceC specifies the signed resource c state for signed resource. + SignedResourceC SignedResource = "c" + // SignedResourceF specifies the signed resource f state for signed resource. + SignedResourceF SignedResource = "f" + // SignedResourceS specifies the signed resource s state for signed resource. + SignedResourceS SignedResource = "s" +) + +// SignedResourceTypes enumerates the values for signed resource types. +type SignedResourceTypes string + +const ( + // SignedResourceTypesC specifies the signed resource types c state for signed resource types. + SignedResourceTypesC SignedResourceTypes = "c" + // SignedResourceTypesO specifies the signed resource types o state for signed resource types. + SignedResourceTypesO SignedResourceTypes = "o" + // SignedResourceTypesS specifies the signed resource types s state for signed resource types. + SignedResourceTypesS SignedResourceTypes = "s" +) + +// SkuName enumerates the values for sku name. +type SkuName string + +const ( + // PremiumLRS specifies the premium lrs state for sku name. + PremiumLRS SkuName = "Premium_LRS" + // StandardGRS specifies the standard grs state for sku name. + StandardGRS SkuName = "Standard_GRS" + // StandardLRS specifies the standard lrs state for sku name. + StandardLRS SkuName = "Standard_LRS" + // StandardRAGRS specifies the standard ragrs state for sku name. + StandardRAGRS SkuName = "Standard_RAGRS" + // StandardZRS specifies the standard zrs state for sku name. + StandardZRS SkuName = "Standard_ZRS" +) + +// SkuTier enumerates the values for sku tier. +type SkuTier string + +const ( + // Premium specifies the premium state for sku tier. + Premium SkuTier = "Premium" + // Standard specifies the standard state for sku tier. + Standard SkuTier = "Standard" +) + +// State enumerates the values for state. +type State string + +const ( + // StateDeprovisioning specifies the state deprovisioning state for state. + StateDeprovisioning State = "deprovisioning" + // StateFailed specifies the state failed state for state. + StateFailed State = "failed" + // StateNetworkSourceDeleted specifies the state network source deleted state for state. + StateNetworkSourceDeleted State = "networkSourceDeleted" + // StateProvisioning specifies the state provisioning state for state. + StateProvisioning State = "provisioning" + // StateSucceeded specifies the state succeeded state for state. + StateSucceeded State = "succeeded" +) + +// UsageUnit enumerates the values for usage unit. +type UsageUnit string + +const ( + // Bytes specifies the bytes state for usage unit. + Bytes UsageUnit = "Bytes" + // BytesPerSecond specifies the bytes per second state for usage unit. + BytesPerSecond UsageUnit = "BytesPerSecond" + // Count specifies the count state for usage unit. + Count UsageUnit = "Count" + // CountsPerSecond specifies the counts per second state for usage unit. + CountsPerSecond UsageUnit = "CountsPerSecond" + // Percent specifies the percent state for usage unit. + Percent UsageUnit = "Percent" + // Seconds specifies the seconds state for usage unit. + Seconds UsageUnit = "Seconds" +) + +// Account is the storage account. +type Account struct { + autorest.Response `json:"-"` + ID *string `json:"id,omitempty"` + Name *string `json:"name,omitempty"` + Type *string `json:"type,omitempty"` + Location *string `json:"location,omitempty"` + Tags *map[string]*string `json:"tags,omitempty"` + Sku *Sku `json:"sku,omitempty"` + Kind Kind `json:"kind,omitempty"` + Identity *Identity `json:"identity,omitempty"` + *AccountProperties `json:"properties,omitempty"` +} + +// AccountCheckNameAvailabilityParameters is the parameters used to check the availabity of the storage account name. +type AccountCheckNameAvailabilityParameters struct { + Name *string `json:"name,omitempty"` + Type *string `json:"type,omitempty"` +} + +// AccountCreateParameters is the parameters used when creating a storage account. +type AccountCreateParameters struct { + Sku *Sku `json:"sku,omitempty"` + Kind Kind `json:"kind,omitempty"` + Location *string `json:"location,omitempty"` + Tags *map[string]*string `json:"tags,omitempty"` + Identity *Identity `json:"identity,omitempty"` + *AccountPropertiesCreateParameters `json:"properties,omitempty"` +} + +// AccountKey is an access key for the storage account. +type AccountKey struct { + KeyName *string `json:"keyName,omitempty"` + Value *string `json:"value,omitempty"` + Permissions KeyPermission `json:"permissions,omitempty"` +} + +// AccountListKeysResult is the response from the ListKeys operation. +type AccountListKeysResult struct { + autorest.Response `json:"-"` + Keys *[]AccountKey `json:"keys,omitempty"` +} + +// AccountListResult is the response from the List Storage Accounts operation. +type AccountListResult struct { + autorest.Response `json:"-"` + Value *[]Account `json:"value,omitempty"` +} + +// AccountProperties is properties of the storage account. +type AccountProperties struct { + ProvisioningState ProvisioningState `json:"provisioningState,omitempty"` + PrimaryEndpoints *Endpoints `json:"primaryEndpoints,omitempty"` + PrimaryLocation *string `json:"primaryLocation,omitempty"` + StatusOfPrimary AccountStatus `json:"statusOfPrimary,omitempty"` + LastGeoFailoverTime *date.Time `json:"lastGeoFailoverTime,omitempty"` + SecondaryLocation *string `json:"secondaryLocation,omitempty"` + StatusOfSecondary AccountStatus `json:"statusOfSecondary,omitempty"` + CreationTime *date.Time `json:"creationTime,omitempty"` + CustomDomain *CustomDomain `json:"customDomain,omitempty"` + SecondaryEndpoints *Endpoints `json:"secondaryEndpoints,omitempty"` + Encryption *Encryption `json:"encryption,omitempty"` + AccessTier AccessTier `json:"accessTier,omitempty"` + EnableHTTPSTrafficOnly *bool `json:"supportsHttpsTrafficOnly,omitempty"` + NetworkRuleSet *NetworkRuleSet `json:"networkAcls,omitempty"` +} + +// AccountPropertiesCreateParameters is the parameters used to create the storage account. +type AccountPropertiesCreateParameters struct { + CustomDomain *CustomDomain `json:"customDomain,omitempty"` + Encryption *Encryption `json:"encryption,omitempty"` + NetworkRuleSet *NetworkRuleSet `json:"networkAcls,omitempty"` + AccessTier AccessTier `json:"accessTier,omitempty"` + EnableHTTPSTrafficOnly *bool `json:"supportsHttpsTrafficOnly,omitempty"` +} + +// AccountPropertiesUpdateParameters is the parameters used when updating a storage account. +type AccountPropertiesUpdateParameters struct { + CustomDomain *CustomDomain `json:"customDomain,omitempty"` + Encryption *Encryption `json:"encryption,omitempty"` + AccessTier AccessTier `json:"accessTier,omitempty"` + EnableHTTPSTrafficOnly *bool `json:"supportsHttpsTrafficOnly,omitempty"` + NetworkRuleSet *NetworkRuleSet `json:"networkAcls,omitempty"` +} + +// AccountRegenerateKeyParameters is the parameters used to regenerate the storage account key. +type AccountRegenerateKeyParameters struct { + KeyName *string `json:"keyName,omitempty"` +} + +// AccountSasParameters is the parameters to list SAS credentials of a storage account. +type AccountSasParameters struct { + Services Services `json:"signedServices,omitempty"` + ResourceTypes SignedResourceTypes `json:"signedResourceTypes,omitempty"` + Permissions Permissions `json:"signedPermission,omitempty"` + IPAddressOrRange *string `json:"signedIp,omitempty"` + Protocols HTTPProtocol `json:"signedProtocol,omitempty"` + SharedAccessStartTime *date.Time `json:"signedStart,omitempty"` + SharedAccessExpiryTime *date.Time `json:"signedExpiry,omitempty"` + KeyToSign *string `json:"keyToSign,omitempty"` +} + +// AccountUpdateParameters is the parameters that can be provided when updating the storage account properties. +type AccountUpdateParameters struct { + Sku *Sku `json:"sku,omitempty"` + Tags *map[string]*string `json:"tags,omitempty"` + Identity *Identity `json:"identity,omitempty"` + *AccountPropertiesUpdateParameters `json:"properties,omitempty"` + Kind Kind `json:"kind,omitempty"` +} + +// CheckNameAvailabilityResult is the CheckNameAvailability operation response. +type CheckNameAvailabilityResult struct { + autorest.Response `json:"-"` + NameAvailable *bool `json:"nameAvailable,omitempty"` + Reason Reason `json:"reason,omitempty"` + Message *string `json:"message,omitempty"` +} + +// CustomDomain is the custom domain assigned to this storage account. This can be set via Update. +type CustomDomain struct { + Name *string `json:"name,omitempty"` + UseSubDomain *bool `json:"useSubDomain,omitempty"` +} + +// Dimension is dimension of blobs, possiblly be blob type or access tier. +type Dimension struct { + Name *string `json:"name,omitempty"` + DisplayName *string `json:"displayName,omitempty"` +} + +// Encryption is the encryption settings on the storage account. +type Encryption struct { + Services *EncryptionServices `json:"services,omitempty"` + KeySource KeySource `json:"keySource,omitempty"` + KeyVaultProperties *KeyVaultProperties `json:"keyvaultproperties,omitempty"` +} + +// EncryptionService is a service that allows server-side encryption to be used. +type EncryptionService struct { + Enabled *bool `json:"enabled,omitempty"` + LastEnabledTime *date.Time `json:"lastEnabledTime,omitempty"` +} + +// EncryptionServices is a list of services that support encryption. +type EncryptionServices struct { + Blob *EncryptionService `json:"blob,omitempty"` + File *EncryptionService `json:"file,omitempty"` + Table *EncryptionService `json:"table,omitempty"` + Queue *EncryptionService `json:"queue,omitempty"` +} + +// Endpoints is the URIs that are used to perform a retrieval of a public blob, queue, or table object. +type Endpoints struct { + Blob *string `json:"blob,omitempty"` + Queue *string `json:"queue,omitempty"` + Table *string `json:"table,omitempty"` + File *string `json:"file,omitempty"` +} + +// Identity is identity for the resource. +type Identity struct { + PrincipalID *string `json:"principalId,omitempty"` + TenantID *string `json:"tenantId,omitempty"` + Type *string `json:"type,omitempty"` +} + +// IPRule is IP rule with specific IP or IP range in CIDR format. +type IPRule struct { + IPAddressOrRange *string `json:"value,omitempty"` + Action Action `json:"action,omitempty"` +} + +// KeyVaultProperties is properties of key vault. +type KeyVaultProperties struct { + KeyName *string `json:"keyname,omitempty"` + KeyVersion *string `json:"keyversion,omitempty"` + KeyVaultURI *string `json:"keyvaulturi,omitempty"` +} + +// ListAccountSasResponse is the List SAS credentials operation response. +type ListAccountSasResponse struct { + autorest.Response `json:"-"` + AccountSasToken *string `json:"accountSasToken,omitempty"` +} + +// ListServiceSasResponse is the List service SAS credentials operation response. +type ListServiceSasResponse struct { + autorest.Response `json:"-"` + ServiceSasToken *string `json:"serviceSasToken,omitempty"` +} + +// MetricSpecification is metric specification of operation. +type MetricSpecification struct { + Name *string `json:"name,omitempty"` + DisplayName *string `json:"displayName,omitempty"` + DisplayDescription *string `json:"displayDescription,omitempty"` + Unit *string `json:"unit,omitempty"` + Dimensions *[]Dimension `json:"dimensions,omitempty"` + AggregationType *string `json:"aggregationType,omitempty"` + FillGapWithZero *bool `json:"fillGapWithZero,omitempty"` + Category *string `json:"category,omitempty"` + ResourceIDDimensionNameOverride *string `json:"resourceIdDimensionNameOverride,omitempty"` +} + +// NetworkRuleSet is network rule set +type NetworkRuleSet struct { + Bypass Bypass `json:"bypass,omitempty"` + VirtualNetworkRules *[]VirtualNetworkRule `json:"virtualNetworkRules,omitempty"` + IPRules *[]IPRule `json:"ipRules,omitempty"` + DefaultAction DefaultAction `json:"defaultAction,omitempty"` +} + +// Operation is storage REST API operation definition. +type Operation struct { + Name *string `json:"name,omitempty"` + Display *OperationDisplay `json:"display,omitempty"` + Origin *string `json:"origin,omitempty"` + *OperationProperties `json:"properties,omitempty"` +} + +// OperationDisplay is display metadata associated with the operation. +type OperationDisplay struct { + Provider *string `json:"provider,omitempty"` + Resource *string `json:"resource,omitempty"` + Operation *string `json:"operation,omitempty"` +} + +// OperationListResult is result of the request to list Storage operations. It contains a list of operations and a URL +// link to get the next set of results. +type OperationListResult struct { + autorest.Response `json:"-"` + Value *[]Operation `json:"value,omitempty"` +} + +// OperationProperties is properties of operation, include metric specifications. +type OperationProperties struct { + ServiceSpecification *ServiceSpecification `json:"serviceSpecification,omitempty"` +} + +// Resource is describes a storage resource. +type Resource struct { + ID *string `json:"id,omitempty"` + Name *string `json:"name,omitempty"` + Type *string `json:"type,omitempty"` + Location *string `json:"location,omitempty"` + Tags *map[string]*string `json:"tags,omitempty"` +} + +// Restriction is the restriction because of which SKU cannot be used. +type Restriction struct { + Type *string `json:"type,omitempty"` + Values *[]string `json:"values,omitempty"` + ReasonCode ReasonCode `json:"reasonCode,omitempty"` +} + +// ServiceSasParameters is the parameters to list service SAS credentials of a speicific resource. +type ServiceSasParameters struct { + CanonicalizedResource *string `json:"canonicalizedResource,omitempty"` + Resource SignedResource `json:"signedResource,omitempty"` + Permissions Permissions `json:"signedPermission,omitempty"` + IPAddressOrRange *string `json:"signedIp,omitempty"` + Protocols HTTPProtocol `json:"signedProtocol,omitempty"` + SharedAccessStartTime *date.Time `json:"signedStart,omitempty"` + SharedAccessExpiryTime *date.Time `json:"signedExpiry,omitempty"` + Identifier *string `json:"signedIdentifier,omitempty"` + PartitionKeyStart *string `json:"startPk,omitempty"` + PartitionKeyEnd *string `json:"endPk,omitempty"` + RowKeyStart *string `json:"startRk,omitempty"` + RowKeyEnd *string `json:"endRk,omitempty"` + KeyToSign *string `json:"keyToSign,omitempty"` + CacheControl *string `json:"rscc,omitempty"` + ContentDisposition *string `json:"rscd,omitempty"` + ContentEncoding *string `json:"rsce,omitempty"` + ContentLanguage *string `json:"rscl,omitempty"` + ContentType *string `json:"rsct,omitempty"` +} + +// ServiceSpecification is one property of operation, include metric specifications. +type ServiceSpecification struct { + MetricSpecifications *[]MetricSpecification `json:"metricSpecifications,omitempty"` +} + +// Sku is the SKU of the storage account. +type Sku struct { + Name SkuName `json:"name,omitempty"` + Tier SkuTier `json:"tier,omitempty"` + ResourceType *string `json:"resourceType,omitempty"` + Kind Kind `json:"kind,omitempty"` + Locations *[]string `json:"locations,omitempty"` + Capabilities *[]SKUCapability `json:"capabilities,omitempty"` + Restrictions *[]Restriction `json:"restrictions,omitempty"` +} + +// SKUCapability is the capability information in the specified sku, including file encryption, network acls, change +// notification, etc. +type SKUCapability struct { + Name *string `json:"name,omitempty"` + Value *string `json:"value,omitempty"` +} + +// SkuListResult is the response from the List Storage SKUs operation. +type SkuListResult struct { + autorest.Response `json:"-"` + Value *[]Sku `json:"value,omitempty"` +} + +// Usage is describes Storage Resource Usage. +type Usage struct { + Unit UsageUnit `json:"unit,omitempty"` + CurrentValue *int32 `json:"currentValue,omitempty"` + Limit *int32 `json:"limit,omitempty"` + Name *UsageName `json:"name,omitempty"` +} + +// UsageListResult is the response from the List Usages operation. +type UsageListResult struct { + autorest.Response `json:"-"` + Value *[]Usage `json:"value,omitempty"` +} + +// UsageName is the usage names that can be used; currently limited to StorageAccount. +type UsageName struct { + Value *string `json:"value,omitempty"` + LocalizedValue *string `json:"localizedValue,omitempty"` +} + +// VirtualNetworkRule is virtual Network rule. +type VirtualNetworkRule struct { + VirtualNetworkResourceID *string `json:"id,omitempty"` + Action Action `json:"action,omitempty"` + State State `json:"state,omitempty"` +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/operations.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/operations.go new file mode 100644 index 000000000..e6636e909 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/operations.go @@ -0,0 +1,98 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "net/http" +) + +// OperationsClient is the the Azure Storage Management API. +type OperationsClient struct { + ManagementClient +} + +// NewOperationsClient creates an instance of the OperationsClient client. +func NewOperationsClient(subscriptionID string) OperationsClient { + return NewOperationsClientWithBaseURI(DefaultBaseURI, subscriptionID) +} + +// NewOperationsClientWithBaseURI creates an instance of the OperationsClient client. +func NewOperationsClientWithBaseURI(baseURI string, subscriptionID string) OperationsClient { + return OperationsClient{NewWithBaseURI(baseURI, subscriptionID)} +} + +// List lists all of the available Storage Rest API operations. +func (client OperationsClient) List() (result OperationListResult, err error) { + req, err := client.ListPreparer() + if err != nil { + err = autorest.NewErrorWithError(err, "storage.OperationsClient", "List", nil, "Failure preparing request") + return + } + + resp, err := client.ListSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.OperationsClient", "List", resp, "Failure sending request") + return + } + + result, err = client.ListResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.OperationsClient", "List", resp, "Failure responding to request") + } + + return +} + +// ListPreparer prepares the List request. +func (client OperationsClient) ListPreparer() (*http.Request, error) { + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPath("/providers/Microsoft.Storage/operations"), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListSender sends the List request. The method will close the +// http.Response Body if it receives an error. +func (client OperationsClient) ListSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + autorest.DoRetryForStatusCodes(client.RetryAttempts, client.RetryDuration, autorest.StatusCodesForRetry...)) +} + +// ListResponder handles the response to the List request. The method always +// closes the http.Response Body. +func (client OperationsClient) ListResponder(resp *http.Response) (result OperationListResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/skus.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/skus.go new file mode 100644 index 000000000..1caca57d0 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/skus.go @@ -0,0 +1,102 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "net/http" +) + +// SkusClient is the the Azure Storage Management API. +type SkusClient struct { + ManagementClient +} + +// NewSkusClient creates an instance of the SkusClient client. +func NewSkusClient(subscriptionID string) SkusClient { + return NewSkusClientWithBaseURI(DefaultBaseURI, subscriptionID) +} + +// NewSkusClientWithBaseURI creates an instance of the SkusClient client. +func NewSkusClientWithBaseURI(baseURI string, subscriptionID string) SkusClient { + return SkusClient{NewWithBaseURI(baseURI, subscriptionID)} +} + +// List lists the available SKUs supported by Microsoft.Storage for given subscription. +func (client SkusClient) List() (result SkuListResult, err error) { + req, err := client.ListPreparer() + if err != nil { + err = autorest.NewErrorWithError(err, "storage.SkusClient", "List", nil, "Failure preparing request") + return + } + + resp, err := client.ListSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.SkusClient", "List", resp, "Failure sending request") + return + } + + result, err = client.ListResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.SkusClient", "List", resp, "Failure responding to request") + } + + return +} + +// ListPreparer prepares the List request. +func (client SkusClient) ListPreparer() (*http.Request, error) { + pathParameters := map[string]interface{}{ + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/providers/Microsoft.Storage/skus", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListSender sends the List request. The method will close the +// http.Response Body if it receives an error. +func (client SkusClient) ListSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListResponder handles the response to the List request. The method always +// closes the http.Response Body. +func (client SkusClient) ListResponder(resp *http.Response) (result SkuListResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/usage.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/usage.go new file mode 100644 index 000000000..007725e1e --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/usage.go @@ -0,0 +1,102 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" + "net/http" +) + +// UsageClient is the the Azure Storage Management API. +type UsageClient struct { + ManagementClient +} + +// NewUsageClient creates an instance of the UsageClient client. +func NewUsageClient(subscriptionID string) UsageClient { + return NewUsageClientWithBaseURI(DefaultBaseURI, subscriptionID) +} + +// NewUsageClientWithBaseURI creates an instance of the UsageClient client. +func NewUsageClientWithBaseURI(baseURI string, subscriptionID string) UsageClient { + return UsageClient{NewWithBaseURI(baseURI, subscriptionID)} +} + +// List gets the current usage count and the limit for the resources under the subscription. +func (client UsageClient) List() (result UsageListResult, err error) { + req, err := client.ListPreparer() + if err != nil { + err = autorest.NewErrorWithError(err, "storage.UsageClient", "List", nil, "Failure preparing request") + return + } + + resp, err := client.ListSender(req) + if err != nil { + result.Response = autorest.Response{Response: resp} + err = autorest.NewErrorWithError(err, "storage.UsageClient", "List", resp, "Failure sending request") + return + } + + result, err = client.ListResponder(resp) + if err != nil { + err = autorest.NewErrorWithError(err, "storage.UsageClient", "List", resp, "Failure responding to request") + } + + return +} + +// ListPreparer prepares the List request. +func (client UsageClient) ListPreparer() (*http.Request, error) { + pathParameters := map[string]interface{}{ + "subscriptionId": autorest.Encode("path", client.SubscriptionID), + } + + const APIVersion = "2017-10-01" + queryParameters := map[string]interface{}{ + "api-version": APIVersion, + } + + preparer := autorest.CreatePreparer( + autorest.AsGet(), + autorest.WithBaseURL(client.BaseURI), + autorest.WithPathParameters("/subscriptions/{subscriptionId}/providers/Microsoft.Storage/usages", pathParameters), + autorest.WithQueryParameters(queryParameters)) + return preparer.Prepare(&http.Request{}) +} + +// ListSender sends the List request. The method will close the +// http.Response Body if it receives an error. +func (client UsageClient) ListSender(req *http.Request) (*http.Response, error) { + return autorest.SendWithSender(client, + req, + azure.DoRetryWithRegistration(client.Client)) +} + +// ListResponder handles the response to the List request. The method always +// closes the http.Response Body. +func (client UsageClient) ListResponder(resp *http.Response) (result UsageListResult, err error) { + err = autorest.Respond( + resp, + client.ByInspecting(), + azure.WithErrorUnlessStatusCode(http.StatusOK), + autorest.ByUnmarshallingJSON(&result), + autorest.ByClosing()) + result.Response = autorest.Response{Response: resp} + return +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/version.go b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/version.go new file mode 100644 index 000000000..3c4fa5c01 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage/version.go @@ -0,0 +1,28 @@ +package storage + +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +// UserAgent returns the UserAgent string to use when sending http.Requests. +func UserAgent() string { + return "Azure-SDK-For-Go/v11.3.0-beta arm-storage/2017-10-01" +} + +// Version returns the semantic version (see http://semver.org) of the client. +func Version() string { + return "v11.3.0-beta" +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go index c13d409b7..8b5b96d48 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go @@ -1,7 +1,23 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" + "crypto/md5" + "encoding/base64" "fmt" "net/http" "net/url" @@ -31,8 +47,7 @@ func (b *Blob) PutAppendBlob(options *PutBlobOptions) error { if err != nil { return err } - readAndCloseBody(resp.body) - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) + return b.respondCreation(resp, BlobTypeAppend) } // AppendBlockOptions includes the options for an append block operation @@ -46,6 +61,7 @@ type AppendBlockOptions struct { IfMatch string `header:"If-Match"` IfNoneMatch string `header:"If-None-Match"` RequestID string `header:"x-ms-client-request-id"` + ContentMD5 bool } // AppendBlock appends a block to an append blob. @@ -60,6 +76,10 @@ func (b *Blob) AppendBlock(chunk []byte, options *AppendBlockOptions) error { if options != nil { params = addTimeout(params, options.Timeout) headers = mergeHeaders(headers, headersFromStruct(*options)) + if options.ContentMD5 { + md5sum := md5.Sum(chunk) + headers[headerContentMD5] = base64.StdEncoding.EncodeToString(md5sum[:]) + } } uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params) @@ -67,6 +87,5 @@ func (b *Blob) AppendBlock(chunk []byte, options *AppendBlockOptions) error { if err != nil { return err } - readAndCloseBody(resp.body) - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) + return b.respondCreation(resp, BlobTypeAppend) } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/authorization.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/authorization.go index 608bf3133..76794c305 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/authorization.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/authorization.go @@ -1,6 +1,20 @@ // Package storage provides clients for Microsoft Azure Storage Services. package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "fmt" @@ -41,16 +55,18 @@ const ( ) func (c *Client) addAuthorizationHeader(verb, url string, headers map[string]string, auth authentication) (map[string]string, error) { - authHeader, err := c.getSharedKey(verb, url, headers, auth) - if err != nil { - return nil, err + if !c.sasClient { + authHeader, err := c.getSharedKey(verb, url, headers, auth) + if err != nil { + return nil, err + } + headers[headerAuthorization] = authHeader } - headers[headerAuthorization] = authHeader return headers, nil } func (c *Client) getSharedKey(verb, url string, headers map[string]string, auth authentication) (string, error) { - canRes, err := c.buildCanonicalizedResource(url, auth) + canRes, err := c.buildCanonicalizedResource(url, auth, false) if err != nil { return "", err } @@ -62,15 +78,18 @@ func (c *Client) getSharedKey(verb, url string, headers map[string]string, auth return c.createAuthorizationHeader(canString, auth), nil } -func (c *Client) buildCanonicalizedResource(uri string, auth authentication) (string, error) { +func (c *Client) buildCanonicalizedResource(uri string, auth authentication, sas bool) (string, error) { errMsg := "buildCanonicalizedResource error: %s" u, err := url.Parse(uri) if err != nil { return "", fmt.Errorf(errMsg, err.Error()) } - cr := bytes.NewBufferString("/") - cr.WriteString(c.getCanonicalizedAccountName()) + cr := bytes.NewBufferString("") + if c.accountName != StorageEmulatorAccountName || !sas { + cr.WriteString("/") + cr.WriteString(c.getCanonicalizedAccountName()) + } if len(u.Path) > 0 { // Any portion of the CanonicalizedResource string that is derived from diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go index 12f61ac2a..5047bfbb2 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "errors" @@ -90,7 +104,7 @@ type BlobProperties struct { CacheControl string `xml:"Cache-Control" header:"x-ms-blob-cache-control"` ContentLanguage string `xml:"Cache-Language" header:"x-ms-blob-content-language"` ContentDisposition string `xml:"Content-Disposition" header:"x-ms-blob-content-disposition"` - BlobType BlobType `xml:"x-ms-blob-blob-type"` + BlobType BlobType `xml:"BlobType"` SequenceNumber int64 `xml:"x-ms-blob-sequence-number"` CopyID string `xml:"CopyId"` CopyStatus string `xml:"CopyStatus"` @@ -135,8 +149,7 @@ func (b *Blob) Exists() (bool, error) { } // GetURL gets the canonical URL to the blob with the specified name in the -// specified container. If name is not specified, the canonical URL for the entire -// container is obtained. +// specified container. // This method does not create a publicly accessible URL if the blob or container // is private and this method does not check if the blob exists. func (b *Blob) GetURL() string { @@ -437,8 +450,8 @@ func (b *Blob) SetProperties(options *SetBlobPropertiesOptions) error { uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params) if b.Properties.BlobType == BlobTypePage { - headers = addToHeaders(headers, "x-ms-blob-content-length", fmt.Sprintf("byte %v", b.Properties.ContentLength)) - if options != nil || options.SequenceNumberAction != nil { + headers = addToHeaders(headers, "x-ms-blob-content-length", fmt.Sprintf("%v", b.Properties.ContentLength)) + if options != nil && options.SequenceNumberAction != nil { headers = addToHeaders(headers, "x-ms-sequence-number-action", string(*options.SequenceNumberAction)) if *options.SequenceNumberAction != SequenceNumberActionIncrement { headers = addToHeaders(headers, "x-ms-blob-sequence-number", fmt.Sprintf("%v", b.Properties.SequenceNumber)) @@ -536,27 +549,7 @@ func (b *Blob) GetMetadata(options *GetBlobMetadataOptions) error { } func (b *Blob) writeMetadata(h http.Header) { - metadata := make(map[string]string) - for k, v := range h { - // Can't trust CanonicalHeaderKey() to munge case - // reliably. "_" is allowed in identifiers: - // https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx - // https://msdn.microsoft.com/library/aa664670(VS.71).aspx - // http://tools.ietf.org/html/rfc7230#section-3.2 - // ...but "_" is considered invalid by - // CanonicalMIMEHeaderKey in - // https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542 - // so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl". - k = strings.ToLower(k) - if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) { - continue - } - // metadata["lol"] = content of the last X-Ms-Meta-Lol header - k = k[len(userDefinedMetadataHeaderPrefix):] - metadata[k] = v[len(v)-1] - } - - b.Metadata = BlobMetadata(metadata) + b.Metadata = BlobMetadata(writeMetadata(h)) } // DeleteBlobOptions includes the options for a delete blob operation @@ -627,3 +620,13 @@ func pathForResource(container, name string) string { } return fmt.Sprintf("/%s", container) } + +func (b *Blob) respondCreation(resp *storageResponse, bt BlobType) error { + readAndCloseBody(resp.body) + err := checkRespCode(resp.statusCode, []int{http.StatusCreated}) + if err != nil { + return err + } + b.Properties.BlobType = bt + return nil +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blobsasuri.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blobsasuri.go index 43173d3a4..e11af7744 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blobsasuri.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blobsasuri.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "errors" "fmt" @@ -8,68 +22,122 @@ import ( "time" ) -// GetSASURIWithSignedIPAndProtocol creates an URL to the specified blob which contains the Shared -// Access Signature with specified permissions and expiration time. Also includes signedIPRange and allowed protocols. -// If old API version is used but no signedIP is passed (ie empty string) then this should still work. -// We only populate the signedIP when it non-empty. +// OverrideHeaders defines overridable response heaedrs in +// a request using a SAS URI. +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +type OverrideHeaders struct { + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string +} + +// BlobSASOptions are options to construct a blob SAS +// URI. +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +type BlobSASOptions struct { + BlobServiceSASPermissions + OverrideHeaders + SASOptions +} + +// BlobServiceSASPermissions includes the available permissions for +// blob service SAS URI. +type BlobServiceSASPermissions struct { + Read bool + Add bool + Create bool + Write bool + Delete bool +} + +func (p BlobServiceSASPermissions) buildString() string { + permissions := "" + if p.Read { + permissions += "r" + } + if p.Add { + permissions += "a" + } + if p.Create { + permissions += "c" + } + if p.Write { + permissions += "w" + } + if p.Delete { + permissions += "d" + } + return permissions +} + +// GetSASURI creates an URL to the blob which contains the Shared +// Access Signature with the specified options. // -// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx -func (b *Blob) GetSASURIWithSignedIPAndProtocol(expiry time.Time, permissions string, signedIPRange string, HTTPSOnly bool) (string, error) { - var ( - signedPermissions = permissions - blobURL = b.GetURL() - ) - canonicalizedResource, err := b.Container.bsc.client.buildCanonicalizedResource(blobURL, b.Container.bsc.auth) +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +func (b *Blob) GetSASURI(options BlobSASOptions) (string, error) { + uri := b.GetURL() + signedResource := "b" + canonicalizedResource, err := b.Container.bsc.client.buildCanonicalizedResource(uri, b.Container.bsc.auth, true) if err != nil { return "", err } - // "The canonicalizedresouce portion of the string is a canonical path to the signed resource. - // It must include the service name (blob, table, queue or file) for version 2015-02-21 or - // later, the storage account name, and the resource name, and must be URL-decoded. - // -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx + permissions := options.BlobServiceSASPermissions.buildString() + return b.Container.bsc.client.blobAndFileSASURI(options.SASOptions, uri, permissions, canonicalizedResource, signedResource, options.OverrideHeaders) +} + +func (c *Client) blobAndFileSASURI(options SASOptions, uri, permissions, canonicalizedResource, signedResource string, headers OverrideHeaders) (string, error) { + start := "" + if options.Start != (time.Time{}) { + start = options.Start.UTC().Format(time.RFC3339) + } + + expiry := options.Expiry.UTC().Format(time.RFC3339) // We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component). canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1) - canonicalizedResource, err = url.QueryUnescape(canonicalizedResource) + canonicalizedResource, err := url.QueryUnescape(canonicalizedResource) if err != nil { return "", err } - signedExpiry := expiry.UTC().Format(time.RFC3339) - - //If blob name is missing, resource is a container - signedResource := "c" - if len(b.Name) > 0 { - signedResource = "b" - } - - protocols := "https,http" - if HTTPSOnly { + protocols := "" + if options.UseHTTPS { protocols = "https" } - stringToSign, err := blobSASStringToSign(b.Container.bsc.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions, signedIPRange, protocols) + stringToSign, err := blobSASStringToSign(permissions, start, expiry, canonicalizedResource, options.Identifier, options.IP, protocols, c.apiVersion, headers) if err != nil { return "", err } - sig := b.Container.bsc.client.computeHmac256(stringToSign) + sig := c.computeHmac256(stringToSign) sasParams := url.Values{ - "sv": {b.Container.bsc.client.apiVersion}, - "se": {signedExpiry}, + "sv": {c.apiVersion}, + "se": {expiry}, "sr": {signedResource}, - "sp": {signedPermissions}, + "sp": {permissions}, "sig": {sig}, } - if b.Container.bsc.client.apiVersion >= "2015-04-05" { - sasParams.Add("spr", protocols) - if signedIPRange != "" { - sasParams.Add("sip", signedIPRange) + if c.apiVersion >= "2015-04-05" { + if protocols != "" { + sasParams.Add("spr", protocols) + } + if options.IP != "" { + sasParams.Add("sip", options.IP) } } - sasURL, err := url.Parse(blobURL) + // Add override response hedaers + addQueryParameter(sasParams, "rscc", headers.CacheControl) + addQueryParameter(sasParams, "rscd", headers.ContentDisposition) + addQueryParameter(sasParams, "rsce", headers.ContentEncoding) + addQueryParameter(sasParams, "rscl", headers.ContentLanguage) + addQueryParameter(sasParams, "rsct", headers.ContentType) + + sasURL, err := url.Parse(uri) if err != nil { return "", err } @@ -77,16 +145,12 @@ func (b *Blob) GetSASURIWithSignedIPAndProtocol(expiry time.Time, permissions st return sasURL.String(), nil } -// GetSASURI creates an URL to the specified blob which contains the Shared -// Access Signature with specified permissions and expiration time. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx -func (b *Blob) GetSASURI(expiry time.Time, permissions string) (string, error) { - return b.GetSASURIWithSignedIPAndProtocol(expiry, permissions, "", false) -} - -func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string, signedIP string, protocols string) (string, error) { - var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string +func blobSASStringToSign(signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion string, headers OverrideHeaders) (string, error) { + rscc := headers.CacheControl + rscd := headers.ContentDisposition + rsce := headers.ContentEncoding + rscl := headers.ContentLanguage + rsct := headers.ContentType if signedVersion >= "2015-02-21" { canonicalizedResource = "/blob" + canonicalizedResource diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blobserviceclient.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blobserviceclient.go index 450b20f96..e6b9704ee 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blobserviceclient.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blobserviceclient.go @@ -1,9 +1,26 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( + "encoding/xml" + "fmt" "net/http" "net/url" "strconv" + "strings" ) // BlobStorageClient contains operations for Microsoft Azure Blob Storage @@ -45,6 +62,21 @@ func (b *BlobStorageClient) GetContainerReference(name string) *Container { } } +// GetContainerReferenceFromSASURI returns a Container object for the specified +// container SASURI +func GetContainerReferenceFromSASURI(sasuri url.URL) (*Container, error) { + path := strings.Split(sasuri.Path, "/") + if len(path) <= 1 { + return nil, fmt.Errorf("could not find a container in URI: %s", sasuri.String()) + } + cli := newSASClient().GetBlobService() + return &Container{ + bsc: &cli, + Name: path[1], + sasuri: sasuri, + }, nil +} + // ListContainers returns the list of containers in a storage account along with // pagination token and other response details. // @@ -54,21 +86,53 @@ func (b BlobStorageClient) ListContainers(params ListContainersParameters) (*Con uri := b.client.getEndpoint(blobServiceName, "", q) headers := b.client.getStandardHeaders() - var out ContainerListResponse + type ContainerAlias struct { + bsc *BlobStorageClient + Name string `xml:"Name"` + Properties ContainerProperties `xml:"Properties"` + Metadata BlobMetadata + sasuri url.URL + } + type ContainerListResponseAlias struct { + XMLName xml.Name `xml:"EnumerationResults"` + Xmlns string `xml:"xmlns,attr"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + MaxResults int64 `xml:"MaxResults"` + Containers []ContainerAlias `xml:"Containers>Container"` + } + + var outAlias ContainerListResponseAlias resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) if err != nil { return nil, err } defer resp.body.Close() - err = xmlUnmarshal(resp.body, &out) + err = xmlUnmarshal(resp.body, &outAlias) if err != nil { return nil, err } - // assign our client to the newly created Container objects - for i := range out.Containers { - out.Containers[i].bsc = &b + out := ContainerListResponse{ + XMLName: outAlias.XMLName, + Xmlns: outAlias.Xmlns, + Prefix: outAlias.Prefix, + Marker: outAlias.Marker, + NextMarker: outAlias.NextMarker, + MaxResults: outAlias.MaxResults, + Containers: make([]Container, len(outAlias.Containers)), } + for i, cnt := range outAlias.Containers { + out.Containers[i] = Container{ + bsc: &b, + Name: cnt.Name, + Properties: cnt.Properties, + Metadata: map[string]string(cnt.Metadata), + sasuri: cnt.sasuri, + } + } + return &out, err } @@ -93,3 +157,26 @@ func (p ListContainersParameters) getParameters() url.Values { return out } + +func writeMetadata(h http.Header) map[string]string { + metadata := make(map[string]string) + for k, v := range h { + // Can't trust CanonicalHeaderKey() to munge case + // reliably. "_" is allowed in identifiers: + // https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx + // https://msdn.microsoft.com/library/aa664670(VS.71).aspx + // http://tools.ietf.org/html/rfc7230#section-3.2 + // ...but "_" is considered invalid by + // CanonicalMIMEHeaderKey in + // https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542 + // so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl". + k = strings.ToLower(k) + if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) { + continue + } + // metadata["lol"] = content of the last X-Ms-Meta-Lol header + k = k[len(userDefinedMetadataHeaderPrefix):] + metadata[k] = v[len(v)-1] + } + return metadata +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go index 5258f24fd..e0176d664 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "encoding/xml" @@ -132,8 +146,7 @@ func (b *Blob) CreateBlockBlobFromReader(blob io.Reader, options *PutBlobOptions if err != nil { return err } - readAndCloseBody(resp.body) - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) + return b.respondCreation(resp, BlobTypeBlock) } // PutBlockOptions includes the options for a put block operation @@ -181,8 +194,7 @@ func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, o if err != nil { return err } - readAndCloseBody(resp.body) - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) + return b.respondCreation(resp, BlobTypeBlock) } // PutBlockListOptions includes the options for a put block list operation diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go index a701c4b2c..a9ae9d11f 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go @@ -1,6 +1,20 @@ // Package storage provides clients for Microsoft Azure Storage Services. package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bufio" "bytes" @@ -17,6 +31,7 @@ import ( "net/url" "regexp" "runtime" + "strconv" "strings" "time" @@ -33,7 +48,9 @@ const ( // basic client is created. DefaultAPIVersion = "2016-05-31" - defaultUseHTTPS = true + defaultUseHTTPS = true + defaultRetryAttempts = 5 + defaultRetryDuration = time.Second * 5 // StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator StorageEmulatorAccountName = "devstoreaccount1" @@ -53,10 +70,22 @@ const ( userAgentHeader = "User-Agent" userDefinedMetadataHeaderPrefix = "x-ms-meta-" + + connectionStringAccountName = "accountname" + connectionStringAccountKey = "accountkey" + connectionStringEndpointSuffix = "endpointsuffix" + connectionStringEndpointProtocol = "defaultendpointsprotocol" ) var ( - validStorageAccount = regexp.MustCompile("^[0-9a-z]{3,24}$") + validStorageAccount = regexp.MustCompile("^[0-9a-z]{3,24}$") + defaultValidStatusCodes = []int{ + http.StatusRequestTimeout, // 408 + http.StatusInternalServerError, // 500 + http.StatusBadGateway, // 502 + http.StatusServiceUnavailable, // 503 + http.StatusGatewayTimeout, // 504 + } ) // Sender sends a request @@ -112,6 +141,8 @@ type Client struct { baseURL string apiVersion string userAgent string + sasClient bool + accountSASToken url.Values } type storageResponse struct { @@ -179,6 +210,45 @@ func (e UnexpectedStatusCodeError) Got() int { return e.got } +// NewClientFromConnectionString creates a Client from the connection string. +func NewClientFromConnectionString(input string) (Client, error) { + var ( + accountName, accountKey, endpointSuffix string + useHTTPS = defaultUseHTTPS + ) + + for _, pair := range strings.Split(input, ";") { + if pair == "" { + continue + } + + equalDex := strings.IndexByte(pair, '=') + if equalDex <= 0 { + return Client{}, fmt.Errorf("Invalid connection segment %q", pair) + } + + value := pair[equalDex+1:] + key := strings.ToLower(pair[:equalDex]) + switch key { + case connectionStringAccountName: + accountName = value + case connectionStringAccountKey: + accountKey = value + case connectionStringEndpointSuffix: + endpointSuffix = value + case connectionStringEndpointProtocol: + useHTTPS = value == "https" + default: + // ignored + } + } + + if accountName == StorageEmulatorAccountName { + return NewEmulatorClient() + } + return NewClient(accountName, accountKey, endpointSuffix, DefaultAPIVersion, useHTTPS) +} + // NewBasicClient constructs a Client with given storage service name and // key. func NewBasicClient(accountName, accountKey string) (Client, error) { @@ -206,13 +276,13 @@ func NewEmulatorClient() (Client, error) { // NewClient constructs a Client. This should be used if the caller wants // to specify whether to use HTTPS, a specific REST API version or a custom // storage endpoint than Azure Public Cloud. -func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) { +func NewClient(accountName, accountKey, serviceBaseURL, apiVersion string, useHTTPS bool) (Client, error) { var c Client if !IsValidStorageAccount(accountName) { return c, fmt.Errorf("azure: account name is not valid: it must be between 3 and 24 characters, and only may contain numbers and lowercase letters: %v", accountName) } else if accountKey == "" { return c, fmt.Errorf("azure: account key required") - } else if blobServiceBaseURL == "" { + } else if serviceBaseURL == "" { return c, fmt.Errorf("azure: base storage service url required") } @@ -226,19 +296,14 @@ func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, u accountName: accountName, accountKey: key, useHTTPS: useHTTPS, - baseURL: blobServiceBaseURL, + baseURL: serviceBaseURL, apiVersion: apiVersion, + sasClient: false, UseSharedKeyLite: false, Sender: &DefaultSender{ - RetryAttempts: 5, - ValidStatusCodes: []int{ - http.StatusRequestTimeout, // 408 - http.StatusInternalServerError, // 500 - http.StatusBadGateway, // 502 - http.StatusServiceUnavailable, // 503 - http.StatusGatewayTimeout, // 504 - }, - RetryDuration: time.Second * 5, + RetryAttempts: defaultRetryAttempts, + ValidStatusCodes: defaultValidStatusCodes, + RetryDuration: defaultRetryDuration, }, } c.userAgent = c.getDefaultUserAgent() @@ -251,6 +316,43 @@ func IsValidStorageAccount(account string) bool { return validStorageAccount.MatchString(account) } +// NewAccountSASClient contructs a client that uses accountSAS authorization +// for its operations. +func NewAccountSASClient(account string, token url.Values, env azure.Environment) Client { + c := newSASClient() + c.accountSASToken = token + c.accountName = account + c.baseURL = env.StorageEndpointSuffix + + // Get API version and protocol from token + c.apiVersion = token.Get("sv") + c.useHTTPS = token.Get("spr") == "https" + return c +} + +func newSASClient() Client { + c := Client{ + HTTPClient: http.DefaultClient, + apiVersion: DefaultAPIVersion, + sasClient: true, + Sender: &DefaultSender{ + RetryAttempts: defaultRetryAttempts, + ValidStatusCodes: defaultValidStatusCodes, + RetryDuration: defaultRetryDuration, + }, + } + c.userAgent = c.getDefaultUserAgent() + return c +} + +func (c Client) isServiceSASClient() bool { + return c.sasClient && c.accountSASToken == nil +} + +func (c Client) isAccountSASClient() bool { + return c.sasClient && c.accountSASToken != nil +} + func (c Client) getDefaultUserAgent() string { return fmt.Sprintf("Go/%s (%s-%s) azure-storage-go/%s api-version/%s", runtime.Version(), @@ -323,6 +425,164 @@ func (c Client) getEndpoint(service, path string, params url.Values) string { return u.String() } +// AccountSASTokenOptions includes options for constructing +// an account SAS token. +// https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas +type AccountSASTokenOptions struct { + APIVersion string + Services Services + ResourceTypes ResourceTypes + Permissions Permissions + Start time.Time + Expiry time.Time + IP string + UseHTTPS bool +} + +// Services specify services accessible with an account SAS. +type Services struct { + Blob bool + Queue bool + Table bool + File bool +} + +// ResourceTypes specify the resources accesible with an +// account SAS. +type ResourceTypes struct { + Service bool + Container bool + Object bool +} + +// Permissions specifies permissions for an accountSAS. +type Permissions struct { + Read bool + Write bool + Delete bool + List bool + Add bool + Create bool + Update bool + Process bool +} + +// GetAccountSASToken creates an account SAS token +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas +func (c Client) GetAccountSASToken(options AccountSASTokenOptions) (url.Values, error) { + if options.APIVersion == "" { + options.APIVersion = c.apiVersion + } + + if options.APIVersion < "2015-04-05" { + return url.Values{}, fmt.Errorf("account SAS does not support API versions prior to 2015-04-05. API version : %s", options.APIVersion) + } + + // build services string + services := "" + if options.Services.Blob { + services += "b" + } + if options.Services.Queue { + services += "q" + } + if options.Services.Table { + services += "t" + } + if options.Services.File { + services += "f" + } + + // build resources string + resources := "" + if options.ResourceTypes.Service { + resources += "s" + } + if options.ResourceTypes.Container { + resources += "c" + } + if options.ResourceTypes.Object { + resources += "o" + } + + // build permissions string + permissions := "" + if options.Permissions.Read { + permissions += "r" + } + if options.Permissions.Write { + permissions += "w" + } + if options.Permissions.Delete { + permissions += "d" + } + if options.Permissions.List { + permissions += "l" + } + if options.Permissions.Add { + permissions += "a" + } + if options.Permissions.Create { + permissions += "c" + } + if options.Permissions.Update { + permissions += "u" + } + if options.Permissions.Process { + permissions += "p" + } + + // build start time, if exists + start := "" + if options.Start != (time.Time{}) { + start = options.Start.Format(time.RFC3339) + // For some reason I don't understand, it fails when the rest of the string is included + start = start[:10] + } + + // build expiry time + expiry := options.Expiry.Format(time.RFC3339) + // For some reason I don't understand, it fails when the rest of the string is included + expiry = expiry[:10] + + protocol := "https,http" + if options.UseHTTPS { + protocol = "https" + } + + stringToSign := strings.Join([]string{ + c.accountName, + permissions, + services, + resources, + start, + expiry, + options.IP, + protocol, + options.APIVersion, + "", + }, "\n") + signature := c.computeHmac256(stringToSign) + + sasParams := url.Values{ + "sv": {options.APIVersion}, + "ss": {services}, + "srt": {resources}, + "sp": {permissions}, + "se": {expiry}, + "spr": {protocol}, + "sig": {signature}, + } + if start != "" { + sasParams.Add("st", start) + } + if options.IP != "" { + sasParams.Add("sip", options.IP) + } + + return sasParams, nil +} + // GetBlobService returns a BlobStorageClient which can operate on the blob // service of the storage account. func (c Client) GetBlobService() BlobStorageClient { @@ -398,16 +658,12 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader return nil, errors.New("azure/storage: error creating request: " + err.Error()) } - // if a body was provided ensure that the content length was set. - // http.NewRequest() will automatically do this for a handful of types - // and for those that it doesn't we will handle here. - if body != nil && req.ContentLength < 1 { - if lr, ok := body.(*io.LimitedReader); ok { - req.ContentLength = lr.N - snapshot := *lr - req.GetBody = func() (io.ReadCloser, error) { - r := snapshot - return ioutil.NopCloser(&r), nil + // http.NewRequest() will automatically set req.ContentLength for a handful of types + // otherwise we will handle here. + if req.ContentLength < 1 { + if clstr, ok := headers["Content-Length"]; ok { + if cl, err := strconv.ParseInt(clstr, 10, 64); err == nil { + req.ContentLength = cl } } } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/commonsasuri.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/commonsasuri.go new file mode 100644 index 000000000..e898e9bfa --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/commonsasuri.go @@ -0,0 +1,38 @@ +package storage + +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + "net/url" + "time" +) + +// SASOptions includes options used by SAS URIs for different +// services and resources. +type SASOptions struct { + APIVersion string + Start time.Time + Expiry time.Time + IP string + UseHTTPS bool + Identifier string +} + +func addQueryParameter(query url.Values, key, value string) url.Values { + if value != "" { + query.Add(key, value) + } + return query +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go index c2c9c055b..9f2324883 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "errors" @@ -18,12 +32,66 @@ type Container struct { Name string `xml:"Name"` Properties ContainerProperties `xml:"Properties"` Metadata map[string]string + sasuri url.URL +} + +// Client returns the HTTP client used by the Container reference. +func (c *Container) Client() *Client { + return &c.bsc.client } func (c *Container) buildPath() string { return fmt.Sprintf("/%s", c.Name) } +// GetURL gets the canonical URL to the container. +// This method does not create a publicly accessible URL if the container +// is private and this method does not check if the blob exists. +func (c *Container) GetURL() string { + container := c.Name + if container == "" { + container = "$root" + } + return c.bsc.client.getEndpoint(blobServiceName, pathForResource(container, ""), nil) +} + +// ContainerSASOptions are options to construct a container SAS +// URI. +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +type ContainerSASOptions struct { + ContainerSASPermissions + OverrideHeaders + SASOptions +} + +// ContainerSASPermissions includes the available permissions for +// a container SAS URI. +type ContainerSASPermissions struct { + BlobServiceSASPermissions + List bool +} + +// GetSASURI creates an URL to the container which contains the Shared +// Access Signature with the specified options. +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +func (c *Container) GetSASURI(options ContainerSASOptions) (string, error) { + uri := c.GetURL() + signedResource := "c" + canonicalizedResource, err := c.bsc.client.buildCanonicalizedResource(uri, c.bsc.auth, true) + if err != nil { + return "", err + } + + // build permissions string + permissions := options.BlobServiceSASPermissions.buildString() + if options.List { + permissions += "l" + } + + return c.bsc.client.blobAndFileSASURI(options.SASOptions, uri, permissions, canonicalizedResource, signedResource, options.OverrideHeaders) +} + // ContainerProperties contains various properties of a container returned from // various endpoints like ListContainers. type ContainerProperties struct { @@ -224,7 +292,20 @@ func (c *Container) create(options *CreateContainerOptions) (*storageResponse, e // Exists returns true if a container with given name exists // on the storage account, otherwise returns false. func (c *Container) Exists() (bool, error) { - uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), url.Values{"restype": {"container"}}) + q := url.Values{"restype": {"container"}} + var uri string + if c.bsc.client.isServiceSASClient() { + q = mergeParams(q, c.sasuri.Query()) + newURI := c.sasuri + newURI.RawQuery = q.Encode() + uri = newURI.String() + + } else { + if c.bsc.client.isAccountSASClient() { + q = mergeParams(q, c.bsc.client.accountSASToken) + } + uri = c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q) + } headers := c.bsc.client.getStandardHeaders() resp, err := c.bsc.client.exec(http.MethodHead, uri, headers, nil, c.bsc.auth) @@ -399,9 +480,20 @@ func (c *Container) delete(options *DeleteContainerOptions) (*storageResponse, e func (c *Container) ListBlobs(params ListBlobsParameters) (BlobListResponse, error) { q := mergeParams(params.getParameters(), url.Values{ "restype": {"container"}, - "comp": {"list"}}, - ) - uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q) + "comp": {"list"}, + }) + var uri string + if c.bsc.client.isServiceSASClient() { + q = mergeParams(q, c.sasuri.Query()) + newURI := c.sasuri + newURI.RawQuery = q.Encode() + uri = newURI.String() + } else { + if c.bsc.client.isAccountSASClient() { + q = mergeParams(q, c.bsc.client.accountSASToken) + } + uri = c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q) + } headers := c.bsc.client.getStandardHeaders() headers = addToHeaders(headers, "x-ms-client-request-id", params.RequestID) @@ -420,6 +512,81 @@ func (c *Container) ListBlobs(params ListBlobsParameters) (BlobListResponse, err return out, err } +// ContainerMetadataOptions includes options for container metadata operations +type ContainerMetadataOptions struct { + Timeout uint + LeaseID string `header:"x-ms-lease-id"` + RequestID string `header:"x-ms-client-request-id"` +} + +// SetMetadata replaces the metadata for the specified container. +// +// Some keys may be converted to Camel-Case before sending. All keys +// are returned in lower case by GetBlobMetadata. HTTP header names +// are case-insensitive so case munging should not matter to other +// applications either. +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/set-container-metadata +func (c *Container) SetMetadata(options *ContainerMetadataOptions) error { + params := url.Values{ + "comp": {"metadata"}, + "restype": {"container"}, + } + headers := c.bsc.client.getStandardHeaders() + headers = c.bsc.client.addMetadataToHeaders(headers, c.Metadata) + + if options != nil { + params = addTimeout(params, options.Timeout) + headers = mergeHeaders(headers, headersFromStruct(*options)) + } + + uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params) + + resp, err := c.bsc.client.exec(http.MethodPut, uri, headers, nil, c.bsc.auth) + if err != nil { + return err + } + readAndCloseBody(resp.body) + return checkRespCode(resp.statusCode, []int{http.StatusOK}) +} + +// GetMetadata returns all user-defined metadata for the specified container. +// +// All metadata keys will be returned in lower case. (HTTP header +// names are case-insensitive.) +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/get-container-metadata +func (c *Container) GetMetadata(options *ContainerMetadataOptions) error { + params := url.Values{ + "comp": {"metadata"}, + "restype": {"container"}, + } + headers := c.bsc.client.getStandardHeaders() + + if options != nil { + params = addTimeout(params, options.Timeout) + headers = mergeHeaders(headers, headersFromStruct(*options)) + } + + uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params) + + resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth) + if err != nil { + return err + } + readAndCloseBody(resp.body) + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return err + } + + c.writeMetadata(resp.headers) + return nil +} + +func (c *Container) writeMetadata(h http.Header) { + c.Metadata = writeMetadata(h) +} + func generateContainerACLpayload(policies []ContainerAccessPolicy) (io.Reader, int, error) { sil := SignedIdentifiers{ SignedIdentifiers: []SignedIdentifier{}, diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go index f14342618..a4cc2527b 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "errors" "fmt" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go index 57053efd1..189e03802 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "net/http" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/entity.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/entity.go index 13e947507..9668ea669 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/entity.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/entity.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "encoding/json" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go index 27dbdd1fc..5fb516c55 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "errors" "fmt" @@ -426,7 +440,7 @@ func (f *File) URL() string { return f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), nil) } -// WriteRangeOptions includes opptions for a write file range operation +// WriteRangeOptions includes options for a write file range operation type WriteRangeOptions struct { Timeout uint ContentMD5 string diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/fileserviceclient.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/fileserviceclient.go index 81217bdfa..295e3d3e2 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/fileserviceclient.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/fileserviceclient.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "fmt" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/leaseblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/leaseblob.go index 415b74018..3d9d52d8e 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/leaseblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/leaseblob.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "errors" "net/http" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/message.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/message.go index 3ededcd42..7d9038a5f 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/message.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/message.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "fmt" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/odata.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/odata.go index 41d832e2b..800adf129 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/odata.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/odata.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // MetadataLevel determines if operations should return a paylod, // and it level of detail. type MetadataLevel string diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go index 468b3868a..f07166521 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "errors" @@ -73,10 +87,10 @@ func (b *Blob) modifyRange(blobRange BlobRange, bytes io.Reader, options *PutPag return errors.New("the value for rangeEnd must be greater than or equal to rangeStart") } if blobRange.Start%512 != 0 { - return errors.New("the value for rangeStart must be a modulus of 512") + return errors.New("the value for rangeStart must be a multiple of 512") } if blobRange.End%512 != 511 { - return errors.New("the value for rangeEnd must be a modulus of 511") + return errors.New("the value for rangeEnd must be a multiple of 512 - 1") } params := url.Values{"comp": {"page"}} @@ -133,7 +147,7 @@ func (b *Blob) GetPageRanges(options *GetPageRangesOptions) (GetPageRangesRespon params = addTimeout(params, options.Timeout) params = addSnapshot(params, options.Snapshot) if options.PreviousSnapshot != nil { - params.Add("prevsnapshot", timeRfc1123Formatted(*options.PreviousSnapshot)) + params.Add("prevsnapshot", timeRFC3339Formatted(*options.PreviousSnapshot)) } if options.Range != nil { headers["Range"] = options.Range.String() @@ -186,6 +200,5 @@ func (b *Blob) PutPageBlob(options *PutBlobOptions) error { if err != nil { return err } - readAndCloseBody(resp.body) - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) + return b.respondCreation(resp, BlobTypePage) } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go index c2c7f742c..499592ebd 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/xml" "errors" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/queuesasuri.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/queuesasuri.go new file mode 100644 index 000000000..28d9ab937 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/queuesasuri.go @@ -0,0 +1,146 @@ +package storage + +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +// QueueSASOptions are options to construct a blob SAS +// URI. +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +type QueueSASOptions struct { + QueueSASPermissions + SASOptions +} + +// QueueSASPermissions includes the available permissions for +// a queue SAS URI. +type QueueSASPermissions struct { + Read bool + Add bool + Update bool + Process bool +} + +func (q QueueSASPermissions) buildString() string { + permissions := "" + + if q.Read { + permissions += "r" + } + if q.Add { + permissions += "a" + } + if q.Update { + permissions += "u" + } + if q.Process { + permissions += "p" + } + return permissions +} + +// GetSASURI creates an URL to the specified queue which contains the Shared +// Access Signature with specified permissions and expiration time. +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +func (q *Queue) GetSASURI(options QueueSASOptions) (string, error) { + canonicalizedResource, err := q.qsc.client.buildCanonicalizedResource(q.buildPath(), q.qsc.auth, true) + if err != nil { + return "", err + } + + // "The canonicalizedresouce portion of the string is a canonical path to the signed resource. + // It must include the service name (blob, table, queue or file) for version 2015-02-21 or + // later, the storage account name, and the resource name, and must be URL-decoded. + // -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx + // We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component). + canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1) + canonicalizedResource, err = url.QueryUnescape(canonicalizedResource) + if err != nil { + return "", err + } + + signedStart := "" + if options.Start != (time.Time{}) { + signedStart = options.Start.UTC().Format(time.RFC3339) + } + signedExpiry := options.Expiry.UTC().Format(time.RFC3339) + + protocols := "https,http" + if options.UseHTTPS { + protocols = "https" + } + + permissions := options.QueueSASPermissions.buildString() + stringToSign, err := queueSASStringToSign(q.qsc.client.apiVersion, canonicalizedResource, signedStart, signedExpiry, options.IP, permissions, protocols, options.Identifier) + if err != nil { + return "", err + } + + sig := q.qsc.client.computeHmac256(stringToSign) + sasParams := url.Values{ + "sv": {q.qsc.client.apiVersion}, + "se": {signedExpiry}, + "sp": {permissions}, + "sig": {sig}, + } + + if q.qsc.client.apiVersion >= "2015-04-05" { + sasParams.Add("spr", protocols) + addQueryParameter(sasParams, "sip", options.IP) + } + + uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), nil) + sasURL, err := url.Parse(uri) + if err != nil { + return "", err + } + sasURL.RawQuery = sasParams.Encode() + return sasURL.String(), nil +} + +func queueSASStringToSign(signedVersion, canonicalizedResource, signedStart, signedExpiry, signedIP, signedPermissions, protocols, signedIdentifier string) (string, error) { + + if signedVersion >= "2015-02-21" { + canonicalizedResource = "/queue" + canonicalizedResource + } + + // https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12 + if signedVersion >= "2015-04-05" { + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", + signedPermissions, + signedStart, + signedExpiry, + canonicalizedResource, + signedIdentifier, + signedIP, + protocols, + signedVersion), nil + + } + + // reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx + if signedVersion >= "2013-08-15" { + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion), nil + } + + return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/queueserviceclient.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/queueserviceclient.go index 19b44941c..29febe146 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/queueserviceclient.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/queueserviceclient.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // QueueServiceClient contains operations for Microsoft Azure Queue Storage // Service. type QueueServiceClient struct { diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/share.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/share.go index e6a868081..a14d9d324 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/share.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/share.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "fmt" "net/http" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/storagepolicy.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/storagepolicy.go index bee1c31ad..056ab398a 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/storagepolicy.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/storagepolicy.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "strings" "time" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/storageservice.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/storageservice.go index 88700fbc9..c102619c9 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/storageservice.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/storageservice.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "net/http" "net/url" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/table.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/table.go index 4eae3af9d..6c01d32ee 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/table.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/table.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "encoding/json" @@ -174,11 +188,7 @@ func (t *Table) Delete(timeout uint, options *TableOptions) error { } defer readAndCloseBody(resp.body) - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - - } - return nil + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) } // QueryOptions includes options for a query entities operation. @@ -261,10 +271,7 @@ func (t *Table) SetPermissions(tap []TableAccessPolicy, timeout uint, options *T } defer readAndCloseBody(resp.body) - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - } - return nil + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) } func generateTableACLPayload(policies []TableAccessPolicy) (io.Reader, int, error) { diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/table_batch.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/table_batch.go index 7a0f0915c..3f882417c 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/table_batch.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/table_batch.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "encoding/json" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/tableserviceclient.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/tableserviceclient.go index 895dcfded..456bee773 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/tableserviceclient.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/tableserviceclient.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "encoding/json" "fmt" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go index d3ae9d092..089a74a8c 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + import ( "bytes" "crypto/hmac" @@ -18,7 +32,29 @@ import ( ) var ( - fixedTime = time.Date(2050, time.December, 20, 21, 55, 0, 0, time.FixedZone("GMT", -6)) + fixedTime = time.Date(2050, time.December, 20, 21, 55, 0, 0, time.FixedZone("GMT", -6)) + accountSASOptions = AccountSASTokenOptions{ + Services: Services{ + Blob: true, + }, + ResourceTypes: ResourceTypes{ + Service: true, + Container: true, + Object: true, + }, + Permissions: Permissions{ + Read: true, + Write: true, + Delete: true, + List: true, + Add: true, + Create: true, + Update: true, + Process: true, + }, + Expiry: fixedTime, + UseHTTPS: true, + } ) func (c Client) computeHmac256(message string) string { @@ -35,6 +71,10 @@ func timeRfc1123Formatted(t time.Time) string { return t.Format(http.TimeFormat) } +func timeRFC3339Formatted(t time.Time) string { + return t.Format("2006-01-02T15:04:05.0000000Z") +} + func mergeParams(v1, v2 url.Values) url.Values { out := url.Values{} for k, v := range v1 { @@ -136,7 +176,7 @@ func addTimeout(params url.Values, timeout uint) url.Values { func addSnapshot(params url.Values, snapshot *time.Time) url.Values { if snapshot != nil { - params.Add("snapshot", snapshot.Format("2006-01-02T15:04:05.0000000Z")) + params.Add("snapshot", timeRFC3339Formatted(*snapshot)) } return params } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/version.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/version.go index a23fff1e2..ced3f7ed8 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/version.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/version.go @@ -1,5 +1,19 @@ package storage +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + var ( - sdkVersion = "10.0.2" + sdkVersion = "v11.3.0-beta" ) From cd4e9f5336cb7089a98c030c973cd24a574efef3 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 23 Aug 2018 16:14:37 -0700 Subject: [PATCH 26/29] azure: fix for breaking change in blob.GetSASURI Signed-off-by: Steve Kriss --- pkg/cloudprovider/azure/object_store.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 392030b2a..11cf232c0 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -147,8 +147,6 @@ func (o *objectStore) DeleteObject(bucket string, key string) error { return errors.WithStack(blob.Delete(nil)) } -const sasURIReadPermission = "r" - func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { container, err := getContainerReference(o.blobClient, bucket) if err != nil { @@ -160,7 +158,16 @@ func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (st return "", err } - return blob.GetSASURI(time.Now().Add(ttl), sasURIReadPermission) + opts := storage.BlobSASOptions{ + SASOptions: storage.SASOptions{ + Expiry: time.Now().Add(ttl), + }, + BlobServiceSASPermissions: storage.BlobServiceSASPermissions{ + Read: true, + }, + } + + return blob.GetSASURI(opts) } func getContainerReference(blobClient *storage.BlobStorageClient, bucket string) (*storage.Container, error) { From 9d7ea7483cbc4b9acb6d7885b3c051b7a0dbfc41 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 23 Aug 2018 16:24:44 -0700 Subject: [PATCH 27/29] azure: support different RGs/storage accounts per backup location Signed-off-by: Steve Kriss --- pkg/cloudprovider/azure/block_store.go | 34 +++++++-------- pkg/cloudprovider/azure/object_store.go | 57 +++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/pkg/cloudprovider/azure/block_store.go b/pkg/cloudprovider/azure/block_store.go index dd7d34d7f..feecfb928 100644 --- a/pkg/cloudprovider/azure/block_store.go +++ b/pkg/cloudprovider/azure/block_store.go @@ -40,16 +40,14 @@ import ( ) const ( - azureClientIDKey = "AZURE_CLIENT_ID" - azureClientSecretKey = "AZURE_CLIENT_SECRET" - azureSubscriptionIDKey = "AZURE_SUBSCRIPTION_ID" - azureTenantIDKey = "AZURE_TENANT_ID" - azureStorageAccountIDKey = "AZURE_STORAGE_ACCOUNT_ID" - azureStorageKeyKey = "AZURE_STORAGE_KEY" - azureResourceGroupKey = "AZURE_RESOURCE_GROUP" - apiTimeoutKey = "apiTimeout" - snapshotsResource = "snapshots" - disksResource = "disks" + azureTenantIDKey = "AZURE_TENANT_ID" + azureSubscriptionIDKey = "AZURE_SUBSCRIPTION_ID" + azureClientIDKey = "AZURE_CLIENT_ID" + azureClientSecretKey = "AZURE_CLIENT_SECRET" + azureResourceGroupKey = "AZURE_RESOURCE_GROUP" + apiTimeoutKey = "apiTimeout" + snapshotsResource = "snapshots" + disksResource = "disks" ) type blockStore struct { @@ -71,15 +69,13 @@ func (si *snapshotIdentifier) String() string { return getComputeResourceName(si.subscription, si.resourceGroup, snapshotsResource, si.name) } -func getConfig() map[string]string { +func getAzureEnvVars() map[string]string { cfg := map[string]string{ - azureClientIDKey: "", - azureClientSecretKey: "", - azureSubscriptionIDKey: "", - azureTenantIDKey: "", - azureStorageAccountIDKey: "", - azureStorageKeyKey: "", - azureResourceGroupKey: "", + azureTenantIDKey: "", + azureSubscriptionIDKey: "", + azureClientIDKey: "", + azureClientSecretKey: "", + azureResourceGroupKey: "", } for key := range cfg { @@ -108,7 +104,7 @@ func (b *blockStore) Init(config map[string]string) error { apiTimeout = 2 * time.Minute } - cfg := getConfig() + cfg := getAzureEnvVars() spt, err := helpers.NewServicePrincipalTokenFromCredentials(cfg, azure.PublicCloud.ResourceManagerEndpoint) if err != nil { diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 11cf232c0..2673408ac 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -21,7 +21,11 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/arm/examples/helpers" + storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage" "github.com/Azure/azure-sdk-for-go/storage" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -37,10 +41,57 @@ func NewObjectStore(logger logrus.FieldLogger) cloudprovider.ObjectStore { return &objectStore{log: logger} } -func (o *objectStore) Init(config map[string]string) error { - cfg := getConfig() +func getStorageAccountsClient(envVars map[string]string) (*storagemgmt.AccountsClient, error) { + spt, err := helpers.NewServicePrincipalTokenFromCredentials(envVars, azure.PublicCloud.ResourceManagerEndpoint) + if err != nil { + return nil, errors.Wrap(err, "error creating new service principal token") + } - storageClient, err := storage.NewBasicClient(cfg[azureStorageAccountIDKey], cfg[azureStorageKeyKey]) + accountsClient := storagemgmt.NewAccountsClient(envVars[azureSubscriptionIDKey]) + accountsClient.Authorizer = autorest.NewBearerAuthorizer(spt) + + return &accountsClient, nil +} + +func getStorageAccountKey(client *storagemgmt.AccountsClient, resourceGroup, storageAccount string) (string, error) { + res, err := client.ListKeys(resourceGroup, storageAccount) + if err != nil { + return "", errors.WithStack(err) + } + if res.Keys == nil || len(*res.Keys) == 0 { + return "", errors.New("No storage keys found") + } + + var storageKey string + + for _, key := range *res.Keys { + // uppercase both strings for comparison because the ListKeys call returns e.g. "FULL" but + // the storagemgmt.Full constant in the SDK is defined as "Full". + if strings.ToUpper(string(key.Permissions)) == strings.ToUpper(string(storagemgmt.Full)) { + storageKey = *key.Value + break + } + } + + if storageKey == "" { + return "", errors.New("No storage key with Full permissions found") + } + + return storageKey, nil +} + +func (o *objectStore) Init(config map[string]string) error { + storageAccountsClient, err := getStorageAccountsClient(getAzureEnvVars()) + if err != nil { + return err + } + + storageAccountKey, err := getStorageAccountKey(storageAccountsClient, config["resourceGroup"], config["storageAccount"]) + if err != nil { + return err + } + + storageClient, err := storage.NewBasicClient(config["storageAccount"], storageAccountKey) if err != nil { return errors.WithStack(err) } From cb321db21f61594335dbb02cbb67b3ca9602ae4d Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 24 Aug 2018 10:19:40 -0700 Subject: [PATCH 28/29] azure: refactor to not use helpers/ pkg, validate all env/config inputs Signed-off-by: Steve Kriss --- Gopkg.lock | 3 +- pkg/cloudprovider/azure/block_store.go | 76 +++++++------------ pkg/cloudprovider/azure/common.go | 60 +++++++++++++++ pkg/cloudprovider/azure/object_store.go | 60 +++++++++------ .../arm/examples/helpers/helpers.go | 64 ---------------- 5 files changed, 128 insertions(+), 135 deletions(-) create mode 100644 pkg/cloudprovider/azure/common.go delete mode 100644 vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go diff --git a/Gopkg.lock b/Gopkg.lock index 88fe09b1e..d8c7fa49a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -18,7 +18,6 @@ name = "github.com/Azure/azure-sdk-for-go" packages = [ "arm/disk", - "arm/examples/helpers", "services/storage/mgmt/2017-10-01/storage", "storage" ] @@ -807,6 +806,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "84d160fa2e769b80040762566acadbe7c23ee774124dfdf7a498c0e65cd8011a" + inputs-digest = "4706135745ec21274791f454998d264dd167c78b472674ba813dca08cc962d7d" solver-name = "gps-cdcl" solver-version = 1 diff --git a/pkg/cloudprovider/azure/block_store.go b/pkg/cloudprovider/azure/block_store.go index feecfb928..8112c1583 100644 --- a/pkg/cloudprovider/azure/block_store.go +++ b/pkg/cloudprovider/azure/block_store.go @@ -26,7 +26,6 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/arm/disk" - "github.com/Azure/azure-sdk-for-go/arm/examples/helpers" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/pkg/errors" @@ -40,14 +39,10 @@ import ( ) const ( - azureTenantIDKey = "AZURE_TENANT_ID" - azureSubscriptionIDKey = "AZURE_SUBSCRIPTION_ID" - azureClientIDKey = "AZURE_CLIENT_ID" - azureClientSecretKey = "AZURE_CLIENT_SECRET" - azureResourceGroupKey = "AZURE_RESOURCE_GROUP" - apiTimeoutKey = "apiTimeout" - snapshotsResource = "snapshots" - disksResource = "disks" + resourceGroupEnvVar = "AZURE_RESOURCE_GROUP" + apiTimeoutConfigKey = "apiTimeout" + snapshotsResource = "snapshots" + disksResource = "disks" ) type blockStore struct { @@ -69,50 +64,37 @@ func (si *snapshotIdentifier) String() string { return getComputeResourceName(si.subscription, si.resourceGroup, snapshotsResource, si.name) } -func getAzureEnvVars() map[string]string { - cfg := map[string]string{ - azureTenantIDKey: "", - azureSubscriptionIDKey: "", - azureClientIDKey: "", - azureClientSecretKey: "", - azureResourceGroupKey: "", - } - - for key := range cfg { - cfg[key] = os.Getenv(key) - } - - return cfg -} - func NewBlockStore(logger logrus.FieldLogger) cloudprovider.BlockStore { return &blockStore{log: logger} } func (b *blockStore) Init(config map[string]string) error { - var ( - apiTimeoutVal = config[apiTimeoutKey] - apiTimeout time.Duration - err error - ) - - if apiTimeout, err = time.ParseDuration(apiTimeoutVal); err != nil { - return errors.Wrapf(err, "could not parse %s (expected time.Duration)", apiTimeoutKey) - } - - if apiTimeout == 0 { - apiTimeout = 2 * time.Minute - } - - cfg := getAzureEnvVars() - - spt, err := helpers.NewServicePrincipalTokenFromCredentials(cfg, azure.PublicCloud.ResourceManagerEndpoint) + // 1. we need AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP + envVars, err := getRequiredValues(os.Getenv, tenantIDEnvVar, clientIDEnvVar, clientSecretEnvVar, subscriptionIDEnvVar, resourceGroupEnvVar) if err != nil { - return errors.Wrap(err, "error creating new service principal token") + return errors.Wrap(err, "unable to get all required environment variables") } - disksClient := disk.NewDisksClient(cfg[azureSubscriptionIDKey]) - snapsClient := disk.NewSnapshotsClient(cfg[azureSubscriptionIDKey]) + // 2. if config["apiTimeout"] is empty, default to 2m; otherwise, parse it + var apiTimeout time.Duration + if val := config[apiTimeoutConfigKey]; val == "" { + apiTimeout = 2 * time.Minute + } else { + apiTimeout, err = time.ParseDuration(val) + if err != nil { + return errors.Wrapf(err, "unable to parse value %q for config key %q (expected a duration string)", val, apiTimeoutConfigKey) + } + } + + // 3. get SPT + spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) + if err != nil { + return errors.Wrap(err, "error getting service principal token") + } + + // 4. set up clients + disksClient := disk.NewDisksClient(envVars[subscriptionIDEnvVar]) + snapsClient := disk.NewSnapshotsClient(envVars[subscriptionIDEnvVar]) disksClient.PollingDelay = 5 * time.Second snapsClient.PollingDelay = 5 * time.Second @@ -123,8 +105,8 @@ func (b *blockStore) Init(config map[string]string) error { b.disks = &disksClient b.snaps = &snapsClient - b.subscription = cfg[azureSubscriptionIDKey] - b.resourceGroup = cfg[azureResourceGroupKey] + b.subscription = envVars[subscriptionIDEnvVar] + b.resourceGroup = envVars[resourceGroupEnvVar] b.apiTimeout = apiTimeout return nil diff --git a/pkg/cloudprovider/azure/common.go b/pkg/cloudprovider/azure/common.go new file mode 100644 index 000000000..d40c77ac5 --- /dev/null +++ b/pkg/cloudprovider/azure/common.go @@ -0,0 +1,60 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "strings" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/pkg/errors" +) + +const ( + tenantIDEnvVar = "AZURE_TENANT_ID" + subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" + clientIDEnvVar = "AZURE_CLIENT_ID" + clientSecretEnvVar = "AZURE_CLIENT_SECRET" +) + +func newServicePrincipalToken(tenantID, clientID, clientSecret, scope string) (*adal.ServicePrincipalToken, error) { + oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, tenantID) + if err != nil { + return nil, errors.Wrap(err, "error getting OAuthConfig") + } + + return adal.NewServicePrincipalToken(*oauthConfig, clientID, clientSecret, scope) +} + +func getRequiredValues(getValue func(string) string, keys ...string) (map[string]string, error) { + missing := []string{} + results := map[string]string{} + + for _, key := range keys { + if val := getValue(key); val == "" { + missing = append(missing, key) + } else { + results[key] = val + } + } + + if len(missing) > 0 { + return nil, errors.Errorf("the following keys do not have values: %s", strings.Join(missing, ", ")) + } + + return results, nil +} diff --git a/pkg/cloudprovider/azure/object_store.go b/pkg/cloudprovider/azure/object_store.go index 2673408ac..1b2a39288 100644 --- a/pkg/cloudprovider/azure/object_store.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -18,10 +18,10 @@ package azure import ( "io" + "os" "strings" "time" - "github.com/Azure/azure-sdk-for-go/arm/examples/helpers" storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage" "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/go-autorest/autorest" @@ -32,6 +32,11 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) +const ( + resourceGroupConfigKey = "resourceGroup" + storageAccountConfigKey = "storageAccount" +) + type objectStore struct { blobClient *storage.BlobStorageClient log logrus.FieldLogger @@ -41,19 +46,7 @@ func NewObjectStore(logger logrus.FieldLogger) cloudprovider.ObjectStore { return &objectStore{log: logger} } -func getStorageAccountsClient(envVars map[string]string) (*storagemgmt.AccountsClient, error) { - spt, err := helpers.NewServicePrincipalTokenFromCredentials(envVars, azure.PublicCloud.ResourceManagerEndpoint) - if err != nil { - return nil, errors.Wrap(err, "error creating new service principal token") - } - - accountsClient := storagemgmt.NewAccountsClient(envVars[azureSubscriptionIDKey]) - accountsClient.Authorizer = autorest.NewBearerAuthorizer(spt) - - return &accountsClient, nil -} - -func getStorageAccountKey(client *storagemgmt.AccountsClient, resourceGroup, storageAccount string) (string, error) { +func getStorageAccountKey(client storagemgmt.AccountsClient, resourceGroup, storageAccount string) (string, error) { res, err := client.ListKeys(resourceGroup, storageAccount) if err != nil { return "", errors.WithStack(err) @@ -80,24 +73,47 @@ func getStorageAccountKey(client *storagemgmt.AccountsClient, resourceGroup, sto return storageKey, nil } +func mapLookup(data map[string]string) func(string) string { + return func(key string) string { + return data[key] + } +} + func (o *objectStore) Init(config map[string]string) error { - storageAccountsClient, err := getStorageAccountsClient(getAzureEnvVars()) + // 1. we need AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID + envVars, err := getRequiredValues(os.Getenv, tenantIDEnvVar, clientIDEnvVar, clientSecretEnvVar, subscriptionIDEnvVar) if err != nil { - return err + return errors.Wrap(err, "unable to get all required environment variables") } - storageAccountKey, err := getStorageAccountKey(storageAccountsClient, config["resourceGroup"], config["storageAccount"]) - if err != nil { - return err + // 2. we need config["resourceGroup"], config["storageAccount"] + if _, err := getRequiredValues(mapLookup(config), resourceGroupConfigKey, storageAccountConfigKey); err != nil { + return errors.Wrap(err, "unable to get all required config values") } - storageClient, err := storage.NewBasicClient(config["storageAccount"], storageAccountKey) + // 3. get SPT + spt, err := newServicePrincipalToken(envVars[tenantIDEnvVar], envVars[clientIDEnvVar], envVars[clientSecretEnvVar], azure.PublicCloud.ResourceManagerEndpoint) if err != nil { - return errors.WithStack(err) + return errors.Wrap(err, "error getting service principal token") + } + + // 4. get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClient(envVars[subscriptionIDEnvVar]) + storageAccountsClient.Authorizer = autorest.NewBearerAuthorizer(spt) + + // 5. get storage key + storageAccountKey, err := getStorageAccountKey(storageAccountsClient, config[resourceGroupConfigKey], config[storageAccountConfigKey]) + if err != nil { + return errors.Wrap(err, "error getting storage account key") + } + + // 6. get storageClient and blobClient + storageClient, err := storage.NewBasicClient(config[storageAccountConfigKey], storageAccountKey) + if err != nil { + return errors.Wrap(err, "error getting storage client") } blobClient := storageClient.GetBlobService() - o.blobClient = &blobClient return nil diff --git a/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go b/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go deleted file mode 100644 index de9d55166..000000000 --- a/vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go +++ /dev/null @@ -1,64 +0,0 @@ -package helpers - -// Copyright 2017 Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import ( - "encoding/json" - "fmt" - - "github.com/Azure/go-autorest/autorest/adal" - "github.com/Azure/go-autorest/autorest/azure" -) - -const ( - credentialsPath = "/.azure/credentials.json" -) - -// ToJSON returns the passed item as a pretty-printed JSON string. If any JSON error occurs, -// it returns the empty string. -func ToJSON(v interface{}) (string, error) { - j, err := json.MarshalIndent(v, "", " ") - return string(j), err -} - -// NewServicePrincipalTokenFromCredentials creates a new ServicePrincipalToken using values of the -// passed credentials map. -func NewServicePrincipalTokenFromCredentials(c map[string]string, scope string) (*adal.ServicePrincipalToken, error) { - oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, c["AZURE_TENANT_ID"]) - if err != nil { - panic(err) - } - return adal.NewServicePrincipalToken(*oauthConfig, c["AZURE_CLIENT_ID"], c["AZURE_CLIENT_SECRET"], scope) -} - -func ensureValueStrings(mapOfInterface map[string]interface{}) map[string]string { - mapOfStrings := make(map[string]string) - for key, value := range mapOfInterface { - mapOfStrings[key] = ensureValueString(value) - } - return mapOfStrings -} - -func ensureValueString(value interface{}) string { - if value == nil { - return "" - } - switch v := value.(type) { - case string: - return v - default: - return fmt.Sprintf("%v", v) - } -} From df69b274a0c744d60410b631a80813b4624b334d Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Mon, 27 Aug 2018 12:21:50 -0700 Subject: [PATCH 29/29] azure: update documentation and examples Signed-off-by: Steve Kriss --- docs/azure-config.md | 24 ++++++------------- docs/backupstoragelocation-definition.md | 13 ++++++---- docs/config-definition.md | 13 +++++----- docs/debugging-install.md | 2 +- .../azure/05-ark-backupstoragelocation.yaml | 5 +++- 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/azure-config.md b/docs/azure-config.md index b02d29229..04e9cdbe8 100644 --- a/docs/azure-config.md +++ b/docs/azure-config.md @@ -51,23 +51,15 @@ az storage account create \ --access-tier Hot ``` -Create the blob container named `ark`. Feel free to use a different name, preferrably unique to a single Kubernetes cluster. See the [FAQ][20] for more details. You'll need to -adjust the `bucket` field under `backupStorageProvider` in the Ark Config accordingly if you do. +Create the blob container named `ark`. Feel free to use a different name, preferably unique to a single Kubernetes cluster. See the [FAQ][20] for more details. ```bash az storage container create -n ark --public-access off --account-name $AZURE_STORAGE_ACCOUNT_ID - -# Obtain the storage access key for the storage account just created -AZURE_STORAGE_KEY=`az storage account keys list \ - --account-name $AZURE_STORAGE_ACCOUNT_ID \ - --resource-group $AZURE_BACKUP_RESOURCE_GROUP \ - --query '[0].value' \ - -o tsv` ``` ## Create service principal -To integrate Ark with Azure, you must create an Ark-specific [service principal][17]. Note that seven environment variables must be set for Ark to work properly. +To integrate Ark with Azure, you must create an Ark-specific [service principal][17]. 1. Obtain your Azure Account Subscription ID and Tenant ID: @@ -79,11 +71,11 @@ To integrate Ark with Azure, you must create an Ark-specific [service principal] 1. Set the name of the Resource Group that contains your Kubernetes cluster. ```bash - # Make sure this is the name of the second resource group. See warning. + # Make sure this is the name of the auto-generated resource group. See warning. AZURE_RESOURCE_GROUP= ``` - WARNING: `AZURE_RESOURCE_GROUP` must be set to the name of the second resource group that is created when you provision your cluster in Azure. Your cluster is provisioned in the resource group that you specified when you created the cluster. Your disks, however, are provisioned in the second resource group. + WARNING: `AZURE_RESOURCE_GROUP` must be set to the name of the auto-generated resource group that is created when you provision your cluster in Azure. Your cluster is provisioned in the resource group that you specified when you created the cluster. Your disks, however, are provisioned in the second resource group. If you are unsure of the Resource Group name, run the following command to get a list that you can select from. Then set the `AZURE_RESOURCE_GROUP` environment variable to the appropriate value. @@ -117,18 +109,16 @@ In the Ark root directory, run the following to first set up namespaces, RBAC, a kubectl apply -f examples/common/00-prereqs.yaml ``` -Now you need to create a Secret that contains all the seven environment variables you just set. The command looks like the following: +Now you need to create a Secret that contains all the environment variables you just set. The command looks like the following: ```bash kubectl create secret generic cloud-credentials \ --namespace \ --from-literal AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} \ --from-literal AZURE_TENANT_ID=${AZURE_TENANT_ID} \ - --from-literal AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} \ --from-literal AZURE_CLIENT_ID=${AZURE_CLIENT_ID} \ --from-literal AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} \ - --from-literal AZURE_STORAGE_ACCOUNT_ID=${AZURE_STORAGE_ACCOUNT_ID} \ - --from-literal AZURE_STORAGE_KEY=${AZURE_STORAGE_KEY} + --from-literal AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP} ``` Now that you have your Azure credentials stored in a Secret, you need to replace some placeholder values in the template files. Specifically, you need to change the following: @@ -139,7 +129,7 @@ Now that you have your Azure credentials stored in a Secret, you need to replace * In file `examples/azure/05-ark-backupstoragelocation.yaml`: - * Replace ``. See the [BackupStorageLocation definition][21] for details. + * Replace ``, ``, and ``. See the [BackupStorageLocation definition][21] for details. Here is an example of a completed config file. diff --git a/docs/backupstoragelocation-definition.md b/docs/backupstoragelocation-definition.md index f8529b0fc..9b5e9dd1f 100644 --- a/docs/backupstoragelocation-definition.md +++ b/docs/backupstoragelocation-definition.md @@ -51,12 +51,17 @@ The configurable parameters are as follows: | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000

You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.| | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/``"

Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.| -#### GCP - -No parameters required. - #### Azure +##### objectStorage/config + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `resourceGroup` | string | Required Field | Name of the resource group containing the storage account for this backup storage location. | +| `storageAccount` | string | Required Field | Name of the storage account for this backup storage location. | + +#### GCP + No parameters required. [0]: #aws diff --git a/docs/config-definition.md b/docs/config-definition.md index 4b4bf0942..d17e086bf 100644 --- a/docs/config-definition.md +++ b/docs/config-definition.md @@ -53,12 +53,6 @@ The configurable parameters are as follows: | --- | --- | --- | --- | | `region` | string | Required Field | *Example*: "us-east-1"

See [AWS documentation][3] for the full list. | -#### GCP - -##### persistentVolumeProvider/config - -No parameters required. - #### Azure ##### persistentVolumeProvider/config @@ -67,6 +61,13 @@ No parameters required. | --- | --- | --- | --- | | `apiTimeout` | metav1.Duration | 2m0s | How long to wait for an Azure API request to complete before timeout. | +#### GCP + +##### persistentVolumeProvider/config + +No parameters required. + + ## Deployment Heptio Ark also defines its own Deployment object for starting the Ark server on Kubernetes. When the Ark server is deployed, there are specific configurations that might be changed. diff --git a/docs/debugging-install.md b/docs/debugging-install.md index 7969f1bcd..ac31e6d69 100644 --- a/docs/debugging-install.md +++ b/docs/debugging-install.md @@ -41,7 +41,7 @@ into the Ark server pod. Ensure the following: This means that the secrets containing the Azure service principal credentials for Ark has not been created/mounted properly into the Ark server pod. Ensure the following: * The `cloud-credentials` secret exists in the Ark server's namespace -* The `cloud-credentials` secret has seven keys and each one has the correct value (see [setup instructions](0)) +* The `cloud-credentials` secret has all of the expected keys and each one has the correct value (see [setup instructions](0)) * The `cloud-credentials` secret is defined as a volume for the Ark deployment * The `cloud-credentials` secret is being mounted into the Ark server pod at `/credentials` diff --git a/examples/azure/05-ark-backupstoragelocation.yaml b/examples/azure/05-ark-backupstoragelocation.yaml index 76c178206..9a94d2a2e 100644 --- a/examples/azure/05-ark-backupstoragelocation.yaml +++ b/examples/azure/05-ark-backupstoragelocation.yaml @@ -21,7 +21,10 @@ metadata: spec: provider: azure objectStorage: - bucket: + bucket: + config: + resourceGroup: + storageAccount: # Uncomment the below line to enable restic integration. # The format for resticLocation is [/], # e.g. "my-restic-bucket" or "my-restic-bucket/repos".