Extract providers (#1985)

* Remove cloud providers and reorg code Signed-off-by: Carlisia <carlisia@vmware.com> * Update dependencies Signed-off-by: Carlisia <carlisia@vmware.com> * Fix tests Signed-off-by: Carlisia <carlisia@vmware.com> * fix dependency issues Signed-off-by: Carlisia <carlisia@vmware.com> * Delete dup test Signed-off-by: Carlisia <carlisia@vmware.com> * Add back spaces to file Signed-off-by: Carlisia <carlisia@vmware.com> * Remove and update docs Signed-off-by: Carlisia <carlisia@vmware.com> * Make the plugins flag required Signed-off-by: Carlisia <carlisia@vmware.com> * Add changelog Signed-off-by: Carlisia <carlisia@vmware.com> * Make the plugins flag conditional Signed-off-by: Carlisia <carlisia@vmware.com>
2026-09-19 22:44:21 +00:00 · 2019-10-22 15:31:27 -07:00
parent 69f993aebd
commit d26bf05b33
214 changed files with 441 additions and 250600 deletions
@@ -0,0 +1,178 @@
+/*
+
+Copyright 2018, 2019 the Velero contributors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package persistence
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+)
+
+type BucketData map[string][]byte
+
+// inMemoryObjectStore is a simple implementation of the ObjectStore interface
+// that stores its data in-memory/in-proc. This is mainly intended to be used
+// as a test fake.
+type inMemoryObjectStore struct {
+	Data map[string]BucketData
+}
+
+func newInMemoryObjectStore(buckets ...string) *inMemoryObjectStore {
+	o := &inMemoryObjectStore{
+		Data: make(map[string]BucketData),
+	}
+
+	for _, bucket := range buckets {
+		o.Data[bucket] = make(map[string][]byte)
+	}
+
+	return o
+}
+
+//
+// Interface Implementation
+//
+
+func (o *inMemoryObjectStore) Init(config map[string]string) error {
+	return nil
+}
+
+func (o *inMemoryObjectStore) PutObject(bucket, key string, body io.Reader) error {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return errors.New("bucket not found")
+	}
+
+	obj, err := ioutil.ReadAll(body)
+	if err != nil {
+		return err
+	}
+
+	bucketData[key] = obj
+
+	return nil
+}
+
+func (o *inMemoryObjectStore) ObjectExists(bucket, key string) (bool, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return false, errors.New("bucket not found")
+	}
+
+	_, ok = bucketData[key]
+	return ok, nil
+}
+
+func (o *inMemoryObjectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return nil, errors.New("bucket not found")
+	}
+
+	obj, ok := bucketData[key]
+	if !ok {
+		return nil, errors.New("key not found")
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(obj)), nil
+}
+
+func (o *inMemoryObjectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]string, error) {
+	keys, err := o.ListObjects(bucket, prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	// For each key, check if it has an instance of the delimiter *after* the prefix.
+	// If not, skip it; if so, return the prefix of the key up to/including the delimiter.
+
+	var prefixes []string
+	for _, key := range keys {
+		// everything after 'prefix'
+		afterPrefix := key[len(prefix):]
+
+		// index of the *start* of 'delimiter' in 'afterPrefix'
+		delimiterStart := strings.Index(afterPrefix, delimiter)
+		if delimiterStart == -1 {
+			continue
+		}
+
+		// return the prefix, plus everything after the prefix and before
+		// the delimiter, plus the delimiter
+		fullPrefix := prefix + afterPrefix[0:delimiterStart] + delimiter
+
+		prefixes = append(prefixes, fullPrefix)
+	}
+
+	return prefixes, nil
+}
+
+func (o *inMemoryObjectStore) ListObjects(bucket, prefix string) ([]string, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return nil, errors.New("bucket not found")
+	}
+
+	var objs []string
+	for key := range bucketData {
+		if strings.HasPrefix(key, prefix) {
+			objs = append(objs, key)
+		}
+	}
+
+	return objs, nil
+}
+
+func (o *inMemoryObjectStore) DeleteObject(bucket, key string) error {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return errors.New("bucket not found")
+	}
+
+	delete(bucketData, key)
+
+	return nil
+}
+
+func (o *inMemoryObjectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
+	bucketData, ok := o.Data[bucket]
+	if !ok {
+		return "", errors.New("bucket not found")
+	}
+
+	_, ok = bucketData[key]
+	if !ok {
+		return "", errors.New("key not found")
+	}
+
+	return "a-url", nil
+}
+
+//
+// Test Helper Methods
+//
+
+func (o *inMemoryObjectStore) ClearBucket(bucket string) {
+	if _, ok := o.Data[bucket]; !ok {
+		return
+	}
+
+	o.Data[bucket] = make(map[string][]byte)
+}
@@ -0,0 +1,180 @@
+/*
+Copyright 2017 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package mocks
+
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import time "time"
+
+// ObjectStore is an autogenerated mock type for the ObjectStore type
+type ObjectStore struct {
+	mock.Mock
+}
+
+// CreateSignedURL provides a mock function with given fields: bucket, key, ttl
+func (_m *ObjectStore) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) {
+	ret := _m.Called(bucket, key, ttl)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, time.Duration) string); ok {
+		r0 = rf(bucket, key, ttl)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, time.Duration) error); ok {
+		r1 = rf(bucket, key, ttl)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DeleteObject provides a mock function with given fields: bucket, key
+func (_m *ObjectStore) DeleteObject(bucket string, key string) error {
+	ret := _m.Called(bucket, key)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, string) error); ok {
+		r0 = rf(bucket, key)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// GetObject provides a mock function with given fields: bucket, key
+func (_m *ObjectStore) GetObject(bucket string, key string) (io.ReadCloser, error) {
+	ret := _m.Called(bucket, key)
+
+	var r0 io.ReadCloser
+	if rf, ok := ret.Get(0).(func(string, string) io.ReadCloser); ok {
+		r0 = rf(bucket, key)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(io.ReadCloser)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string) error); ok {
+		r1 = rf(bucket, key)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Init provides a mock function with given fields: config
+func (_m *ObjectStore) Init(config map[string]string) error {
+	ret := _m.Called(config)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(map[string]string) error); ok {
+		r0 = rf(config)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// ListCommonPrefixes provides a mock function with given fields: bucket, prefix, delimiter
+func (_m *ObjectStore) ListCommonPrefixes(bucket string, prefix string, delimiter string) ([]string, error) {
+	ret := _m.Called(bucket, prefix, delimiter)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string, string, string) []string); ok {
+		r0 = rf(bucket, prefix, delimiter)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, string) error); ok {
+		r1 = rf(bucket, prefix, delimiter)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListObjects provides a mock function with given fields: bucket, prefix
+func (_m *ObjectStore) ListObjects(bucket string, prefix string) ([]string, error) {
+	ret := _m.Called(bucket, prefix)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string, string) []string); ok {
+		r0 = rf(bucket, prefix)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string) error); ok {
+		r1 = rf(bucket, prefix)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ObjectExists provides a mock function with given fields: bucket, key
+func (_m *ObjectStore) ObjectExists(bucket string, key string) (bool, error) {
+	ret := _m.Called(bucket, key)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(string, string) bool); ok {
+		r0 = rf(bucket, key)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string) error); ok {
+		r1 = rf(bucket, key)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// PutObject provides a mock function with given fields: bucket, key, body
+func (_m *ObjectStore) PutObject(bucket string, key string, body io.Reader) error {
+	ret := _m.Called(bucket, key, body)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok {
+		r0 = rf(bucket, key, body)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
@@ -34,9 +34,8 @@ import (

 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	"github.com/vmware-tanzu/velero/pkg/cloudprovider"
-	cloudprovidermocks "github.com/vmware-tanzu/velero/pkg/cloudprovider/mocks"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/encode"
 	"github.com/vmware-tanzu/velero/pkg/volume"
@@ -46,12 +45,12 @@ type objectBackupStoreTestHarness struct {
 	// embedded to reduce verbosity when calling methods
 	*objectBackupStore

-	objectStore    *cloudprovider.InMemoryObjectStore
+	objectStore    *inMemoryObjectStore
 	bucket, prefix string
 }

 func newObjectBackupStoreTestHarness(bucket, prefix string) *objectBackupStoreTestHarness {
-	objectStore := cloudprovider.NewInMemoryObjectStore(bucket)
+	objectStore := newInMemoryObjectStore(bucket)

 	return &objectBackupStoreTestHarness{
 		objectBackupStore: &objectBackupStore{
@@ -70,7 +69,7 @@ func TestIsValid(t *testing.T) {
 	tests := []struct {
 		name        string
 		prefix      string
-		storageData cloudprovider.BucketData
+		storageData BucketData
 		expectErr   bool
 	}{
 		{
@@ -163,7 +162,7 @@ func TestListBackups(t *testing.T) {
 	tests := []struct {
 		name        string
 		prefix      string
-		storageData cloudprovider.BucketData
+		storageData BucketData
 		expectedRes []string
 		expectedErr string
 	}{
@@ -456,7 +455,7 @@ func TestDeleteBackup(t *testing.T) {

 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			objectStore := new(cloudprovidermocks.ObjectStore)
+			objectStore := new(providermocks.ObjectStore)
 			backupStore := &objectBackupStore{
 				objectStore: objectStore,
 				bucket:      "test-bucket",
@@ -629,7 +628,7 @@ func TestNewObjectBackupStore(t *testing.T) {
 			name:     "when Bucket has a leading and trailing slash, they are both stripped",
 			location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("/bucket/").Result(),
 			objectStoreGetter: objectStoreGetter{
-				"provider-1": cloudprovider.NewInMemoryObjectStore("bucket"),
+				"provider-1": newInMemoryObjectStore("bucket"),
 			},
 			wantBucket: "bucket",
 		},
@@ -637,7 +636,7 @@ func TestNewObjectBackupStore(t *testing.T) {
 			name:     "when Prefix has a leading and trailing slash, the leading slash is stripped and the trailing slash is left",
 			location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("bucket").Prefix("/prefix/").Result(),
 			objectStoreGetter: objectStoreGetter{
-				"provider-1": cloudprovider.NewInMemoryObjectStore("bucket"),
+				"provider-1": newInMemoryObjectStore("bucket"),
 			},
 			wantBucket: "bucket",
 			wantPrefix: "prefix/",
@@ -646,7 +645,7 @@ func TestNewObjectBackupStore(t *testing.T) {
 			name:     "when Prefix has no leading or trailing slash, a trailing slash is added",
 			location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("bucket").Prefix("prefix").Result(),
 			objectStoreGetter: objectStoreGetter{
-				"provider-1": cloudprovider.NewInMemoryObjectStore("bucket"),
+				"provider-1": newInMemoryObjectStore("bucket"),
 			},
 			wantBucket: "bucket",
 			wantPrefix: "prefix/",