From 5ad21854f74f016711dd820d1600b9dc9c53584b Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 21 Jun 2018 17:20:53 -0700 Subject: [PATCH] restic: if S3, get bucket's region up-front Signed-off-by: Steve Kriss --- pkg/cloudprovider/aws/helpers.go | 53 +++++++++++++++++++++++++++ pkg/cloudprovider/aws/object_store.go | 28 +------------- pkg/restic/config.go | 36 ++++++++++-------- 3 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 pkg/cloudprovider/aws/helpers.go diff --git a/pkg/cloudprovider/aws/helpers.go b/pkg/cloudprovider/aws/helpers.go new file mode 100644 index 000000000..b7747c73b --- /dev/null +++ b/pkg/cloudprovider/aws/helpers.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +// GetBucketRegion returns the AWS region that a bucket is in, or an error +// if the region cannot be determined. +func GetBucketRegion(bucket string) (string, error) { + var region string + + session, err := session.NewSession() + if err != nil { + return "", errors.WithStack(err) + } + + for _, partition := range endpoints.DefaultPartitions() { + for regionHint := range partition.Regions() { + region, _ = s3manager.GetBucketRegion(context.Background(), session, bucket, regionHint) + + // we only need to try a single region hint per partition, so break after the first + break + } + + if region != "" { + return region, nil + } + } + + return "", errors.New("unable to determine bucket's region") +} diff --git a/pkg/cloudprovider/aws/object_store.go b/pkg/cloudprovider/aws/object_store.go index 4e27e5ee8..ba8ee5b55 100644 --- a/pkg/cloudprovider/aws/object_store.go +++ b/pkg/cloudprovider/aws/object_store.go @@ -17,14 +17,12 @@ limitations under the License. package aws import ( - "context" "io" "strconv" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/endpoints" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/pkg/errors" @@ -49,30 +47,6 @@ func NewObjectStore() cloudprovider.ObjectStore { return &objectStore{} } -func getBucketRegion(bucket string) (string, error) { - var region string - - session, err := session.NewSession() - if err != nil { - return "", errors.WithStack(err) - } - - for _, partition := range endpoints.DefaultPartitions() { - for regionHint := range partition.Regions() { - region, _ = s3manager.GetBucketRegion(context.Background(), session, bucket, regionHint) - - // we only need to try a single region hint per partition, so break after the first - break - } - - if region != "" { - return region, nil - } - } - - return "", errors.New("unable to determine bucket's region") -} - func (o *objectStore) Init(config map[string]string) error { var ( region = config[regionKey] @@ -100,7 +74,7 @@ func (o *objectStore) Init(config map[string]string) error { if s3URL == "" && region == "" { var err error - region, err = getBucketRegion(bucket) + region, err = GetBucketRegion(bucket) if err != nil { return err } diff --git a/pkg/restic/config.go b/pkg/restic/config.go index 9ab8891e7..0e8d9fccf 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -21,6 +21,7 @@ import ( "strings" arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/cloudprovider/aws" ) type BackendType string @@ -34,22 +35,9 @@ const ( // getRepoPrefix returns the prefix of the value of the --repo flag for // restic commands, i.e. everything except the "/". func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { - if BackendType(config.Name) == AWSBackend { - var url string - switch { - // non-AWS, S3-compatible object store - case config.Config["s3Url"] != "": - url = config.Config["s3Url"] - default: - url = "s3.amazonaws.com" - } - - return fmt.Sprintf("s3:%s/%s", url, config.ResticLocation) - } - var ( - parts = strings.SplitN(config.ResticLocation, "/", 2) - bucket, path string + parts = strings.SplitN(config.ResticLocation, "/", 2) + bucket, path, prefix string ) if len(parts) >= 1 { @@ -59,8 +47,24 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string { path = parts[1] } - var prefix string switch BackendType(config.Name) { + case AWSBackend: + var url string + switch { + // non-AWS, S3-compatible object store + case config.Config["s3Url"] != "": + url = config.Config["s3Url"] + default: + region, err := aws.GetBucketRegion(bucket) + if err != nil { + url = "s3.amazonaws.com" + break + } + + url = fmt.Sprintf("s3-%s.amazonaws.com", region) + } + + return fmt.Sprintf("s3:%s/%s", url, config.ResticLocation) case AzureBackend: prefix = "azure" case GCPBackend: