Merge pull request #455 from skriss/remove-aws-region

AWS: remove region requirement from config for backupStorageProvider

docs/aws-config.md

@@ -139,7 +139,7 @@ Specify the following values in the example files:
 
 * In `examples/aws/00-ark-config.yaml`:
 
-  * Replace `<YOUR_BUCKET>` and `<YOUR_REGION>`. See the [Config definition][6] for details.
+  * Replace `<YOUR_BUCKET>` and `<YOUR_REGION>` (for S3, region is optional and will be queried from the AWS S3 API if not provided). See the [Config definition][6] for details.
 
 * (Optional) If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`:
 

docs/config-definition.md

@@ -67,7 +67,7 @@ The configurable parameters are as follows:
 
 | Key | Type | Default | Meaning |
 | --- | --- | --- | --- |
-| `region` | string | Required Field | *Example*: "us-east-1"<br><br>See [AWS documentation][3] for the full list. |
+| `region` | string | Empty | *Example*: "us-east-1"<br><br>See [AWS documentation][3] for the full list.<br><br>Queried from the AWS S3 API if not provided. |
 | `s3ForcePathStyle` | bool | `false` | Set this to `true` if you are using a local storage service like Minio. |
 | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000<br><br>You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.|
 | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/`<KMS-Key-Alias-Name>`"<br><br>Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.|

pkg/cloudprovider/aws/object_store.go

@@ -17,12 +17,14 @@ limitations under the License.
 package aws
 
 import (
+	"context"
 	"io"
 	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
@@ -34,6 +36,7 @@ const (
 	s3URLKey            = "s3Url"
 	kmsKeyIDKey         = "kmsKeyId"
 	s3ForcePathStyleKey = "s3ForcePathStyle"
+	bucketKey           = "bucket"
 )
 
 type objectStore struct {
@@ -46,19 +49,45 @@ func NewObjectStore() cloudprovider.ObjectStore {
 	return &objectStore{}
 }
 
+func getBucketRegion(bucket string) (string, error) {
+	var region string
+
+	session, err := session.NewSession()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	for _, partition := range endpoints.DefaultPartitions() {
+		for regionHint := range partition.Regions() {
+			region, _ = s3manager.GetBucketRegion(context.Background(), session, bucket, regionHint)
+
+			// we only need to try a single region hint per partition, so break after the first
+			break
+		}
+
+		if region != "" {
+			return region, nil
+		}
+	}
+
+	return "", errors.New("unable to determine bucket's region")
+}
+
 func (o *objectStore) Init(config map[string]string) error {
 	var (
 		region              = config[regionKey]
 		s3URL               = config[s3URLKey]
 		kmsKeyID            = config[kmsKeyIDKey]
 		s3ForcePathStyleVal = config[s3ForcePathStyleKey]
-		s3ForcePathStyle    bool
-		err                 error
-	)
 
-	if region == "" {
-		return errors.Errorf("missing %s in aws configuration", regionKey)
-	}
+		// note that bucket is automatically added to the config map
+		// by the server from the ObjectStorageProviderConfig so
+		// doesn't need to be explicitly set by the user within
+		// config.
+		bucket           = config[bucketKey]
+		s3ForcePathStyle bool
+		err              error
+	)
 
 	if s3ForcePathStyleVal != "" {
 		if s3ForcePathStyle, err = strconv.ParseBool(s3ForcePathStyleVal); err != nil {
@@ -66,6 +95,17 @@ func (o *objectStore) Init(config map[string]string) error {
 		}
 	}
 
+	// AWS (not an alternate S3-compatible API) and region not
+	// explicitly specified: determine the bucket's region
+	if s3URL == "" && region == "" {
+		var err error
+
+		region, err = getBucketRegion(bucket)
+		if err != nil {
+			return err
+		}
+	}
+
 	awsConfig := aws.NewConfig().
 		WithRegion(region).
 		WithS3ForcePathStyle(s3ForcePathStyle)

pkg/cmd/server/server.go

@@ -335,6 +335,15 @@ func applyConfigDefaults(c *api.Config, logger logrus.FieldLogger) {
 	} else {
 		logger.WithField("priorities", c.ResourcePriorities).Info("Using resource priorities from config")
 	}
+
+	if c.BackupStorageProvider.Config == nil {
+		c.BackupStorageProvider.Config = make(map[string]string)
+	}
+
+	// add the bucket name to the config map so that object stores can use
+	// it when initializing. The AWS object store uses this to determine the
+	// bucket's region when setting up its client.
+	c.BackupStorageProvider.Config["bucket"] = c.BackupStorageProvider.Bucket
 }
 
 // watchConfig adds an update event handler to the Config shared informer, invoking s.cancelFunc