Allow to use AWS Signature v1 for creating signed AWS urls

Some aws implementations, for example the quobyte object storage, do not
support the v4 signing algorithm, but only v1.
This makes it possible to configure the signatureVersion.

The algorithm implementation was ported from d6c1be296e/botocore/auth.py (L860-L862)
which is used by the aws CLI client.

This fixes https://github.com/heptio/ark/issues/811.

Signed-off-by: Bastian Hofmann <bashofmann@gmail.com>

CHANGELOG.md

@@ -10,7 +10,8 @@
   * Initialize empty schedule metrics on server init (#1054, @cbeneke)
   * Update CHANGELOGs (#1063, @wwitzel3)
   * Remove default token from all service accounts (#1048, @ncdc)
-
+  * Allow to use AWS Signature v1 for creating signed AWS urls (#811, @bashofmann)
+  
 ## Current release:
   * [CHANGELOG-0.10.md][8]
 

docs/api-types/backupstoragelocation.md

@@ -51,6 +51,7 @@ The configurable parameters are as follows:
 | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000<br><br>You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.|
 | `publicUrl` | string | Empty | *Example*: https://minio.mycluster.com<br><br>If specified, use this instead of `s3Url` when generating download URLs (e.g., for logs). This field is primarily for local storage services like Minio.|
 | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/`<KMS-Key-Alias-Name>`"<br><br>Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.|
+| `signatureVersion` | string | `"4"` | Version of the signature algorithm used to create signed URLs that are used by ark cli to download backups or fetch logs. Possible versions are "1" and "4". Usually the default version 4 is correct, but some S3-compatible providers like Quobyte only support version 1.|
 
 #### Azure
 

docs/support-matrix.md

@@ -20,6 +20,9 @@ _Note that these providers are not regularly tested by the Ark team._
  * [Minio][9]
  * Ceph RADOS v12.2.7
  * [DigitalOcean][7]
+ * Quobyte
+
+_Some storage providers, like Quobyte, may need a different [signature algorithm version][15]._
 
 ## Volume Snapshot Providers
 
@@ -52,3 +55,4 @@ After you publish your plugin, open a PR that adds your plugin to the appropriat
 [12]: https://github.com/aws/aws-sdk-go/aws
 [13]: https://portworx.slack.com/messages/px-k8s
 [14]: https://github.com/portworx/ark-plugin/issues
+[15]: api-types/backupstoragelocation.md#aws

pkg/cloudprovider/aws/object_store.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
@@ -38,20 +39,30 @@ const (
 	kmsKeyIDKey         = "kmsKeyId"
 	s3ForcePathStyleKey = "s3ForcePathStyle"
 	bucketKey           = "bucket"
+	signatureVersionKey = "signatureVersion"
 )
 
 type objectStore struct {
-	log        logrus.FieldLogger
-	s3         *s3.S3
-	preSignS3  *s3.S3
-	s3Uploader *s3manager.Uploader
-	kmsKeyID   string
+	log              logrus.FieldLogger
+	s3               *s3.S3
+	preSignS3        *s3.S3
+	s3Uploader       *s3manager.Uploader
+	kmsKeyID         string
+	signatureVersion string
 }
 
 func NewObjectStore(logger logrus.FieldLogger) cloudprovider.ObjectStore {
 	return &objectStore{log: logger}
 }
 
+func isValidSignatureVersion(signatureVersion string) bool {
+	switch signatureVersion {
+	case "1", "4":
+		return true
+	}
+	return false
+}
+
 func (o *objectStore) Init(config map[string]string) error {
 	var (
 		region              = config[regionKey]
@@ -59,6 +70,7 @@ func (o *objectStore) Init(config map[string]string) error {
 		publicURL           = config[publicURLKey]
 		kmsKeyID            = config[kmsKeyIDKey]
 		s3ForcePathStyleVal = config[s3ForcePathStyleKey]
+		signatureVersion    = config[signatureVersionKey]
 
 		// note that bucket is automatically added to the config map
 		// by the server from the ObjectStorageProviderConfig so
@@ -100,6 +112,13 @@ func (o *objectStore) Init(config map[string]string) error {
 	o.s3Uploader = s3manager.NewUploader(serverSession)
 	o.kmsKeyID = kmsKeyID
 
+	if signatureVersion != "" {
+		if !isValidSignatureVersion(signatureVersion) {
+			return errors.Errorf("invalid signature version: %s", signatureVersion)
+		}
+		o.signatureVersion = signatureVersion
+	}
+
 	if publicURL != "" {
 		publicConfig, err := newAWSConfig(publicURL, region, s3ForcePathStyle)
 		if err != nil {
@@ -239,5 +258,10 @@ func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (st
 		Key:    aws.String(key),
 	})
 
+	if o.signatureVersion == "1" {
+		req.Handlers.Sign.Remove(v4.SignRequestHandler)
+		req.Handlers.Sign.PushBackNamed(v1SignRequestHandler)
+	}
+
 	return req.Presign(ttl)
 }

pkg/cloudprovider/aws/object_store_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsValidSignatureVersion(t *testing.T) {
+	assert.True(t, isValidSignatureVersion("1"))
+	assert.True(t, isValidSignatureVersion("4"))
+	assert.False(t, isValidSignatureVersion("3"))
+}

pkg/cloudprovider/aws/v1_sign_request_handler.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/pkg/errors"
+)
+
+var (
+	errInvalidMethod = errors.New("v1 signer only handles HTTP GET")
+)
+
+type signer struct {
+	// Values that must be populated from the request
+	request     *request.Request
+	time        time.Time
+	credentials *credentials.Credentials
+	debug       aws.LogLevelType
+	logger      aws.Logger
+
+	query        url.Values
+	stringToSign string
+	signature    string
+}
+
+// SignRequestHandler is a named request handler the SDK will use to sign
+// service client request with using the V4 signature.
+var v1SignRequestHandler = request.NamedHandler{
+	Name: "v1.SignRequestHandler", Fn: signSDKRequest,
+}
+
+func signSDKRequest(req *request.Request) {
+	// If the request does not need to be signed ignore the signing of the
+	// request if the AnonymousCredentials object is used.
+	if req.Config.Credentials == credentials.AnonymousCredentials {
+		return
+	}
+
+	if req.HTTPRequest.Method != "GET" {
+		// The V1 signer only supports GET
+		req.Error = errInvalidMethod
+		return
+	}
+
+	v1 := signer{
+		request:     req,
+		time:        req.Time,
+		credentials: req.Config.Credentials,
+		debug:       req.Config.LogLevel.Value(),
+		logger:      req.Config.Logger,
+	}
+
+	req.Error = v1.sign()
+
+	if req.Error != nil {
+		return
+	}
+
+	req.HTTPRequest.URL.RawQuery = v1.query.Encode()
+}
+
+func (v1 *signer) sign() error {
+	credentialsValue, err := v1.credentials.Get()
+	if err != nil {
+		return errors.Wrap(err, "error getting credentials")
+	}
+
+	httpRequest := v1.request.HTTPRequest
+
+	v1.query = httpRequest.URL.Query()
+
+	// Set new query parameters
+	v1.query.Set("AWSAccessKeyId", credentialsValue.AccessKeyID)
+	if credentialsValue.SessionToken != "" {
+		v1.query.Set("SecurityToken", credentialsValue.SessionToken)
+	}
+
+	// in case this is a retry, ensure no signature present
+	v1.query.Del("Signature")
+
+	method := httpRequest.Method
+	path := httpRequest.URL.Path
+	if path == "" {
+		path = "/"
+	}
+
+	duration := int64(v1.request.ExpireTime / time.Second)
+	expires := strconv.FormatInt(duration, 10)
+	// build the canonical string for the v1 signature
+	v1.stringToSign = strings.Join([]string{
+		method,
+		"",
+		"",
+		expires,
+		path,
+	}, "\n")
+
+	hash := hmac.New(sha1.New, []byte(credentialsValue.SecretAccessKey))
+	hash.Write([]byte(v1.stringToSign))
+	v1.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))
+	v1.query.Set("Signature", v1.signature)
+	v1.query.Set("Expires", expires)
+
+	if v1.debug.Matches(aws.LogDebugWithSigning) {
+		v1.logSigningInfo()
+	}
+
+	return nil
+}
+
+const logSignInfoMsg = `DEBUG: Request Signature:
+---[ STRING TO SIGN ]--------------------------------
+%s
+---[ SIGNATURE ]-------------------------------------
+%s
+-----------------------------------------------------`
+
+func (v1 *signer) logSigningInfo() {
+	msg := fmt.Sprintf(logSignInfoMsg, v1.stringToSign, v1.query.Get("Signature"))
+	v1.logger.Log(msg)
+}