diff --git a/Gopkg.lock b/Gopkg.lock index 2697367cc..84a3110de 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -157,12 +157,30 @@ packages = [".","diskcache"] revision = "c1f8028e62adb3d518b823a2f8e6a95c38bdd3aa" +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-hclog" + packages = ["."] + revision = "ca137eb4b4389c9bc6f1a6d887f056bf16c00510" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-plugin" + packages = ["."] + revision = "e2fbc6864d18d3c37b6cde4297ec9fca266d28f1" + [[projects]] branch = "master" name = "github.com/hashicorp/golang-lru" packages = [".","simplelru"] revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6" +[[projects]] + branch = "master" + name = "github.com/hashicorp/yamux" + packages = ["."] + revision = "f5742cb6b85602e7fa834e9d5d91a7d7fa850824" + [[projects]] branch = "master" name = "github.com/howeyc/gopass" @@ -205,6 +223,12 @@ packages = ["buffer","jlexer","jwriter"] revision = "2f5df55504ebc322e4d52d34df6a1f5b503bf26d" +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-testing-interface" + packages = ["."] + revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" + [[projects]] branch = "master" name = "github.com/petar/GoLLRB" @@ -343,7 +367,7 @@ [[projects]] name = "google.golang.org/grpc" - packages = [".","codes","connectivity","credentials","grpclb/grpc_lb_v1","grpclog","internal","keepalive","metadata","naming","peer","stats","status","tap","transport"] + packages = [".","codes","connectivity","credentials","grpclb/grpc_lb_v1","grpclog","health","health/grpc_health_v1","internal","keepalive","metadata","naming","peer","stats","status","tap","transport"] revision = "b3ddf786825de56a4178401b7e174ee332173b66" version = "v1.5.2" @@ -402,6 +426,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "c726286fac3935d4b5e3d39b37c1c5137f0f4714e24716b32fcb82e61e827541" + inputs-digest = "9d8af931a171a23e1a04151e37c15ebadf7457512a6b149da20e228b6469314d" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index e6af9d2e6..04da96875 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -114,3 +114,7 @@ required = [ [[override]] name = "k8s.io/gengo" revision = "9e661e9308f078838e266cca1c673922088c0ea4" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-plugin" diff --git a/docs/build-from-scratch.md b/docs/build-from-scratch.md index fa5e6aee8..7e81697ed 100644 --- a/docs/build-from-scratch.md +++ b/docs/build-from-scratch.md @@ -44,6 +44,7 @@ These include: * Listers * Shared informers * Documentation +* Protobuf/gRPC types If you make any of the following changes, you will need to run `make update` to regenerate automatically generated files: @@ -51,6 +52,10 @@ automatically generated files: * Add/edit/remove commands or subcommands * Add new API types +If you make the following change, you will need to run [generate-proto.sh][13] to regenerate +automatically generated files (note that this requires the [proto compiler][14] to be installed): +* Add/edit/remove protobuf message or service definitions + ### Cross compiling By default, `make` will build an `ark` binary that runs on your host operating system and @@ -109,3 +114,5 @@ If you need to add or update the vendored dependencies, please see [Vendoring de [10]: #4-vendoring-dependencies [11]: vendoring-dependencies.md [12]: #3-test +[13]: ../hack/generate-proto.sh +[14]: https://grpc.io/docs/quickstart/go.html#install-protocol-buffers-v3 \ No newline at end of file diff --git a/docs/config-definition.md b/docs/config-definition.md index e30e7ecaa..ed22bc87b 100644 --- a/docs/config-definition.md +++ b/docs/config-definition.md @@ -24,11 +24,13 @@ metadata: namespace: heptio-ark name: default persistentVolumeProvider: - aws: + name: aws + config: region: us-west-2 backupStorageProvider: + name: aws bucket: ark - aws: + config: region: us-west-2 backupSyncPeriod: 60m gcSyncPeriod: 60m @@ -44,9 +46,13 @@ The configurable parameters are as follows: | Key | Type | Default | Meaning | | --- | --- | --- | --- | -| `persistentVolumeProvider` | CloudProviderConfig

(Supported key values are `aws`, `gcp`, and `azure`, but only one can be present. See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs.) | None (Optional) | The specification for whichever cloud provider the cluster is using for persistent volumes (to be snapshotted), if any.

If not specified, Backups and Restores requesting PV snapshots & restores, respectively, are considered invalid.

*NOTE*: For Azure, your Kubernetes cluster needs to be version 1.7.2+ in order to support PV snapshotting of its managed disks. | -| `backupStorageProvider`/(inline) | CloudProviderConfig

(Supported key values are `aws`, `gcp`, and `azure`, but only one can be present. See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs.) | Required Field | The specification for whichever cloud provider will be used to actually store the backups. | +| `persistentVolumeProvider` | CloudProviderConfig | None (Optional) | The specification for whichever cloud provider the cluster is using for persistent volumes (to be snapshotted), if any.

If not specified, Backups and Restores requesting PV snapshots & restores, respectively, are considered invalid.

*NOTE*: For Azure, your Kubernetes cluster needs to be version 1.7.2+ in order to support PV snapshotting of its managed disks. | +| `persistentVolumeProvider/name` | String

(Ark natively supports `aws`, `gcp`, and `azure`. Other providers may be available via external plugins.) | None (Optional) | The name of the cloud provider the cluster is using for persistent volumes, if any. | +| `persistentVolumeProvider/config` | map[string]string

(See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs or your provider's documentation.) | None (Optional) | Configuration keys/values to be passed to the cloud provider for persistent volumes. | +| `backupStorageProvider` | CloudProviderConfig | Required Field | The specification for whichever cloud provider will be used to actually store the backups. | +| `backupStorageProvider/name` | String

(Ark natively supports `aws`, `gcp`, and `azure`. Other providers may be available via external plugins.) | Required Field | The name of the cloud provider that will be used to actually store the backups. | | `backupStorageProvider/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | +| `backupStorageProvider/config` | map[string]string

(See the corresponding [AWS][0], [GCP][1], and [Azure][2]-specific configs or your provider's documentation.) | None (Optional) | Configuration keys/values to be passed to the cloud provider for backup storage. | | `backupSyncPeriod` | metav1.Duration | 60m0s | How frequently Ark queries the object storage to make sure that the appropriate Backup resources have been created for existing backup files. | | `gcSyncPeriod` | metav1.Duration | 60m0s | How frequently Ark queries the object storage to delete backup files that have passed their TTL. | | `scheduleSyncPeriod` | metav1.Duration | 1m0s | How frequently Ark checks its Schedule resource objects to see if a backup needs to be initiated. | @@ -57,17 +63,16 @@ The configurable parameters are as follows: **(Or other S3-compatible storage)** -#### backupStorageProvider +#### backupStorageProvider/config | Key | Type | Default | Meaning | | --- | --- | --- | --- | | `region` | string | Required Field | *Example*: "us-east-1"

See [AWS documentation][3] for the full list. | -| `disableSSL` | bool | `false` | Set this to `true` if you are using Minio (or another local, S3-compatible storage service) and your deployment is not secured. | | `s3ForcePathStyle` | bool | `false` | Set this to `true` if you are using a local storage service like Minio. | | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000

You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.| | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/``"

Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.| -#### persistentVolumeProvider (AWS Only) +#### persistentVolumeProvider/config (AWS Only) | Key | Type | Default | Meaning | | --- | --- | --- | --- | @@ -75,11 +80,11 @@ The configurable parameters are as follows: ### GCP -#### backupStorageProvider +#### backupStorageProvider/config -No parameters required; specify an empty object per [example file][11]. +No parameters required. -#### persistentVolumeProvider +#### persistentVolumeProvider/config | Key | Type | Default | Meaning | | --- | --- | --- | --- | @@ -87,11 +92,11 @@ No parameters required; specify an empty object per [example file][11]. ### Azure -#### backupStorageProvider +#### backupStorageProvider/config -No parameters required; specify an empty object per [example file][12]. +No parameters required. -#### persistentVolumeProvider +#### persistentVolumeProvider/config | Key | Type | Default | Meaning | | --- | --- | --- | --- | diff --git a/examples/aws/00-ark-config.yaml b/examples/aws/00-ark-config.yaml index 6b668fb8e..3780e2a01 100644 --- a/examples/aws/00-ark-config.yaml +++ b/examples/aws/00-ark-config.yaml @@ -19,11 +19,13 @@ metadata: namespace: heptio-ark name: default persistentVolumeProvider: - aws: + name: aws + config: region: backupStorageProvider: + name: aws bucket: - aws: + config: region: backupSyncPeriod: 30m gcSyncPeriod: 30m diff --git a/examples/azure/10-ark-config.yaml b/examples/azure/10-ark-config.yaml index 4edcdfe37..75c8fe221 100644 --- a/examples/azure/10-ark-config.yaml +++ b/examples/azure/10-ark-config.yaml @@ -19,12 +19,13 @@ metadata: namespace: heptio-ark name: default persistentVolumeProvider: - azure: + name: azure + config: location: apiTimeout: backupStorageProvider: + name: azure bucket: - azure: {} backupSyncPeriod: 30m gcSyncPeriod: 30m scheduleSyncPeriod: 1m diff --git a/examples/gcp/00-ark-config.yaml b/examples/gcp/00-ark-config.yaml index 6616e9656..2ee4cc119 100644 --- a/examples/gcp/00-ark-config.yaml +++ b/examples/gcp/00-ark-config.yaml @@ -19,11 +19,12 @@ metadata: namespace: heptio-ark name: default persistentVolumeProvider: - gcp: + name: gcp + config: project: backupStorageProvider: + name: gcp bucket: - gcp: {} backupSyncPeriod: 30m gcSyncPeriod: 30m scheduleSyncPeriod: 1m diff --git a/examples/minio/10-ark-config.yaml b/examples/minio/10-ark-config.yaml index 66a6dce55..4b3451177 100644 --- a/examples/minio/10-ark-config.yaml +++ b/examples/minio/10-ark-config.yaml @@ -19,8 +19,9 @@ metadata: namespace: heptio-ark name: default backupStorageProvider: + name: aws bucket: ark - aws: + config: region: minio s3ForcePathStyle: true s3Url: http://minio:9000 diff --git a/hack/generate-proto.sh b/hack/generate-proto.sh new file mode 100755 index 000000000..eea338831 --- /dev/null +++ b/hack/generate-proto.sh @@ -0,0 +1,23 @@ +#!/bin/bash -e +# +# Copyright 2017 Heptio Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HACK_DIR=$(dirname "${BASH_SOURCE}") + +echo "Running protoc" + +protoc pkg/plugin/proto/*.proto --go_out=plugins=grpc:pkg/plugin/generated/ -I pkg/plugin/proto/ + +echo "Success!" diff --git a/pkg/apis/ark/v1/config.go b/pkg/apis/ark/v1/config.go index fa0cec374..097c39dee 100644 --- a/pkg/apis/ark/v1/config.go +++ b/pkg/apis/ark/v1/config.go @@ -69,17 +69,11 @@ type Config struct { } // CloudProviderConfig is configuration information about how to connect -// to a particular cloud. Only one of the members (AWS, GCP, Azure) may -// be present. +// to a particular cloud. type CloudProviderConfig struct { - // AWS is configuration information for connecting to AWS. - AWS *AWSConfig `json:"aws"` + Name string `json:"name"` - // GCP is configuration information for connecting to GCP. - GCP *GCPConfig `json:"gcp"` - - // Azure is configuration information for connecting to Azure. - Azure *AzureConfig `json:"azure"` + Config map[string]string `json:"config"` } // ObjectStorageProviderConfig is configuration information for connecting to @@ -93,23 +87,3 @@ type ObjectStorageProviderConfig struct { // are stored. Bucket string `json:"bucket"` } - -// AWSConfig is configuration information for connecting to AWS. -type AWSConfig struct { - Region string `json:"region"` - DisableSSL bool `json:"disableSSL"` - S3ForcePathStyle bool `json:"s3ForcePathStyle"` - S3Url string `json:"s3Url"` - KMSKeyID string `json:"kmsKeyId"` -} - -// GCPConfig is configuration information for connecting to GCP. -type GCPConfig struct { - Project string `json:"project"` -} - -// AzureConfig is configuration information for connecting to Azure. -type AzureConfig struct { - Location string `json:"location"` - APITimeout metav1.Duration `json:"apiTimeout"` -} diff --git a/pkg/apis/ark/v1/zz_generated.deepcopy.go b/pkg/apis/ark/v1/zz_generated.deepcopy.go index f31a42244..e73389538 100644 --- a/pkg/apis/ark/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ark/v1/zz_generated.deepcopy.go @@ -32,14 +32,6 @@ import ( // Deprecated: deepcopy registration will go away when static deepcopy is fully implemented. func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc { return []conversion.GeneratedDeepCopyFunc{ - {Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { - in.(*AWSConfig).DeepCopyInto(out.(*AWSConfig)) - return nil - }, InType: reflect.TypeOf(&AWSConfig{})}, - {Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { - in.(*AzureConfig).DeepCopyInto(out.(*AzureConfig)) - return nil - }, InType: reflect.TypeOf(&AzureConfig{})}, {Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*Backup).DeepCopyInto(out.(*Backup)) return nil @@ -104,10 +96,6 @@ func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc { in.(*ExecHook).DeepCopyInto(out.(*ExecHook)) return nil }, InType: reflect.TypeOf(&ExecHook{})}, - {Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { - in.(*GCPConfig).DeepCopyInto(out.(*GCPConfig)) - return nil - }, InType: reflect.TypeOf(&GCPConfig{})}, {Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*ObjectStorageProviderConfig).DeepCopyInto(out.(*ObjectStorageProviderConfig)) return nil @@ -155,39 +143,6 @@ func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc { } } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AWSConfig) DeepCopyInto(out *AWSConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSConfig. -func (in *AWSConfig) DeepCopy() *AWSConfig { - if in == nil { - return nil - } - out := new(AWSConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureConfig) DeepCopyInto(out *AzureConfig) { - *out = *in - out.APITimeout = in.APITimeout - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureConfig. -func (in *AzureConfig) DeepCopy() *AzureConfig { - if in == nil { - return nil - } - out := new(AzureConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Backup) DeepCopyInto(out *Backup) { *out = *in @@ -453,31 +408,11 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProviderConfig) DeepCopyInto(out *CloudProviderConfig) { *out = *in - if in.AWS != nil { - in, out := &in.AWS, &out.AWS - if *in == nil { - *out = nil - } else { - *out = new(AWSConfig) - **out = **in - } - } - if in.GCP != nil { - in, out := &in.GCP, &out.GCP - if *in == nil { - *out = nil - } else { - *out = new(GCPConfig) - **out = **in - } - } - if in.Azure != nil { - in, out := &in.Azure, &out.Azure - if *in == nil { - *out = nil - } else { - *out = new(AzureConfig) - **out = **in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val } } return @@ -707,22 +642,6 @@ func (in *ExecHook) DeepCopy() *ExecHook { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GCPConfig) DeepCopyInto(out *GCPConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPConfig. -func (in *GCPConfig) DeepCopy() *GCPConfig { - if in == nil { - return nil - } - out := new(GCPConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStorageProviderConfig) DeepCopyInto(out *ObjectStorageProviderConfig) { *out = *in diff --git a/pkg/cloudprovider/aws/block_storage_adapter.go b/pkg/cloudprovider/aws/block_store.go similarity index 73% rename from pkg/cloudprovider/aws/block_storage_adapter.go rename to pkg/cloudprovider/aws/block_store.go index d611ac512..5c41ea6f3 100644 --- a/pkg/cloudprovider/aws/block_storage_adapter.go +++ b/pkg/cloudprovider/aws/block_store.go @@ -27,9 +27,14 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) -var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{} +const regionKey = "region" -type blockStorageAdapter struct { +// iopsVolumeTypes is a set of AWS EBS volume types for which IOPS should +// be captured during snapshot and provided when creating a new volume +// from snapshot. +var iopsVolumeTypes = sets.NewString("io1") + +type blockStore struct { ec2 *ec2.EC2 } @@ -46,29 +51,29 @@ func getSession(config *aws.Config) (*session.Session, error) { return sess, nil } -func NewBlockStorageAdapter(region string) (cloudprovider.BlockStorageAdapter, error) { +func NewBlockStore() cloudprovider.BlockStore { + return &blockStore{} +} + +func (b *blockStore) Init(config map[string]string) error { + region := config[regionKey] if region == "" { - return nil, errors.New("missing region in aws configuration in config file") + return errors.Errorf("missing %s in aws configuration", regionKey) } awsConfig := aws.NewConfig().WithRegion(region) sess, err := getSession(awsConfig) if err != nil { - return nil, err + return err } - return &blockStorageAdapter{ - ec2: ec2.New(sess), - }, nil + b.ec2 = ec2.New(sess) + + return nil } -// iopsVolumeTypes is a set of AWS EBS volume types for which IOPS should -// be captured during snapshot and provided when creating a new volume -// from snapshot. -var iopsVolumeTypes = sets.NewString("io1") - -func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { +func (b *blockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { req := &ec2.CreateVolumeInput{ SnapshotId: &snapshotID, AvailabilityZone: &volumeAZ, @@ -79,7 +84,7 @@ func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, req.Iops = iops } - res, err := op.ec2.CreateVolume(req) + res, err := b.ec2.CreateVolume(req) if err != nil { return "", errors.WithStack(err) } @@ -87,12 +92,12 @@ func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, return *res.VolumeId, nil } -func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { +func (b *blockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { req := &ec2.DescribeVolumesInput{ VolumeIds: []*string{&volumeID}, } - res, err := op.ec2.DescribeVolumes(req) + res, err := b.ec2.DescribeVolumes(req) if err != nil { return "", nil, errors.WithStack(err) } @@ -119,12 +124,12 @@ func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, return volumeType, iops, nil } -func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { +func (b *blockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { req := &ec2.DescribeVolumesInput{ VolumeIds: []*string{&volumeID}, } - res, err := op.ec2.DescribeVolumes(req) + res, err := b.ec2.DescribeVolumes(req) if err != nil { return false, errors.WithStack(err) } @@ -135,7 +140,7 @@ func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready b return *res.Volumes[0].State == ec2.VolumeStateAvailable, nil } -func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) { +func (b *blockStore) ListSnapshots(tagFilters map[string]string) ([]string, error) { req := &ec2.DescribeSnapshotsInput{} for k, v := range tagFilters { @@ -147,7 +152,7 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st } var ret []string - err := op.ec2.DescribeSnapshotsPages(req, func(res *ec2.DescribeSnapshotsOutput, lastPage bool) bool { + err := b.ec2.DescribeSnapshotsPages(req, func(res *ec2.DescribeSnapshotsOutput, lastPage bool) bool { for _, snapshot := range res.Snapshots { ret = append(ret, *snapshot.SnapshotId) } @@ -161,12 +166,12 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st return ret, nil } -func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { +func (b *blockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { req := &ec2.CreateSnapshotInput{ VolumeId: &volumeID, } - res, err := op.ec2.CreateSnapshot(req) + res, err := b.ec2.CreateSnapshot(req) if err != nil { return "", errors.WithStack(err) } @@ -186,17 +191,17 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma tagsReq.SetTags(ec2Tags) - _, err = op.ec2.CreateTags(tagsReq) + _, err = b.ec2.CreateTags(tagsReq) return *res.SnapshotId, errors.WithStack(err) } -func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error { +func (b *blockStore) DeleteSnapshot(snapshotID string) error { req := &ec2.DeleteSnapshotInput{ SnapshotId: &snapshotID, } - _, err := op.ec2.DeleteSnapshot(req) + _, err := b.ec2.DeleteSnapshot(req) return errors.WithStack(err) } diff --git a/pkg/cloudprovider/aws/object_storage_adapter.go b/pkg/cloudprovider/aws/object_store.go similarity index 59% rename from pkg/cloudprovider/aws/object_storage_adapter.go rename to pkg/cloudprovider/aws/object_store.go index ca3eb1aea..6aa1874a3 100644 --- a/pkg/cloudprovider/aws/object_storage_adapter.go +++ b/pkg/cloudprovider/aws/object_store.go @@ -18,6 +18,7 @@ package aws import ( "io" + "strconv" "time" "github.com/aws/aws-sdk-go/aws" @@ -29,17 +30,40 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) -var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{} +const ( + s3URLKey = "s3Url" + kmsKeyIDKey = "kmsKeyId" + s3ForcePathStyleKey = "s3ForcePathStyle" +) -type objectStorageAdapter struct { +type objectStore struct { s3 *s3.S3 s3Uploader *s3manager.Uploader kmsKeyID string } -func NewObjectStorageAdapter(region, s3URL, kmsKeyID string, s3ForcePathStyle bool) (cloudprovider.ObjectStorageAdapter, error) { +func NewObjectStore() cloudprovider.ObjectStore { + return &objectStore{} +} + +func (o *objectStore) Init(config map[string]string) error { + var ( + region = config[regionKey] + s3URL = config[s3URLKey] + kmsKeyID = config[kmsKeyIDKey] + s3ForcePathStyleVal = config[s3ForcePathStyleKey] + s3ForcePathStyle bool + err error + ) + if region == "" { - return nil, errors.New("missing region in aws configuration in config file") + return errors.Errorf("missing %s in aws configuration", regionKey) + } + + if s3ForcePathStyleVal != "" { + if s3ForcePathStyle, err = strconv.ParseBool(s3ForcePathStyleVal); err != nil { + return errors.Wrapf(err, "could not parse %s (expected bool)", s3ForcePathStyleKey) + } } awsConfig := aws.NewConfig(). @@ -62,17 +86,17 @@ func NewObjectStorageAdapter(region, s3URL, kmsKeyID string, s3ForcePathStyle bo sess, err := getSession(awsConfig) if err != nil { - return nil, err + return err } - return &objectStorageAdapter{ - s3: s3.New(sess), - s3Uploader: s3manager.NewUploader(sess), - kmsKeyID: kmsKeyID, - }, nil + o.s3 = s3.New(sess) + o.s3Uploader = s3manager.NewUploader(sess) + o.kmsKeyID = kmsKeyID + + return nil } -func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.Reader) error { +func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error { req := &s3manager.UploadInput{ Bucket: &bucket, Key: &key, @@ -80,23 +104,23 @@ func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.Rea } // if kmsKeyID is not empty, enable "aws:kms" encryption - if op.kmsKeyID != "" { + if o.kmsKeyID != "" { req.ServerSideEncryption = aws.String("aws:kms") - req.SSEKMSKeyId = &op.kmsKeyID + req.SSEKMSKeyId = &o.kmsKeyID } - _, err := op.s3Uploader.Upload(req) + _, err := o.s3Uploader.Upload(req) return errors.Wrapf(err, "error putting object %s", key) } -func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) { +func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { req := &s3.GetObjectInput{ Bucket: &bucket, Key: &key, } - res, err := op.s3.GetObject(req) + res, err := o.s3.GetObject(req) if err != nil { return nil, errors.Wrapf(err, "error getting object %s", key) } @@ -104,14 +128,14 @@ func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadClo return res.Body, nil } -func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { +func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { req := &s3.ListObjectsV2Input{ Bucket: &bucket, Delimiter: &delimiter, } var ret []string - err := op.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + err := o.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool { for _, prefix := range page.CommonPrefixes { ret = append(ret, *prefix.Prefix) } @@ -125,14 +149,14 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri return ret, nil } -func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) { +func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) { req := &s3.ListObjectsV2Input{ Bucket: &bucket, Prefix: &prefix, } var ret []string - err := op.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + err := o.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool { for _, obj := range page.Contents { ret = append(ret, *obj.Key) } @@ -146,19 +170,19 @@ func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, er return ret, nil } -func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error { +func (o *objectStore) DeleteObject(bucket string, key string) error { req := &s3.DeleteObjectInput{ Bucket: &bucket, Key: &key, } - _, err := op.s3.DeleteObject(req) + _, err := o.s3.DeleteObject(req) return errors.Wrapf(err, "error deleting object %s", key) } -func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { - req, _ := op.s3.GetObjectRequest(&s3.GetObjectInput{ +func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { + req, _ := o.s3.GetObjectRequest(&s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) diff --git a/pkg/cloudprovider/azure/block_storage_adapter.go b/pkg/cloudprovider/azure/block_store.go similarity index 70% rename from pkg/cloudprovider/azure/block_storage_adapter.go rename to pkg/cloudprovider/azure/block_store.go index 5efbbb856..8824dedc6 100644 --- a/pkg/cloudprovider/azure/block_storage_adapter.go +++ b/pkg/cloudprovider/azure/block_store.go @@ -33,17 +33,6 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) -type blockStorageAdapter struct { - disks *disk.DisksClient - snaps *disk.SnapshotsClient - subscription string - resourceGroup string - location string - apiTimeout time.Duration -} - -var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{} - const ( azureClientIDKey string = "AZURE_CLIENT_ID" azureClientSecretKey string = "AZURE_CLIENT_SECRET" @@ -52,8 +41,20 @@ const ( azureStorageAccountIDKey string = "AZURE_STORAGE_ACCOUNT_ID" azureStorageKeyKey string = "AZURE_STORAGE_KEY" azureResourceGroupKey string = "AZURE_RESOURCE_GROUP" + + locationKey = "location" + apiTimeoutKey = "apiTimeout" ) +type blockStore struct { + disks *disk.DisksClient + snaps *disk.SnapshotsClient + subscription string + resourceGroup string + location string + apiTimeout time.Duration +} + func getConfig() map[string]string { cfg := map[string]string{ azureClientIDKey: "", @@ -72,9 +73,24 @@ func getConfig() map[string]string { return cfg } -func NewBlockStorageAdapter(location string, apiTimeout time.Duration) (cloudprovider.BlockStorageAdapter, error) { +func NewBlockStore() cloudprovider.BlockStore { + return &blockStore{} +} + +func (b *blockStore) Init(config map[string]string) error { + var ( + location = config[locationKey] + apiTimeoutVal = config[apiTimeoutKey] + apiTimeout time.Duration + err error + ) + if location == "" { - return nil, errors.New("missing location in azure configuration in config file") + return errors.Errorf("missing %s in azure configuration", locationKey) + } + + if apiTimeout, err = time.ParseDuration(apiTimeoutVal); err != nil { + return errors.Wrapf(err, "could not parse %s (expected time.Duration)", apiTimeoutKey) } if apiTimeout == 0 { @@ -85,7 +101,7 @@ func NewBlockStorageAdapter(location string, apiTimeout time.Duration) (cloudpro spt, err := helpers.NewServicePrincipalTokenFromCredentials(cfg, azure.PublicCloud.ResourceManagerEndpoint) if err != nil { - return nil, errors.Wrap(err, "error creating new service principal token") + return errors.Wrap(err, "error creating new service principal token") } disksClient := disk.NewDisksClient(cfg[azureSubscriptionIDKey]) @@ -101,11 +117,11 @@ func NewBlockStorageAdapter(location string, apiTimeout time.Duration) (cloudpro locs, err := groupClient.ListLocations(cfg[azureSubscriptionIDKey]) if err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } if locs.Value == nil { - return nil, errors.New("no locations returned from Azure API") + return errors.New("no locations returned from Azure API") } locationExists := false @@ -117,26 +133,26 @@ func NewBlockStorageAdapter(location string, apiTimeout time.Duration) (cloudpro } if !locationExists { - return nil, errors.Errorf("location %q not found", location) + return errors.Errorf("location %q not found", location) } - return &blockStorageAdapter{ - disks: &disksClient, - snaps: &snapsClient, - subscription: cfg[azureSubscriptionIDKey], - resourceGroup: cfg[azureResourceGroupKey], - location: location, - apiTimeout: apiTimeout, - }, nil + b.disks = &disksClient + b.snaps = &snapsClient + b.subscription = cfg[azureSubscriptionIDKey] + b.resourceGroup = cfg[azureResourceGroupKey] + b.location = location + b.apiTimeout = apiTimeout + + return nil } -func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { - fullSnapshotName := getFullSnapshotName(op.subscription, op.resourceGroup, snapshotID) +func (b *blockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { + fullSnapshotName := getFullSnapshotName(b.subscription, b.resourceGroup, snapshotID) diskName := "restore-" + uuid.NewV4().String() disk := disk.Model{ Name: &diskName, - Location: &op.location, + Location: &b.location, Properties: &disk.Properties{ CreationData: &disk.CreationData{ CreateOption: disk.Copy, @@ -146,10 +162,10 @@ func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, }, } - ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout) + ctx, cancel := context.WithTimeout(context.Background(), b.apiTimeout) defer cancel() - _, errChan := op.disks.CreateOrUpdate(op.resourceGroup, *disk.Name, disk, ctx.Done()) + _, errChan := b.disks.CreateOrUpdate(b.resourceGroup, *disk.Name, disk, ctx.Done()) err := <-errChan @@ -159,8 +175,8 @@ func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, return diskName, nil } -func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { - res, err := op.disks.Get(op.resourceGroup, volumeID) +func (b *blockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { + res, err := b.disks.Get(b.resourceGroup, volumeID) if err != nil { return "", nil, errors.WithStack(err) } @@ -168,8 +184,8 @@ func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, return string(res.AccountType), nil, nil } -func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { - res, err := op.disks.Get(op.resourceGroup, volumeID) +func (b *blockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { + res, err := b.disks.Get(b.resourceGroup, volumeID) if err != nil { return false, errors.WithStack(err) } @@ -181,8 +197,8 @@ func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready b return *res.ProvisioningState == "Succeeded", nil } -func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) { - res, err := op.snaps.ListByResourceGroup(op.resourceGroup) +func (b *blockStore) ListSnapshots(tagFilters map[string]string) ([]string, error) { + res, err := b.snaps.ListByResourceGroup(b.resourceGroup) if err != nil { return nil, errors.WithStack(err) } @@ -215,8 +231,8 @@ Snapshot: return ret, nil } -func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { - fullDiskName := getFullDiskName(op.subscription, op.resourceGroup, volumeID) +func (b *blockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { + fullDiskName := getFullDiskName(b.subscription, b.resourceGroup, volumeID) // snapshot names must be <= 80 characters long var snapshotName string suffix := "-" + uuid.NewV4().String() @@ -236,7 +252,7 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma }, }, Tags: &map[string]*string{}, - Location: &op.location, + Location: &b.location, } for k, v := range tags { @@ -244,10 +260,10 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma (*snap.Tags)[k] = &val } - ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout) + ctx, cancel := context.WithTimeout(context.Background(), b.apiTimeout) defer cancel() - _, errChan := op.snaps.CreateOrUpdate(op.resourceGroup, *snap.Name, snap, ctx.Done()) + _, errChan := b.snaps.CreateOrUpdate(b.resourceGroup, *snap.Name, snap, ctx.Done()) err := <-errChan @@ -258,11 +274,11 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma return snapshotName, nil } -func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error { - ctx, cancel := context.WithTimeout(context.Background(), op.apiTimeout) +func (b *blockStore) DeleteSnapshot(snapshotID string) error { + ctx, cancel := context.WithTimeout(context.Background(), b.apiTimeout) defer cancel() - _, errChan := op.snaps.Delete(op.resourceGroup, snapshotID, ctx.Done()) + _, errChan := b.snaps.Delete(b.resourceGroup, snapshotID, ctx.Done()) err := <-errChan diff --git a/pkg/cloudprovider/azure/object_storage_adapter.go b/pkg/cloudprovider/azure/object_store.go similarity index 70% rename from pkg/cloudprovider/azure/object_storage_adapter.go rename to pkg/cloudprovider/azure/object_store.go index 8e6a5f147..24f24ca41 100644 --- a/pkg/cloudprovider/azure/object_storage_adapter.go +++ b/pkg/cloudprovider/azure/object_store.go @@ -27,31 +27,31 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) -// ref. https://github.com/Azure-Samples/storage-blob-go-getting-started/blob/master/storageExample.go - -type objectStorageAdapter struct { +type objectStore struct { blobClient *storage.BlobStorageClient } -var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{} +func NewObjectStore() cloudprovider.ObjectStore { + return &objectStore{} +} -func NewObjectStorageAdapter() (cloudprovider.ObjectStorageAdapter, error) { +func (o *objectStore) Init(config map[string]string) error { cfg := getConfig() storageClient, err := storage.NewBasicClient(cfg[azureStorageAccountIDKey], cfg[azureStorageKeyKey]) if err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } blobClient := storageClient.GetBlobService() - return &objectStorageAdapter{ - blobClient: &blobClient, - }, nil + o.blobClient = &blobClient + + return nil } -func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.Reader) error { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return err } @@ -64,8 +64,8 @@ func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.Rea return errors.WithStack(blob.CreateBlockBlobFromReader(body, nil)) } -func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return nil, err } @@ -83,8 +83,8 @@ func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadClo return res, nil } -func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return nil, err } @@ -108,8 +108,8 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri return ret, nil } -func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return nil, err } @@ -131,8 +131,8 @@ func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, er return ret, nil } -func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) DeleteObject(bucket string, key string) error { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return err } @@ -147,8 +147,8 @@ func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error { const sasURIReadPermission = "r" -func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { - container, err := getContainerReference(op.blobClient, bucket) +func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { + container, err := getContainerReference(o.blobClient, bucket) if err != nil { return "", err } diff --git a/pkg/cloudprovider/backup_service.go b/pkg/cloudprovider/backup_service.go index b5d75a722..b2dd8ae79 100644 --- a/pkg/cloudprovider/backup_service.go +++ b/pkg/cloudprovider/backup_service.go @@ -99,35 +99,35 @@ func getRestoreResultsKey(backup, restore string) string { } type backupService struct { - objectStorage ObjectStorageAdapter - decoder runtime.Decoder - logger *logrus.Logger + objectStore ObjectStore + decoder runtime.Decoder + logger *logrus.Logger } var _ BackupService = &backupService{} var _ BackupGetter = &backupService{} -// NewBackupService creates a backup service using the provided object storage adapter -func NewBackupService(objectStorage ObjectStorageAdapter, logger *logrus.Logger) BackupService { +// NewBackupService creates a backup service using the provided object store +func NewBackupService(objectStore ObjectStore, logger *logrus.Logger) BackupService { return &backupService{ - objectStorage: objectStorage, - decoder: scheme.Codecs.UniversalDecoder(api.SchemeGroupVersion), - logger: logger, + objectStore: objectStore, + decoder: scheme.Codecs.UniversalDecoder(api.SchemeGroupVersion), + logger: logger, } } func (br *backupService) UploadBackup(bucket, backupName string, metadata, backup, log io.Reader) error { // upload metadata file metadataKey := getMetadataKey(backupName) - if err := br.objectStorage.PutObject(bucket, metadataKey, metadata); err != nil { + if err := br.objectStore.PutObject(bucket, metadataKey, metadata); err != nil { // failure to upload metadata file is a hard-stop return err } // upload tar file - if err := br.objectStorage.PutObject(bucket, getBackupContentsKey(backupName), backup); err != nil { + if err := br.objectStore.PutObject(bucket, getBackupContentsKey(backupName), backup); err != nil { // try to delete the metadata file since the data upload failed - deleteErr := br.objectStorage.DeleteObject(bucket, metadataKey) + deleteErr := br.objectStore.DeleteObject(bucket, metadataKey) return kerrors.NewAggregate([]error{err, deleteErr}) } @@ -135,7 +135,7 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu // uploading log file is best-effort; if it fails, we log the error but call the overall upload a // success logKey := getBackupLogKey(backupName) - if err := br.objectStorage.PutObject(bucket, logKey, log); err != nil { + if err := br.objectStore.PutObject(bucket, logKey, log); err != nil { br.logger.WithError(err).WithFields(logrus.Fields{ "bucket": bucket, "key": logKey, @@ -146,11 +146,11 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu } func (br *backupService) DownloadBackup(bucket, backupName string) (io.ReadCloser, error) { - return br.objectStorage.GetObject(bucket, getBackupContentsKey(backupName)) + return br.objectStore.GetObject(bucket, getBackupContentsKey(backupName)) } func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) { - prefixes, err := br.objectStorage.ListCommonPrefixes(bucket, "/") + prefixes, err := br.objectStore.ListCommonPrefixes(bucket, "/") if err != nil { return nil, err } @@ -176,7 +176,7 @@ func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) { func (br *backupService) GetBackup(bucket, name string) (*api.Backup, error) { key := fmt.Sprintf(metadataFileFormatString, name) - res, err := br.objectStorage.GetObject(bucket, key) + res, err := br.objectStore.GetObject(bucket, key) if err != nil { return nil, err } @@ -201,7 +201,7 @@ func (br *backupService) GetBackup(bucket, name string) (*api.Backup, error) { } func (br *backupService) DeleteBackupDir(bucket, backupName string) error { - objects, err := br.objectStorage.ListObjects(bucket, backupName+"/") + objects, err := br.objectStore.ListObjects(bucket, backupName+"/") if err != nil { return err } @@ -212,7 +212,7 @@ func (br *backupService) DeleteBackupDir(bucket, backupName string) error { "bucket": bucket, "key": key, }).Debug("Trying to delete object") - if err := br.objectStorage.DeleteObject(bucket, key); err != nil { + if err := br.objectStore.DeleteObject(bucket, key); err != nil { errs = append(errs, err) } } @@ -223,15 +223,15 @@ func (br *backupService) DeleteBackupDir(bucket, backupName string) error { func (br *backupService) CreateSignedURL(target api.DownloadTarget, bucket string, ttl time.Duration) (string, error) { switch target.Kind { case api.DownloadTargetKindBackupContents: - return br.objectStorage.CreateSignedURL(bucket, getBackupContentsKey(target.Name), ttl) + return br.objectStore.CreateSignedURL(bucket, getBackupContentsKey(target.Name), ttl) case api.DownloadTargetKindBackupLog: - return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(target.Name), ttl) + return br.objectStore.CreateSignedURL(bucket, getBackupLogKey(target.Name), ttl) case api.DownloadTargetKindRestoreLog: backup := extractBackupName(target.Name) - return br.objectStorage.CreateSignedURL(bucket, getRestoreLogKey(backup, target.Name), ttl) + return br.objectStore.CreateSignedURL(bucket, getRestoreLogKey(backup, target.Name), ttl) case api.DownloadTargetKindRestoreResults: backup := extractBackupName(target.Name) - return br.objectStorage.CreateSignedURL(bucket, getRestoreResultsKey(backup, target.Name), ttl) + return br.objectStore.CreateSignedURL(bucket, getRestoreResultsKey(backup, target.Name), ttl) default: return "", errors.Errorf("unsupported download target kind %q", target.Kind) } @@ -248,12 +248,12 @@ func extractBackupName(s string) string { func (br *backupService) UploadRestoreLog(bucket, backup, restore string, log io.Reader) error { key := getRestoreLogKey(backup, restore) - return br.objectStorage.PutObject(bucket, key, log) + return br.objectStore.PutObject(bucket, key, log) } func (br *backupService) UploadRestoreResults(bucket, backup, restore string, results io.Reader) error { key := getRestoreResultsKey(backup, restore) - return br.objectStorage.PutObject(bucket, key, results) + return br.objectStore.PutObject(bucket, key, results) } // cachedBackupService wraps a real backup service with a cache for getting cloud backups. diff --git a/pkg/cloudprovider/backup_service_test.go b/pkg/cloudprovider/backup_service_test.go index 6fd857c67..f037d195f 100644 --- a/pkg/cloudprovider/backup_service_test.go +++ b/pkg/cloudprovider/backup_service_test.go @@ -82,7 +82,7 @@ func TestUploadBackup(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( - objStore = &testutil.ObjectStorageAdapter{} + objStore = &testutil.ObjectStore{} bucket = "test-bucket" backupName = "test-backup" logger, _ = testlogger.NewNullLogger() @@ -118,7 +118,7 @@ func TestUploadBackup(t *testing.T) { func TestDownloadBackup(t *testing.T) { var ( - o = &testutil.ObjectStorageAdapter{} + o = &testutil.ObjectStore{} bucket = "b" backup = "bak" logger, _ = testlogger.NewNullLogger() @@ -158,7 +158,7 @@ func TestDeleteBackup(t *testing.T) { bucket = "bucket" backup = "bak" objects = []string{"bak/ark-backup.json", "bak/bak.tar.gz", "bak/bak.log.gz"} - objStore = &testutil.ObjectStorageAdapter{} + objStore = &testutil.ObjectStore{} logger, _ = testlogger.NewNullLogger() ) @@ -230,7 +230,7 @@ func TestGetAllBackups(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( bucket = "bucket" - objStore = &testutil.ObjectStorageAdapter{} + objStore = &testutil.ObjectStore{} logger, _ = testlogger.NewNullLogger() ) @@ -327,7 +327,7 @@ func TestCreateSignedURL(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { var ( - objectStorage = &testutil.ObjectStorageAdapter{} + objectStorage = &testutil.ObjectStore{} logger, _ = testlogger.NewNullLogger() backupService = NewBackupService(objectStorage, logger) ) diff --git a/pkg/cloudprovider/gcp/block_storage_adapter.go b/pkg/cloudprovider/gcp/block_store.go similarity index 63% rename from pkg/cloudprovider/gcp/block_storage_adapter.go rename to pkg/cloudprovider/gcp/block_store.go index b01631837..f012a82f0 100644 --- a/pkg/cloudprovider/gcp/block_storage_adapter.go +++ b/pkg/cloudprovider/gcp/block_store.go @@ -31,46 +31,52 @@ import ( "github.com/heptio/ark/pkg/cloudprovider" ) -type blockStorageAdapter struct { +const projectKey = "project" + +type blockStore struct { gce *compute.Service project string } -var _ cloudprovider.BlockStorageAdapter = &blockStorageAdapter{} +func NewBlockStore() cloudprovider.BlockStore { + return &blockStore{} +} + +func (b *blockStore) Init(config map[string]string) error { + project := config[projectKey] -func NewBlockStorageAdapter(project string) (cloudprovider.BlockStorageAdapter, error) { if project == "" { - return nil, errors.New("missing project in gcp configuration in config file") + return errors.Errorf("missing %s in gcp configuration", projectKey) } client, err := google.DefaultClient(oauth2.NoContext, compute.ComputeScope) if err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } gce, err := compute.New(client) if err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } // validate project res, err := gce.Projects.Get(project).Do() if err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } if res == nil { - return nil, errors.Errorf("error getting project %q", project) + return errors.Errorf("error getting project %q", project) } - return &blockStorageAdapter{ - gce: gce, - project: project, - }, nil + b.gce = gce + b.project = project + + return nil } -func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { - res, err := op.gce.Snapshots.Get(op.project, snapshotID).Do() +func (b *blockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) { + res, err := b.gce.Snapshots.Get(b.project, snapshotID).Do() if err != nil { return "", errors.WithStack(err) } @@ -81,15 +87,15 @@ func (op *blockStorageAdapter) CreateVolumeFromSnapshot(snapshotID, volumeType, Type: volumeType, } - if _, err = op.gce.Disks.Insert(op.project, volumeAZ, disk).Do(); err != nil { + if _, err = b.gce.Disks.Insert(b.project, volumeAZ, disk).Do(); err != nil { return "", errors.WithStack(err) } return disk.Name, nil } -func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { - res, err := op.gce.Disks.Get(op.project, volumeAZ, volumeID).Do() +func (b *blockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { + res, err := b.gce.Disks.Get(b.project, volumeAZ, volumeID).Do() if err != nil { return "", nil, errors.WithStack(err) } @@ -97,8 +103,8 @@ func (op *blockStorageAdapter) GetVolumeInfo(volumeID, volumeAZ string) (string, return res.Type, nil, nil } -func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { - disk, err := op.gce.Disks.Get(op.project, volumeAZ, volumeID).Do() +func (b *blockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { + disk, err := b.gce.Disks.Get(b.project, volumeAZ, volumeID).Do() if err != nil { return false, errors.WithStack(err) } @@ -107,7 +113,7 @@ func (op *blockStorageAdapter) IsVolumeReady(volumeID, volumeAZ string) (ready b return disk.Status == "READY", nil } -func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]string, error) { +func (b *blockStore) ListSnapshots(tagFilters map[string]string) ([]string, error) { useParentheses := len(tagFilters) > 1 subFilters := make([]string, 0, len(tagFilters)) @@ -121,7 +127,7 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st filter := strings.Join(subFilters, " ") - res, err := op.gce.Snapshots.List(op.project).Filter(filter).Do() + res, err := b.gce.Snapshots.List(b.project).Filter(filter).Do() if err != nil { return nil, errors.WithStack(err) } @@ -134,7 +140,7 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st return ret, nil } -func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { +func (b *blockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { // snapshot names must adhere to RFC1035 and be 1-63 characters // long var snapshotName string @@ -150,7 +156,7 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma Name: snapshotName, } - _, err := op.gce.Disks.CreateSnapshot(op.project, volumeAZ, volumeID, &gceSnap).Do() + _, err := b.gce.Disks.CreateSnapshot(b.project, volumeAZ, volumeID, &gceSnap).Do() if err != nil { return "", errors.WithStack(err) } @@ -158,7 +164,7 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma // the snapshot is not immediately available after creation for putting labels // on it. poll for a period of time. if pollErr := wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) { - if res, err := op.gce.Snapshots.Get(op.project, gceSnap.Name).Do(); err == nil { + if res, err := b.gce.Snapshots.Get(b.project, gceSnap.Name).Do(); err == nil { gceSnap = *res return true, nil } @@ -172,7 +178,7 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma LabelFingerprint: gceSnap.LabelFingerprint, } - _, err = op.gce.Snapshots.SetLabels(op.project, gceSnap.Name, labels).Do() + _, err = b.gce.Snapshots.SetLabels(b.project, gceSnap.Name, labels).Do() if err != nil { return "", errors.WithStack(err) } @@ -180,8 +186,8 @@ func (op *blockStorageAdapter) CreateSnapshot(volumeID, volumeAZ string, tags ma return gceSnap.Name, nil } -func (op *blockStorageAdapter) DeleteSnapshot(snapshotID string) error { - _, err := op.gce.Snapshots.Delete(op.project, snapshotID).Do() +func (b *blockStore) DeleteSnapshot(snapshotID string) error { + _, err := b.gce.Snapshots.Delete(b.project, snapshotID).Do() return errors.WithStack(err) } diff --git a/pkg/cloudprovider/gcp/object_storage_adapter.go b/pkg/cloudprovider/gcp/object_storage_adapter.go deleted file mode 100644 index 89a1e64ab..000000000 --- a/pkg/cloudprovider/gcp/object_storage_adapter.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2017 Heptio Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gcp - -import ( - "io" - "strings" - "time" - - "github.com/pkg/errors" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - // TODO switch to using newstorage - newstorage "cloud.google.com/go/storage" - storage "google.golang.org/api/storage/v1" - - "github.com/heptio/ark/pkg/cloudprovider" -) - -type objectStorageAdapter struct { - gcs *storage.Service - googleAccessID string - privateKey []byte -} - -var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{} - -func NewObjectStorageAdapter(googleAccessID string, privateKey []byte) (cloudprovider.ObjectStorageAdapter, error) { - client, err := google.DefaultClient(oauth2.NoContext, storage.DevstorageReadWriteScope) - if err != nil { - return nil, errors.WithStack(err) - } - - gcs, err := storage.New(client) - if err != nil { - return nil, errors.WithStack(err) - } - - return &objectStorageAdapter{ - gcs: gcs, - googleAccessID: googleAccessID, - privateKey: privateKey, - }, nil -} - -func (op *objectStorageAdapter) PutObject(bucket string, key string, body io.Reader) error { - obj := &storage.Object{ - Name: key, - } - - _, err := op.gcs.Objects.Insert(bucket, obj).Media(body).Do() - - return errors.WithStack(err) -} - -func (op *objectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) { - res, err := op.gcs.Objects.Get(bucket, key).Download() - if err != nil { - return nil, errors.WithStack(err) - } - - return res.Body, nil -} - -func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { - res, err := op.gcs.Objects.List(bucket).Delimiter(delimiter).Do() - if err != nil { - return nil, errors.WithStack(err) - } - - // GCP returns prefixes inclusive of the last delimiter. We need to strip - // it. - ret := make([]string, 0, len(res.Prefixes)) - for _, prefix := range res.Prefixes { - ret = append(ret, prefix[0:strings.LastIndex(prefix, delimiter)]) - } - - return ret, nil -} - -func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) { - res, err := op.gcs.Objects.List(bucket).Prefix(prefix).Do() - if err != nil { - return nil, errors.WithStack(err) - } - - ret := make([]string, 0, len(res.Items)) - for _, item := range res.Items { - ret = append(ret, item.Name) - } - - return ret, nil -} - -func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error { - return errors.Wrapf(op.gcs.Objects.Delete(bucket, key).Do(), "error deleting object %s", key) -} - -func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { - if op.googleAccessID == "" { - return "", errors.New("unable to create a pre-signed URL - make sure GOOGLE_APPLICATION_CREDENTIALS points to a valid GCE service account file (missing email address)") - } - if len(op.privateKey) == 0 { - return "", errors.New("unable to create a pre-signed URL - make sure GOOGLE_APPLICATION_CREDENTIALS points to a valid GCE service account file (missing private key)") - } - - return newstorage.SignedURL(bucket, key, &newstorage.SignedURLOptions{ - GoogleAccessID: op.googleAccessID, - PrivateKey: op.privateKey, - Method: "GET", - Expires: time.Now().Add(ttl), - }) -} diff --git a/pkg/cloudprovider/gcp/object_store.go b/pkg/cloudprovider/gcp/object_store.go new file mode 100644 index 000000000..34fac5372 --- /dev/null +++ b/pkg/cloudprovider/gcp/object_store.go @@ -0,0 +1,147 @@ +/* +Copyright 2017 Heptio Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + "io" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + // TODO switch to using newstorage + newstorage "cloud.google.com/go/storage" + storage "google.golang.org/api/storage/v1" + + "github.com/heptio/ark/pkg/cloudprovider" +) + +const credentialsEnvVar = "GOOGLE_APPLICATION_CREDENTIALS" + +type objectStore struct { + gcs *storage.Service + googleAccessID string + privateKey []byte +} + +func NewObjectStore() cloudprovider.ObjectStore { + return &objectStore{} +} + +func (o *objectStore) Init(config map[string]string) error { + credentialsFile := os.Getenv(credentialsEnvVar) + if credentialsFile == "" { + return errors.Errorf("%s is undefined", credentialsEnvVar) + } + + // Get the email and private key from the credentials file so we can pre-sign download URLs + creds, err := ioutil.ReadFile(credentialsFile) + if err != nil { + return errors.WithStack(err) + } + jwtConfig, err := google.JWTConfigFromJSON(creds) + if err != nil { + return errors.WithStack(err) + } + if jwtConfig.Email == "" { + return errors.Errorf("credentials file pointed to by %s does not contain an email", credentialsEnvVar) + } + if len(jwtConfig.PrivateKey) == 0 { + return errors.Errorf("credentials file pointed to by %s does not contain a private key", credentialsEnvVar) + } + + client, err := google.DefaultClient(oauth2.NoContext, storage.DevstorageReadWriteScope) + if err != nil { + return errors.WithStack(err) + } + + gcs, err := storage.New(client) + if err != nil { + return errors.WithStack(err) + } + + o.gcs = gcs + o.googleAccessID = jwtConfig.Email + o.privateKey = jwtConfig.PrivateKey + + return nil +} + +func (o *objectStore) PutObject(bucket string, key string, body io.Reader) error { + obj := &storage.Object{ + Name: key, + } + + _, err := o.gcs.Objects.Insert(bucket, obj).Media(body).Do() + + return errors.WithStack(err) +} + +func (o *objectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { + res, err := o.gcs.Objects.Get(bucket, key).Download() + if err != nil { + return nil, errors.WithStack(err) + } + + return res.Body, nil +} + +func (o *objectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { + res, err := o.gcs.Objects.List(bucket).Delimiter(delimiter).Do() + if err != nil { + return nil, errors.WithStack(err) + } + + // GCP returns prefixes inclusive of the last delimiter. We need to strip + // it. + ret := make([]string, 0, len(res.Prefixes)) + for _, prefix := range res.Prefixes { + ret = append(ret, prefix[0:strings.LastIndex(prefix, delimiter)]) + } + + return ret, nil +} + +func (o *objectStore) ListObjects(bucket, prefix string) ([]string, error) { + res, err := o.gcs.Objects.List(bucket).Prefix(prefix).Do() + if err != nil { + return nil, errors.WithStack(err) + } + + ret := make([]string, 0, len(res.Items)) + for _, item := range res.Items { + ret = append(ret, item.Name) + } + + return ret, nil +} + +func (o *objectStore) DeleteObject(bucket string, key string) error { + return errors.Wrapf(o.gcs.Objects.Delete(bucket, key).Do(), "error deleting object %s", key) +} + +func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { + return newstorage.SignedURL(bucket, key, &newstorage.SignedURLOptions{ + GoogleAccessID: o.googleAccessID, + PrivateKey: o.privateKey, + Method: "GET", + Expires: time.Now().Add(ttl), + }) +} diff --git a/pkg/cloudprovider/snapshot_service.go b/pkg/cloudprovider/snapshot_service.go index 076898f1d..8d1ba9a1d 100644 --- a/pkg/cloudprovider/snapshot_service.go +++ b/pkg/cloudprovider/snapshot_service.go @@ -56,20 +56,20 @@ const ( ) type snapshotService struct { - blockStorage BlockStorageAdapter + blockStore BlockStore } var _ SnapshotService = &snapshotService{} -// NewSnapshotService creates a snapshot service using the provided block storage adapter -func NewSnapshotService(blockStorage BlockStorageAdapter) SnapshotService { +// NewSnapshotService creates a snapshot service using the provided block store +func NewSnapshotService(blockStore BlockStore) SnapshotService { return &snapshotService{ - blockStorage: blockStorage, + blockStore: blockStore, } } func (sr *snapshotService) CreateVolumeFromSnapshot(snapshotID string, volumeType string, volumeAZ string, iops *int64) (string, error) { - volumeID, err := sr.blockStorage.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) + volumeID, err := sr.blockStore.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) if err != nil { return "", err } @@ -85,7 +85,7 @@ func (sr *snapshotService) CreateVolumeFromSnapshot(snapshotID string, volumeTyp case <-timeout.C: return "", errors.Errorf("timeout reached waiting for volume %v to be ready", volumeID) case <-ticker.C: - if ready, err := sr.blockStorage.IsVolumeReady(volumeID, volumeAZ); err == nil && ready { + if ready, err := sr.blockStore.IsVolumeReady(volumeID, volumeAZ); err == nil && ready { return volumeID, nil } } @@ -97,7 +97,7 @@ func (sr *snapshotService) GetAllSnapshots() ([]string, error) { snapshotTagKey: snapshotTagVal, } - res, err := sr.blockStorage.ListSnapshots(tags) + res, err := sr.blockStore.ListSnapshots(tags) if err != nil { return nil, err } @@ -110,13 +110,13 @@ func (sr *snapshotService) CreateSnapshot(volumeID, volumeAZ string) (string, er snapshotTagKey: snapshotTagVal, } - return sr.blockStorage.CreateSnapshot(volumeID, volumeAZ, tags) + return sr.blockStore.CreateSnapshot(volumeID, volumeAZ, tags) } func (sr *snapshotService) DeleteSnapshot(snapshotID string) error { - return sr.blockStorage.DeleteSnapshot(snapshotID) + return sr.blockStore.DeleteSnapshot(snapshotID) } func (sr *snapshotService) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { - return sr.blockStorage.GetVolumeInfo(volumeID, volumeAZ) + return sr.blockStore.GetVolumeInfo(volumeID, volumeAZ) } diff --git a/pkg/cloudprovider/storage_interfaces.go b/pkg/cloudprovider/storage_interfaces.go index eec4e1b0a..7f4a7bb38 100644 --- a/pkg/cloudprovider/storage_interfaces.go +++ b/pkg/cloudprovider/storage_interfaces.go @@ -21,9 +21,14 @@ import ( "time" ) -// ObjectStorageAdapter exposes basic object-storage operations required +// ObjectStore exposes basic object-storage operations required // by Ark. -type ObjectStorageAdapter interface { +type ObjectStore interface { + // Init prepares the ObjectStore for usage using the provided map of + // configuration key-value pairs. It returns an error if the ObjectStore + // cannot be initialized from the provided config. + Init(config map[string]string) error + // PutObject creates a new object using the data in body within the specified // object storage bucket with the given key. PutObject(bucket string, key string, body io.Reader) error @@ -48,9 +53,14 @@ type ObjectStorageAdapter interface { CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) } -// BlockStorageAdapter exposes basic block-storage operations required +// BlockStore exposes basic block-storage operations required // by Ark. -type BlockStorageAdapter interface { +type BlockStore interface { + // Init prepares the BlockStore for usage using the provided map of + // configuration key-value pairs. It returns an error if the BlockStore + // cannot be initialized from the provided config. + Init(config map[string]string) error + // CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot, // and with the specified type and IOPS (if using provisioned IOPS). CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (volumeID string, err error) diff --git a/pkg/cmd/ark/ark.go b/pkg/cmd/ark/ark.go index 0dffd21be..7c3137c65 100644 --- a/pkg/cmd/ark/ark.go +++ b/pkg/cmd/ark/ark.go @@ -29,6 +29,7 @@ import ( "github.com/heptio/ark/pkg/cmd/cli/restore" "github.com/heptio/ark/pkg/cmd/cli/schedule" "github.com/heptio/ark/pkg/cmd/server" + "github.com/heptio/ark/pkg/cmd/server/plugin" "github.com/heptio/ark/pkg/cmd/version" ) @@ -57,6 +58,7 @@ operations can also be performed as 'ark backup get' and 'ark schedule create'.` get.NewCommand(f), describe.NewCommand(f), create.NewCommand(f), + plugin.NewCommand(), ) // add the glog flags diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go new file mode 100644 index 000000000..ee7862c21 --- /dev/null +++ b/pkg/cmd/server/plugin/plugin.go @@ -0,0 +1,86 @@ +/* +Copyright 2017 Heptio Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + plugin "github.com/hashicorp/go-plugin" + "github.com/spf13/cobra" + + "github.com/heptio/ark/pkg/cloudprovider" + "github.com/heptio/ark/pkg/cloudprovider/aws" + "github.com/heptio/ark/pkg/cloudprovider/azure" + "github.com/heptio/ark/pkg/cloudprovider/gcp" + arkplugin "github.com/heptio/ark/pkg/plugin" +) + +func NewCommand() *cobra.Command { + logger := arkplugin.NewPluginLogger() + + objectStores := map[string]cloudprovider.ObjectStore{ + "aws": aws.NewObjectStore(), + "gcp": gcp.NewObjectStore(), + "azure": azure.NewObjectStore(), + } + + blockStores := map[string]cloudprovider.BlockStore{ + "aws": aws.NewBlockStore(), + "gcp": gcp.NewBlockStore(), + "azure": azure.NewBlockStore(), + } + + c := &cobra.Command{ + Use: "plugin [KIND] [NAME]", + Hidden: true, + Short: "INTERNAL COMMAND ONLY - not intended to be run directly by users", + Run: func(c *cobra.Command, args []string) { + if len(args) != 2 { + logger.Fatal("You must specify exactly two arguments, the plugin kind and the plugin name") + } + + kind := args[0] + name := args[1] + + logger.Debugf("Running plugin command for kind=%s, name=%s", kind, name) + + switch kind { + case "cloudprovider": + objectStore, found := objectStores[name] + if !found { + logger.Fatalf("Unrecognized plugin name %q", name) + } + + blockStore, found := blockStores[name] + if !found { + logger.Fatalf("Unrecognized plugin name %q", name) + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: arkplugin.Handshake, + Plugins: map[string]plugin.Plugin{ + string(arkplugin.PluginKindObjectStore): arkplugin.NewObjectStorePlugin(objectStore), + string(arkplugin.PluginKindBlockStore): arkplugin.NewBlockStorePlugin(blockStore), + }, + GRPCServer: plugin.DefaultGRPCServer, + }) + default: + logger.Fatalf("Unsupported plugin kind %q", kind) + } + }, + } + + return c +} diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 98a533c99..f0235ceaf 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -19,8 +19,6 @@ package server import ( "context" "fmt" - "io/ioutil" - "os" "reflect" "sort" "strings" @@ -30,7 +28,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/oauth2/google" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -47,9 +44,6 @@ import ( "github.com/heptio/ark/pkg/backup" "github.com/heptio/ark/pkg/client" "github.com/heptio/ark/pkg/cloudprovider" - arkaws "github.com/heptio/ark/pkg/cloudprovider/aws" - "github.com/heptio/ark/pkg/cloudprovider/azure" - "github.com/heptio/ark/pkg/cloudprovider/gcp" "github.com/heptio/ark/pkg/cmd" "github.com/heptio/ark/pkg/cmd/util/flag" "github.com/heptio/ark/pkg/controller" @@ -57,6 +51,7 @@ import ( clientset "github.com/heptio/ark/pkg/generated/clientset/versioned" arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1" informers "github.com/heptio/ark/pkg/generated/informers/externalversions" + "github.com/heptio/ark/pkg/plugin" "github.com/heptio/ark/pkg/restore" "github.com/heptio/ark/pkg/restore/restorers" "github.com/heptio/ark/pkg/util/kube" @@ -146,6 +141,7 @@ type server struct { ctx context.Context cancelFunc context.CancelFunc logger *logrus.Logger + pluginManager plugin.Manager } func newServer(kubeconfig, baseName string, logger *logrus.Logger) (*server, error) { @@ -173,9 +169,10 @@ func newServer(kubeconfig, baseName string, logger *logrus.Logger) (*server, err discoveryClient: arkClient.Discovery(), clientPool: dynamic.NewDynamicClientPool(clientConfig), sharedInformerFactory: informers.NewSharedInformerFactory(arkClient, 0), - ctx: ctx, - cancelFunc: cancelFunc, - logger: logger, + ctx: ctx, + cancelFunc: cancelFunc, + logger: logger, + pluginManager: plugin.NewManager(logger, logger.Level), } return s, nil @@ -325,12 +322,12 @@ func (s *server) watchConfig(config *api.Config) { func (s *server) initBackupService(config *api.Config) error { s.logger.Info("Configuring cloud provider for backup service") - objectStorage, err := getObjectStorageProvider(config.BackupStorageProvider.CloudProviderConfig, "backupStorageProvider", s.logger) + objectStore, err := getObjectStore(config.BackupStorageProvider.CloudProviderConfig, s.pluginManager) if err != nil { return err } - s.backupService = cloudprovider.NewBackupService(objectStorage, s.logger) + s.backupService = cloudprovider.NewBackupService(objectStore, s.logger) return nil } @@ -341,112 +338,46 @@ func (s *server) initSnapshotService(config *api.Config) error { } s.logger.Info("Configuring cloud provider for snapshot service") - blockStorage, err := getBlockStorageProvider(*config.PersistentVolumeProvider, "persistentVolumeProvider") + blockStore, err := getBlockStore(*config.PersistentVolumeProvider, s.pluginManager) if err != nil { return err } - s.snapshotService = cloudprovider.NewSnapshotService(blockStorage) + s.snapshotService = cloudprovider.NewSnapshotService(blockStore) return nil } -func hasOneCloudProvider(cloudConfig api.CloudProviderConfig) bool { - found := false - - if cloudConfig.AWS != nil { - found = true - } - - if cloudConfig.GCP != nil { - if found { - return false - } - found = true - } - - if cloudConfig.Azure != nil { - if found { - return false - } - found = true - } - - return found -} - -func getObjectStorageProvider(cloudConfig api.CloudProviderConfig, field string, logger *logrus.Logger) (cloudprovider.ObjectStorageAdapter, error) { - var ( - objectStorage cloudprovider.ObjectStorageAdapter - err error - ) - - if !hasOneCloudProvider(cloudConfig) { - return nil, errors.Errorf("you must specify exactly one of aws, gcp, or azure for %s", field) - } - - switch { - case cloudConfig.AWS != nil: - objectStorage, err = arkaws.NewObjectStorageAdapter( - cloudConfig.AWS.Region, - cloudConfig.AWS.S3Url, - cloudConfig.AWS.KMSKeyID, - cloudConfig.AWS.S3ForcePathStyle) - case cloudConfig.GCP != nil: - var email string - var privateKey []byte - - credentialsFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") - if credentialsFile != "" { - // Get the email and private key from the credentials file so we can pre-sign download URLs - creds, err := ioutil.ReadFile(credentialsFile) - if err != nil { - return nil, errors.WithStack(err) - } - jwtConfig, err := google.JWTConfigFromJSON(creds) - if err != nil { - return nil, errors.WithStack(err) - } - email = jwtConfig.Email - privateKey = jwtConfig.PrivateKey - } else { - logger.Warning("GOOGLE_APPLICATION_CREDENTIALS is undefined; some features such as downloading log files will not work") - } - - objectStorage, err = gcp.NewObjectStorageAdapter(email, privateKey) - case cloudConfig.Azure != nil: - objectStorage, err = azure.NewObjectStorageAdapter() +func getObjectStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) (cloudprovider.ObjectStore, error) { + if cloudConfig.Name == "" { + return nil, errors.New("object storage provider name must not be empty") } + objectStore, err := manager.GetObjectStore(cloudConfig.Name) if err != nil { return nil, err } - return objectStorage, nil + if err := objectStore.Init(cloudConfig.Config); err != nil { + return nil, err + } + + return objectStore, nil } -func getBlockStorageProvider(cloudConfig api.CloudProviderConfig, field string) (cloudprovider.BlockStorageAdapter, error) { - var ( - blockStorage cloudprovider.BlockStorageAdapter - err error - ) - - if !hasOneCloudProvider(cloudConfig) { - return nil, errors.Errorf("you must specify exactly one of aws, gcp, or azure for %s", field) - } - - switch { - case cloudConfig.AWS != nil: - blockStorage, err = arkaws.NewBlockStorageAdapter(cloudConfig.AWS.Region) - case cloudConfig.GCP != nil: - blockStorage, err = gcp.NewBlockStorageAdapter(cloudConfig.GCP.Project) - case cloudConfig.Azure != nil: - blockStorage, err = azure.NewBlockStorageAdapter(cloudConfig.Azure.Location, cloudConfig.Azure.APITimeout.Duration) +func getBlockStore(cloudConfig api.CloudProviderConfig, manager plugin.Manager) (cloudprovider.BlockStore, error) { + if cloudConfig.Name == "" { + return nil, errors.New("block storage provider name must not be empty") } + blockStore, err := manager.GetBlockStore(cloudConfig.Name) if err != nil { return nil, err } - return blockStorage, nil + if err := blockStore.Init(cloudConfig.Config); err != nil { + return nil, err + } + + return blockStore, nil } func durationMin(a, b time.Duration) time.Duration { diff --git a/pkg/plugin/block_store.go b/pkg/plugin/block_store.go new file mode 100644 index 000000000..401f8c264 --- /dev/null +++ b/pkg/plugin/block_store.go @@ -0,0 +1,247 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "github.com/hashicorp/go-plugin" + "golang.org/x/net/context" + "google.golang.org/grpc" + + "github.com/heptio/ark/pkg/cloudprovider" + proto "github.com/heptio/ark/pkg/plugin/generated" +) + +// BlockStorePlugin is an implementation of go-plugin's Plugin +// interface with support for gRPC for the cloudprovider/BlockStore +// interface. +type BlockStorePlugin struct { + plugin.NetRPCUnsupportedPlugin + + impl cloudprovider.BlockStore +} + +// NewBlockStorePlugin constructs a BlockStorePlugin. +func NewBlockStorePlugin(blockStore cloudprovider.BlockStore) *BlockStorePlugin { + return &BlockStorePlugin{ + impl: blockStore, + } +} + +// GRPCServer registers a BlockStore gRPC server. +func (p *BlockStorePlugin) GRPCServer(s *grpc.Server) error { + proto.RegisterBlockStoreServer(s, &BlockStoreGRPCServer{impl: p.impl}) + return nil +} + +// GRPCClient returns a BlockStore gRPC client. +func (p *BlockStorePlugin) GRPCClient(c *grpc.ClientConn) (interface{}, error) { + return &BlockStoreGRPCClient{grpcClient: proto.NewBlockStoreClient(c)}, nil +} + +// BlockStoreGRPCClient implements the cloudprovider.BlockStore interface and uses a +// gRPC client to make calls to the plugin server. +type BlockStoreGRPCClient struct { + grpcClient proto.BlockStoreClient +} + +// Init prepares the BlockStore for usage using the provided map of +// configuration key-value pairs. It returns an error if the BlockStore +// cannot be initialized from the provided config. +func (c *BlockStoreGRPCClient) Init(config map[string]string) error { + _, err := c.grpcClient.Init(context.Background(), &proto.InitRequest{Config: config}) + + return err +} + +// CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot, +// and with the specified type and IOPS (if using provisioned IOPS). +func (c *BlockStoreGRPCClient) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { + req := &proto.CreateVolumeRequest{ + SnapshotID: snapshotID, + VolumeType: volumeType, + VolumeAZ: volumeAZ, + } + + if iops == nil { + req.Iops = 0 + } else { + req.Iops = *iops + } + + res, err := c.grpcClient.CreateVolumeFromSnapshot(context.Background(), req) + if err != nil { + return "", err + } + + return res.VolumeID, nil +} + +// GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block +// volume. +func (c *BlockStoreGRPCClient) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { + res, err := c.grpcClient.GetVolumeInfo(context.Background(), &proto.GetVolumeInfoRequest{VolumeID: volumeID, VolumeAZ: volumeAZ}) + if err != nil { + return "", nil, err + } + + var iops *int64 + if res.Iops != 0 { + iops = &res.Iops + } + + return res.VolumeType, iops, nil +} + +// IsVolumeReady returns whether the specified volume is ready to be used. +func (c *BlockStoreGRPCClient) IsVolumeReady(volumeID, volumeAZ string) (bool, error) { + res, err := c.grpcClient.IsVolumeReady(context.Background(), &proto.IsVolumeReadyRequest{VolumeID: volumeID, VolumeAZ: volumeAZ}) + if err != nil { + return false, err + } + + return res.Ready, nil +} + +// ListSnapshots returns a list of all snapshots matching the specified set of tag key/values. +func (c *BlockStoreGRPCClient) ListSnapshots(tagFilters map[string]string) ([]string, error) { + res, err := c.grpcClient.ListSnapshots(context.Background(), &proto.ListSnapshotsRequest{TagFilters: tagFilters}) + if err != nil { + return nil, err + } + + return res.SnapshotIDs, nil +} + +// CreateSnapshot creates a snapshot of the specified block volume, and applies the provided +// set of tags to the snapshot. +func (c *BlockStoreGRPCClient) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { + req := &proto.CreateSnapshotRequest{ + VolumeID: volumeID, + VolumeAZ: volumeAZ, + Tags: tags, + } + + res, err := c.grpcClient.CreateSnapshot(context.Background(), req) + if err != nil { + return "", err + } + + return res.SnapshotID, nil +} + +// DeleteSnapshot deletes the specified volume snapshot. +func (c *BlockStoreGRPCClient) DeleteSnapshot(snapshotID string) error { + _, err := c.grpcClient.DeleteSnapshot(context.Background(), &proto.DeleteSnapshotRequest{SnapshotID: snapshotID}) + + return err +} + +// BlockStoreGRPCServer implements the proto-generated BlockStoreServer interface, and accepts +// gRPC calls and forwards them to an implementation of the pluggable interface. +type BlockStoreGRPCServer struct { + impl cloudprovider.BlockStore +} + +// Init prepares the BlockStore for usage using the provided map of +// configuration key-value pairs. It returns an error if the BlockStore +// cannot be initialized from the provided config. +func (s *BlockStoreGRPCServer) Init(ctx context.Context, req *proto.InitRequest) (*proto.Empty, error) { + if err := s.impl.Init(req.Config); err != nil { + return nil, err + } + + return &proto.Empty{}, nil +} + +// CreateVolumeFromSnapshot creates a new block volume, initialized from the provided snapshot, +// and with the specified type and IOPS (if using provisioned IOPS). +func (s *BlockStoreGRPCServer) CreateVolumeFromSnapshot(ctx context.Context, req *proto.CreateVolumeRequest) (*proto.CreateVolumeResponse, error) { + snapshotID := req.SnapshotID + volumeType := req.VolumeType + volumeAZ := req.VolumeAZ + var iops *int64 + + if req.Iops != 0 { + iops = &req.Iops + } + + volumeID, err := s.impl.CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ, iops) + if err != nil { + return nil, err + } + + return &proto.CreateVolumeResponse{VolumeID: volumeID}, nil +} + +// GetVolumeInfo returns the type and IOPS (if using provisioned IOPS) for a specified block +// volume. +func (s *BlockStoreGRPCServer) GetVolumeInfo(ctx context.Context, req *proto.GetVolumeInfoRequest) (*proto.GetVolumeInfoResponse, error) { + volumeType, iops, err := s.impl.GetVolumeInfo(req.VolumeID, req.VolumeAZ) + if err != nil { + return nil, err + } + + res := &proto.GetVolumeInfoResponse{ + VolumeType: volumeType, + } + + if iops != nil { + res.Iops = *iops + } + + return res, nil +} + +// IsVolumeReady returns whether the specified volume is ready to be used. +func (s *BlockStoreGRPCServer) IsVolumeReady(ctx context.Context, req *proto.IsVolumeReadyRequest) (*proto.IsVolumeReadyResponse, error) { + ready, err := s.impl.IsVolumeReady(req.VolumeID, req.VolumeAZ) + if err != nil { + return nil, err + } + + return &proto.IsVolumeReadyResponse{Ready: ready}, nil +} + +// ListSnapshots returns a list of all snapshots matching the specified set of tag key/values. +func (s *BlockStoreGRPCServer) ListSnapshots(ctx context.Context, req *proto.ListSnapshotsRequest) (*proto.ListSnapshotsResponse, error) { + snapshotIDs, err := s.impl.ListSnapshots(req.TagFilters) + if err != nil { + return nil, err + } + + return &proto.ListSnapshotsResponse{SnapshotIDs: snapshotIDs}, nil +} + +// CreateSnapshot creates a snapshot of the specified block volume, and applies the provided +// set of tags to the snapshot. +func (s *BlockStoreGRPCServer) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*proto.CreateSnapshotResponse, error) { + snapshotID, err := s.impl.CreateSnapshot(req.VolumeID, req.VolumeAZ, req.Tags) + if err != nil { + return nil, err + } + + return &proto.CreateSnapshotResponse{SnapshotID: snapshotID}, nil +} + +// DeleteSnapshot deletes the specified volume snapshot. +func (s *BlockStoreGRPCServer) DeleteSnapshot(ctx context.Context, req *proto.DeleteSnapshotRequest) (*proto.Empty, error) { + if err := s.impl.DeleteSnapshot(req.SnapshotID); err != nil { + return nil, err + } + + return &proto.Empty{}, nil +} diff --git a/pkg/plugin/generated/BlockStore.pb.go b/pkg/plugin/generated/BlockStore.pb.go new file mode 100644 index 000000000..aad719fab --- /dev/null +++ b/pkg/plugin/generated/BlockStore.pb.go @@ -0,0 +1,621 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: BlockStore.proto + +/* +Package generated is a generated protocol buffer package. + +It is generated from these files: + BlockStore.proto + ObjectStore.proto + Shared.proto + +It has these top-level messages: + CreateVolumeRequest + CreateVolumeResponse + GetVolumeInfoRequest + GetVolumeInfoResponse + IsVolumeReadyRequest + IsVolumeReadyResponse + ListSnapshotsRequest + ListSnapshotsResponse + CreateSnapshotRequest + CreateSnapshotResponse + DeleteSnapshotRequest + PutObjectRequest + GetObjectRequest + Bytes + ListCommonPrefixesRequest + ListCommonPrefixesResponse + ListObjectsRequest + ListObjectsResponse + DeleteObjectRequest + CreateSignedURLRequest + CreateSignedURLResponse + Empty + InitRequest +*/ +package generated + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type CreateVolumeRequest struct { + SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID" json:"snapshotID,omitempty"` + VolumeType string `protobuf:"bytes,2,opt,name=volumeType" json:"volumeType,omitempty"` + VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ" json:"volumeAZ,omitempty"` + Iops int64 `protobuf:"varint,4,opt,name=iops" json:"iops,omitempty"` +} + +func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } +func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeRequest) ProtoMessage() {} +func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *CreateVolumeRequest) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func (m *CreateVolumeRequest) GetVolumeType() string { + if m != nil { + return m.VolumeType + } + return "" +} + +func (m *CreateVolumeRequest) GetVolumeAZ() string { + if m != nil { + return m.VolumeAZ + } + return "" +} + +func (m *CreateVolumeRequest) GetIops() int64 { + if m != nil { + return m.Iops + } + return 0 +} + +type CreateVolumeResponse struct { + VolumeID string `protobuf:"bytes,1,opt,name=volumeID" json:"volumeID,omitempty"` +} + +func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } +func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeResponse) ProtoMessage() {} +func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *CreateVolumeResponse) GetVolumeID() string { + if m != nil { + return m.VolumeID + } + return "" +} + +type GetVolumeInfoRequest struct { + VolumeID string `protobuf:"bytes,1,opt,name=volumeID" json:"volumeID,omitempty"` + VolumeAZ string `protobuf:"bytes,2,opt,name=volumeAZ" json:"volumeAZ,omitempty"` +} + +func (m *GetVolumeInfoRequest) Reset() { *m = GetVolumeInfoRequest{} } +func (m *GetVolumeInfoRequest) String() string { return proto.CompactTextString(m) } +func (*GetVolumeInfoRequest) ProtoMessage() {} +func (*GetVolumeInfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *GetVolumeInfoRequest) GetVolumeID() string { + if m != nil { + return m.VolumeID + } + return "" +} + +func (m *GetVolumeInfoRequest) GetVolumeAZ() string { + if m != nil { + return m.VolumeAZ + } + return "" +} + +type GetVolumeInfoResponse struct { + VolumeType string `protobuf:"bytes,1,opt,name=volumeType" json:"volumeType,omitempty"` + Iops int64 `protobuf:"varint,2,opt,name=iops" json:"iops,omitempty"` +} + +func (m *GetVolumeInfoResponse) Reset() { *m = GetVolumeInfoResponse{} } +func (m *GetVolumeInfoResponse) String() string { return proto.CompactTextString(m) } +func (*GetVolumeInfoResponse) ProtoMessage() {} +func (*GetVolumeInfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + +func (m *GetVolumeInfoResponse) GetVolumeType() string { + if m != nil { + return m.VolumeType + } + return "" +} + +func (m *GetVolumeInfoResponse) GetIops() int64 { + if m != nil { + return m.Iops + } + return 0 +} + +type IsVolumeReadyRequest struct { + VolumeID string `protobuf:"bytes,1,opt,name=volumeID" json:"volumeID,omitempty"` + VolumeAZ string `protobuf:"bytes,2,opt,name=volumeAZ" json:"volumeAZ,omitempty"` +} + +func (m *IsVolumeReadyRequest) Reset() { *m = IsVolumeReadyRequest{} } +func (m *IsVolumeReadyRequest) String() string { return proto.CompactTextString(m) } +func (*IsVolumeReadyRequest) ProtoMessage() {} +func (*IsVolumeReadyRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } + +func (m *IsVolumeReadyRequest) GetVolumeID() string { + if m != nil { + return m.VolumeID + } + return "" +} + +func (m *IsVolumeReadyRequest) GetVolumeAZ() string { + if m != nil { + return m.VolumeAZ + } + return "" +} + +type IsVolumeReadyResponse struct { + Ready bool `protobuf:"varint,1,opt,name=ready" json:"ready,omitempty"` +} + +func (m *IsVolumeReadyResponse) Reset() { *m = IsVolumeReadyResponse{} } +func (m *IsVolumeReadyResponse) String() string { return proto.CompactTextString(m) } +func (*IsVolumeReadyResponse) ProtoMessage() {} +func (*IsVolumeReadyResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } + +func (m *IsVolumeReadyResponse) GetReady() bool { + if m != nil { + return m.Ready + } + return false +} + +type ListSnapshotsRequest struct { + TagFilters map[string]string `protobuf:"bytes,1,rep,name=tagFilters" json:"tagFilters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *ListSnapshotsRequest) Reset() { *m = ListSnapshotsRequest{} } +func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) } +func (*ListSnapshotsRequest) ProtoMessage() {} +func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } + +func (m *ListSnapshotsRequest) GetTagFilters() map[string]string { + if m != nil { + return m.TagFilters + } + return nil +} + +type ListSnapshotsResponse struct { + SnapshotIDs []string `protobuf:"bytes,2,rep,name=snapshotIDs" json:"snapshotIDs,omitempty"` +} + +func (m *ListSnapshotsResponse) Reset() { *m = ListSnapshotsResponse{} } +func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) } +func (*ListSnapshotsResponse) ProtoMessage() {} +func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } + +func (m *ListSnapshotsResponse) GetSnapshotIDs() []string { + if m != nil { + return m.SnapshotIDs + } + return nil +} + +type CreateSnapshotRequest struct { + VolumeID string `protobuf:"bytes,1,opt,name=volumeID" json:"volumeID,omitempty"` + VolumeAZ string `protobuf:"bytes,2,opt,name=volumeAZ" json:"volumeAZ,omitempty"` + Tags map[string]string `protobuf:"bytes,3,rep,name=tags" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } +func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotRequest) ProtoMessage() {} +func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } + +func (m *CreateSnapshotRequest) GetVolumeID() string { + if m != nil { + return m.VolumeID + } + return "" +} + +func (m *CreateSnapshotRequest) GetVolumeAZ() string { + if m != nil { + return m.VolumeAZ + } + return "" +} + +func (m *CreateSnapshotRequest) GetTags() map[string]string { + if m != nil { + return m.Tags + } + return nil +} + +type CreateSnapshotResponse struct { + SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID" json:"snapshotID,omitempty"` +} + +func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} } +func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotResponse) ProtoMessage() {} +func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } + +func (m *CreateSnapshotResponse) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +type DeleteSnapshotRequest struct { + SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID" json:"snapshotID,omitempty"` +} + +func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } +func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteSnapshotRequest) ProtoMessage() {} +func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } + +func (m *DeleteSnapshotRequest) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func init() { + proto.RegisterType((*CreateVolumeRequest)(nil), "generated.CreateVolumeRequest") + proto.RegisterType((*CreateVolumeResponse)(nil), "generated.CreateVolumeResponse") + proto.RegisterType((*GetVolumeInfoRequest)(nil), "generated.GetVolumeInfoRequest") + proto.RegisterType((*GetVolumeInfoResponse)(nil), "generated.GetVolumeInfoResponse") + proto.RegisterType((*IsVolumeReadyRequest)(nil), "generated.IsVolumeReadyRequest") + proto.RegisterType((*IsVolumeReadyResponse)(nil), "generated.IsVolumeReadyResponse") + proto.RegisterType((*ListSnapshotsRequest)(nil), "generated.ListSnapshotsRequest") + proto.RegisterType((*ListSnapshotsResponse)(nil), "generated.ListSnapshotsResponse") + proto.RegisterType((*CreateSnapshotRequest)(nil), "generated.CreateSnapshotRequest") + proto.RegisterType((*CreateSnapshotResponse)(nil), "generated.CreateSnapshotResponse") + proto.RegisterType((*DeleteSnapshotRequest)(nil), "generated.DeleteSnapshotRequest") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for BlockStore service + +type BlockStoreClient interface { + Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) + CreateVolumeFromSnapshot(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) + GetVolumeInfo(ctx context.Context, in *GetVolumeInfoRequest, opts ...grpc.CallOption) (*GetVolumeInfoResponse, error) + IsVolumeReady(ctx context.Context, in *IsVolumeReadyRequest, opts ...grpc.CallOption) (*IsVolumeReadyResponse, error) + ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) + CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) + DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) +} + +type blockStoreClient struct { + cc *grpc.ClientConn +} + +func NewBlockStoreClient(cc *grpc.ClientConn) BlockStoreClient { + return &blockStoreClient{cc} +} + +func (c *blockStoreClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.BlockStore/Init", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) CreateVolumeFromSnapshot(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) { + out := new(CreateVolumeResponse) + err := grpc.Invoke(ctx, "/generated.BlockStore/CreateVolumeFromSnapshot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) GetVolumeInfo(ctx context.Context, in *GetVolumeInfoRequest, opts ...grpc.CallOption) (*GetVolumeInfoResponse, error) { + out := new(GetVolumeInfoResponse) + err := grpc.Invoke(ctx, "/generated.BlockStore/GetVolumeInfo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) IsVolumeReady(ctx context.Context, in *IsVolumeReadyRequest, opts ...grpc.CallOption) (*IsVolumeReadyResponse, error) { + out := new(IsVolumeReadyResponse) + err := grpc.Invoke(ctx, "/generated.BlockStore/IsVolumeReady", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) { + out := new(ListSnapshotsResponse) + err := grpc.Invoke(ctx, "/generated.BlockStore/ListSnapshots", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) { + out := new(CreateSnapshotResponse) + err := grpc.Invoke(ctx, "/generated.BlockStore/CreateSnapshot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockStoreClient) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.BlockStore/DeleteSnapshot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for BlockStore service + +type BlockStoreServer interface { + Init(context.Context, *InitRequest) (*Empty, error) + CreateVolumeFromSnapshot(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) + GetVolumeInfo(context.Context, *GetVolumeInfoRequest) (*GetVolumeInfoResponse, error) + IsVolumeReady(context.Context, *IsVolumeReadyRequest) (*IsVolumeReadyResponse, error) + ListSnapshots(context.Context, *ListSnapshotsRequest) (*ListSnapshotsResponse, error) + CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotResponse, error) + DeleteSnapshot(context.Context, *DeleteSnapshotRequest) (*Empty, error) +} + +func RegisterBlockStoreServer(s *grpc.Server, srv BlockStoreServer) { + s.RegisterService(&_BlockStore_serviceDesc, srv) +} + +func _BlockStore_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InitRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/Init", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).Init(ctx, req.(*InitRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_CreateVolumeFromSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).CreateVolumeFromSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/CreateVolumeFromSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).CreateVolumeFromSnapshot(ctx, req.(*CreateVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_GetVolumeInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVolumeInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).GetVolumeInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/GetVolumeInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).GetVolumeInfo(ctx, req.(*GetVolumeInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_IsVolumeReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IsVolumeReadyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).IsVolumeReady(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/IsVolumeReady", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).IsVolumeReady(ctx, req.(*IsVolumeReadyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_ListSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListSnapshotsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).ListSnapshots(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/ListSnapshots", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).ListSnapshots(ctx, req.(*ListSnapshotsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_CreateSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).CreateSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/CreateSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).CreateSnapshot(ctx, req.(*CreateSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockStore_DeleteSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockStoreServer).DeleteSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.BlockStore/DeleteSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockStoreServer).DeleteSnapshot(ctx, req.(*DeleteSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _BlockStore_serviceDesc = grpc.ServiceDesc{ + ServiceName: "generated.BlockStore", + HandlerType: (*BlockStoreServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Init", + Handler: _BlockStore_Init_Handler, + }, + { + MethodName: "CreateVolumeFromSnapshot", + Handler: _BlockStore_CreateVolumeFromSnapshot_Handler, + }, + { + MethodName: "GetVolumeInfo", + Handler: _BlockStore_GetVolumeInfo_Handler, + }, + { + MethodName: "IsVolumeReady", + Handler: _BlockStore_IsVolumeReady_Handler, + }, + { + MethodName: "ListSnapshots", + Handler: _BlockStore_ListSnapshots_Handler, + }, + { + MethodName: "CreateSnapshot", + Handler: _BlockStore_CreateSnapshot_Handler, + }, + { + MethodName: "DeleteSnapshot", + Handler: _BlockStore_DeleteSnapshot_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "BlockStore.proto", +} + +func init() { proto.RegisterFile("BlockStore.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 539 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0xd5, 0xc6, 0x06, 0x35, 0x53, 0x5a, 0xa2, 0xc5, 0xae, 0x2c, 0x1f, 0x8a, 0xf1, 0x29, 0x42, + 0x22, 0xa0, 0x70, 0x68, 0x41, 0x02, 0x09, 0x48, 0x8b, 0x22, 0x50, 0x91, 0x9c, 0xc2, 0x01, 0x4e, + 0x86, 0x2c, 0x69, 0x54, 0xc7, 0x6b, 0x76, 0x37, 0x95, 0xfc, 0x01, 0xfc, 0x0a, 0x5f, 0xc0, 0x47, + 0xf0, 0x59, 0xc8, 0xf6, 0xda, 0xde, 0xb5, 0xdd, 0x54, 0x55, 0x6e, 0x9e, 0x19, 0xcf, 0xdb, 0x37, + 0xcf, 0x6f, 0xd6, 0x30, 0x78, 0x1b, 0xd1, 0x1f, 0x97, 0x33, 0x41, 0x19, 0x19, 0x25, 0x8c, 0x0a, + 0x8a, 0xfb, 0x0b, 0x12, 0x13, 0x16, 0x0a, 0x32, 0x77, 0xef, 0xcd, 0x2e, 0x42, 0x46, 0xe6, 0x45, + 0xc1, 0xff, 0x8d, 0xe0, 0xc1, 0x3b, 0x46, 0x42, 0x41, 0xbe, 0xd0, 0x68, 0xbd, 0x22, 0x01, 0xf9, + 0xb5, 0x26, 0x5c, 0xe0, 0x43, 0x00, 0x1e, 0x87, 0x09, 0xbf, 0xa0, 0x62, 0x3a, 0x71, 0x90, 0x87, + 0x86, 0xfd, 0x40, 0xc9, 0x64, 0xf5, 0xab, 0xbc, 0xe1, 0x3c, 0x4d, 0x88, 0xd3, 0x2b, 0xea, 0x75, + 0x06, 0xbb, 0xb0, 0x53, 0x44, 0x6f, 0xbe, 0x3a, 0x46, 0x5e, 0xad, 0x62, 0x8c, 0xc1, 0x5c, 0xd2, + 0x84, 0x3b, 0xa6, 0x87, 0x86, 0x46, 0x90, 0x3f, 0xfb, 0x63, 0xb0, 0x74, 0x1a, 0x3c, 0xa1, 0x31, + 0x57, 0x70, 0x2a, 0x16, 0x55, 0xec, 0x9f, 0x81, 0xf5, 0x9e, 0x88, 0xa2, 0x61, 0x1a, 0xff, 0xa4, + 0x25, 0xf7, 0x0d, 0x3d, 0x1a, 0xaf, 0x9e, 0xce, 0xcb, 0xff, 0x00, 0x76, 0x03, 0x4f, 0x92, 0xd0, + 0x87, 0x45, 0xad, 0x61, 0xcb, 0x81, 0x7a, 0xca, 0x40, 0x67, 0x60, 0x4d, 0x79, 0x39, 0x4c, 0x38, + 0x4f, 0xb7, 0x25, 0xf7, 0x04, 0xec, 0x06, 0x9e, 0x24, 0x67, 0xc1, 0x1d, 0x96, 0x25, 0x72, 0xb4, + 0x9d, 0xa0, 0x08, 0xfc, 0x3f, 0x08, 0xac, 0x8f, 0x4b, 0x2e, 0x66, 0xf2, 0x93, 0xf1, 0xf2, 0xfc, + 0x4f, 0x00, 0x22, 0x5c, 0x9c, 0x2e, 0x23, 0x41, 0x18, 0x77, 0x90, 0x67, 0x0c, 0x77, 0xc7, 0x4f, + 0x47, 0x95, 0x3d, 0x46, 0x5d, 0x4d, 0xa3, 0xf3, 0xaa, 0xe3, 0x24, 0x16, 0x2c, 0x0d, 0x14, 0x08, + 0xf7, 0x15, 0xdc, 0x6f, 0x94, 0xf1, 0x00, 0x8c, 0x4b, 0x92, 0xca, 0xf1, 0xb2, 0xc7, 0x8c, 0xe4, + 0x55, 0x18, 0xad, 0x4b, 0xa7, 0x14, 0xc1, 0xcb, 0xde, 0x31, 0xf2, 0x5f, 0x80, 0xdd, 0x38, 0x52, + 0xce, 0xe5, 0xc1, 0x6e, 0xed, 0xb7, 0x4c, 0x5b, 0x63, 0xd8, 0x0f, 0xd4, 0x94, 0xff, 0x0f, 0x81, + 0x5d, 0x98, 0xa6, 0xec, 0xde, 0x52, 0x64, 0xfc, 0x1a, 0x4c, 0x11, 0x2e, 0xb8, 0x63, 0xe4, 0xb2, + 0x3c, 0x56, 0x64, 0xe9, 0x3c, 0x27, 0xd3, 0x45, 0x2a, 0x92, 0xf7, 0xb9, 0x47, 0xd0, 0xaf, 0x52, + 0xb7, 0x52, 0xe1, 0x18, 0x0e, 0x9a, 0x27, 0xd4, 0xde, 0xdb, 0xb4, 0x88, 0xfe, 0x11, 0xd8, 0x13, + 0x12, 0x91, 0xb6, 0x06, 0x37, 0x34, 0x8e, 0xff, 0x9a, 0x00, 0xf5, 0x3d, 0x81, 0x9f, 0x81, 0x39, + 0x8d, 0x97, 0x02, 0x1f, 0x28, 0x43, 0x67, 0x09, 0x09, 0xe7, 0x0e, 0x94, 0xfc, 0xc9, 0x2a, 0x11, + 0x29, 0xfe, 0x06, 0x8e, 0xba, 0xb2, 0xa7, 0x8c, 0xae, 0x4a, 0x0e, 0xf8, 0xb0, 0x25, 0x9d, 0x76, + 0xbd, 0xb8, 0x0f, 0xaf, 0xad, 0xcb, 0xb1, 0x03, 0xd8, 0xd3, 0x76, 0x11, 0xab, 0x1d, 0x5d, 0x5b, + 0xef, 0x7a, 0xd7, 0xbf, 0x50, 0x63, 0x6a, 0x2b, 0xa4, 0x61, 0x76, 0x2d, 0xab, 0x86, 0xd9, 0xbd, + 0x7d, 0x01, 0xec, 0x69, 0xf6, 0xd5, 0x30, 0xbb, 0x76, 0x49, 0xc3, 0xec, 0x76, 0xfe, 0x67, 0xd8, + 0xd7, 0xcd, 0x80, 0xbd, 0x9b, 0x9c, 0xe8, 0x3e, 0xda, 0xf0, 0x86, 0x84, 0x9d, 0xc0, 0xbe, 0xee, + 0x14, 0x0d, 0xb6, 0xd3, 0x44, 0xed, 0xaf, 0xfe, 0xfd, 0x6e, 0xfe, 0xdf, 0x78, 0xfe, 0x3f, 0x00, + 0x00, 0xff, 0xff, 0xd7, 0x15, 0x7f, 0x32, 0x64, 0x06, 0x00, 0x00, +} diff --git a/pkg/plugin/generated/ObjectStore.pb.go b/pkg/plugin/generated/ObjectStore.pb.go new file mode 100644 index 000000000..4e755d257 --- /dev/null +++ b/pkg/plugin/generated/ObjectStore.pb.go @@ -0,0 +1,621 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: ObjectStore.proto + +package generated + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type PutObjectRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key" json:"key,omitempty"` + Body []byte `protobuf:"bytes,3,opt,name=body,proto3" json:"body,omitempty"` +} + +func (m *PutObjectRequest) Reset() { *m = PutObjectRequest{} } +func (m *PutObjectRequest) String() string { return proto.CompactTextString(m) } +func (*PutObjectRequest) ProtoMessage() {} +func (*PutObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *PutObjectRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *PutObjectRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *PutObjectRequest) GetBody() []byte { + if m != nil { + return m.Body + } + return nil +} + +type GetObjectRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key" json:"key,omitempty"` +} + +func (m *GetObjectRequest) Reset() { *m = GetObjectRequest{} } +func (m *GetObjectRequest) String() string { return proto.CompactTextString(m) } +func (*GetObjectRequest) ProtoMessage() {} +func (*GetObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{1} } + +func (m *GetObjectRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *GetObjectRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +type Bytes struct { + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *Bytes) Reset() { *m = Bytes{} } +func (m *Bytes) String() string { return proto.CompactTextString(m) } +func (*Bytes) ProtoMessage() {} +func (*Bytes) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{2} } + +func (m *Bytes) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type ListCommonPrefixesRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Delimiter string `protobuf:"bytes,2,opt,name=delimiter" json:"delimiter,omitempty"` +} + +func (m *ListCommonPrefixesRequest) Reset() { *m = ListCommonPrefixesRequest{} } +func (m *ListCommonPrefixesRequest) String() string { return proto.CompactTextString(m) } +func (*ListCommonPrefixesRequest) ProtoMessage() {} +func (*ListCommonPrefixesRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{3} } + +func (m *ListCommonPrefixesRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *ListCommonPrefixesRequest) GetDelimiter() string { + if m != nil { + return m.Delimiter + } + return "" +} + +type ListCommonPrefixesResponse struct { + Prefixes []string `protobuf:"bytes,1,rep,name=prefixes" json:"prefixes,omitempty"` +} + +func (m *ListCommonPrefixesResponse) Reset() { *m = ListCommonPrefixesResponse{} } +func (m *ListCommonPrefixesResponse) String() string { return proto.CompactTextString(m) } +func (*ListCommonPrefixesResponse) ProtoMessage() {} +func (*ListCommonPrefixesResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{4} } + +func (m *ListCommonPrefixesResponse) GetPrefixes() []string { + if m != nil { + return m.Prefixes + } + return nil +} + +type ListObjectsRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Prefix string `protobuf:"bytes,2,opt,name=prefix" json:"prefix,omitempty"` +} + +func (m *ListObjectsRequest) Reset() { *m = ListObjectsRequest{} } +func (m *ListObjectsRequest) String() string { return proto.CompactTextString(m) } +func (*ListObjectsRequest) ProtoMessage() {} +func (*ListObjectsRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{5} } + +func (m *ListObjectsRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *ListObjectsRequest) GetPrefix() string { + if m != nil { + return m.Prefix + } + return "" +} + +type ListObjectsResponse struct { + Keys []string `protobuf:"bytes,1,rep,name=keys" json:"keys,omitempty"` +} + +func (m *ListObjectsResponse) Reset() { *m = ListObjectsResponse{} } +func (m *ListObjectsResponse) String() string { return proto.CompactTextString(m) } +func (*ListObjectsResponse) ProtoMessage() {} +func (*ListObjectsResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{6} } + +func (m *ListObjectsResponse) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + +type DeleteObjectRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key" json:"key,omitempty"` +} + +func (m *DeleteObjectRequest) Reset() { *m = DeleteObjectRequest{} } +func (m *DeleteObjectRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteObjectRequest) ProtoMessage() {} +func (*DeleteObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{7} } + +func (m *DeleteObjectRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *DeleteObjectRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +type CreateSignedURLRequest struct { + Bucket string `protobuf:"bytes,1,opt,name=bucket" json:"bucket,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key" json:"key,omitempty"` + Ttl int64 `protobuf:"varint,3,opt,name=ttl" json:"ttl,omitempty"` +} + +func (m *CreateSignedURLRequest) Reset() { *m = CreateSignedURLRequest{} } +func (m *CreateSignedURLRequest) String() string { return proto.CompactTextString(m) } +func (*CreateSignedURLRequest) ProtoMessage() {} +func (*CreateSignedURLRequest) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{8} } + +func (m *CreateSignedURLRequest) GetBucket() string { + if m != nil { + return m.Bucket + } + return "" +} + +func (m *CreateSignedURLRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *CreateSignedURLRequest) GetTtl() int64 { + if m != nil { + return m.Ttl + } + return 0 +} + +type CreateSignedURLResponse struct { + Url string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"` +} + +func (m *CreateSignedURLResponse) Reset() { *m = CreateSignedURLResponse{} } +func (m *CreateSignedURLResponse) String() string { return proto.CompactTextString(m) } +func (*CreateSignedURLResponse) ProtoMessage() {} +func (*CreateSignedURLResponse) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{9} } + +func (m *CreateSignedURLResponse) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func init() { + proto.RegisterType((*PutObjectRequest)(nil), "generated.PutObjectRequest") + proto.RegisterType((*GetObjectRequest)(nil), "generated.GetObjectRequest") + proto.RegisterType((*Bytes)(nil), "generated.Bytes") + proto.RegisterType((*ListCommonPrefixesRequest)(nil), "generated.ListCommonPrefixesRequest") + proto.RegisterType((*ListCommonPrefixesResponse)(nil), "generated.ListCommonPrefixesResponse") + proto.RegisterType((*ListObjectsRequest)(nil), "generated.ListObjectsRequest") + proto.RegisterType((*ListObjectsResponse)(nil), "generated.ListObjectsResponse") + proto.RegisterType((*DeleteObjectRequest)(nil), "generated.DeleteObjectRequest") + proto.RegisterType((*CreateSignedURLRequest)(nil), "generated.CreateSignedURLRequest") + proto.RegisterType((*CreateSignedURLResponse)(nil), "generated.CreateSignedURLResponse") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for ObjectStore service + +type ObjectStoreClient interface { + Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) + PutObject(ctx context.Context, opts ...grpc.CallOption) (ObjectStore_PutObjectClient, error) + GetObject(ctx context.Context, in *GetObjectRequest, opts ...grpc.CallOption) (ObjectStore_GetObjectClient, error) + ListCommonPrefixes(ctx context.Context, in *ListCommonPrefixesRequest, opts ...grpc.CallOption) (*ListCommonPrefixesResponse, error) + ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) + DeleteObject(ctx context.Context, in *DeleteObjectRequest, opts ...grpc.CallOption) (*Empty, error) + CreateSignedURL(ctx context.Context, in *CreateSignedURLRequest, opts ...grpc.CallOption) (*CreateSignedURLResponse, error) +} + +type objectStoreClient struct { + cc *grpc.ClientConn +} + +func NewObjectStoreClient(cc *grpc.ClientConn) ObjectStoreClient { + return &objectStoreClient{cc} +} + +func (c *objectStoreClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.ObjectStore/Init", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *objectStoreClient) PutObject(ctx context.Context, opts ...grpc.CallOption) (ObjectStore_PutObjectClient, error) { + stream, err := grpc.NewClientStream(ctx, &_ObjectStore_serviceDesc.Streams[0], c.cc, "/generated.ObjectStore/PutObject", opts...) + if err != nil { + return nil, err + } + x := &objectStorePutObjectClient{stream} + return x, nil +} + +type ObjectStore_PutObjectClient interface { + Send(*PutObjectRequest) error + CloseAndRecv() (*Empty, error) + grpc.ClientStream +} + +type objectStorePutObjectClient struct { + grpc.ClientStream +} + +func (x *objectStorePutObjectClient) Send(m *PutObjectRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *objectStorePutObjectClient) CloseAndRecv() (*Empty, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Empty) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *objectStoreClient) GetObject(ctx context.Context, in *GetObjectRequest, opts ...grpc.CallOption) (ObjectStore_GetObjectClient, error) { + stream, err := grpc.NewClientStream(ctx, &_ObjectStore_serviceDesc.Streams[1], c.cc, "/generated.ObjectStore/GetObject", opts...) + if err != nil { + return nil, err + } + x := &objectStoreGetObjectClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type ObjectStore_GetObjectClient interface { + Recv() (*Bytes, error) + grpc.ClientStream +} + +type objectStoreGetObjectClient struct { + grpc.ClientStream +} + +func (x *objectStoreGetObjectClient) Recv() (*Bytes, error) { + m := new(Bytes) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *objectStoreClient) ListCommonPrefixes(ctx context.Context, in *ListCommonPrefixesRequest, opts ...grpc.CallOption) (*ListCommonPrefixesResponse, error) { + out := new(ListCommonPrefixesResponse) + err := grpc.Invoke(ctx, "/generated.ObjectStore/ListCommonPrefixes", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *objectStoreClient) ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) { + out := new(ListObjectsResponse) + err := grpc.Invoke(ctx, "/generated.ObjectStore/ListObjects", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *objectStoreClient) DeleteObject(ctx context.Context, in *DeleteObjectRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.ObjectStore/DeleteObject", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *objectStoreClient) CreateSignedURL(ctx context.Context, in *CreateSignedURLRequest, opts ...grpc.CallOption) (*CreateSignedURLResponse, error) { + out := new(CreateSignedURLResponse) + err := grpc.Invoke(ctx, "/generated.ObjectStore/CreateSignedURL", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for ObjectStore service + +type ObjectStoreServer interface { + Init(context.Context, *InitRequest) (*Empty, error) + PutObject(ObjectStore_PutObjectServer) error + GetObject(*GetObjectRequest, ObjectStore_GetObjectServer) error + ListCommonPrefixes(context.Context, *ListCommonPrefixesRequest) (*ListCommonPrefixesResponse, error) + ListObjects(context.Context, *ListObjectsRequest) (*ListObjectsResponse, error) + DeleteObject(context.Context, *DeleteObjectRequest) (*Empty, error) + CreateSignedURL(context.Context, *CreateSignedURLRequest) (*CreateSignedURLResponse, error) +} + +func RegisterObjectStoreServer(s *grpc.Server, srv ObjectStoreServer) { + s.RegisterService(&_ObjectStore_serviceDesc, srv) +} + +func _ObjectStore_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InitRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectStoreServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ObjectStore/Init", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectStoreServer).Init(ctx, req.(*InitRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ObjectStore_PutObject_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ObjectStoreServer).PutObject(&objectStorePutObjectServer{stream}) +} + +type ObjectStore_PutObjectServer interface { + SendAndClose(*Empty) error + Recv() (*PutObjectRequest, error) + grpc.ServerStream +} + +type objectStorePutObjectServer struct { + grpc.ServerStream +} + +func (x *objectStorePutObjectServer) SendAndClose(m *Empty) error { + return x.ServerStream.SendMsg(m) +} + +func (x *objectStorePutObjectServer) Recv() (*PutObjectRequest, error) { + m := new(PutObjectRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _ObjectStore_GetObject_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(GetObjectRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(ObjectStoreServer).GetObject(m, &objectStoreGetObjectServer{stream}) +} + +type ObjectStore_GetObjectServer interface { + Send(*Bytes) error + grpc.ServerStream +} + +type objectStoreGetObjectServer struct { + grpc.ServerStream +} + +func (x *objectStoreGetObjectServer) Send(m *Bytes) error { + return x.ServerStream.SendMsg(m) +} + +func _ObjectStore_ListCommonPrefixes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListCommonPrefixesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectStoreServer).ListCommonPrefixes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ObjectStore/ListCommonPrefixes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectStoreServer).ListCommonPrefixes(ctx, req.(*ListCommonPrefixesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ObjectStore_ListObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListObjectsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectStoreServer).ListObjects(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ObjectStore/ListObjects", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectStoreServer).ListObjects(ctx, req.(*ListObjectsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ObjectStore_DeleteObject_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteObjectRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectStoreServer).DeleteObject(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ObjectStore/DeleteObject", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectStoreServer).DeleteObject(ctx, req.(*DeleteObjectRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ObjectStore_CreateSignedURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateSignedURLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectStoreServer).CreateSignedURL(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ObjectStore/CreateSignedURL", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectStoreServer).CreateSignedURL(ctx, req.(*CreateSignedURLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _ObjectStore_serviceDesc = grpc.ServiceDesc{ + ServiceName: "generated.ObjectStore", + HandlerType: (*ObjectStoreServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Init", + Handler: _ObjectStore_Init_Handler, + }, + { + MethodName: "ListCommonPrefixes", + Handler: _ObjectStore_ListCommonPrefixes_Handler, + }, + { + MethodName: "ListObjects", + Handler: _ObjectStore_ListObjects_Handler, + }, + { + MethodName: "DeleteObject", + Handler: _ObjectStore_DeleteObject_Handler, + }, + { + MethodName: "CreateSignedURL", + Handler: _ObjectStore_CreateSignedURL_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "PutObject", + Handler: _ObjectStore_PutObject_Handler, + ClientStreams: true, + }, + { + StreamName: "GetObject", + Handler: _ObjectStore_GetObject_Handler, + ServerStreams: true, + }, + }, + Metadata: "ObjectStore.proto", +} + +func init() { proto.RegisterFile("ObjectStore.proto", fileDescriptor1) } + +var fileDescriptor1 = []byte{ + // 444 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0xdf, 0x8b, 0xd3, 0x40, + 0x10, 0xc7, 0x89, 0xa9, 0xc5, 0xcc, 0x15, 0x8c, 0x73, 0x50, 0x6b, 0x4e, 0xa5, 0x2e, 0x0a, 0x15, + 0xa1, 0x1c, 0xfa, 0xe2, 0xc3, 0x81, 0xe2, 0x9d, 0x88, 0x50, 0xb0, 0xa6, 0x0a, 0xbe, 0xa6, 0x97, + 0xf1, 0x8c, 0xcd, 0x2f, 0x37, 0x13, 0x30, 0xff, 0x81, 0x7f, 0xb6, 0x64, 0xb3, 0xc6, 0x4d, 0x2e, + 0x67, 0xf1, 0xde, 0x66, 0x67, 0xe7, 0xfb, 0x9d, 0xc9, 0xce, 0x87, 0xc0, 0x9d, 0x0f, 0xdb, 0xef, + 0x74, 0xce, 0x1b, 0xce, 0x24, 0x2d, 0x73, 0x99, 0x71, 0x86, 0xce, 0x05, 0xa5, 0x24, 0x03, 0xa6, + 0xd0, 0x9b, 0x6c, 0xbe, 0x05, 0x92, 0xc2, 0xe6, 0x42, 0xac, 0xc1, 0x5d, 0x97, 0xdc, 0x08, 0x7c, + 0xfa, 0x51, 0x52, 0xc1, 0x38, 0x85, 0xf1, 0xb6, 0x3c, 0xdf, 0x11, 0xcf, 0xac, 0xb9, 0xb5, 0x70, + 0x7c, 0x7d, 0x42, 0x17, 0xec, 0x1d, 0x55, 0xb3, 0x1b, 0x2a, 0x59, 0x87, 0x88, 0x30, 0xda, 0x66, + 0x61, 0x35, 0xb3, 0xe7, 0xd6, 0x62, 0xe2, 0xab, 0x58, 0x9c, 0x80, 0xfb, 0x8e, 0xae, 0xeb, 0x28, + 0x8e, 0xe0, 0xe6, 0x9b, 0x8a, 0xa9, 0xa8, 0xad, 0xc3, 0x80, 0x03, 0x25, 0x98, 0xf8, 0x2a, 0x16, + 0x1f, 0xe1, 0xde, 0x2a, 0x2a, 0xf8, 0x34, 0x4b, 0x92, 0x2c, 0x5d, 0x4b, 0xfa, 0x1a, 0xfd, 0xa4, + 0x62, 0x5f, 0x8f, 0xfb, 0xe0, 0x84, 0x14, 0x47, 0x49, 0xc4, 0x24, 0x75, 0xa7, 0xbf, 0x09, 0xf1, + 0x12, 0xbc, 0x21, 0xcb, 0x22, 0xcf, 0xd2, 0x82, 0xd0, 0x83, 0x5b, 0xb9, 0xce, 0xcd, 0xac, 0xb9, + 0xbd, 0x70, 0xfc, 0xf6, 0x2c, 0xce, 0x00, 0x6b, 0x65, 0xf3, 0xa1, 0x7b, 0xa7, 0x98, 0xc2, 0xb8, + 0x51, 0xea, 0x11, 0xf4, 0x49, 0x3c, 0x85, 0xc3, 0x8e, 0x8b, 0x6e, 0x8c, 0x30, 0xda, 0x51, 0xf5, + 0xa7, 0xa9, 0x8a, 0xc5, 0x2b, 0x38, 0x3c, 0xa3, 0x98, 0x98, 0xae, 0xfb, 0xb6, 0x9f, 0x60, 0x7a, + 0x2a, 0x29, 0x60, 0xda, 0x44, 0x17, 0x29, 0x85, 0x9f, 0xfd, 0xd5, 0xff, 0x6f, 0xdc, 0x05, 0x9b, + 0x39, 0x56, 0x0b, 0xb7, 0xfd, 0x3a, 0x14, 0xcf, 0xe0, 0xee, 0x25, 0x57, 0xfd, 0x15, 0x2e, 0xd8, + 0xa5, 0x8c, 0xb5, 0x67, 0x1d, 0x3e, 0xff, 0x35, 0x82, 0x03, 0x83, 0x4e, 0x3c, 0x86, 0xd1, 0xfb, + 0x34, 0x62, 0x9c, 0x2e, 0x5b, 0x40, 0x97, 0x75, 0x42, 0x0f, 0xe6, 0xb9, 0x46, 0xfe, 0x6d, 0x92, + 0x73, 0x85, 0x27, 0xe0, 0xb4, 0xc0, 0xe2, 0x91, 0x71, 0xdd, 0xc7, 0xf8, 0xb2, 0x76, 0x61, 0xd5, + 0xea, 0x16, 0xce, 0x8e, 0xba, 0x8f, 0x6c, 0x47, 0xad, 0x88, 0x3c, 0xb6, 0x30, 0x68, 0x56, 0xde, + 0x85, 0x05, 0x1f, 0x1b, 0x95, 0x57, 0xe2, 0xe9, 0x3d, 0xd9, 0x53, 0xa5, 0x9f, 0x6c, 0x05, 0x07, + 0x06, 0x0f, 0xf8, 0xa0, 0xa7, 0xea, 0xd2, 0xe6, 0x3d, 0xbc, 0xea, 0x5a, 0xbb, 0xbd, 0x86, 0x89, + 0x89, 0x0c, 0x9a, 0xf5, 0x03, 0x2c, 0x0d, 0x3c, 0xf7, 0x17, 0xb8, 0xdd, 0xdb, 0x2e, 0x3e, 0x32, + 0x8a, 0x86, 0x79, 0xf2, 0xc4, 0xbf, 0x4a, 0x9a, 0xd9, 0xb6, 0x63, 0xf5, 0x03, 0x7a, 0xf1, 0x3b, + 0x00, 0x00, 0xff, 0xff, 0xf3, 0xb2, 0x85, 0x9a, 0xae, 0x04, 0x00, 0x00, +} diff --git a/pkg/plugin/generated/Shared.pb.go b/pkg/plugin/generated/Shared.pb.go new file mode 100644 index 000000000..81ea2a973 --- /dev/null +++ b/pkg/plugin/generated/Shared.pb.go @@ -0,0 +1,58 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: Shared.proto + +package generated + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Empty struct { +} + +func (m *Empty) Reset() { *m = Empty{} } +func (m *Empty) String() string { return proto.CompactTextString(m) } +func (*Empty) ProtoMessage() {} +func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +type InitRequest struct { + Config map[string]string `protobuf:"bytes,1,rep,name=config" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *InitRequest) Reset() { *m = InitRequest{} } +func (m *InitRequest) String() string { return proto.CompactTextString(m) } +func (*InitRequest) ProtoMessage() {} +func (*InitRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{1} } + +func (m *InitRequest) GetConfig() map[string]string { + if m != nil { + return m.Config + } + return nil +} + +func init() { + proto.RegisterType((*Empty)(nil), "generated.Empty") + proto.RegisterType((*InitRequest)(nil), "generated.InitRequest") +} + +func init() { proto.RegisterFile("Shared.proto", fileDescriptor2) } + +var fileDescriptor2 = []byte{ + // 156 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x09, 0xce, 0x48, 0x2c, + 0x4a, 0x4d, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4c, 0x4f, 0xcd, 0x4b, 0x2d, 0x4a, + 0x2c, 0x49, 0x4d, 0x51, 0x62, 0xe7, 0x62, 0x75, 0xcd, 0x2d, 0x28, 0xa9, 0x54, 0x6a, 0x61, 0xe4, + 0xe2, 0xf6, 0xcc, 0xcb, 0x2c, 0x09, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0xb2, 0xe2, 0x62, + 0x4b, 0xce, 0xcf, 0x4b, 0xcb, 0x4c, 0x97, 0x60, 0x54, 0x60, 0xd6, 0xe0, 0x36, 0x52, 0xd2, 0x83, + 0x6b, 0xd2, 0x43, 0x52, 0xa7, 0xe7, 0x0c, 0x56, 0xe4, 0x9a, 0x57, 0x52, 0x54, 0x19, 0x04, 0xd5, + 0x21, 0x65, 0xc9, 0xc5, 0x8d, 0x24, 0x2c, 0x24, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, 0xa8, + 0xc0, 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a, 0x89, 0x70, 0xb1, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x4a, + 0x30, 0x81, 0xc5, 0x20, 0x1c, 0x2b, 0x26, 0x0b, 0xc6, 0x24, 0x36, 0xb0, 0x0b, 0x8d, 0x01, 0x01, + 0x00, 0x00, 0xff, 0xff, 0x85, 0xab, 0x54, 0x37, 0xb1, 0x00, 0x00, 0x00, +} diff --git a/pkg/plugin/logger.go b/pkg/plugin/logger.go new file mode 100644 index 000000000..616a2eef2 --- /dev/null +++ b/pkg/plugin/logger.go @@ -0,0 +1,52 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "github.com/sirupsen/logrus" + + "github.com/heptio/ark/pkg/util/logging" +) + +// NewPluginLogger returns a logger that is suitable for use within an +// Ark plugin. +func NewPluginLogger() logrus.FieldLogger { + logger := logrus.New() + + // we use the JSON formatter because go-plugin will parse incoming + // JSON on stderr and use it to create structured log entries. + logger.Formatter = &logrus.JSONFormatter{ + FieldMap: logrus.FieldMap{ + // this is the hclog-compatible message field + logrus.FieldKeyMsg: "@message", + }, + // Ark server already adds timestamps when emitting logs, so + // don't do it within the plugin. + DisableTimestamp: true, + } + + // set a logger name for the location hook which will signal to the Ark + // server logger that the location has been set within a hook. + logger.Hooks.Add((&logging.LogLocationHook{}).WithLoggerName("plugin")) + + // this hook adjusts the string representation of WarnLevel to "warn" + // rather than "warning" to make it parseable by go-plugin within the + // Ark server code + logger.Hooks.Add(&logging.HcLogLevelHook{}) + + return logger +} diff --git a/pkg/plugin/logrus_adapter.go b/pkg/plugin/logrus_adapter.go new file mode 100644 index 000000000..a8b3bc256 --- /dev/null +++ b/pkg/plugin/logrus_adapter.go @@ -0,0 +1,158 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "log" + + hclog "github.com/hashicorp/go-hclog" + "github.com/sirupsen/logrus" +) + +const pluginNameField = "pluginName" + +// logrusAdapter implements the hclog.Logger interface and +// delegates all calls to a logrus logger. +type logrusAdapter struct { + impl logrus.FieldLogger + level logrus.Level + name string +} + +// args are alternating key, value pairs, where the keys +// are expected to be strings, and values can be any type. +func argsToFields(args ...interface{}) logrus.Fields { + fields := make(map[string]interface{}) + + for i := 0; i < len(args); i += 2 { + switch args[i] { + case "time", "timestamp", "level": + // remove `time` & `timestamp` because this info will be added + // by the Ark logger and we don't want to have duplicated + // fields. + // + // remove `level` because it'll be added by the Ark logger based + // on the call we make (and go-plugin is determining which level + // to log at based on the hclog-compatible `@level` field which + // we're adding via HcLogLevelHook). + default: + var val interface{} + if i+1 < len(args) { + val = args[i+1] + } + + fields[fmt.Sprintf("%v", args[i])] = val + } + } + + return logrus.Fields(fields) +} + +// Trace emits a message and key/value pairs at the DEBUG level +// (logrus doesn't have a TRACE level) +func (l *logrusAdapter) Trace(msg string, args ...interface{}) { + l.Debug(msg, args...) +} + +// Debug emits a message and key/value pairs at the DEBUG level +func (l *logrusAdapter) Debug(msg string, args ...interface{}) { + l.impl.WithFields(argsToFields(args...)).Debug(msg) +} + +// Info emits a message and key/value pairs at the INFO level +func (l *logrusAdapter) Info(msg string, args ...interface{}) { + l.impl.WithFields(argsToFields(args...)).Info(msg) +} + +// Warn emits a message and key/value pairs at the WARN level +func (l *logrusAdapter) Warn(msg string, args ...interface{}) { + l.impl.WithFields(argsToFields(args...)).Warn(msg) +} + +// Error emits a message and key/value pairs at the ERROR level +func (l *logrusAdapter) Error(msg string, args ...interface{}) { + l.impl.WithFields(argsToFields(args...)).Error(msg) +} + +// IsTrace indicates if TRACE logs would be emitted. This and the other Is* guards +// are used to elide expensive logging code based on the current level. +func (l *logrusAdapter) IsTrace() bool { + return l.IsDebug() +} + +// IsDebug indicates if DEBUG logs would be emitted. This and the other Is* guards +// are used to elide expensive logging code based on the current level. +func (l *logrusAdapter) IsDebug() bool { + return l.level <= logrus.DebugLevel +} + +// IsInfo indicates if INFO logs would be emitted. This and the other Is* guards +// are used to elide expensive logging code based on the current level. +func (l *logrusAdapter) IsInfo() bool { + return l.level <= logrus.InfoLevel +} + +// IsWarn indicates if WARN logs would be emitted. This and the other Is* guards +// are used to elide expensive logging code based on the current level. +func (l *logrusAdapter) IsWarn() bool { + return l.level <= logrus.WarnLevel +} + +// IsError indicates if ERROR logs would be emitted. This and the other Is* guards +// are used to elide expensive logging code based on the current level. +func (l *logrusAdapter) IsError() bool { + return l.level <= logrus.ErrorLevel +} + +// With creates a sublogger that will always have the given key/value pairs +func (l *logrusAdapter) With(args ...interface{}) hclog.Logger { + return &logrusAdapter{ + impl: l.impl.WithFields(argsToFields(args...)), + level: l.level, + } +} + +// Named creates a logger that will add a `pluginName` field with the name string +// as the value. If the logger already has a name, the new value will be appended +// to the current name. +func (l *logrusAdapter) Named(name string) hclog.Logger { + var newName string + if l.name == "" { + newName = name + } else { + newName = l.name + "." + name + } + + return l.ResetNamed(newName) +} + +// ResetNamed creates a logger that will add a `pluginName` field with the name string +// as the value. This sets the name of the logger to the value directly, unlike `Named` +// which appends the given value to the current name. +func (l *logrusAdapter) ResetNamed(name string) hclog.Logger { + return &logrusAdapter{ + impl: l.impl.WithField(pluginNameField, name), + level: l.level, + name: name, + } +} + +// StandardLogger returns a value that conforms to the stdlib log.Logger interface +func (l *logrusAdapter) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger { + panic("not implemented") +} diff --git a/pkg/plugin/logrus_adapter_test.go b/pkg/plugin/logrus_adapter_test.go new file mode 100644 index 000000000..98e2872ef --- /dev/null +++ b/pkg/plugin/logrus_adapter_test.go @@ -0,0 +1,55 @@ +package plugin + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/sirupsen/logrus" +) + +func TestArgsToFields(t *testing.T) { + tests := []struct { + name string + args []interface{} + expectedFields logrus.Fields + }{ + { + name: "empty args results in empty map of fields", + args: []interface{}{}, + expectedFields: logrus.Fields(map[string]interface{}{}), + }, + { + name: "matching string keys/values are correctly set as fields", + args: []interface{}{"key-1", "value-1", "key-2", "value-2"}, + expectedFields: logrus.Fields(map[string]interface{}{ + "key-1": "value-1", + "key-2": "value-2", + }), + }, + { + name: "time/timestamp/level entries are removed", + args: []interface{}{"time", time.Now(), "key-1", "value-1", "timestamp", time.Now(), "key-2", "value-2", "level", "WARN"}, + expectedFields: logrus.Fields(map[string]interface{}{ + "key-1": "value-1", + "key-2": "value-2", + }), + }, + { + name: "odd number of args adds the last arg as a field with a nil value", + args: []interface{}{"key-1", "value-1", "key-2", "value-2", "key-3"}, + expectedFields: logrus.Fields(map[string]interface{}{ + "key-1": "value-1", + "key-2": "value-2", + "key-3": nil, + }), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expectedFields, argsToFields(test.args...)) + }) + } +} diff --git a/pkg/plugin/manager.go b/pkg/plugin/manager.go new file mode 100644 index 000000000..cff434467 --- /dev/null +++ b/pkg/plugin/manager.go @@ -0,0 +1,185 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/heptio/ark/pkg/cloudprovider" +) + +// PluginKind is a type alias for a string that describes +// the kind of an Ark-supported plugin. +type PluginKind string + +func (k PluginKind) String() string { + return string(k) +} + +const ( + // PluginKindObjectStore is the Kind string for + // an Object Store plugin. + PluginKindObjectStore PluginKind = "objectstore" + + // PluginKindBlockStore is the Kind string for + // a Block Store plugin. + PluginKindBlockStore PluginKind = "blockstore" + + pluginDir = "/plugins" +) + +type pluginInfo struct { + kind PluginKind + name string +} + +// Manager exposes functions for getting implementations of the pluggable +// Ark interfaces. +type Manager interface { + // GetObjectStore returns the plugin implementation of the + // cloudprovider.ObjectStore interface with the specified name. + GetObjectStore(name string) (cloudprovider.ObjectStore, error) + + // GetBlockStore returns the plugin implementation of the + // cloudprovider.BlockStore interface with the specified name. + GetBlockStore(name string) (cloudprovider.BlockStore, error) +} + +type manager struct { + logger hclog.Logger + clients map[pluginInfo]*plugin.Client + internalPlugins map[pluginInfo]interface{} +} + +// NewManager constructs a manager for getting plugin implementations. +func NewManager(logger logrus.FieldLogger, level logrus.Level) Manager { + return &manager{ + logger: (&logrusAdapter{impl: logger, level: level}), + clients: make(map[pluginInfo]*plugin.Client), + internalPlugins: map[pluginInfo]interface{}{ + {kind: PluginKindObjectStore, name: "aws"}: struct{}{}, + {kind: PluginKindBlockStore, name: "aws"}: struct{}{}, + + {kind: PluginKindObjectStore, name: "gcp"}: struct{}{}, + {kind: PluginKindBlockStore, name: "gcp"}: struct{}{}, + + {kind: PluginKindObjectStore, name: "azure"}: struct{}{}, + {kind: PluginKindBlockStore, name: "azure"}: struct{}{}, + }, + } +} + +func addPlugins(config *plugin.ClientConfig, kinds ...PluginKind) { + for _, kind := range kinds { + if kind == PluginKindObjectStore { + config.Plugins[kind.String()] = &ObjectStorePlugin{} + } else if kind == PluginKindBlockStore { + config.Plugins[kind.String()] = &BlockStorePlugin{} + } + } +} + +func (m *manager) getPlugin(descriptor pluginInfo, logger hclog.Logger) (interface{}, error) { + client, found := m.clients[descriptor] + if !found { + var ( + externalPath = filepath.Join(pluginDir, fmt.Sprintf("ark-%s-%s", descriptor.kind, descriptor.name)) + config = &plugin.ClientConfig{ + HandshakeConfig: Handshake, + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Plugins: make(map[string]plugin.Plugin), + Logger: logger, + } + ) + + // First check to see if there's an external plugin for this kind and name. this + // is so users can override the built-in plugins if they want. If it doesn't exist, + // see if there's an internal one. + if _, err := os.Stat(externalPath); err == nil { + addPlugins(config, descriptor.kind) + config.Cmd = exec.Command(externalPath) + + client = plugin.NewClient(config) + + m.clients[descriptor] = client + } else if _, found := m.internalPlugins[descriptor]; found { + addPlugins(config, PluginKindObjectStore, PluginKindBlockStore) + config.Cmd = exec.Command("/ark", "plugin", "cloudprovider", descriptor.name) + + client = plugin.NewClient(config) + + // since a single sub-process will serve both an object and block store + // for a given cloud-provider, record this client as being valid for both + m.clients[pluginInfo{PluginKindObjectStore, descriptor.name}] = client + m.clients[pluginInfo{PluginKindBlockStore, descriptor.name}] = client + } else { + return nil, errors.Errorf("plugin not found for kind=%s, name=%s", descriptor.kind, descriptor.name) + } + } + + protocolClient, err := client.Client() + if err != nil { + return nil, errors.WithStack(err) + } + + plugin, err := protocolClient.Dispense(descriptor.kind.String()) + if err != nil { + return nil, errors.WithStack(err) + } + + return plugin, nil +} + +// GetObjectStore returns the plugin implementation of the cloudprovider.ObjectStore +// interface with the specified name. +func (m *manager) GetObjectStore(name string) (cloudprovider.ObjectStore, error) { + pluginObj, err := m.getPlugin(pluginInfo{PluginKindObjectStore, name}, m.logger) + if err != nil { + return nil, err + } + + objStore, ok := pluginObj.(cloudprovider.ObjectStore) + if !ok { + return nil, errors.New("could not convert gRPC client to cloudprovider.ObjectStore") + } + + return objStore, nil +} + +// GetBlockStore returns the plugin implementation of the cloudprovider.BlockStore +// interface with the specified name. +func (m *manager) GetBlockStore(name string) (cloudprovider.BlockStore, error) { + pluginObj, err := m.getPlugin(pluginInfo{PluginKindBlockStore, name}, m.logger) + if err != nil { + return nil, err + } + + blockStore, ok := pluginObj.(cloudprovider.BlockStore) + if !ok { + return nil, errors.New("could not convert gRPC client to cloudprovider.BlockStore") + } + + return blockStore, nil +} diff --git a/pkg/plugin/object_store.go b/pkg/plugin/object_store.go new file mode 100644 index 000000000..17108bedc --- /dev/null +++ b/pkg/plugin/object_store.go @@ -0,0 +1,291 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "io" + "time" + + "github.com/hashicorp/go-plugin" + "golang.org/x/net/context" + "google.golang.org/grpc" + + "github.com/heptio/ark/pkg/cloudprovider" + proto "github.com/heptio/ark/pkg/plugin/generated" +) + +const byteChunkSize = 16384 + +// ObjectStorePlugin is an implementation of go-plugin's Plugin +// interface with support for gRPC for the cloudprovider/ObjectStore +// interface. +type ObjectStorePlugin struct { + plugin.NetRPCUnsupportedPlugin + + impl cloudprovider.ObjectStore +} + +// NewObjectStorePlugin construct an ObjectStorePlugin. +func NewObjectStorePlugin(objectStore cloudprovider.ObjectStore) *ObjectStorePlugin { + return &ObjectStorePlugin{ + impl: objectStore, + } +} + +// GRPCServer registers an ObjectStore gRPC server. +func (p *ObjectStorePlugin) GRPCServer(s *grpc.Server) error { + proto.RegisterObjectStoreServer(s, &ObjectStoreGRPCServer{impl: p.impl}) + return nil +} + +// GRPCClient returns an ObjectStore gRPC client. +func (p *ObjectStorePlugin) GRPCClient(c *grpc.ClientConn) (interface{}, error) { + return &ObjectStoreGRPCClient{grpcClient: proto.NewObjectStoreClient(c)}, nil +} + +// ObjectStoreGRPCClient implements the cloudprovider.ObjectStore interface and uses a +// gRPC client to make calls to the plugin server. +type ObjectStoreGRPCClient struct { + grpcClient proto.ObjectStoreClient +} + +// Init prepares the ObjectStore for usage using the provided map of +// configuration key-value pairs. It returns an error if the ObjectStore +// cannot be initialized from the provided config. +func (c *ObjectStoreGRPCClient) Init(config map[string]string) error { + _, err := c.grpcClient.Init(context.Background(), &proto.InitRequest{Config: config}) + + return err +} + +// PutObject creates a new object using the data in body within the specified +// object storage bucket with the given key. +func (c *ObjectStoreGRPCClient) PutObject(bucket, key string, body io.Reader) error { + stream, err := c.grpcClient.PutObject(context.Background()) + if err != nil { + return err + } + + // read from the provider io.Reader into chunks, and send each one over + // the gRPC stream + chunk := make([]byte, byteChunkSize) + for { + n, err := body.Read(chunk) + if err == io.EOF { + _, resErr := stream.CloseAndRecv() + return resErr + } + if err != nil { + stream.CloseSend() + return err + } + + if err := stream.Send(&proto.PutObjectRequest{Bucket: bucket, Key: key, Body: chunk[0:n]}); err != nil { + return err + } + } +} + +// GetObject retrieves the object with the given key from the specified +// bucket in object storage. +func (c *ObjectStoreGRPCClient) GetObject(bucket, key string) (io.ReadCloser, error) { + stream, err := c.grpcClient.GetObject(context.Background(), &proto.GetObjectRequest{Bucket: bucket, Key: key}) + if err != nil { + return nil, err + } + + receive := func() ([]byte, error) { + data, err := stream.Recv() + if err != nil { + return nil, err + } + + return data.Data, nil + } + + close := func() error { + return stream.CloseSend() + } + + return &StreamReadCloser{receive: receive, close: close}, nil +} + +// ListCommonPrefixes gets a list of all object key prefixes that come +// before the provided delimiter (this is often used to simulate a directory +// hierarchy in object storage). +func (c *ObjectStoreGRPCClient) ListCommonPrefixes(bucket, delimiter string) ([]string, error) { + res, err := c.grpcClient.ListCommonPrefixes(context.Background(), &proto.ListCommonPrefixesRequest{Bucket: bucket, Delimiter: delimiter}) + if err != nil { + return nil, err + } + + return res.Prefixes, nil +} + +// ListObjects gets a list of all objects in bucket that have the same prefix. +func (c *ObjectStoreGRPCClient) ListObjects(bucket, prefix string) ([]string, error) { + res, err := c.grpcClient.ListObjects(context.Background(), &proto.ListObjectsRequest{Bucket: bucket, Prefix: prefix}) + if err != nil { + return nil, err + } + + return res.Keys, nil +} + +// DeleteObject removes object with the specified key from the given +// bucket. +func (c *ObjectStoreGRPCClient) DeleteObject(bucket, key string) error { + _, err := c.grpcClient.DeleteObject(context.Background(), &proto.DeleteObjectRequest{Bucket: bucket, Key: key}) + + return err +} + +// CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl. +func (c *ObjectStoreGRPCClient) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) { + res, err := c.grpcClient.CreateSignedURL(context.Background(), &proto.CreateSignedURLRequest{ + Bucket: bucket, + Key: key, + Ttl: int64(ttl), + }) + if err != nil { + return "", nil + } + + return res.Url, nil +} + +// ObjectStoreGRPCServer implements the proto-generated ObjectStoreServer interface, and accepts +// gRPC calls and forwards them to an implementation of the pluggable interface. +type ObjectStoreGRPCServer struct { + impl cloudprovider.ObjectStore +} + +// Init prepares the ObjectStore for usage using the provided map of +// configuration key-value pairs. It returns an error if the ObjectStore +// cannot be initialized from the provided config. +func (s *ObjectStoreGRPCServer) Init(ctx context.Context, req *proto.InitRequest) (*proto.Empty, error) { + if err := s.impl.Init(req.Config); err != nil { + return nil, err + } + + return &proto.Empty{}, nil +} + +// PutObject creates a new object using the data in body within the specified +// object storage bucket with the given key. +func (s *ObjectStoreGRPCServer) PutObject(stream proto.ObjectStore_PutObjectServer) error { + // we need to read the first chunk ahead of time to get the bucket and key; + // in our receive method, we'll use `first` on the first call + firstChunk, err := stream.Recv() + if err != nil { + return err + } + + bucket := firstChunk.Bucket + key := firstChunk.Key + + receive := func() ([]byte, error) { + if firstChunk != nil { + res := firstChunk.Body + firstChunk = nil + return res, nil + } + + data, err := stream.Recv() + if err != nil { + return nil, err + } + return data.Body, nil + } + + close := func() error { + return nil + } + + if err := s.impl.PutObject(bucket, key, &StreamReadCloser{receive: receive, close: close}); err != nil { + return err + } + + return stream.SendAndClose(&proto.Empty{}) +} + +// GetObject retrieves the object with the given key from the specified +// bucket in object storage. +func (s *ObjectStoreGRPCServer) GetObject(req *proto.GetObjectRequest, stream proto.ObjectStore_GetObjectServer) error { + rdr, err := s.impl.GetObject(req.Bucket, req.Key) + if err != nil { + return err + } + + chunk := make([]byte, byteChunkSize) + for { + n, err := rdr.Read(chunk) + if err != nil && err != io.EOF { + return err + } + if n == 0 { + return nil + } + + if err := stream.Send(&proto.Bytes{Data: chunk[0:n]}); err != nil { + return err + } + } +} + +// ListCommonPrefixes gets a list of all object key prefixes that come +// before the provided delimiter (this is often used to simulate a directory +// hierarchy in object storage). +func (s *ObjectStoreGRPCServer) ListCommonPrefixes(ctx context.Context, req *proto.ListCommonPrefixesRequest) (*proto.ListCommonPrefixesResponse, error) { + prefixes, err := s.impl.ListCommonPrefixes(req.Bucket, req.Delimiter) + if err != nil { + return nil, err + } + + return &proto.ListCommonPrefixesResponse{Prefixes: prefixes}, nil +} + +// ListObjects gets a list of all objects in bucket that have the same prefix. +func (s *ObjectStoreGRPCServer) ListObjects(ctx context.Context, req *proto.ListObjectsRequest) (*proto.ListObjectsResponse, error) { + keys, err := s.impl.ListObjects(req.Bucket, req.Prefix) + if err != nil { + return nil, err + } + + return &proto.ListObjectsResponse{Keys: keys}, nil +} + +// DeleteObject removes object with the specified key from the given +// bucket. +func (s *ObjectStoreGRPCServer) DeleteObject(ctx context.Context, req *proto.DeleteObjectRequest) (*proto.Empty, error) { + err := s.impl.DeleteObject(req.Bucket, req.Key) + if err != nil { + return nil, err + } + + return &proto.Empty{}, nil +} + +// CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl. +func (s *ObjectStoreGRPCServer) CreateSignedURL(ctx context.Context, req *proto.CreateSignedURLRequest) (*proto.CreateSignedURLResponse, error) { + url, err := s.impl.CreateSignedURL(req.Bucket, req.Key, time.Duration(req.Ttl)) + if err != nil { + return nil, err + } + + return &proto.CreateSignedURLResponse{Url: url}, nil +} diff --git a/pkg/plugin/proto/BlockStore.proto b/pkg/plugin/proto/BlockStore.proto new file mode 100644 index 000000000..c126ad51c --- /dev/null +++ b/pkg/plugin/proto/BlockStore.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; +package generated; + +import "Shared.proto"; + +message CreateVolumeRequest { + string snapshotID = 1; + string volumeType = 2; + string volumeAZ = 3; + int64 iops = 4; +} + +message CreateVolumeResponse { + string volumeID = 1; +} + +message GetVolumeInfoRequest { + string volumeID = 1; + string volumeAZ = 2; +} + +message GetVolumeInfoResponse { + string volumeType = 1; + int64 iops = 2; +} + +message IsVolumeReadyRequest { + string volumeID = 1; + string volumeAZ = 2; +} + +message IsVolumeReadyResponse { + bool ready = 1; +} + +message ListSnapshotsRequest { + map tagFilters = 1; +} + +message ListSnapshotsResponse { + repeated string snapshotIDs = 2; +} + +message CreateSnapshotRequest { + string volumeID = 1; + string volumeAZ = 2; + map tags = 3; +} + +message CreateSnapshotResponse { + string snapshotID = 1; +} + +message DeleteSnapshotRequest { + string snapshotID = 1; +} + +service BlockStore { + rpc Init(InitRequest) returns (Empty); + rpc CreateVolumeFromSnapshot(CreateVolumeRequest) returns (CreateVolumeResponse); + rpc GetVolumeInfo(GetVolumeInfoRequest) returns (GetVolumeInfoResponse); + rpc IsVolumeReady(IsVolumeReadyRequest) returns (IsVolumeReadyResponse); + rpc ListSnapshots(ListSnapshotsRequest) returns (ListSnapshotsResponse); + rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse); + rpc DeleteSnapshot(DeleteSnapshotRequest) returns (Empty); +} diff --git a/pkg/plugin/proto/ObjectStore.proto b/pkg/plugin/proto/ObjectStore.proto new file mode 100644 index 000000000..072d17b6d --- /dev/null +++ b/pkg/plugin/proto/ObjectStore.proto @@ -0,0 +1,63 @@ +syntax = "proto3"; +package generated; + +import "Shared.proto"; + +message PutObjectRequest { + string bucket = 1; + string key = 2; + bytes body = 3; +} + +message GetObjectRequest { + string bucket = 1; + string key = 2; +} + +message Bytes { + bytes data = 1; +} + +message ListCommonPrefixesRequest { + string bucket = 1; + string delimiter = 2; +} + +message ListCommonPrefixesResponse { + repeated string prefixes = 1; +} + +message ListObjectsRequest { + string bucket = 1; + string prefix = 2; +} + +message ListObjectsResponse { + repeated string keys = 1; +} + +message DeleteObjectRequest { + string bucket = 1; + string key = 2; +} + + +message CreateSignedURLRequest { + string bucket = 1; + string key = 2; + int64 ttl = 3; +} + +message CreateSignedURLResponse { + string url = 1; +} + +service ObjectStore { + rpc Init(InitRequest) returns (Empty); + rpc PutObject(stream PutObjectRequest) returns (Empty); + rpc GetObject(GetObjectRequest) returns (stream Bytes); + rpc ListCommonPrefixes(ListCommonPrefixesRequest) returns (ListCommonPrefixesResponse); + rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse); + rpc DeleteObject(DeleteObjectRequest) returns (Empty); + rpc CreateSignedURL(CreateSignedURLRequest) returns (CreateSignedURLResponse); +} diff --git a/pkg/plugin/proto/Shared.proto b/pkg/plugin/proto/Shared.proto new file mode 100644 index 000000000..f1b1bc284 --- /dev/null +++ b/pkg/plugin/proto/Shared.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package generated; + +message Empty {} + +message InitRequest { + map config = 1; +} \ No newline at end of file diff --git a/pkg/plugin/shared.go b/pkg/plugin/shared.go new file mode 100644 index 000000000..0b92621cb --- /dev/null +++ b/pkg/plugin/shared.go @@ -0,0 +1,27 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import plugin "github.com/hashicorp/go-plugin" + +// Handshake is configuration information that allows go-plugin +// clients and servers to perform a handshake. +var Handshake = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARK_PLUGIN", + MagicCookieValue: "hello", +} diff --git a/pkg/plugin/stream_reader.go b/pkg/plugin/stream_reader.go new file mode 100644 index 000000000..5d6cfe405 --- /dev/null +++ b/pkg/plugin/stream_reader.go @@ -0,0 +1,75 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "bytes" + "io" +) + +// ReceiveFunc is a function that either returns a slice +// of an arbitrary number of bytes OR an error. Returning +// an io.EOF means there is no more data to be read; any +// other error is considered an actual error. +type ReceiveFunc func() ([]byte, error) + +// CloseFunc is used to signal to the source of data that +// the StreamReadCloser has been closed. +type CloseFunc func() error + +// StreamReadCloser wraps a ReceiveFunc and a CloseSendFunc +// to implement io.ReadCloser. +type StreamReadCloser struct { + buf *bytes.Buffer + receive ReceiveFunc + close CloseFunc +} + +func (s *StreamReadCloser) Read(p []byte) (n int, err error) { + for { + // if buf exists and holds at least as much as we're trying to read, + // read from the buffer + if s.buf != nil && s.buf.Len() >= len(p) { + return s.buf.Read(p) + } + + // if buf is nil, create it + if s.buf == nil { + s.buf = new(bytes.Buffer) + } + + // buf exists but doesn't hold enough data to fill p, so + // receive again. If we get an EOF, return what's in the + // buffer; else, write the new data to the buffer and + // try another read. + data, err := s.receive() + if err == io.EOF { + return s.buf.Read(p) + } + if err != nil { + return 0, err + } + + if _, err := s.buf.Write(data); err != nil { + return 0, err + } + } +} + +func (s *StreamReadCloser) Close() error { + return s.close() +} diff --git a/pkg/plugin/stream_reader_test.go b/pkg/plugin/stream_reader_test.go new file mode 100644 index 000000000..73cc83e58 --- /dev/null +++ b/pkg/plugin/stream_reader_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type stringByteReceiver struct { + buf *bytes.Buffer + chunkSize int +} + +func (r *stringByteReceiver) Receive() ([]byte, error) { + chunk := make([]byte, r.chunkSize) + + n, err := r.buf.Read(chunk) + if err != nil { + return nil, err + } + + return chunk[0:n], nil +} + +func (r *stringByteReceiver) CloseSend() error { + r.buf = nil + return nil +} + +func TestStreamReader(t *testing.T) { + s := "hello world, it's me, streamreader!!!!!" + + rdr := &stringByteReceiver{ + buf: bytes.NewBufferString(s), + chunkSize: 3, + } + + sr := &StreamReadCloser{ + receive: rdr.Receive, + close: rdr.CloseSend, + } + + res, err := ioutil.ReadAll(sr) + + require.Nil(t, err) + assert.Equal(t, s, string(res)) +} diff --git a/pkg/util/logging/hclog_level_hook.go b/pkg/util/logging/hclog_level_hook.go new file mode 100644 index 000000000..9b8772d76 --- /dev/null +++ b/pkg/util/logging/hclog_level_hook.go @@ -0,0 +1,45 @@ +/* +Copyright 2017 Heptio Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +import ( + "github.com/sirupsen/logrus" +) + +// HcLogLevelHook adds an hclog-compatible field ("@level") containing +// the log level. Note that if you use this, you SHOULD NOT use +// logrus.JSONFormatter's FieldMap to set the level key to "@level" because +// that will result in the hclog-compatible info written here being +// overwritten. +type HcLogLevelHook struct{} + +func (h *HcLogLevelHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *HcLogLevelHook) Fire(entry *logrus.Entry) error { + switch entry.Level { + // logrus uses "warning" to represent WarnLevel, + // which is not compatible with hclog's "warn". + case logrus.WarnLevel: + entry.Data["@level"] = "warn" + default: + entry.Data["@level"] = entry.Level.String() + } + + return nil +} diff --git a/pkg/util/logging/log_location_hook.go b/pkg/util/logging/log_location_hook.go index 784e3499b..6362a9471 100644 --- a/pkg/util/logging/log_location_hook.go +++ b/pkg/util/logging/log_location_hook.go @@ -24,11 +24,28 @@ import ( "github.com/sirupsen/logrus" ) -const logLocationField = "logSource" +const ( + logSourceField = "logSource" + logSourceSetMarkerField = "@logSourceSetBy" +) // LogLocationHook is a logrus hook that attaches location information // to log entries, i.e. the file and line number of the logrus log call. +// This hook is designed for use in both the Ark server and Ark plugin +// implementations. When triggered within a plugin, a marker field will +// be set on the log entry indicating that the location came from a plugin. +// The Ark server instance will not overwrite location information if +// it sees this marker. type LogLocationHook struct { + loggerName string +} + +// WithLoggerName gives the hook a name to use when setting the marker field +// on a log entry indicating the location has been recorded by a plugin. This +// should only be used when setting up a hook for a logger used in a plugin. +func (h *LogLocationHook) WithLoggerName(name string) *LogLocationHook { + h.loggerName = name + return h } func (h *LogLocationHook) Levels() []logrus.Level { @@ -60,9 +77,40 @@ func (h *LogLocationHook) Fire(entry *logrus.Entry) error { continue } - entry.Data[logLocationField] = fmt.Sprintf("%s:%d", frame.File, frame.Line) + // set the marker field if we're within a plugin indicating that + // the location comes from the plugin. + if h.loggerName != "" { + entry.Data[logSourceSetMarkerField] = h.loggerName + } + + // record the log statement location if we're within a plugin OR if + // we're in Ark server and not logging something that has the marker + // set (which would indicate the log statement is coming from a plugin). + if h.loggerName != "" || getLogSourceSetMarker(entry) == "" { + entry.Data[logSourceField] = fmt.Sprintf("%s:%d", frame.File, frame.Line) + } + + // if we're in the Ark server, remove the marker field since we don't + // want to record it in the actual log. + if h.loggerName == "" { + delete(entry.Data, logSourceSetMarkerField) + } + break } return nil } + +func getLogSourceSetMarker(entry *logrus.Entry) string { + nameVal, found := entry.Data[logSourceSetMarkerField] + if !found { + return "" + } + + if name, ok := nameVal.(string); ok { + return name + } + + return fmt.Sprintf("%s", nameVal) +} diff --git a/pkg/util/test/object_storage_adapter.go b/pkg/util/test/object_store.go similarity index 76% rename from pkg/util/test/object_storage_adapter.go rename to pkg/util/test/object_store.go index e3c6d830c..57616167e 100644 --- a/pkg/util/test/object_storage_adapter.go +++ b/pkg/util/test/object_store.go @@ -21,13 +21,13 @@ import io "io" import mock "github.com/stretchr/testify/mock" import time "time" -// ObjectStorageAdapter is an autogenerated mock type for the ObjectStorageAdapter type -type ObjectStorageAdapter struct { +// ObjectStore is an autogenerated mock type for the ObjectStore type +type ObjectStore struct { mock.Mock } // CreateSignedURL provides a mock function with given fields: bucket, key, ttl -func (_m *ObjectStorageAdapter) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) { +func (_m *ObjectStore) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) { ret := _m.Called(bucket, key, ttl) var r0 string @@ -48,7 +48,7 @@ func (_m *ObjectStorageAdapter) CreateSignedURL(bucket string, key string, ttl t } // DeleteObject provides a mock function with given fields: bucket, key -func (_m *ObjectStorageAdapter) DeleteObject(bucket string, key string) error { +func (_m *ObjectStore) DeleteObject(bucket string, key string) error { ret := _m.Called(bucket, key) var r0 error @@ -62,7 +62,7 @@ func (_m *ObjectStorageAdapter) DeleteObject(bucket string, key string) error { } // GetObject provides a mock function with given fields: bucket, key -func (_m *ObjectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) { +func (_m *ObjectStore) GetObject(bucket string, key string) (io.ReadCloser, error) { ret := _m.Called(bucket, key) var r0 io.ReadCloser @@ -84,8 +84,22 @@ func (_m *ObjectStorageAdapter) GetObject(bucket string, key string) (io.ReadClo return r0, r1 } +// Init provides a mock function with given fields: config +func (_m *ObjectStore) Init(config map[string]string) error { + ret := _m.Called(config) + + var r0 error + if rf, ok := ret.Get(0).(func(map[string]string) error); ok { + r0 = rf(config) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // ListCommonPrefixes provides a mock function with given fields: bucket, delimiter -func (_m *ObjectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { +func (_m *ObjectStore) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) { ret := _m.Called(bucket, delimiter) var r0 []string @@ -108,7 +122,7 @@ func (_m *ObjectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri } // ListObjects provides a mock function with given fields: bucket, prefix -func (_m *ObjectStorageAdapter) ListObjects(bucket string, prefix string) ([]string, error) { +func (_m *ObjectStore) ListObjects(bucket string, prefix string) ([]string, error) { ret := _m.Called(bucket, prefix) var r0 []string @@ -131,7 +145,7 @@ func (_m *ObjectStorageAdapter) ListObjects(bucket string, prefix string) ([]str } // PutObject provides a mock function with given fields: bucket, key, body -func (_m *ObjectStorageAdapter) PutObject(bucket string, key string, body io.Reader) error { +func (_m *ObjectStore) PutObject(bucket string, key string, body io.Reader) error { ret := _m.Called(bucket, key, body) var r0 error diff --git a/vendor/github.com/hashicorp/go-hclog/LICENSE b/vendor/github.com/hashicorp/go-hclog/LICENSE new file mode 100644 index 000000000..abaf1e45f --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 HashiCorp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/hashicorp/go-hclog/README.md b/vendor/github.com/hashicorp/go-hclog/README.md new file mode 100644 index 000000000..614342b2d --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/README.md @@ -0,0 +1,123 @@ +# go-hclog + +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[godocs]: https://godoc.org/github.com/hashicorp/go-hclog + +`go-hclog` is a package for Go that provides a simple key/value logging +interface for use in development and production environments. + +It provides logging levels that provide decreased output based upon the +desired amount of output, unlike the standard library `log` package. + +It does not provide `Printf` style logging, only key/value logging that is +exposed as arguments to the logging functions for simplicity. + +It provides a human readable output mode for use in development as well as +JSON output mode for production. + +## Stability Note + +While this library is fully open source and HashiCorp will be maintaining it +(since we are and will be making extensive use of it), the API and output +format is subject to minor changes as we fully bake and vet it in our projects. +This notice will be removed once it's fully integrated into our major projects +and no further changes are anticipated. + +## Installation and Docs + +Install using `go get github.com/hashicorp/go-hclog`. + +Full documentation is available at +http://godoc.org/github.com/hashicorp/go-hclog + +## Usage + +### Use the global logger + +```go +hclog.Default().Info("hello world") +``` + +```text +2017-07-05T16:15:55.167-0700 [INFO ] hello world +``` + +(Note timestamps are removed in future examples for brevity.) + +### Create a new logger + +```go +appLogger := hclog.New(&hclog.LoggerOptions{ + Name: "my-app", + Level: hclog.LevelFromString("DEBUG"), +}) +``` + +### Emit an Info level message with 2 key/value pairs + +```go +input := "5.5" +_, err := strconv.ParseInt(input, 10, 32) +if err != nil { + appLogger.Info("Invalid input for ParseInt", "input", input, "error", err) +} +``` + +```text +... [INFO ] my-app: Invalid input for ParseInt: input=5.5 error="strconv.ParseInt: parsing "5.5": invalid syntax" +``` + +### Create a new Logger for a major subsystem + +```go +subsystemLogger := appLogger.Named("transport") +subsystemLogger.Info("we are transporting something") +``` + +```text +... [INFO ] my-app.transport: we are transporting something +``` + +Notice that logs emitted by `subsystemLogger` contain `my-app.transport`, +reflecting both the application and subsystem names. + +### Create a new Logger with fixed key/value pairs + +Using `With()` will include a specific key-value pair in all messages emitted +by that logger. + +```go +requestID := "5fb446b6-6eba-821d-df1b-cd7501b6a363" +requestLogger := subsystemLogger.With("request", requestID) +requestLogger.Info("we are transporting a request") +``` + +```text +... [INFO ] my-app.transport: we are transporting a request: request=5fb446b6-6eba-821d-df1b-cd7501b6a363 +``` + +This allows sub Loggers to be context specific without having to thread that +into all the callers. + +### Use this with code that uses the standard library logger + +If you want to use the standard library's `log.Logger` interface you can wrap +`hclog.Logger` by calling the `StandardLogger()` method. This allows you to use +it with the familiar `Println()`, `Printf()`, etc. For example: + +```go +stdLogger := appLogger.StandardLogger(&hclog.StandardLoggerOptions{ + InferLevels: true, +}) +// Printf() is provided by stdlib log.Logger interface, not hclog.Logger +stdLogger.Printf("[DEBUG] %+v", stdLogger) +``` + +```text +... [DEBUG] my-app: &{mu:{state:0 sema:0} prefix: flag:0 out:0xc42000a0a0 buf:[]} +``` + +Notice that if `appLogger` is initialized with the `INFO` log level _and_ you +specify `InferLevels: true`, you will not see any output here. You must change +`appLogger` to `DEBUG` to see output. See the docs for more information. diff --git a/vendor/github.com/hashicorp/go-hclog/global.go b/vendor/github.com/hashicorp/go-hclog/global.go new file mode 100644 index 000000000..55ce43960 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/global.go @@ -0,0 +1,34 @@ +package hclog + +import ( + "sync" +) + +var ( + protect sync.Once + def Logger + + // The options used to create the Default logger. These are + // read only when the Default logger is created, so set them + // as soon as the process starts. + DefaultOptions = &LoggerOptions{ + Level: DefaultLevel, + Output: DefaultOutput, + } +) + +// Return a logger that is held globally. This can be a good starting +// place, and then you can use .With() and .Name() to create sub-loggers +// to be used in more specific contexts. +func Default() Logger { + protect.Do(func() { + def = New(DefaultOptions) + }) + + return def +} + +// A short alias for Default() +func L() Logger { + return Default() +} diff --git a/vendor/github.com/hashicorp/go-hclog/int.go b/vendor/github.com/hashicorp/go-hclog/int.go new file mode 100644 index 000000000..20adcfbb9 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/int.go @@ -0,0 +1,404 @@ +package hclog + +import ( + "bufio" + "encoding" + "encoding/json" + "fmt" + "log" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +var ( + _levelToBracket = map[Level]string{ + Debug: "[DEBUG]", + Trace: "[TRACE]", + Info: "[INFO ]", + Warn: "[WARN ]", + Error: "[ERROR]", + } +) + +// Given the options (nil for defaults), create a new Logger +func New(opts *LoggerOptions) Logger { + if opts == nil { + opts = &LoggerOptions{} + } + + output := opts.Output + if output == nil { + output = os.Stderr + } + + level := opts.Level + if level == NoLevel { + level = DefaultLevel + } + + mtx := opts.Mutex + if mtx == nil { + mtx = new(sync.Mutex) + } + + return &intLogger{ + m: mtx, + json: opts.JSONFormat, + caller: opts.IncludeLocation, + name: opts.Name, + w: bufio.NewWriter(output), + level: level, + } +} + +// The internal logger implementation. Internal in that it is defined entirely +// by this package. +type intLogger struct { + json bool + caller bool + name string + + // this is a pointer so that it's shared by any derived loggers, since + // those derived loggers share the bufio.Writer as well. + m *sync.Mutex + w *bufio.Writer + level Level + + implied []interface{} +} + +// Make sure that intLogger is a Logger +var _ Logger = &intLogger{} + +// The time format to use for logging. This is a version of RFC3339 that +// contains millisecond precision +const TimeFormat = "2006-01-02T15:04:05.000Z0700" + +// Log a message and a set of key/value pairs if the given level is at +// or more severe that the threshold configured in the Logger. +func (z *intLogger) Log(level Level, msg string, args ...interface{}) { + if level < z.level { + return + } + + t := time.Now() + + z.m.Lock() + defer z.m.Unlock() + + if z.json { + z.logJson(t, level, msg, args...) + } else { + z.log(t, level, msg, args...) + } + + z.w.Flush() +} + +// Cleanup a path by returning the last 2 segments of the path only. +func trimCallerPath(path string) string { + // lovely borrowed from zap + // nb. To make sure we trim the path correctly on Windows too, we + // counter-intuitively need to use '/' and *not* os.PathSeparator here, + // because the path given originates from Go stdlib, specifically + // runtime.Caller() which (as of Mar/17) returns forward slashes even on + // Windows. + // + // See https://github.com/golang/go/issues/3335 + // and https://github.com/golang/go/issues/18151 + // + // for discussion on the issue on Go side. + // + + // Find the last separator. + // + idx := strings.LastIndexByte(path, '/') + if idx == -1 { + return path + } + + // Find the penultimate separator. + idx = strings.LastIndexByte(path[:idx], '/') + if idx == -1 { + return path + } + + return path[idx+1:] +} + +// Non-JSON logging format function +func (z *intLogger) log(t time.Time, level Level, msg string, args ...interface{}) { + z.w.WriteString(t.Format(TimeFormat)) + z.w.WriteByte(' ') + + s, ok := _levelToBracket[level] + if ok { + z.w.WriteString(s) + } else { + z.w.WriteString("[UNKN ]") + } + + if z.caller { + if _, file, line, ok := runtime.Caller(3); ok { + z.w.WriteByte(' ') + z.w.WriteString(trimCallerPath(file)) + z.w.WriteByte(':') + z.w.WriteString(strconv.Itoa(line)) + z.w.WriteByte(':') + } + } + + z.w.WriteByte(' ') + + if z.name != "" { + z.w.WriteString(z.name) + z.w.WriteString(": ") + } + + z.w.WriteString(msg) + + args = append(z.implied, args...) + + var stacktrace CapturedStacktrace + + if args != nil && len(args) > 0 { + if len(args)%2 != 0 { + cs, ok := args[len(args)-1].(CapturedStacktrace) + if ok { + args = args[:len(args)-1] + stacktrace = cs + } else { + args = append(args, "") + } + } + + z.w.WriteByte(':') + + FOR: + for i := 0; i < len(args); i = i + 2 { + var val string + + switch st := args[i+1].(type) { + case string: + val = st + case int: + val = strconv.FormatInt(int64(st), 10) + case int64: + val = strconv.FormatInt(int64(st), 10) + case int32: + val = strconv.FormatInt(int64(st), 10) + case int16: + val = strconv.FormatInt(int64(st), 10) + case int8: + val = strconv.FormatInt(int64(st), 10) + case uint: + val = strconv.FormatUint(uint64(st), 10) + case uint64: + val = strconv.FormatUint(uint64(st), 10) + case uint32: + val = strconv.FormatUint(uint64(st), 10) + case uint16: + val = strconv.FormatUint(uint64(st), 10) + case uint8: + val = strconv.FormatUint(uint64(st), 10) + case CapturedStacktrace: + stacktrace = st + continue FOR + default: + val = fmt.Sprintf("%v", st) + } + + z.w.WriteByte(' ') + z.w.WriteString(args[i].(string)) + z.w.WriteByte('=') + + if strings.ContainsAny(val, " \t\n\r") { + z.w.WriteByte('"') + z.w.WriteString(val) + z.w.WriteByte('"') + } else { + z.w.WriteString(val) + } + } + } + + z.w.WriteString("\n") + + if stacktrace != "" { + z.w.WriteString(string(stacktrace)) + } +} + +// JSON logging function +func (z *intLogger) logJson(t time.Time, level Level, msg string, args ...interface{}) { + vals := map[string]interface{}{ + "@message": msg, + "@timestamp": t.Format("2006-01-02T15:04:05.000000Z07:00"), + } + + var levelStr string + switch level { + case Error: + levelStr = "error" + case Warn: + levelStr = "warn" + case Info: + levelStr = "info" + case Debug: + levelStr = "debug" + case Trace: + levelStr = "trace" + default: + levelStr = "all" + } + + vals["@level"] = levelStr + + if z.name != "" { + vals["@module"] = z.name + } + + if z.caller { + if _, file, line, ok := runtime.Caller(3); ok { + vals["@caller"] = fmt.Sprintf("%s:%d", file, line) + } + } + + if args != nil && len(args) > 0 { + if len(args)%2 != 0 { + cs, ok := args[len(args)-1].(CapturedStacktrace) + if ok { + args = args[:len(args)-1] + vals["stacktrace"] = cs + } else { + args = append(args, "") + } + } + + for i := 0; i < len(args); i = i + 2 { + if _, ok := args[i].(string); !ok { + // As this is the logging function not much we can do here + // without injecting into logs... + continue + } + val := args[i+1] + // Check if val is of type error. If error type doesn't + // implement json.Marshaler or encoding.TextMarshaler + // then set val to err.Error() so that it gets marshaled + if err, ok := val.(error); ok { + switch err.(type) { + case json.Marshaler, encoding.TextMarshaler: + default: + val = err.Error() + } + } + vals[args[i].(string)] = val + } + } + + err := json.NewEncoder(z.w).Encode(vals) + if err != nil { + panic(err) + } +} + +// Emit the message and args at DEBUG level +func (z *intLogger) Debug(msg string, args ...interface{}) { + z.Log(Debug, msg, args...) +} + +// Emit the message and args at TRACE level +func (z *intLogger) Trace(msg string, args ...interface{}) { + z.Log(Trace, msg, args...) +} + +// Emit the message and args at INFO level +func (z *intLogger) Info(msg string, args ...interface{}) { + z.Log(Info, msg, args...) +} + +// Emit the message and args at WARN level +func (z *intLogger) Warn(msg string, args ...interface{}) { + z.Log(Warn, msg, args...) +} + +// Emit the message and args at ERROR level +func (z *intLogger) Error(msg string, args ...interface{}) { + z.Log(Error, msg, args...) +} + +// Indicate that the logger would emit TRACE level logs +func (z *intLogger) IsTrace() bool { + return z.level == Trace +} + +// Indicate that the logger would emit DEBUG level logs +func (z *intLogger) IsDebug() bool { + return z.level <= Debug +} + +// Indicate that the logger would emit INFO level logs +func (z *intLogger) IsInfo() bool { + return z.level <= Info +} + +// Indicate that the logger would emit WARN level logs +func (z *intLogger) IsWarn() bool { + return z.level <= Warn +} + +// Indicate that the logger would emit ERROR level logs +func (z *intLogger) IsError() bool { + return z.level <= Error +} + +// Return a sub-Logger for which every emitted log message will contain +// the given key/value pairs. This is used to create a context specific +// Logger. +func (z *intLogger) With(args ...interface{}) Logger { + var nz intLogger = *z + + nz.implied = append(nz.implied, args...) + + return &nz +} + +// Create a new sub-Logger that a name decending from the current name. +// This is used to create a subsystem specific Logger. +func (z *intLogger) Named(name string) Logger { + var nz intLogger = *z + + if nz.name != "" { + nz.name = nz.name + "." + name + } else { + nz.name = name + } + + return &nz +} + +// Create a new sub-Logger with an explicit name. This ignores the current +// name. This is used to create a standalone logger that doesn't fall +// within the normal hierarchy. +func (z *intLogger) ResetNamed(name string) Logger { + var nz intLogger = *z + + nz.name = name + + return &nz +} + +// Create a *log.Logger that will send it's data through this Logger. This +// allows packages that expect to be using the standard library log to actually +// use this logger. +func (z *intLogger) StandardLogger(opts *StandardLoggerOptions) *log.Logger { + if opts == nil { + opts = &StandardLoggerOptions{} + } + + return log.New(&stdlogAdapter{z, opts.InferLevels}, "", 0) +} diff --git a/vendor/github.com/hashicorp/go-hclog/log.go b/vendor/github.com/hashicorp/go-hclog/log.go new file mode 100644 index 000000000..dbc4198a5 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/log.go @@ -0,0 +1,142 @@ +package hclog + +import ( + "io" + "log" + "os" + "strings" + "sync" +) + +var ( + DefaultOutput = os.Stderr + DefaultLevel = Info +) + +type Level int + +const ( + // This is a special level used to indicate that no level has been + // set and allow for a default to be used. + NoLevel Level = 0 + + // The most verbose level. Intended to be used for the tracing of actions + // in code, such as function enters/exits, etc. + Trace Level = 1 + + // For programmer lowlevel analysis. + Debug Level = 2 + + // For information about steady state operations. + Info Level = 3 + + // For information about rare but handled events. + Warn Level = 4 + + // For information about unrecoverable events. + Error Level = 5 +) + +// LevelFromString returns a Level type for the named log level, or "NoLevel" if +// the level string is invalid. This facilitates setting the log level via +// config or environment variable by name in a predictable way. +func LevelFromString(levelStr string) Level { + // We don't care about case. Accept "INFO" or "info" + levelStr = strings.ToLower(strings.TrimSpace(levelStr)) + switch levelStr { + case "trace": + return Trace + case "debug": + return Debug + case "info": + return Info + case "warn": + return Warn + case "error": + return Error + default: + return NoLevel + } +} + +// The main Logger interface. All code should code against this interface only. +type Logger interface { + // Args are alternating key, val pairs + // keys must be strings + // vals can be any type, but display is implementation specific + // Emit a message and key/value pairs at the TRACE level + Trace(msg string, args ...interface{}) + + // Emit a message and key/value pairs at the DEBUG level + Debug(msg string, args ...interface{}) + + // Emit a message and key/value pairs at the INFO level + Info(msg string, args ...interface{}) + + // Emit a message and key/value pairs at the WARN level + Warn(msg string, args ...interface{}) + + // Emit a message and key/value pairs at the ERROR level + Error(msg string, args ...interface{}) + + // Indicate if TRACE logs would be emitted. This and the other Is* guards + // are used to elide expensive logging code based on the current level. + IsTrace() bool + + // Indicate if DEBUG logs would be emitted. This and the other Is* guards + IsDebug() bool + + // Indicate if INFO logs would be emitted. This and the other Is* guards + IsInfo() bool + + // Indicate if WARN logs would be emitted. This and the other Is* guards + IsWarn() bool + + // Indicate if ERROR logs would be emitted. This and the other Is* guards + IsError() bool + + // Creates a sublogger that will always have the given key/value pairs + With(args ...interface{}) Logger + + // Create a logger that will prepend the name string on the front of all messages. + // If the logger already has a name, the new value will be appended to the current + // name. That way, a major subsystem can use this to decorate all it's own logs + // without losing context. + Named(name string) Logger + + // Create a logger that will prepend the name string on the front of all messages. + // This sets the name of the logger to the value directly, unlike Named which honor + // the current name as well. + ResetNamed(name string) Logger + + // Return a value that conforms to the stdlib log.Logger interface + StandardLogger(opts *StandardLoggerOptions) *log.Logger +} + +type StandardLoggerOptions struct { + // Indicate that some minimal parsing should be done on strings to try + // and detect their level and re-emit them. + // This supports the strings like [ERROR], [ERR] [TRACE], [WARN], [INFO], + // [DEBUG] and strip it off before reapplying it. + InferLevels bool +} + +type LoggerOptions struct { + // Name of the subsystem to prefix logs with + Name string + + // The threshold for the logger. Anything less severe is supressed + Level Level + + // Where to write the logs to. Defaults to os.Stdout if nil + Output io.Writer + + // An optional mutex pointer in case Output is shared + Mutex *sync.Mutex + + // Control if the output should be in JSON. + JSONFormat bool + + // Include file and line information in each log line + IncludeLocation bool +} diff --git a/vendor/github.com/hashicorp/go-hclog/stacktrace.go b/vendor/github.com/hashicorp/go-hclog/stacktrace.go new file mode 100644 index 000000000..8af1a3be4 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/stacktrace.go @@ -0,0 +1,108 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package hclog + +import ( + "bytes" + "runtime" + "strconv" + "strings" + "sync" +) + +var ( + _stacktraceIgnorePrefixes = []string{ + "runtime.goexit", + "runtime.main", + } + _stacktracePool = sync.Pool{ + New: func() interface{} { + return newProgramCounters(64) + }, + } +) + +// A stacktrace gathered by a previous call to log.Stacktrace. If passed +// to a logging function, the stacktrace will be appended. +type CapturedStacktrace string + +// Gather a stacktrace of the current goroutine and return it to be passed +// to a logging function. +func Stacktrace() CapturedStacktrace { + return CapturedStacktrace(takeStacktrace()) +} + +func takeStacktrace() string { + programCounters := _stacktracePool.Get().(*programCounters) + defer _stacktracePool.Put(programCounters) + + var buffer bytes.Buffer + + for { + // Skip the call to runtime.Counters and takeStacktrace so that the + // program counters start at the caller of takeStacktrace. + n := runtime.Callers(2, programCounters.pcs) + if n < cap(programCounters.pcs) { + programCounters.pcs = programCounters.pcs[:n] + break + } + // Don't put the too-short counter slice back into the pool; this lets + // the pool adjust if we consistently take deep stacktraces. + programCounters = newProgramCounters(len(programCounters.pcs) * 2) + } + + i := 0 + frames := runtime.CallersFrames(programCounters.pcs) + for frame, more := frames.Next(); more; frame, more = frames.Next() { + if shouldIgnoreStacktraceFunction(frame.Function) { + continue + } + if i != 0 { + buffer.WriteByte('\n') + } + i++ + buffer.WriteString(frame.Function) + buffer.WriteByte('\n') + buffer.WriteByte('\t') + buffer.WriteString(frame.File) + buffer.WriteByte(':') + buffer.WriteString(strconv.Itoa(int(frame.Line))) + } + + return buffer.String() +} + +func shouldIgnoreStacktraceFunction(function string) bool { + for _, prefix := range _stacktraceIgnorePrefixes { + if strings.HasPrefix(function, prefix) { + return true + } + } + return false +} + +type programCounters struct { + pcs []uintptr +} + +func newProgramCounters(size int) *programCounters { + return &programCounters{make([]uintptr, size)} +} diff --git a/vendor/github.com/hashicorp/go-hclog/stdlog.go b/vendor/github.com/hashicorp/go-hclog/stdlog.go new file mode 100644 index 000000000..2bb927fc9 --- /dev/null +++ b/vendor/github.com/hashicorp/go-hclog/stdlog.go @@ -0,0 +1,62 @@ +package hclog + +import ( + "bytes" + "strings" +) + +// Provides a io.Writer to shim the data out of *log.Logger +// and back into our Logger. This is basically the only way to +// build upon *log.Logger. +type stdlogAdapter struct { + hl Logger + inferLevels bool +} + +// Take the data, infer the levels if configured, and send it through +// a regular Logger +func (s *stdlogAdapter) Write(data []byte) (int, error) { + str := string(bytes.TrimRight(data, " \t\n")) + + if s.inferLevels { + level, str := s.pickLevel(str) + switch level { + case Trace: + s.hl.Trace(str) + case Debug: + s.hl.Debug(str) + case Info: + s.hl.Info(str) + case Warn: + s.hl.Warn(str) + case Error: + s.hl.Error(str) + default: + s.hl.Info(str) + } + } else { + s.hl.Info(str) + } + + return len(data), nil +} + +// Detect, based on conventions, what log level this is +func (s *stdlogAdapter) pickLevel(str string) (Level, string) { + switch { + case strings.HasPrefix(str, "[DEBUG]"): + return Debug, strings.TrimSpace(str[7:]) + case strings.HasPrefix(str, "[TRACE]"): + return Trace, strings.TrimSpace(str[7:]) + case strings.HasPrefix(str, "[INFO]"): + return Info, strings.TrimSpace(str[6:]) + case strings.HasPrefix(str, "[WARN]"): + return Warn, strings.TrimSpace(str[7:]) + case strings.HasPrefix(str, "[ERROR]"): + return Error, strings.TrimSpace(str[7:]) + case strings.HasPrefix(str, "[ERR]"): + return Error, strings.TrimSpace(str[5:]) + default: + return Info, str + } +} diff --git a/vendor/github.com/hashicorp/go-plugin/.gitignore b/vendor/github.com/hashicorp/go-plugin/.gitignore new file mode 100644 index 000000000..e43b0f988 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/vendor/github.com/hashicorp/go-plugin/LICENSE b/vendor/github.com/hashicorp/go-plugin/LICENSE new file mode 100644 index 000000000..82b4de97c --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/LICENSE @@ -0,0 +1,353 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/hashicorp/go-plugin/README.md b/vendor/github.com/hashicorp/go-plugin/README.md new file mode 100644 index 000000000..e4558dbc5 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/README.md @@ -0,0 +1,168 @@ +# Go Plugin System over RPC + +`go-plugin` is a Go (golang) plugin system over RPC. It is the plugin system +that has been in use by HashiCorp tooling for over 4 years. While initially +created for [Packer](https://www.packer.io), it is additionally in use by +[Terraform](https://www.terraform.io), [Nomad](https://www.nomadproject.io), and +[Vault](https://www.vaultproject.io). + +While the plugin system is over RPC, it is currently only designed to work +over a local [reliable] network. Plugins over a real network are not supported +and will lead to unexpected behavior. + +This plugin system has been used on millions of machines across many different +projects and has proven to be battle hardened and ready for production use. + +## Features + +The HashiCorp plugin system supports a number of features: + +**Plugins are Go interface implementations.** This makes writing and consuming +plugins feel very natural. To a plugin author: you just implement an +interface as if it were going to run in the same process. For a plugin user: +you just use and call functions on an interface as if it were in the same +process. This plugin system handles the communication in between. + +**Cross-language support.** Plugins can be written (and consumed) by +almost every major language. This library supports serving plugins via +[gRPC](http://www.grpc.io). gRPC-based plugins enable plugins to be written +in any language. + +**Complex arguments and return values are supported.** This library +provides APIs for handling complex arguments and return values such +as interfaces, `io.Reader/Writer`, etc. We do this by giving you a library +(`MuxBroker`) for creating new connections between the client/server to +serve additional interfaces or transfer raw data. + +**Bidirectional communication.** Because the plugin system supports +complex arguments, the host process can send it interface implementations +and the plugin can call back into the host process. + +**Built-in Logging.** Any plugins that use the `log` standard library +will have log data automatically sent to the host process. The host +process will mirror this output prefixed with the path to the plugin +binary. This makes debugging with plugins simple. If the host system +uses [hclog](https://github.com/hashicorp/go-hclog) then the log data +will be structured. If the plugin also uses hclog, logs from the plugin +will be sent to the host hclog and be structured. + +**Protocol Versioning.** A very basic "protocol version" is supported that +can be incremented to invalidate any previous plugins. This is useful when +interface signatures are changing, protocol level changes are necessary, +etc. When a protocol version is incompatible, a human friendly error +message is shown to the end user. + +**Stdout/Stderr Syncing.** While plugins are subprocesses, they can continue +to use stdout/stderr as usual and the output will get mirrored back to +the host process. The host process can control what `io.Writer` these +streams go to to prevent this from happening. + +**TTY Preservation.** Plugin subprocesses are connected to the identical +stdin file descriptor as the host process, allowing software that requires +a TTY to work. For example, a plugin can execute `ssh` and even though there +are multiple subprocesses and RPC happening, it will look and act perfectly +to the end user. + +**Host upgrade while a plugin is running.** Plugins can be "reattached" +so that the host process can be upgraded while the plugin is still running. +This requires the host/plugin to know this is possible and daemonize +properly. `NewClient` takes a `ReattachConfig` to determine if and how to +reattach. + +**Cryptographically Secure Plugins.** Plugins can be verified with an expected +checksum and RPC communications can be configured to use TLS. The host process +must be properly secured to protect this configuration. + +## Architecture + +The HashiCorp plugin system works by launching subprocesses and communicating +over RPC (using standard `net/rpc` or [gRPC](http://www.grpc.io)). A single +connection is made between any plugin and the host process. For net/rpc-based +plugins, we use a [connection multiplexing](https://github.com/hashicorp/yamux) +library to multiplex any other connections on top. For gRPC-based plugins, +the HTTP2 protocol handles multiplexing. + +This architecture has a number of benefits: + + * Plugins can't crash your host process: A panic in a plugin doesn't + panic the plugin user. + + * Plugins are very easy to write: just write a Go application and `go build`. + Or use any other language to write a gRPC server with a tiny amount of + boilerplate to support go-plugin. + + * Plugins are very easy to install: just put the binary in a location where + the host will find it (depends on the host but this library also provides + helpers), and the plugin host handles the rest. + + * Plugins can be relatively secure: The plugin only has access to the + interfaces and args given to it, not to the entire memory space of the + process. Additionally, go-plugin can communicate with the plugin over + TLS. + +## Usage + +To use the plugin system, you must take the following steps. These are +high-level steps that must be done. Examples are available in the +`examples/` directory. + + 1. Choose the interface(s) you want to expose for plugins. + + 2. For each interface, implement an implementation of that interface + that communicates over a `net/rpc` connection or other a + [gRPC](http://www.grpc.io) connection or both. You'll have to implement + both a client and server implementation. + + 3. Create a `Plugin` implementation that knows how to create the RPC + client/server for a given plugin type. + + 4. Plugin authors call `plugin.Serve` to serve a plugin from the + `main` function. + + 5. Plugin users use `plugin.Client` to launch a subprocess and request + an interface implementation over RPC. + +That's it! In practice, step 2 is the most tedious and time consuming step. +Even so, it isn't very difficult and you can see examples in the `examples/` +directory as well as throughout our various open source projects. + +For complete API documentation, see [GoDoc](https://godoc.org/github.com/hashicorp/go-plugin). + +## Roadmap + +Our plugin system is constantly evolving. As we use the plugin system for +new projects or for new features in existing projects, we constantly find +improvements we can make. + +At this point in time, the roadmap for the plugin system is: + +**Semantic Versioning.** Plugins will be able to implement a semantic version. +This plugin system will give host processes a system for constraining +versions. This is in addition to the protocol versioning already present +which is more for larger underlying changes. + +**Plugin fetching.** We will integrate with [go-getter](https://github.com/hashicorp/go-getter) +to support automatic download + install of plugins. Paired with cryptographically +secure plugins (above), we can make this a safe operation for an amazing +user experience. + +## What About Shared Libraries? + +When we started using plugins (late 2012, early 2013), plugins over RPC +were the only option since Go didn't support dynamic library loading. Today, +Go still doesn't support dynamic library loading, but they do intend to. +Since 2012, our plugin system has stabilized from millions of users using it, +and has many benefits we've come to value greatly. + +For example, we intend to use this plugin system in +[Vault](https://www.vaultproject.io), and dynamic library loading will +simply never be acceptable in Vault for security reasons. That is an extreme +example, but we believe our library system has more upsides than downsides +over dynamic library loading and since we've had it built and tested for years, +we'll likely continue to use it. + +Shared libraries have one major advantage over our system which is much +higher performance. In real world scenarios across our various tools, +we've never required any more performance out of our plugin system and it +has seen very high throughput, so this isn't a concern for us at the moment. + diff --git a/vendor/github.com/hashicorp/go-plugin/client.go b/vendor/github.com/hashicorp/go-plugin/client.go new file mode 100644 index 000000000..c3cbc45e7 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/client.go @@ -0,0 +1,772 @@ +package plugin + +import ( + "bufio" + "crypto/subtle" + "crypto/tls" + "errors" + "fmt" + "hash" + "io" + "io/ioutil" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode" + + hclog "github.com/hashicorp/go-hclog" +) + +// If this is 1, then we've called CleanupClients. This can be used +// by plugin RPC implementations to change error behavior since you +// can expected network connection errors at this point. This should be +// read by using sync/atomic. +var Killed uint32 = 0 + +// This is a slice of the "managed" clients which are cleaned up when +// calling Cleanup +var managedClients = make([]*Client, 0, 5) +var managedClientsLock sync.Mutex + +// Error types +var ( + // ErrProcessNotFound is returned when a client is instantiated to + // reattach to an existing process and it isn't found. + ErrProcessNotFound = errors.New("Reattachment process not found") + + // ErrChecksumsDoNotMatch is returned when binary's checksum doesn't match + // the one provided in the SecureConfig. + ErrChecksumsDoNotMatch = errors.New("checksums did not match") + + // ErrSecureNoChecksum is returned when an empty checksum is provided to the + // SecureConfig. + ErrSecureConfigNoChecksum = errors.New("no checksum provided") + + // ErrSecureNoHash is returned when a nil Hash object is provided to the + // SecureConfig. + ErrSecureConfigNoHash = errors.New("no hash implementation provided") + + // ErrSecureConfigAndReattach is returned when both Reattach and + // SecureConfig are set. + ErrSecureConfigAndReattach = errors.New("only one of Reattach or SecureConfig can be set") +) + +// Client handles the lifecycle of a plugin application. It launches +// plugins, connects to them, dispenses interface implementations, and handles +// killing the process. +// +// Plugin hosts should use one Client for each plugin executable. To +// dispense a plugin type, use the `Client.Client` function, and then +// cal `Dispense`. This awkward API is mostly historical but is used to split +// the client that deals with subprocess management and the client that +// does RPC management. +// +// See NewClient and ClientConfig for using a Client. +type Client struct { + config *ClientConfig + exited bool + doneLogging chan struct{} + l sync.Mutex + address net.Addr + process *os.Process + client ClientProtocol + protocol Protocol + logger hclog.Logger +} + +// ClientConfig is the configuration used to initialize a new +// plugin client. After being used to initialize a plugin client, +// that configuration must not be modified again. +type ClientConfig struct { + // HandshakeConfig is the configuration that must match servers. + HandshakeConfig + + // Plugins are the plugins that can be consumed. + Plugins map[string]Plugin + + // One of the following must be set, but not both. + // + // Cmd is the unstarted subprocess for starting the plugin. If this is + // set, then the Client starts the plugin process on its own and connects + // to it. + // + // Reattach is configuration for reattaching to an existing plugin process + // that is already running. This isn't common. + Cmd *exec.Cmd + Reattach *ReattachConfig + + // SecureConfig is configuration for verifying the integrity of the + // executable. It can not be used with Reattach. + SecureConfig *SecureConfig + + // TLSConfig is used to enable TLS on the RPC client. + TLSConfig *tls.Config + + // Managed represents if the client should be managed by the + // plugin package or not. If true, then by calling CleanupClients, + // it will automatically be cleaned up. Otherwise, the client + // user is fully responsible for making sure to Kill all plugin + // clients. By default the client is _not_ managed. + Managed bool + + // The minimum and maximum port to use for communicating with + // the subprocess. If not set, this defaults to 10,000 and 25,000 + // respectively. + MinPort, MaxPort uint + + // StartTimeout is the timeout to wait for the plugin to say it + // has started successfully. + StartTimeout time.Duration + + // If non-nil, then the stderr of the client will be written to here + // (as well as the log). This is the original os.Stderr of the subprocess. + // This isn't the output of synced stderr. + Stderr io.Writer + + // SyncStdout, SyncStderr can be set to override the + // respective os.Std* values in the plugin. Care should be taken to + // avoid races here. If these are nil, then this will automatically be + // hooked up to os.Stdin, Stdout, and Stderr, respectively. + // + // If the default values (nil) are used, then this package will not + // sync any of these streams. + SyncStdout io.Writer + SyncStderr io.Writer + + // AllowedProtocols is a list of allowed protocols. If this isn't set, + // then only netrpc is allowed. This is so that older go-plugin systems + // can show friendly errors if they see a plugin with an unknown + // protocol. + // + // By setting this, you can cause an error immediately on plugin start + // if an unsupported protocol is used with a good error message. + // + // If this isn't set at all (nil value), then only net/rpc is accepted. + // This is done for legacy reasons. You must explicitly opt-in to + // new protocols. + AllowedProtocols []Protocol + + // Logger is the logger that the client will used. If none is provided, + // it will default to hclog's default logger. + Logger hclog.Logger +} + +// ReattachConfig is used to configure a client to reattach to an +// already-running plugin process. You can retrieve this information by +// calling ReattachConfig on Client. +type ReattachConfig struct { + Protocol Protocol + Addr net.Addr + Pid int +} + +// SecureConfig is used to configure a client to verify the integrity of an +// executable before running. It does this by verifying the checksum is +// expected. Hash is used to specify the hashing method to use when checksumming +// the file. The configuration is verified by the client by calling the +// SecureConfig.Check() function. +// +// The host process should ensure the checksum was provided by a trusted and +// authoritative source. The binary should be installed in such a way that it +// can not be modified by an unauthorized user between the time of this check +// and the time of execution. +type SecureConfig struct { + Checksum []byte + Hash hash.Hash +} + +// Check takes the filepath to an executable and returns true if the checksum of +// the file matches the checksum provided in the SecureConfig. +func (s *SecureConfig) Check(filePath string) (bool, error) { + if len(s.Checksum) == 0 { + return false, ErrSecureConfigNoChecksum + } + + if s.Hash == nil { + return false, ErrSecureConfigNoHash + } + + file, err := os.Open(filePath) + if err != nil { + return false, err + } + defer file.Close() + + _, err = io.Copy(s.Hash, file) + if err != nil { + return false, err + } + + sum := s.Hash.Sum(nil) + + return subtle.ConstantTimeCompare(sum, s.Checksum) == 1, nil +} + +// This makes sure all the managed subprocesses are killed and properly +// logged. This should be called before the parent process running the +// plugins exits. +// +// This must only be called _once_. +func CleanupClients() { + // Set the killed to true so that we don't get unexpected panics + atomic.StoreUint32(&Killed, 1) + + // Kill all the managed clients in parallel and use a WaitGroup + // to wait for them all to finish up. + var wg sync.WaitGroup + managedClientsLock.Lock() + for _, client := range managedClients { + wg.Add(1) + + go func(client *Client) { + client.Kill() + wg.Done() + }(client) + } + managedClientsLock.Unlock() + + log.Println("[DEBUG] plugin: waiting for all plugin processes to complete...") + wg.Wait() +} + +// Creates a new plugin client which manages the lifecycle of an external +// plugin and gets the address for the RPC connection. +// +// The client must be cleaned up at some point by calling Kill(). If +// the client is a managed client (created with NewManagedClient) you +// can just call CleanupClients at the end of your program and they will +// be properly cleaned. +func NewClient(config *ClientConfig) (c *Client) { + if config.MinPort == 0 && config.MaxPort == 0 { + config.MinPort = 10000 + config.MaxPort = 25000 + } + + if config.StartTimeout == 0 { + config.StartTimeout = 1 * time.Minute + } + + if config.Stderr == nil { + config.Stderr = ioutil.Discard + } + + if config.SyncStdout == nil { + config.SyncStdout = ioutil.Discard + } + if config.SyncStderr == nil { + config.SyncStderr = ioutil.Discard + } + + if config.AllowedProtocols == nil { + config.AllowedProtocols = []Protocol{ProtocolNetRPC} + } + + if config.Logger == nil { + config.Logger = hclog.New(&hclog.LoggerOptions{ + Output: hclog.DefaultOutput, + Level: hclog.Trace, + Name: "plugin", + }) + } + + c = &Client{ + config: config, + logger: config.Logger, + } + if config.Managed { + managedClientsLock.Lock() + managedClients = append(managedClients, c) + managedClientsLock.Unlock() + } + + return +} + +// Client returns the protocol client for this connection. +// +// Subsequent calls to this will return the same client. +func (c *Client) Client() (ClientProtocol, error) { + _, err := c.Start() + if err != nil { + return nil, err + } + + c.l.Lock() + defer c.l.Unlock() + + if c.client != nil { + return c.client, nil + } + + switch c.protocol { + case ProtocolNetRPC: + c.client, err = newRPCClient(c) + + case ProtocolGRPC: + c.client, err = newGRPCClient(c) + + default: + return nil, fmt.Errorf("unknown server protocol: %s", c.protocol) + } + + if err != nil { + c.client = nil + return nil, err + } + + return c.client, nil +} + +// Tells whether or not the underlying process has exited. +func (c *Client) Exited() bool { + c.l.Lock() + defer c.l.Unlock() + return c.exited +} + +// End the executing subprocess (if it is running) and perform any cleanup +// tasks necessary such as capturing any remaining logs and so on. +// +// This method blocks until the process successfully exits. +// +// This method can safely be called multiple times. +func (c *Client) Kill() { + // Grab a lock to read some private fields. + c.l.Lock() + process := c.process + addr := c.address + doneCh := c.doneLogging + c.l.Unlock() + + // If there is no process, we never started anything. Nothing to kill. + if process == nil { + return + } + + // We need to check for address here. It is possible that the plugin + // started (process != nil) but has no address (addr == nil) if the + // plugin failed at startup. If we do have an address, we need to close + // the plugin net connections. + graceful := false + if addr != nil { + // Close the client to cleanly exit the process. + client, err := c.Client() + if err == nil { + err = client.Close() + + // If there is no error, then we attempt to wait for a graceful + // exit. If there was an error, we assume that graceful cleanup + // won't happen and just force kill. + graceful = err == nil + if err != nil { + // If there was an error just log it. We're going to force + // kill in a moment anyways. + c.logger.Warn("error closing client during Kill", "err", err) + } + } + } + + // If we're attempting a graceful exit, then we wait for a short period + // of time to allow that to happen. To wait for this we just wait on the + // doneCh which would be closed if the process exits. + if graceful { + select { + case <-doneCh: + return + case <-time.After(250 * time.Millisecond): + } + } + + // If graceful exiting failed, just kill it + process.Kill() + + // Wait for the client to finish logging so we have a complete log + <-doneCh +} + +// Starts the underlying subprocess, communicating with it to negotiate +// a port for RPC connections, and returning the address to connect via RPC. +// +// This method is safe to call multiple times. Subsequent calls have no effect. +// Once a client has been started once, it cannot be started again, even if +// it was killed. +func (c *Client) Start() (addr net.Addr, err error) { + c.l.Lock() + defer c.l.Unlock() + + if c.address != nil { + return c.address, nil + } + + // If one of cmd or reattach isn't set, then it is an error. We wrap + // this in a {} for scoping reasons, and hopeful that the escape + // analysis will pop the stock here. + { + cmdSet := c.config.Cmd != nil + attachSet := c.config.Reattach != nil + secureSet := c.config.SecureConfig != nil + if cmdSet == attachSet { + return nil, fmt.Errorf("Only one of Cmd or Reattach must be set") + } + + if secureSet && attachSet { + return nil, ErrSecureConfigAndReattach + } + } + + // Create the logging channel for when we kill + c.doneLogging = make(chan struct{}) + + if c.config.Reattach != nil { + // Verify the process still exists. If not, then it is an error + p, err := os.FindProcess(c.config.Reattach.Pid) + if err != nil { + return nil, err + } + + // Attempt to connect to the addr since on Unix systems FindProcess + // doesn't actually return an error if it can't find the process. + conn, err := net.Dial( + c.config.Reattach.Addr.Network(), + c.config.Reattach.Addr.String()) + if err != nil { + p.Kill() + return nil, ErrProcessNotFound + } + conn.Close() + + // Goroutine to mark exit status + go func(pid int) { + // Wait for the process to die + pidWait(pid) + + // Log so we can see it + c.logger.Debug("reattached plugin process exited") + + // Mark it + c.l.Lock() + defer c.l.Unlock() + c.exited = true + + // Close the logging channel since that doesn't work on reattach + close(c.doneLogging) + }(p.Pid) + + // Set the address and process + c.address = c.config.Reattach.Addr + c.process = p + c.protocol = c.config.Reattach.Protocol + if c.protocol == "" { + // Default the protocol to net/rpc for backwards compatibility + c.protocol = ProtocolNetRPC + } + + return c.address, nil + } + + env := []string{ + fmt.Sprintf("%s=%s", c.config.MagicCookieKey, c.config.MagicCookieValue), + fmt.Sprintf("PLUGIN_MIN_PORT=%d", c.config.MinPort), + fmt.Sprintf("PLUGIN_MAX_PORT=%d", c.config.MaxPort), + } + + stdout_r, stdout_w := io.Pipe() + stderr_r, stderr_w := io.Pipe() + + cmd := c.config.Cmd + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, env...) + cmd.Stdin = os.Stdin + cmd.Stderr = stderr_w + cmd.Stdout = stdout_w + + if c.config.SecureConfig != nil { + if ok, err := c.config.SecureConfig.Check(cmd.Path); err != nil { + return nil, fmt.Errorf("error verifying checksum: %s", err) + } else if !ok { + return nil, ErrChecksumsDoNotMatch + } + } + + c.logger.Debug("starting plugin", "path", cmd.Path, "args", cmd.Args) + err = cmd.Start() + if err != nil { + return + } + + // Set the process + c.process = cmd.Process + + // Make sure the command is properly cleaned up if there is an error + defer func() { + r := recover() + + if err != nil || r != nil { + cmd.Process.Kill() + } + + if r != nil { + panic(r) + } + }() + + // Start goroutine to wait for process to exit + exitCh := make(chan struct{}) + go func() { + // Make sure we close the write end of our stderr/stdout so + // that the readers send EOF properly. + defer stderr_w.Close() + defer stdout_w.Close() + + // Wait for the command to end. + cmd.Wait() + + // Log and make sure to flush the logs write away + c.logger.Debug("plugin process exited", "path", cmd.Path) + os.Stderr.Sync() + + // Mark that we exited + close(exitCh) + + // Set that we exited, which takes a lock + c.l.Lock() + defer c.l.Unlock() + c.exited = true + }() + + // Start goroutine that logs the stderr + go c.logStderr(stderr_r) + + // Start a goroutine that is going to be reading the lines + // out of stdout + linesCh := make(chan []byte) + go func() { + defer close(linesCh) + + buf := bufio.NewReader(stdout_r) + for { + line, err := buf.ReadBytes('\n') + if line != nil { + linesCh <- line + } + + if err == io.EOF { + return + } + } + }() + + // Make sure after we exit we read the lines from stdout forever + // so they don't block since it is an io.Pipe + defer func() { + go func() { + for _ = range linesCh { + } + }() + }() + + // Some channels for the next step + timeout := time.After(c.config.StartTimeout) + + // Start looking for the address + c.logger.Debug("waiting for RPC address", "path", cmd.Path) + select { + case <-timeout: + err = errors.New("timeout while waiting for plugin to start") + case <-exitCh: + err = errors.New("plugin exited before we could connect") + case lineBytes := <-linesCh: + // Trim the line and split by "|" in order to get the parts of + // the output. + line := strings.TrimSpace(string(lineBytes)) + parts := strings.SplitN(line, "|", 6) + if len(parts) < 4 { + err = fmt.Errorf( + "Unrecognized remote plugin message: %s\n\n"+ + "This usually means that the plugin is either invalid or simply\n"+ + "needs to be recompiled to support the latest protocol.", line) + return + } + + // Check the core protocol. Wrapped in a {} for scoping. + { + var coreProtocol int64 + coreProtocol, err = strconv.ParseInt(parts[0], 10, 0) + if err != nil { + err = fmt.Errorf("Error parsing core protocol version: %s", err) + return + } + + if int(coreProtocol) != CoreProtocolVersion { + err = fmt.Errorf("Incompatible core API version with plugin. "+ + "Plugin version: %s, Core version: %d\n\n"+ + "To fix this, the plugin usually only needs to be recompiled.\n"+ + "Please report this to the plugin author.", parts[0], CoreProtocolVersion) + return + } + } + + // Parse the protocol version + var protocol int64 + protocol, err = strconv.ParseInt(parts[1], 10, 0) + if err != nil { + err = fmt.Errorf("Error parsing protocol version: %s", err) + return + } + + // Test the API version + if uint(protocol) != c.config.ProtocolVersion { + err = fmt.Errorf("Incompatible API version with plugin. "+ + "Plugin version: %s, Core version: %d", parts[1], c.config.ProtocolVersion) + return + } + + switch parts[2] { + case "tcp": + addr, err = net.ResolveTCPAddr("tcp", parts[3]) + case "unix": + addr, err = net.ResolveUnixAddr("unix", parts[3]) + default: + err = fmt.Errorf("Unknown address type: %s", parts[3]) + } + + // If we have a server type, then record that. We default to net/rpc + // for backwards compatibility. + c.protocol = ProtocolNetRPC + if len(parts) >= 5 { + c.protocol = Protocol(parts[4]) + } + + found := false + for _, p := range c.config.AllowedProtocols { + if p == c.protocol { + found = true + break + } + } + if !found { + err = fmt.Errorf("Unsupported plugin protocol %q. Supported: %v", + c.protocol, c.config.AllowedProtocols) + return + } + + } + + c.address = addr + return +} + +// ReattachConfig returns the information that must be provided to NewClient +// to reattach to the plugin process that this client started. This is +// useful for plugins that detach from their parent process. +// +// If this returns nil then the process hasn't been started yet. Please +// call Start or Client before calling this. +func (c *Client) ReattachConfig() *ReattachConfig { + c.l.Lock() + defer c.l.Unlock() + + if c.address == nil { + return nil + } + + if c.config.Cmd != nil && c.config.Cmd.Process == nil { + return nil + } + + // If we connected via reattach, just return the information as-is + if c.config.Reattach != nil { + return c.config.Reattach + } + + return &ReattachConfig{ + Protocol: c.protocol, + Addr: c.address, + Pid: c.config.Cmd.Process.Pid, + } +} + +// Protocol returns the protocol of server on the remote end. This will +// start the plugin process if it isn't already started. Errors from +// starting the plugin are surpressed and ProtocolInvalid is returned. It +// is recommended you call Start explicitly before calling Protocol to ensure +// no errors occur. +func (c *Client) Protocol() Protocol { + _, err := c.Start() + if err != nil { + return ProtocolInvalid + } + + return c.protocol +} + +// dialer is compatible with grpc.WithDialer and creates the connection +// to the plugin. +func (c *Client) dialer(_ string, timeout time.Duration) (net.Conn, error) { + // Connect to the client + conn, err := net.Dial(c.address.Network(), c.address.String()) + if err != nil { + return nil, err + } + if tcpConn, ok := conn.(*net.TCPConn); ok { + // Make sure to set keep alive so that the connection doesn't die + tcpConn.SetKeepAlive(true) + } + + // If we have a TLS config we wrap our connection. We only do this + // for net/rpc since gRPC uses its own mechanism for TLS. + if c.protocol == ProtocolNetRPC && c.config.TLSConfig != nil { + conn = tls.Client(conn, c.config.TLSConfig) + } + + return conn, nil +} + +func (c *Client) logStderr(r io.Reader) { + bufR := bufio.NewReader(r) + for { + line, err := bufR.ReadString('\n') + if line != "" { + c.config.Stderr.Write([]byte(line)) + line = strings.TrimRightFunc(line, unicode.IsSpace) + + l := c.logger.Named(filepath.Base(c.config.Cmd.Path)) + + entry, err := parseJSON(line) + // If output is not JSON format, print directly to Debug + if err != nil { + l.Debug(line) + } else { + out := flattenKVPairs(entry.KVPairs) + + l = l.With("timestamp", entry.Timestamp.Format(hclog.TimeFormat)) + switch hclog.LevelFromString(entry.Level) { + case hclog.Trace: + l.Trace(entry.Message, out...) + case hclog.Debug: + l.Debug(entry.Message, out...) + case hclog.Info: + l.Info(entry.Message, out...) + case hclog.Warn: + l.Warn(entry.Message, out...) + case hclog.Error: + l.Error(entry.Message, out...) + } + } + } + + if err == io.EOF { + break + } + } + + // Flag that we've completed logging for others + close(c.doneLogging) +} diff --git a/vendor/github.com/hashicorp/go-plugin/discover.go b/vendor/github.com/hashicorp/go-plugin/discover.go new file mode 100644 index 000000000..d22c566ed --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/discover.go @@ -0,0 +1,28 @@ +package plugin + +import ( + "path/filepath" +) + +// Discover discovers plugins that are in a given directory. +// +// The directory doesn't need to be absolute. For example, "." will work fine. +// +// This currently assumes any file matching the glob is a plugin. +// In the future this may be smarter about checking that a file is +// executable and so on. +// +// TODO: test +func Discover(glob, dir string) ([]string, error) { + var err error + + // Make the directory absolute if it isn't already + if !filepath.IsAbs(dir) { + dir, err = filepath.Abs(dir) + if err != nil { + return nil, err + } + } + + return filepath.Glob(filepath.Join(dir, glob)) +} diff --git a/vendor/github.com/hashicorp/go-plugin/error.go b/vendor/github.com/hashicorp/go-plugin/error.go new file mode 100644 index 000000000..22a7baa6a --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/error.go @@ -0,0 +1,24 @@ +package plugin + +// This is a type that wraps error types so that they can be messaged +// across RPC channels. Since "error" is an interface, we can't always +// gob-encode the underlying structure. This is a valid error interface +// implementer that we will push across. +type BasicError struct { + Message string +} + +// NewBasicError is used to create a BasicError. +// +// err is allowed to be nil. +func NewBasicError(err error) *BasicError { + if err == nil { + return nil + } + + return &BasicError{err.Error()} +} + +func (e *BasicError) Error() string { + return e.Message +} diff --git a/vendor/github.com/hashicorp/go-plugin/grpc_client.go b/vendor/github.com/hashicorp/go-plugin/grpc_client.go new file mode 100644 index 000000000..3bcf95efc --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/grpc_client.go @@ -0,0 +1,83 @@ +package plugin + +import ( + "fmt" + + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health/grpc_health_v1" +) + +// newGRPCClient creates a new GRPCClient. The Client argument is expected +// to be successfully started already with a lock held. +func newGRPCClient(c *Client) (*GRPCClient, error) { + // Build dialing options. + opts := make([]grpc.DialOption, 0, 5) + + // We use a custom dialer so that we can connect over unix domain sockets + opts = append(opts, grpc.WithDialer(c.dialer)) + + // go-plugin expects to block the connection + opts = append(opts, grpc.WithBlock()) + + // Fail right away + opts = append(opts, grpc.FailOnNonTempDialError(true)) + + // If we have no TLS configuration set, we need to explicitly tell grpc + // that we're connecting with an insecure connection. + if c.config.TLSConfig == nil { + opts = append(opts, grpc.WithInsecure()) + } else { + opts = append(opts, grpc.WithTransportCredentials( + credentials.NewTLS(c.config.TLSConfig))) + } + + // Connect. Note the first parameter is unused because we use a custom + // dialer that has the state to see the address. + conn, err := grpc.Dial("unused", opts...) + if err != nil { + return nil, err + } + + return &GRPCClient{ + Conn: conn, + Plugins: c.config.Plugins, + }, nil +} + +// GRPCClient connects to a GRPCServer over gRPC to dispense plugin types. +type GRPCClient struct { + Conn *grpc.ClientConn + Plugins map[string]Plugin +} + +// ClientProtocol impl. +func (c *GRPCClient) Close() error { + return c.Conn.Close() +} + +// ClientProtocol impl. +func (c *GRPCClient) Dispense(name string) (interface{}, error) { + raw, ok := c.Plugins[name] + if !ok { + return nil, fmt.Errorf("unknown plugin type: %s", name) + } + + p, ok := raw.(GRPCPlugin) + if !ok { + return nil, fmt.Errorf("plugin %q doesn't support gRPC", name) + } + + return p.GRPCClient(c.Conn) +} + +// ClientProtocol impl. +func (c *GRPCClient) Ping() error { + client := grpc_health_v1.NewHealthClient(c.Conn) + _, err := client.Check(context.Background(), &grpc_health_v1.HealthCheckRequest{ + Service: GRPCServiceName, + }) + + return err +} diff --git a/vendor/github.com/hashicorp/go-plugin/grpc_server.go b/vendor/github.com/hashicorp/go-plugin/grpc_server.go new file mode 100644 index 000000000..177a0cdd7 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/grpc_server.go @@ -0,0 +1,115 @@ +package plugin + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" +) + +// GRPCServiceName is the name of the service that the health check should +// return as passing. +const GRPCServiceName = "plugin" + +// DefaultGRPCServer can be used with the "GRPCServer" field for Server +// as a default factory method to create a gRPC server with no extra options. +func DefaultGRPCServer(opts []grpc.ServerOption) *grpc.Server { + return grpc.NewServer(opts...) +} + +// GRPCServer is a ServerType implementation that serves plugins over +// gRPC. This allows plugins to easily be written for other languages. +// +// The GRPCServer outputs a custom configuration as a base64-encoded +// JSON structure represented by the GRPCServerConfig config structure. +type GRPCServer struct { + // Plugins are the list of plugins to serve. + Plugins map[string]Plugin + + // Server is the actual server that will accept connections. This + // will be used for plugin registration as well. + Server func([]grpc.ServerOption) *grpc.Server + + // TLS should be the TLS configuration if available. If this is nil, + // the connection will not have transport security. + TLS *tls.Config + + // DoneCh is the channel that is closed when this server has exited. + DoneCh chan struct{} + + // Stdout/StderrLis are the readers for stdout/stderr that will be copied + // to the stdout/stderr connection that is output. + Stdout io.Reader + Stderr io.Reader + + config GRPCServerConfig + server *grpc.Server +} + +// ServerProtocol impl. +func (s *GRPCServer) Init() error { + // Create our server + var opts []grpc.ServerOption + if s.TLS != nil { + opts = append(opts, grpc.Creds(credentials.NewTLS(s.TLS))) + } + s.server = s.Server(opts) + + // Register the health service + healthCheck := health.NewServer() + healthCheck.SetServingStatus( + GRPCServiceName, grpc_health_v1.HealthCheckResponse_SERVING) + grpc_health_v1.RegisterHealthServer(s.server, healthCheck) + + // Register all our plugins onto the gRPC server. + for k, raw := range s.Plugins { + p, ok := raw.(GRPCPlugin) + if !ok { + return fmt.Errorf("%q is not a GRPC-compatibile plugin", k) + } + + if err := p.GRPCServer(s.server); err != nil { + return fmt.Errorf("error registring %q: %s", k, err) + } + } + + return nil +} + +// Config is the GRPCServerConfig encoded as JSON then base64. +func (s *GRPCServer) Config() string { + // Create a buffer that will contain our final contents + var buf bytes.Buffer + + // Wrap the base64 encoding with JSON encoding. + if err := json.NewEncoder(&buf).Encode(s.config); err != nil { + // We panic since ths shouldn't happen under any scenario. We + // carefully control the structure being encoded here and it should + // always be successful. + panic(err) + } + + return buf.String() +} + +func (s *GRPCServer) Serve(lis net.Listener) { + // Start serving in a goroutine + go s.server.Serve(lis) + + // Wait until graceful completion + <-s.DoneCh +} + +// GRPCServerConfig is the extra configuration passed along for consumers +// to facilitate using GRPC plugins. +type GRPCServerConfig struct { + StdoutAddr string `json:"stdout_addr"` + StderrAddr string `json:"stderr_addr"` +} diff --git a/vendor/github.com/hashicorp/go-plugin/log_entry.go b/vendor/github.com/hashicorp/go-plugin/log_entry.go new file mode 100644 index 000000000..2996c14c3 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/log_entry.go @@ -0,0 +1,73 @@ +package plugin + +import ( + "encoding/json" + "time" +) + +// logEntry is the JSON payload that gets sent to Stderr from the plugin to the host +type logEntry struct { + Message string `json:"@message"` + Level string `json:"@level"` + Timestamp time.Time `json:"timestamp"` + KVPairs []*logEntryKV `json:"kv_pairs"` +} + +// logEntryKV is a key value pair within the Output payload +type logEntryKV struct { + Key string `json:"key"` + Value interface{} `json:"value"` +} + +// flattenKVPairs is used to flatten KVPair slice into []interface{} +// for hclog consumption. +func flattenKVPairs(kvs []*logEntryKV) []interface{} { + var result []interface{} + for _, kv := range kvs { + result = append(result, kv.Key) + result = append(result, kv.Value) + } + + return result +} + +// parseJSON handles parsing JSON output +func parseJSON(input string) (*logEntry, error) { + var raw map[string]interface{} + entry := &logEntry{} + + err := json.Unmarshal([]byte(input), &raw) + if err != nil { + return nil, err + } + + // Parse hclog-specific objects + if v, ok := raw["@message"]; ok { + entry.Message = v.(string) + delete(raw, "@message") + } + + if v, ok := raw["@level"]; ok { + entry.Level = v.(string) + delete(raw, "@level") + } + + if v, ok := raw["@timestamp"]; ok { + t, err := time.Parse("2006-01-02T15:04:05.000000Z07:00", v.(string)) + if err != nil { + return nil, err + } + entry.Timestamp = t + delete(raw, "@timestamp") + } + + // Parse dynamic KV args from the hclog payload. + for k, v := range raw { + entry.KVPairs = append(entry.KVPairs, &logEntryKV{ + Key: k, + Value: v, + }) + } + + return entry, nil +} diff --git a/vendor/github.com/hashicorp/go-plugin/mux_broker.go b/vendor/github.com/hashicorp/go-plugin/mux_broker.go new file mode 100644 index 000000000..01c45ad7c --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/mux_broker.go @@ -0,0 +1,204 @@ +package plugin + +import ( + "encoding/binary" + "fmt" + "log" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/yamux" +) + +// MuxBroker is responsible for brokering multiplexed connections by unique ID. +// +// It is used by plugins to multiplex multiple RPC connections and data +// streams on top of a single connection between the plugin process and the +// host process. +// +// This allows a plugin to request a channel with a specific ID to connect to +// or accept a connection from, and the broker handles the details of +// holding these channels open while they're being negotiated. +// +// The Plugin interface has access to these for both Server and Client. +// The broker can be used by either (optionally) to reserve and connect to +// new multiplexed streams. This is useful for complex args and return values, +// or anything else you might need a data stream for. +type MuxBroker struct { + nextId uint32 + session *yamux.Session + streams map[uint32]*muxBrokerPending + + sync.Mutex +} + +type muxBrokerPending struct { + ch chan net.Conn + doneCh chan struct{} +} + +func newMuxBroker(s *yamux.Session) *MuxBroker { + return &MuxBroker{ + session: s, + streams: make(map[uint32]*muxBrokerPending), + } +} + +// Accept accepts a connection by ID. +// +// This should not be called multiple times with the same ID at one time. +func (m *MuxBroker) Accept(id uint32) (net.Conn, error) { + var c net.Conn + p := m.getStream(id) + select { + case c = <-p.ch: + close(p.doneCh) + case <-time.After(5 * time.Second): + m.Lock() + defer m.Unlock() + delete(m.streams, id) + + return nil, fmt.Errorf("timeout waiting for accept") + } + + // Ack our connection + if err := binary.Write(c, binary.LittleEndian, id); err != nil { + c.Close() + return nil, err + } + + return c, nil +} + +// AcceptAndServe is used to accept a specific stream ID and immediately +// serve an RPC server on that stream ID. This is used to easily serve +// complex arguments. +// +// The served interface is always registered to the "Plugin" name. +func (m *MuxBroker) AcceptAndServe(id uint32, v interface{}) { + conn, err := m.Accept(id) + if err != nil { + log.Printf("[ERR] plugin: plugin acceptAndServe error: %s", err) + return + } + + serve(conn, "Plugin", v) +} + +// Close closes the connection and all sub-connections. +func (m *MuxBroker) Close() error { + return m.session.Close() +} + +// Dial opens a connection by ID. +func (m *MuxBroker) Dial(id uint32) (net.Conn, error) { + // Open the stream + stream, err := m.session.OpenStream() + if err != nil { + return nil, err + } + + // Write the stream ID onto the wire. + if err := binary.Write(stream, binary.LittleEndian, id); err != nil { + stream.Close() + return nil, err + } + + // Read the ack that we connected. Then we're off! + var ack uint32 + if err := binary.Read(stream, binary.LittleEndian, &ack); err != nil { + stream.Close() + return nil, err + } + if ack != id { + stream.Close() + return nil, fmt.Errorf("bad ack: %d (expected %d)", ack, id) + } + + return stream, nil +} + +// NextId returns a unique ID to use next. +// +// It is possible for very long-running plugin hosts to wrap this value, +// though it would require a very large amount of RPC calls. In practice +// we've never seen it happen. +func (m *MuxBroker) NextId() uint32 { + return atomic.AddUint32(&m.nextId, 1) +} + +// Run starts the brokering and should be executed in a goroutine, since it +// blocks forever, or until the session closes. +// +// Uses of MuxBroker never need to call this. It is called internally by +// the plugin host/client. +func (m *MuxBroker) Run() { + for { + stream, err := m.session.AcceptStream() + if err != nil { + // Once we receive an error, just exit + break + } + + // Read the stream ID from the stream + var id uint32 + if err := binary.Read(stream, binary.LittleEndian, &id); err != nil { + stream.Close() + continue + } + + // Initialize the waiter + p := m.getStream(id) + select { + case p.ch <- stream: + default: + } + + // Wait for a timeout + go m.timeoutWait(id, p) + } +} + +func (m *MuxBroker) getStream(id uint32) *muxBrokerPending { + m.Lock() + defer m.Unlock() + + p, ok := m.streams[id] + if ok { + return p + } + + m.streams[id] = &muxBrokerPending{ + ch: make(chan net.Conn, 1), + doneCh: make(chan struct{}), + } + return m.streams[id] +} + +func (m *MuxBroker) timeoutWait(id uint32, p *muxBrokerPending) { + // Wait for the stream to either be picked up and connected, or + // for a timeout. + timeout := false + select { + case <-p.doneCh: + case <-time.After(5 * time.Second): + timeout = true + } + + m.Lock() + defer m.Unlock() + + // Delete the stream so no one else can grab it + delete(m.streams, id) + + // If we timed out, then check if we have a channel in the buffer, + // and if so, close it. + if timeout { + select { + case s := <-p.ch: + s.Close() + } + } +} diff --git a/vendor/github.com/hashicorp/go-plugin/plugin.go b/vendor/github.com/hashicorp/go-plugin/plugin.go new file mode 100644 index 000000000..6b7bdd1cf --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/plugin.go @@ -0,0 +1,56 @@ +// The plugin package exposes functions and helpers for communicating to +// plugins which are implemented as standalone binary applications. +// +// plugin.Client fully manages the lifecycle of executing the application, +// connecting to it, and returning the RPC client for dispensing plugins. +// +// plugin.Serve fully manages listeners to expose an RPC server from a binary +// that plugin.Client can connect to. +package plugin + +import ( + "errors" + "net/rpc" + + "google.golang.org/grpc" +) + +// Plugin is the interface that is implemented to serve/connect to an +// inteface implementation. +type Plugin interface { + // Server should return the RPC server compatible struct to serve + // the methods that the Client calls over net/rpc. + Server(*MuxBroker) (interface{}, error) + + // Client returns an interface implementation for the plugin you're + // serving that communicates to the server end of the plugin. + Client(*MuxBroker, *rpc.Client) (interface{}, error) +} + +// GRPCPlugin is the interface that is implemented to serve/connect to +// a plugin over gRPC. +type GRPCPlugin interface { + // GRPCServer should register this plugin for serving with the + // given GRPCServer. Unlike Plugin.Server, this is only called once + // since gRPC plugins serve singletons. + GRPCServer(*grpc.Server) error + + // GRPCClient should return the interface implementation for the plugin + // you're serving via gRPC. + GRPCClient(*grpc.ClientConn) (interface{}, error) +} + +// NetRPCUnsupportedPlugin implements Plugin but returns errors for the +// Server and Client functions. This will effectively disable support for +// net/rpc based plugins. +// +// This struct can be embedded in your struct. +type NetRPCUnsupportedPlugin struct{} + +func (p NetRPCUnsupportedPlugin) Server(*MuxBroker) (interface{}, error) { + return nil, errors.New("net/rpc plugin protocol not supported") +} + +func (p NetRPCUnsupportedPlugin) Client(*MuxBroker, *rpc.Client) (interface{}, error) { + return nil, errors.New("net/rpc plugin protocol not supported") +} diff --git a/vendor/github.com/hashicorp/go-plugin/process.go b/vendor/github.com/hashicorp/go-plugin/process.go new file mode 100644 index 000000000..88c999a58 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/process.go @@ -0,0 +1,24 @@ +package plugin + +import ( + "time" +) + +// pidAlive checks whether a pid is alive. +func pidAlive(pid int) bool { + return _pidAlive(pid) +} + +// pidWait blocks for a process to exit. +func pidWait(pid int) error { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for range ticker.C { + if !pidAlive(pid) { + break + } + } + + return nil +} diff --git a/vendor/github.com/hashicorp/go-plugin/process_posix.go b/vendor/github.com/hashicorp/go-plugin/process_posix.go new file mode 100644 index 000000000..70ba546bf --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/process_posix.go @@ -0,0 +1,19 @@ +// +build !windows + +package plugin + +import ( + "os" + "syscall" +) + +// _pidAlive tests whether a process is alive or not by sending it Signal 0, +// since Go otherwise has no way to test this. +func _pidAlive(pid int) bool { + proc, err := os.FindProcess(pid) + if err == nil { + err = proc.Signal(syscall.Signal(0)) + } + + return err == nil +} diff --git a/vendor/github.com/hashicorp/go-plugin/process_windows.go b/vendor/github.com/hashicorp/go-plugin/process_windows.go new file mode 100644 index 000000000..9f7b01809 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/process_windows.go @@ -0,0 +1,29 @@ +package plugin + +import ( + "syscall" +) + +const ( + // Weird name but matches the MSDN docs + exit_STILL_ACTIVE = 259 + + processDesiredAccess = syscall.STANDARD_RIGHTS_READ | + syscall.PROCESS_QUERY_INFORMATION | + syscall.SYNCHRONIZE +) + +// _pidAlive tests whether a process is alive or not +func _pidAlive(pid int) bool { + h, err := syscall.OpenProcess(processDesiredAccess, false, uint32(pid)) + if err != nil { + return false + } + + var ec uint32 + if e := syscall.GetExitCodeProcess(h, &ec); e != nil { + return false + } + + return ec == exit_STILL_ACTIVE +} diff --git a/vendor/github.com/hashicorp/go-plugin/protocol.go b/vendor/github.com/hashicorp/go-plugin/protocol.go new file mode 100644 index 000000000..0cfc19e52 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/protocol.go @@ -0,0 +1,45 @@ +package plugin + +import ( + "io" + "net" +) + +// Protocol is an enum representing the types of protocols. +type Protocol string + +const ( + ProtocolInvalid Protocol = "" + ProtocolNetRPC Protocol = "netrpc" + ProtocolGRPC Protocol = "grpc" +) + +// ServerProtocol is an interface that must be implemented for new plugin +// protocols to be servers. +type ServerProtocol interface { + // Init is called once to configure and initialize the protocol, but + // not start listening. This is the point at which all validation should + // be done and errors returned. + Init() error + + // Config is extra configuration to be outputted to stdout. This will + // be automatically base64 encoded to ensure it can be parsed properly. + // This can be an empty string if additional configuration is not needed. + Config() string + + // Serve is called to serve connections on the given listener. This should + // continue until the listener is closed. + Serve(net.Listener) +} + +// ClientProtocol is an interface that must be implemented for new plugin +// protocols to be clients. +type ClientProtocol interface { + io.Closer + + // Dispense dispenses a new instance of the plugin with the given name. + Dispense(string) (interface{}, error) + + // Ping checks that the client connection is still healthy. + Ping() error +} diff --git a/vendor/github.com/hashicorp/go-plugin/rpc_client.go b/vendor/github.com/hashicorp/go-plugin/rpc_client.go new file mode 100644 index 000000000..f30a4b1d3 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/rpc_client.go @@ -0,0 +1,170 @@ +package plugin + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/rpc" + + "github.com/hashicorp/yamux" +) + +// RPCClient connects to an RPCServer over net/rpc to dispense plugin types. +type RPCClient struct { + broker *MuxBroker + control *rpc.Client + plugins map[string]Plugin + + // These are the streams used for the various stdout/err overrides + stdout, stderr net.Conn +} + +// newRPCClient creates a new RPCClient. The Client argument is expected +// to be successfully started already with a lock held. +func newRPCClient(c *Client) (*RPCClient, error) { + // Connect to the client + conn, err := net.Dial(c.address.Network(), c.address.String()) + if err != nil { + return nil, err + } + if tcpConn, ok := conn.(*net.TCPConn); ok { + // Make sure to set keep alive so that the connection doesn't die + tcpConn.SetKeepAlive(true) + } + + if c.config.TLSConfig != nil { + conn = tls.Client(conn, c.config.TLSConfig) + } + + // Create the actual RPC client + result, err := NewRPCClient(conn, c.config.Plugins) + if err != nil { + conn.Close() + return nil, err + } + + // Begin the stream syncing so that stdin, out, err work properly + err = result.SyncStreams( + c.config.SyncStdout, + c.config.SyncStderr) + if err != nil { + result.Close() + return nil, err + } + + return result, nil +} + +// NewRPCClient creates a client from an already-open connection-like value. +// Dial is typically used instead. +func NewRPCClient(conn io.ReadWriteCloser, plugins map[string]Plugin) (*RPCClient, error) { + // Create the yamux client so we can multiplex + mux, err := yamux.Client(conn, nil) + if err != nil { + conn.Close() + return nil, err + } + + // Connect to the control stream. + control, err := mux.Open() + if err != nil { + mux.Close() + return nil, err + } + + // Connect stdout, stderr streams + stdstream := make([]net.Conn, 2) + for i, _ := range stdstream { + stdstream[i], err = mux.Open() + if err != nil { + mux.Close() + return nil, err + } + } + + // Create the broker and start it up + broker := newMuxBroker(mux) + go broker.Run() + + // Build the client using our broker and control channel. + return &RPCClient{ + broker: broker, + control: rpc.NewClient(control), + plugins: plugins, + stdout: stdstream[0], + stderr: stdstream[1], + }, nil +} + +// SyncStreams should be called to enable syncing of stdout, +// stderr with the plugin. +// +// This will return immediately and the syncing will continue to happen +// in the background. You do not need to launch this in a goroutine itself. +// +// This should never be called multiple times. +func (c *RPCClient) SyncStreams(stdout io.Writer, stderr io.Writer) error { + go copyStream("stdout", stdout, c.stdout) + go copyStream("stderr", stderr, c.stderr) + return nil +} + +// Close closes the connection. The client is no longer usable after this +// is called. +func (c *RPCClient) Close() error { + // Call the control channel and ask it to gracefully exit. If this + // errors, then we save it so that we always return an error but we + // want to try to close the other channels anyways. + var empty struct{} + returnErr := c.control.Call("Control.Quit", true, &empty) + + // Close the other streams we have + if err := c.control.Close(); err != nil { + return err + } + if err := c.stdout.Close(); err != nil { + return err + } + if err := c.stderr.Close(); err != nil { + return err + } + if err := c.broker.Close(); err != nil { + return err + } + + // Return back the error we got from Control.Quit. This is very important + // since we MUST return non-nil error if this fails so that Client.Kill + // will properly try a process.Kill. + return returnErr +} + +func (c *RPCClient) Dispense(name string) (interface{}, error) { + p, ok := c.plugins[name] + if !ok { + return nil, fmt.Errorf("unknown plugin type: %s", name) + } + + var id uint32 + if err := c.control.Call( + "Dispenser.Dispense", name, &id); err != nil { + return nil, err + } + + conn, err := c.broker.Dial(id) + if err != nil { + return nil, err + } + + return p.Client(c.broker, rpc.NewClient(conn)) +} + +// Ping pings the connection to ensure it is still alive. +// +// The error from the RPC call is returned exactly if you want to inspect +// it for further error analysis. Any error returned from here would indicate +// that the connection to the plugin is not healthy. +func (c *RPCClient) Ping() error { + var empty struct{} + return c.control.Call("Control.Ping", true, &empty) +} diff --git a/vendor/github.com/hashicorp/go-plugin/rpc_server.go b/vendor/github.com/hashicorp/go-plugin/rpc_server.go new file mode 100644 index 000000000..5bb18dd5d --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/rpc_server.go @@ -0,0 +1,197 @@ +package plugin + +import ( + "errors" + "fmt" + "io" + "log" + "net" + "net/rpc" + "sync" + + "github.com/hashicorp/yamux" +) + +// RPCServer listens for network connections and then dispenses interface +// implementations over net/rpc. +// +// After setting the fields below, they shouldn't be read again directly +// from the structure which may be reading/writing them concurrently. +type RPCServer struct { + Plugins map[string]Plugin + + // Stdout, Stderr are what this server will use instead of the + // normal stdin/out/err. This is because due to the multi-process nature + // of our plugin system, we can't use the normal process values so we + // make our own custom one we pipe across. + Stdout io.Reader + Stderr io.Reader + + // DoneCh should be set to a non-nil channel that will be closed + // when the control requests the RPC server to end. + DoneCh chan<- struct{} + + lock sync.Mutex +} + +// ServerProtocol impl. +func (s *RPCServer) Init() error { return nil } + +// ServerProtocol impl. +func (s *RPCServer) Config() string { return "" } + +// ServerProtocol impl. +func (s *RPCServer) Serve(lis net.Listener) { + for { + conn, err := lis.Accept() + if err != nil { + log.Printf("[ERR] plugin: plugin server: %s", err) + return + } + + go s.ServeConn(conn) + } +} + +// ServeConn runs a single connection. +// +// ServeConn blocks, serving the connection until the client hangs up. +func (s *RPCServer) ServeConn(conn io.ReadWriteCloser) { + // First create the yamux server to wrap this connection + mux, err := yamux.Server(conn, nil) + if err != nil { + conn.Close() + log.Printf("[ERR] plugin: error creating yamux server: %s", err) + return + } + + // Accept the control connection + control, err := mux.Accept() + if err != nil { + mux.Close() + if err != io.EOF { + log.Printf("[ERR] plugin: error accepting control connection: %s", err) + } + + return + } + + // Connect the stdstreams (in, out, err) + stdstream := make([]net.Conn, 2) + for i, _ := range stdstream { + stdstream[i], err = mux.Accept() + if err != nil { + mux.Close() + log.Printf("[ERR] plugin: accepting stream %d: %s", i, err) + return + } + } + + // Copy std streams out to the proper place + go copyStream("stdout", stdstream[0], s.Stdout) + go copyStream("stderr", stdstream[1], s.Stderr) + + // Create the broker and start it up + broker := newMuxBroker(mux) + go broker.Run() + + // Use the control connection to build the dispenser and serve the + // connection. + server := rpc.NewServer() + server.RegisterName("Control", &controlServer{ + server: s, + }) + server.RegisterName("Dispenser", &dispenseServer{ + broker: broker, + plugins: s.Plugins, + }) + server.ServeConn(control) +} + +// done is called internally by the control server to trigger the +// doneCh to close which is listened to by the main process to cleanly +// exit. +func (s *RPCServer) done() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.DoneCh != nil { + close(s.DoneCh) + s.DoneCh = nil + } +} + +// dispenseServer dispenses variousinterface implementations for Terraform. +type controlServer struct { + server *RPCServer +} + +// Ping can be called to verify the connection (and likely the binary) +// is still alive to a plugin. +func (c *controlServer) Ping( + null bool, response *struct{}) error { + *response = struct{}{} + return nil +} + +func (c *controlServer) Quit( + null bool, response *struct{}) error { + // End the server + c.server.done() + + // Always return true + *response = struct{}{} + + return nil +} + +// dispenseServer dispenses variousinterface implementations for Terraform. +type dispenseServer struct { + broker *MuxBroker + plugins map[string]Plugin +} + +func (d *dispenseServer) Dispense( + name string, response *uint32) error { + // Find the function to create this implementation + p, ok := d.plugins[name] + if !ok { + return fmt.Errorf("unknown plugin type: %s", name) + } + + // Create the implementation first so we know if there is an error. + impl, err := p.Server(d.broker) + if err != nil { + // We turn the error into an errors error so that it works across RPC + return errors.New(err.Error()) + } + + // Reserve an ID for our implementation + id := d.broker.NextId() + *response = id + + // Run the rest in a goroutine since it can only happen once this RPC + // call returns. We wait for a connection for the plugin implementation + // and serve it. + go func() { + conn, err := d.broker.Accept(id) + if err != nil { + log.Printf("[ERR] go-plugin: plugin dispense error: %s: %s", name, err) + return + } + + serve(conn, "Plugin", impl) + }() + + return nil +} + +func serve(conn io.ReadWriteCloser, name string, v interface{}) { + server := rpc.NewServer() + if err := server.RegisterName(name, v); err != nil { + log.Printf("[ERR] go-plugin: plugin dispense error: %s", err) + return + } + + server.ServeConn(conn) +} diff --git a/vendor/github.com/hashicorp/go-plugin/server.go b/vendor/github.com/hashicorp/go-plugin/server.go new file mode 100644 index 000000000..e1543214a --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/server.go @@ -0,0 +1,310 @@ +package plugin + +import ( + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "os/signal" + "runtime" + "strconv" + "sync/atomic" + + "github.com/hashicorp/go-hclog" + + "google.golang.org/grpc" +) + +// CoreProtocolVersion is the ProtocolVersion of the plugin system itself. +// We will increment this whenever we change any protocol behavior. This +// will invalidate any prior plugins but will at least allow us to iterate +// on the core in a safe way. We will do our best to do this very +// infrequently. +const CoreProtocolVersion = 1 + +// HandshakeConfig is the configuration used by client and servers to +// handshake before starting a plugin connection. This is embedded by +// both ServeConfig and ClientConfig. +// +// In practice, the plugin host creates a HandshakeConfig that is exported +// and plugins then can easily consume it. +type HandshakeConfig struct { + // ProtocolVersion is the version that clients must match on to + // agree they can communicate. This should match the ProtocolVersion + // set on ClientConfig when using a plugin. + ProtocolVersion uint + + // MagicCookieKey and value are used as a very basic verification + // that a plugin is intended to be launched. This is not a security + // measure, just a UX feature. If the magic cookie doesn't match, + // we show human-friendly output. + MagicCookieKey string + MagicCookieValue string +} + +// ServeConfig configures what sorts of plugins are served. +type ServeConfig struct { + // HandshakeConfig is the configuration that must match clients. + HandshakeConfig + + // TLSProvider is a function that returns a configured tls.Config. + TLSProvider func() (*tls.Config, error) + + // Plugins are the plugins that are served. + Plugins map[string]Plugin + + // GRPCServer should be non-nil to enable serving the plugins over + // gRPC. This is a function to create the server when needed with the + // given server options. The server options populated by go-plugin will + // be for TLS if set. You may modify the input slice. + // + // Note that the grpc.Server will automatically be registered with + // the gRPC health checking service. This is not optional since go-plugin + // relies on this to implement Ping(). + GRPCServer func([]grpc.ServerOption) *grpc.Server +} + +// Protocol returns the protocol that this server should speak. +func (c *ServeConfig) Protocol() Protocol { + result := ProtocolNetRPC + if c.GRPCServer != nil { + result = ProtocolGRPC + } + + return result +} + +// Serve serves the plugins given by ServeConfig. +// +// Serve doesn't return until the plugin is done being executed. Any +// errors will be outputted to os.Stderr. +// +// This is the method that plugins should call in their main() functions. +func Serve(opts *ServeConfig) { + // Validate the handshake config + if opts.MagicCookieKey == "" || opts.MagicCookieValue == "" { + fmt.Fprintf(os.Stderr, + "Misconfigured ServeConfig given to serve this plugin: no magic cookie\n"+ + "key or value was set. Please notify the plugin author and report\n"+ + "this as a bug.\n") + os.Exit(1) + } + + // First check the cookie + if os.Getenv(opts.MagicCookieKey) != opts.MagicCookieValue { + fmt.Fprintf(os.Stderr, + "This binary is a plugin. These are not meant to be executed directly.\n"+ + "Please execute the program that consumes these plugins, which will\n"+ + "load any plugins automatically\n") + os.Exit(1) + } + + // Logging goes to the original stderr + log.SetOutput(os.Stderr) + + // internal logger to os.Stderr + logger := hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: os.Stderr, + JSONFormat: true, + }) + + // Create our new stdout, stderr files. These will override our built-in + // stdout/stderr so that it works across the stream boundary. + stdout_r, stdout_w, err := os.Pipe() + if err != nil { + fmt.Fprintf(os.Stderr, "Error preparing plugin: %s\n", err) + os.Exit(1) + } + stderr_r, stderr_w, err := os.Pipe() + if err != nil { + fmt.Fprintf(os.Stderr, "Error preparing plugin: %s\n", err) + os.Exit(1) + } + + // Register a listener so we can accept a connection + listener, err := serverListener() + if err != nil { + logger.Error("plugin init error", "error", err) + return + } + + // Close the listener on return. We wrap this in a func() on purpose + // because the "listener" reference may change to TLS. + defer func() { + listener.Close() + }() + + var tlsConfig *tls.Config + if opts.TLSProvider != nil { + tlsConfig, err = opts.TLSProvider() + if err != nil { + logger.Error("plugin tls init", "error", err) + return + } + } + + // Create the channel to tell us when we're done + doneCh := make(chan struct{}) + + // Build the server type + var server ServerProtocol + switch opts.Protocol() { + case ProtocolNetRPC: + // If we have a TLS configuration then we wrap the listener + // ourselves and do it at that level. + if tlsConfig != nil { + listener = tls.NewListener(listener, tlsConfig) + } + + // Create the RPC server to dispense + server = &RPCServer{ + Plugins: opts.Plugins, + Stdout: stdout_r, + Stderr: stderr_r, + DoneCh: doneCh, + } + + case ProtocolGRPC: + // Create the gRPC server + server = &GRPCServer{ + Plugins: opts.Plugins, + Server: opts.GRPCServer, + TLS: tlsConfig, + Stdout: stdout_r, + Stderr: stderr_r, + DoneCh: doneCh, + } + + default: + panic("unknown server protocol: " + opts.Protocol()) + } + + // Initialize the servers + if err := server.Init(); err != nil { + logger.Error("protocol init", "error", err) + return + } + + // Build the extra configuration + extra := "" + if v := server.Config(); v != "" { + extra = base64.StdEncoding.EncodeToString([]byte(v)) + } + if extra != "" { + extra = "|" + extra + } + + logger.Debug("plugin address", "network", listener.Addr().Network(), "address", listener.Addr().String()) + + // Output the address and service name to stdout so that core can bring it up. + fmt.Printf("%d|%d|%s|%s|%s%s\n", + CoreProtocolVersion, + opts.ProtocolVersion, + listener.Addr().Network(), + listener.Addr().String(), + opts.Protocol(), + extra) + os.Stdout.Sync() + + // Eat the interrupts + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + go func() { + var count int32 = 0 + for { + <-ch + newCount := atomic.AddInt32(&count, 1) + logger.Debug("plugin received interrupt signal, ignoring", "count", newCount) + } + }() + + // Set our new out, err + os.Stdout = stdout_w + os.Stderr = stderr_w + + // Accept connections and wait for completion + go server.Serve(listener) + <-doneCh +} + +func serverListener() (net.Listener, error) { + if runtime.GOOS == "windows" { + return serverListener_tcp() + } + + return serverListener_unix() +} + +func serverListener_tcp() (net.Listener, error) { + minPort, err := strconv.ParseInt(os.Getenv("PLUGIN_MIN_PORT"), 10, 32) + if err != nil { + return nil, err + } + + maxPort, err := strconv.ParseInt(os.Getenv("PLUGIN_MAX_PORT"), 10, 32) + if err != nil { + return nil, err + } + + for port := minPort; port <= maxPort; port++ { + address := fmt.Sprintf("127.0.0.1:%d", port) + listener, err := net.Listen("tcp", address) + if err == nil { + return listener, nil + } + } + + return nil, errors.New("Couldn't bind plugin TCP listener") +} + +func serverListener_unix() (net.Listener, error) { + tf, err := ioutil.TempFile("", "plugin") + if err != nil { + return nil, err + } + path := tf.Name() + + // Close the file and remove it because it has to not exist for + // the domain socket. + if err := tf.Close(); err != nil { + return nil, err + } + if err := os.Remove(path); err != nil { + return nil, err + } + + l, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + + // Wrap the listener in rmListener so that the Unix domain socket file + // is removed on close. + return &rmListener{ + Listener: l, + Path: path, + }, nil +} + +// rmListener is an implementation of net.Listener that forwards most +// calls to the listener but also removes a file as part of the close. We +// use this to cleanup the unix domain socket on close. +type rmListener struct { + net.Listener + Path string +} + +func (l *rmListener) Close() error { + // Close the listener itself + if err := l.Listener.Close(); err != nil { + return err + } + + // Remove the file + return os.Remove(l.Path) +} diff --git a/vendor/github.com/hashicorp/go-plugin/server_mux.go b/vendor/github.com/hashicorp/go-plugin/server_mux.go new file mode 100644 index 000000000..033079ea0 --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/server_mux.go @@ -0,0 +1,31 @@ +package plugin + +import ( + "fmt" + "os" +) + +// ServeMuxMap is the type that is used to configure ServeMux +type ServeMuxMap map[string]*ServeConfig + +// ServeMux is like Serve, but serves multiple types of plugins determined +// by the argument given on the command-line. +// +// This command doesn't return until the plugin is done being executed. Any +// errors are logged or output to stderr. +func ServeMux(m ServeMuxMap) { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, + "Invoked improperly. This is an internal command that shouldn't\n"+ + "be manually invoked.\n") + os.Exit(1) + } + + opts, ok := m[os.Args[1]] + if !ok { + fmt.Fprintf(os.Stderr, "Unknown plugin: %s\n", os.Args[1]) + os.Exit(1) + } + + Serve(opts) +} diff --git a/vendor/github.com/hashicorp/go-plugin/stream.go b/vendor/github.com/hashicorp/go-plugin/stream.go new file mode 100644 index 000000000..1d547aaaa --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/stream.go @@ -0,0 +1,18 @@ +package plugin + +import ( + "io" + "log" +) + +func copyStream(name string, dst io.Writer, src io.Reader) { + if src == nil { + panic(name + ": src is nil") + } + if dst == nil { + panic(name + ": dst is nil") + } + if _, err := io.Copy(dst, src); err != nil && err != io.EOF { + log.Printf("[ERR] plugin: stream copy '%s' error: %s", name, err) + } +} diff --git a/vendor/github.com/hashicorp/go-plugin/testing.go b/vendor/github.com/hashicorp/go-plugin/testing.go new file mode 100644 index 000000000..c6bf7c4ed --- /dev/null +++ b/vendor/github.com/hashicorp/go-plugin/testing.go @@ -0,0 +1,120 @@ +package plugin + +import ( + "bytes" + "net" + "net/rpc" + + "github.com/mitchellh/go-testing-interface" + "google.golang.org/grpc" +) + +// The testing file contains test helpers that you can use outside of +// this package for making it easier to test plugins themselves. + +// TestConn is a helper function for returning a client and server +// net.Conn connected to each other. +func TestConn(t testing.T) (net.Conn, net.Conn) { + // Listen to any local port. This listener will be closed + // after a single connection is established. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("err: %s", err) + } + + // Start a goroutine to accept our client connection + var serverConn net.Conn + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + defer l.Close() + var err error + serverConn, err = l.Accept() + if err != nil { + t.Fatalf("err: %s", err) + } + }() + + // Connect to the server + clientConn, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Wait for the server side to acknowledge it has connected + <-doneCh + + return clientConn, serverConn +} + +// TestRPCConn returns a rpc client and server connected to each other. +func TestRPCConn(t testing.T) (*rpc.Client, *rpc.Server) { + clientConn, serverConn := TestConn(t) + + server := rpc.NewServer() + go server.ServeConn(serverConn) + + client := rpc.NewClient(clientConn) + return client, server +} + +// TestPluginRPCConn returns a plugin RPC client and server that are connected +// together and configured. +func TestPluginRPCConn(t testing.T, ps map[string]Plugin) (*RPCClient, *RPCServer) { + // Create two net.Conns we can use to shuttle our control connection + clientConn, serverConn := TestConn(t) + + // Start up the server + server := &RPCServer{Plugins: ps, Stdout: new(bytes.Buffer), Stderr: new(bytes.Buffer)} + go server.ServeConn(serverConn) + + // Connect the client to the server + client, err := NewRPCClient(clientConn, ps) + if err != nil { + t.Fatalf("err: %s", err) + } + + return client, server +} + +// TestPluginGRPCConn returns a plugin gRPC client and server that are connected +// together and configured. This is used to test gRPC connections. +func TestPluginGRPCConn(t testing.T, ps map[string]Plugin) (*GRPCClient, *GRPCServer) { + // Create a listener + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("err: %s", err) + } + + // Start up the server + server := &GRPCServer{ + Plugins: ps, + Server: DefaultGRPCServer, + Stdout: new(bytes.Buffer), + Stderr: new(bytes.Buffer), + } + if err := server.Init(); err != nil { + t.Fatalf("err: %s", err) + } + go server.Serve(l) + + // Connect to the server + conn, err := grpc.Dial( + l.Addr().String(), + grpc.WithBlock(), + grpc.WithInsecure()) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Connection successful, close the listener + l.Close() + + // Create the client + client := &GRPCClient{ + Conn: conn, + Plugins: ps, + } + + return client, server +} diff --git a/vendor/github.com/hashicorp/yamux/.gitignore b/vendor/github.com/hashicorp/yamux/.gitignore new file mode 100644 index 000000000..836562412 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/vendor/github.com/hashicorp/yamux/LICENSE b/vendor/github.com/hashicorp/yamux/LICENSE new file mode 100644 index 000000000..f0e5c79e1 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/LICENSE @@ -0,0 +1,362 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. \ No newline at end of file diff --git a/vendor/github.com/hashicorp/yamux/README.md b/vendor/github.com/hashicorp/yamux/README.md new file mode 100644 index 000000000..d4db7fc99 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/README.md @@ -0,0 +1,86 @@ +# Yamux + +Yamux (Yet another Multiplexer) is a multiplexing library for Golang. +It relies on an underlying connection to provide reliability +and ordering, such as TCP or Unix domain sockets, and provides +stream-oriented multiplexing. It is inspired by SPDY but is not +interoperable with it. + +Yamux features include: + +* Bi-directional streams + * Streams can be opened by either client or server + * Useful for NAT traversal + * Server-side push support +* Flow control + * Avoid starvation + * Back-pressure to prevent overwhelming a receiver +* Keep Alives + * Enables persistent connections over a load balancer +* Efficient + * Enables thousands of logical streams with low overhead + +## Documentation + +For complete documentation, see the associated [Godoc](http://godoc.org/github.com/hashicorp/yamux). + +## Specification + +The full specification for Yamux is provided in the `spec.md` file. +It can be used as a guide to implementors of interoperable libraries. + +## Usage + +Using Yamux is remarkably simple: + +```go + +func client() { + // Get a TCP connection + conn, err := net.Dial(...) + if err != nil { + panic(err) + } + + // Setup client side of yamux + session, err := yamux.Client(conn, nil) + if err != nil { + panic(err) + } + + // Open a new stream + stream, err := session.Open() + if err != nil { + panic(err) + } + + // Stream implements net.Conn + stream.Write([]byte("ping")) +} + +func server() { + // Accept a TCP connection + conn, err := listener.Accept() + if err != nil { + panic(err) + } + + // Setup server side of yamux + session, err := yamux.Server(conn, nil) + if err != nil { + panic(err) + } + + // Accept a stream + stream, err := session.Accept() + if err != nil { + panic(err) + } + + // Listen for a message + buf := make([]byte, 4) + stream.Read(buf) +} + +``` + diff --git a/vendor/github.com/hashicorp/yamux/addr.go b/vendor/github.com/hashicorp/yamux/addr.go new file mode 100644 index 000000000..be6ebca9c --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/addr.go @@ -0,0 +1,60 @@ +package yamux + +import ( + "fmt" + "net" +) + +// hasAddr is used to get the address from the underlying connection +type hasAddr interface { + LocalAddr() net.Addr + RemoteAddr() net.Addr +} + +// yamuxAddr is used when we cannot get the underlying address +type yamuxAddr struct { + Addr string +} + +func (*yamuxAddr) Network() string { + return "yamux" +} + +func (y *yamuxAddr) String() string { + return fmt.Sprintf("yamux:%s", y.Addr) +} + +// Addr is used to get the address of the listener. +func (s *Session) Addr() net.Addr { + return s.LocalAddr() +} + +// LocalAddr is used to get the local address of the +// underlying connection. +func (s *Session) LocalAddr() net.Addr { + addr, ok := s.conn.(hasAddr) + if !ok { + return &yamuxAddr{"local"} + } + return addr.LocalAddr() +} + +// RemoteAddr is used to get the address of remote end +// of the underlying connection +func (s *Session) RemoteAddr() net.Addr { + addr, ok := s.conn.(hasAddr) + if !ok { + return &yamuxAddr{"remote"} + } + return addr.RemoteAddr() +} + +// LocalAddr returns the local address +func (s *Stream) LocalAddr() net.Addr { + return s.session.LocalAddr() +} + +// LocalAddr returns the remote address +func (s *Stream) RemoteAddr() net.Addr { + return s.session.RemoteAddr() +} diff --git a/vendor/github.com/hashicorp/yamux/const.go b/vendor/github.com/hashicorp/yamux/const.go new file mode 100644 index 000000000..4f5293828 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/const.go @@ -0,0 +1,157 @@ +package yamux + +import ( + "encoding/binary" + "fmt" +) + +var ( + // ErrInvalidVersion means we received a frame with an + // invalid version + ErrInvalidVersion = fmt.Errorf("invalid protocol version") + + // ErrInvalidMsgType means we received a frame with an + // invalid message type + ErrInvalidMsgType = fmt.Errorf("invalid msg type") + + // ErrSessionShutdown is used if there is a shutdown during + // an operation + ErrSessionShutdown = fmt.Errorf("session shutdown") + + // ErrStreamsExhausted is returned if we have no more + // stream ids to issue + ErrStreamsExhausted = fmt.Errorf("streams exhausted") + + // ErrDuplicateStream is used if a duplicate stream is + // opened inbound + ErrDuplicateStream = fmt.Errorf("duplicate stream initiated") + + // ErrReceiveWindowExceeded indicates the window was exceeded + ErrRecvWindowExceeded = fmt.Errorf("recv window exceeded") + + // ErrTimeout is used when we reach an IO deadline + ErrTimeout = fmt.Errorf("i/o deadline reached") + + // ErrStreamClosed is returned when using a closed stream + ErrStreamClosed = fmt.Errorf("stream closed") + + // ErrUnexpectedFlag is set when we get an unexpected flag + ErrUnexpectedFlag = fmt.Errorf("unexpected flag") + + // ErrRemoteGoAway is used when we get a go away from the other side + ErrRemoteGoAway = fmt.Errorf("remote end is not accepting connections") + + // ErrConnectionReset is sent if a stream is reset. This can happen + // if the backlog is exceeded, or if there was a remote GoAway. + ErrConnectionReset = fmt.Errorf("connection reset") + + // ErrConnectionWriteTimeout indicates that we hit the "safety valve" + // timeout writing to the underlying stream connection. + ErrConnectionWriteTimeout = fmt.Errorf("connection write timeout") + + // ErrKeepAliveTimeout is sent if a missed keepalive caused the stream close + ErrKeepAliveTimeout = fmt.Errorf("keepalive timeout") +) + +const ( + // protoVersion is the only version we support + protoVersion uint8 = 0 +) + +const ( + // Data is used for data frames. They are followed + // by length bytes worth of payload. + typeData uint8 = iota + + // WindowUpdate is used to change the window of + // a given stream. The length indicates the delta + // update to the window. + typeWindowUpdate + + // Ping is sent as a keep-alive or to measure + // the RTT. The StreamID and Length value are echoed + // back in the response. + typePing + + // GoAway is sent to terminate a session. The StreamID + // should be 0 and the length is an error code. + typeGoAway +) + +const ( + // SYN is sent to signal a new stream. May + // be sent with a data payload + flagSYN uint16 = 1 << iota + + // ACK is sent to acknowledge a new stream. May + // be sent with a data payload + flagACK + + // FIN is sent to half-close the given stream. + // May be sent with a data payload. + flagFIN + + // RST is used to hard close a given stream. + flagRST +) + +const ( + // initialStreamWindow is the initial stream window size + initialStreamWindow uint32 = 256 * 1024 +) + +const ( + // goAwayNormal is sent on a normal termination + goAwayNormal uint32 = iota + + // goAwayProtoErr sent on a protocol error + goAwayProtoErr + + // goAwayInternalErr sent on an internal error + goAwayInternalErr +) + +const ( + sizeOfVersion = 1 + sizeOfType = 1 + sizeOfFlags = 2 + sizeOfStreamID = 4 + sizeOfLength = 4 + headerSize = sizeOfVersion + sizeOfType + sizeOfFlags + + sizeOfStreamID + sizeOfLength +) + +type header []byte + +func (h header) Version() uint8 { + return h[0] +} + +func (h header) MsgType() uint8 { + return h[1] +} + +func (h header) Flags() uint16 { + return binary.BigEndian.Uint16(h[2:4]) +} + +func (h header) StreamID() uint32 { + return binary.BigEndian.Uint32(h[4:8]) +} + +func (h header) Length() uint32 { + return binary.BigEndian.Uint32(h[8:12]) +} + +func (h header) String() string { + return fmt.Sprintf("Vsn:%d Type:%d Flags:%d StreamID:%d Length:%d", + h.Version(), h.MsgType(), h.Flags(), h.StreamID(), h.Length()) +} + +func (h header) encode(msgType uint8, flags uint16, streamID uint32, length uint32) { + h[0] = protoVersion + h[1] = msgType + binary.BigEndian.PutUint16(h[2:4], flags) + binary.BigEndian.PutUint32(h[4:8], streamID) + binary.BigEndian.PutUint32(h[8:12], length) +} diff --git a/vendor/github.com/hashicorp/yamux/mux.go b/vendor/github.com/hashicorp/yamux/mux.go new file mode 100644 index 000000000..7abc7c744 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/mux.go @@ -0,0 +1,87 @@ +package yamux + +import ( + "fmt" + "io" + "os" + "time" +) + +// Config is used to tune the Yamux session +type Config struct { + // AcceptBacklog is used to limit how many streams may be + // waiting an accept. + AcceptBacklog int + + // EnableKeepalive is used to do a period keep alive + // messages using a ping. + EnableKeepAlive bool + + // KeepAliveInterval is how often to perform the keep alive + KeepAliveInterval time.Duration + + // ConnectionWriteTimeout is meant to be a "safety valve" timeout after + // we which will suspect a problem with the underlying connection and + // close it. This is only applied to writes, where's there's generally + // an expectation that things will move along quickly. + ConnectionWriteTimeout time.Duration + + // MaxStreamWindowSize is used to control the maximum + // window size that we allow for a stream. + MaxStreamWindowSize uint32 + + // LogOutput is used to control the log destination + LogOutput io.Writer +} + +// DefaultConfig is used to return a default configuration +func DefaultConfig() *Config { + return &Config{ + AcceptBacklog: 256, + EnableKeepAlive: true, + KeepAliveInterval: 30 * time.Second, + ConnectionWriteTimeout: 10 * time.Second, + MaxStreamWindowSize: initialStreamWindow, + LogOutput: os.Stderr, + } +} + +// VerifyConfig is used to verify the sanity of configuration +func VerifyConfig(config *Config) error { + if config.AcceptBacklog <= 0 { + return fmt.Errorf("backlog must be positive") + } + if config.KeepAliveInterval == 0 { + return fmt.Errorf("keep-alive interval must be positive") + } + if config.MaxStreamWindowSize < initialStreamWindow { + return fmt.Errorf("MaxStreamWindowSize must be larger than %d", initialStreamWindow) + } + return nil +} + +// Server is used to initialize a new server-side connection. +// There must be at most one server-side connection. If a nil config is +// provided, the DefaultConfiguration will be used. +func Server(conn io.ReadWriteCloser, config *Config) (*Session, error) { + if config == nil { + config = DefaultConfig() + } + if err := VerifyConfig(config); err != nil { + return nil, err + } + return newSession(config, conn, false), nil +} + +// Client is used to initialize a new client-side connection. +// There must be at most one client-side connection. +func Client(conn io.ReadWriteCloser, config *Config) (*Session, error) { + if config == nil { + config = DefaultConfig() + } + + if err := VerifyConfig(config); err != nil { + return nil, err + } + return newSession(config, conn, true), nil +} diff --git a/vendor/github.com/hashicorp/yamux/session.go b/vendor/github.com/hashicorp/yamux/session.go new file mode 100644 index 000000000..e17981839 --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/session.go @@ -0,0 +1,623 @@ +package yamux + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "net" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Session is used to wrap a reliable ordered connection and to +// multiplex it into multiple streams. +type Session struct { + // remoteGoAway indicates the remote side does + // not want futher connections. Must be first for alignment. + remoteGoAway int32 + + // localGoAway indicates that we should stop + // accepting futher connections. Must be first for alignment. + localGoAway int32 + + // nextStreamID is the next stream we should + // send. This depends if we are a client/server. + nextStreamID uint32 + + // config holds our configuration + config *Config + + // logger is used for our logs + logger *log.Logger + + // conn is the underlying connection + conn io.ReadWriteCloser + + // bufRead is a buffered reader + bufRead *bufio.Reader + + // pings is used to track inflight pings + pings map[uint32]chan struct{} + pingID uint32 + pingLock sync.Mutex + + // streams maps a stream id to a stream, and inflight has an entry + // for any outgoing stream that has not yet been established. Both are + // protected by streamLock. + streams map[uint32]*Stream + inflight map[uint32]struct{} + streamLock sync.Mutex + + // synCh acts like a semaphore. It is sized to the AcceptBacklog which + // is assumed to be symmetric between the client and server. This allows + // the client to avoid exceeding the backlog and instead blocks the open. + synCh chan struct{} + + // acceptCh is used to pass ready streams to the client + acceptCh chan *Stream + + // sendCh is used to mark a stream as ready to send, + // or to send a header out directly. + sendCh chan sendReady + + // recvDoneCh is closed when recv() exits to avoid a race + // between stream registration and stream shutdown + recvDoneCh chan struct{} + + // shutdown is used to safely close a session + shutdown bool + shutdownErr error + shutdownCh chan struct{} + shutdownLock sync.Mutex +} + +// sendReady is used to either mark a stream as ready +// or to directly send a header +type sendReady struct { + Hdr []byte + Body io.Reader + Err chan error +} + +// newSession is used to construct a new session +func newSession(config *Config, conn io.ReadWriteCloser, client bool) *Session { + s := &Session{ + config: config, + logger: log.New(config.LogOutput, "", log.LstdFlags), + conn: conn, + bufRead: bufio.NewReader(conn), + pings: make(map[uint32]chan struct{}), + streams: make(map[uint32]*Stream), + inflight: make(map[uint32]struct{}), + synCh: make(chan struct{}, config.AcceptBacklog), + acceptCh: make(chan *Stream, config.AcceptBacklog), + sendCh: make(chan sendReady, 64), + recvDoneCh: make(chan struct{}), + shutdownCh: make(chan struct{}), + } + if client { + s.nextStreamID = 1 + } else { + s.nextStreamID = 2 + } + go s.recv() + go s.send() + if config.EnableKeepAlive { + go s.keepalive() + } + return s +} + +// IsClosed does a safe check to see if we have shutdown +func (s *Session) IsClosed() bool { + select { + case <-s.shutdownCh: + return true + default: + return false + } +} + +// NumStreams returns the number of currently open streams +func (s *Session) NumStreams() int { + s.streamLock.Lock() + num := len(s.streams) + s.streamLock.Unlock() + return num +} + +// Open is used to create a new stream as a net.Conn +func (s *Session) Open() (net.Conn, error) { + conn, err := s.OpenStream() + if err != nil { + return nil, err + } + return conn, nil +} + +// OpenStream is used to create a new stream +func (s *Session) OpenStream() (*Stream, error) { + if s.IsClosed() { + return nil, ErrSessionShutdown + } + if atomic.LoadInt32(&s.remoteGoAway) == 1 { + return nil, ErrRemoteGoAway + } + + // Block if we have too many inflight SYNs + select { + case s.synCh <- struct{}{}: + case <-s.shutdownCh: + return nil, ErrSessionShutdown + } + +GET_ID: + // Get an ID, and check for stream exhaustion + id := atomic.LoadUint32(&s.nextStreamID) + if id >= math.MaxUint32-1 { + return nil, ErrStreamsExhausted + } + if !atomic.CompareAndSwapUint32(&s.nextStreamID, id, id+2) { + goto GET_ID + } + + // Register the stream + stream := newStream(s, id, streamInit) + s.streamLock.Lock() + s.streams[id] = stream + s.inflight[id] = struct{}{} + s.streamLock.Unlock() + + // Send the window update to create + if err := stream.sendWindowUpdate(); err != nil { + select { + case <-s.synCh: + default: + s.logger.Printf("[ERR] yamux: aborted stream open without inflight syn semaphore") + } + return nil, err + } + return stream, nil +} + +// Accept is used to block until the next available stream +// is ready to be accepted. +func (s *Session) Accept() (net.Conn, error) { + conn, err := s.AcceptStream() + if err != nil { + return nil, err + } + return conn, err +} + +// AcceptStream is used to block until the next available stream +// is ready to be accepted. +func (s *Session) AcceptStream() (*Stream, error) { + select { + case stream := <-s.acceptCh: + if err := stream.sendWindowUpdate(); err != nil { + return nil, err + } + return stream, nil + case <-s.shutdownCh: + return nil, s.shutdownErr + } +} + +// Close is used to close the session and all streams. +// Attempts to send a GoAway before closing the connection. +func (s *Session) Close() error { + s.shutdownLock.Lock() + defer s.shutdownLock.Unlock() + + if s.shutdown { + return nil + } + s.shutdown = true + if s.shutdownErr == nil { + s.shutdownErr = ErrSessionShutdown + } + close(s.shutdownCh) + s.conn.Close() + <-s.recvDoneCh + + s.streamLock.Lock() + defer s.streamLock.Unlock() + for _, stream := range s.streams { + stream.forceClose() + } + return nil +} + +// exitErr is used to handle an error that is causing the +// session to terminate. +func (s *Session) exitErr(err error) { + s.shutdownLock.Lock() + if s.shutdownErr == nil { + s.shutdownErr = err + } + s.shutdownLock.Unlock() + s.Close() +} + +// GoAway can be used to prevent accepting further +// connections. It does not close the underlying conn. +func (s *Session) GoAway() error { + return s.waitForSend(s.goAway(goAwayNormal), nil) +} + +// goAway is used to send a goAway message +func (s *Session) goAway(reason uint32) header { + atomic.SwapInt32(&s.localGoAway, 1) + hdr := header(make([]byte, headerSize)) + hdr.encode(typeGoAway, 0, 0, reason) + return hdr +} + +// Ping is used to measure the RTT response time +func (s *Session) Ping() (time.Duration, error) { + // Get a channel for the ping + ch := make(chan struct{}) + + // Get a new ping id, mark as pending + s.pingLock.Lock() + id := s.pingID + s.pingID++ + s.pings[id] = ch + s.pingLock.Unlock() + + // Send the ping request + hdr := header(make([]byte, headerSize)) + hdr.encode(typePing, flagSYN, 0, id) + if err := s.waitForSend(hdr, nil); err != nil { + return 0, err + } + + // Wait for a response + start := time.Now() + select { + case <-ch: + case <-time.After(s.config.ConnectionWriteTimeout): + s.pingLock.Lock() + delete(s.pings, id) // Ignore it if a response comes later. + s.pingLock.Unlock() + return 0, ErrTimeout + case <-s.shutdownCh: + return 0, ErrSessionShutdown + } + + // Compute the RTT + return time.Now().Sub(start), nil +} + +// keepalive is a long running goroutine that periodically does +// a ping to keep the connection alive. +func (s *Session) keepalive() { + for { + select { + case <-time.After(s.config.KeepAliveInterval): + _, err := s.Ping() + if err != nil { + s.logger.Printf("[ERR] yamux: keepalive failed: %v", err) + s.exitErr(ErrKeepAliveTimeout) + return + } + case <-s.shutdownCh: + return + } + } +} + +// waitForSendErr waits to send a header, checking for a potential shutdown +func (s *Session) waitForSend(hdr header, body io.Reader) error { + errCh := make(chan error, 1) + return s.waitForSendErr(hdr, body, errCh) +} + +// waitForSendErr waits to send a header with optional data, checking for a +// potential shutdown. Since there's the expectation that sends can happen +// in a timely manner, we enforce the connection write timeout here. +func (s *Session) waitForSendErr(hdr header, body io.Reader, errCh chan error) error { + timer := time.NewTimer(s.config.ConnectionWriteTimeout) + defer timer.Stop() + + ready := sendReady{Hdr: hdr, Body: body, Err: errCh} + select { + case s.sendCh <- ready: + case <-s.shutdownCh: + return ErrSessionShutdown + case <-timer.C: + return ErrConnectionWriteTimeout + } + + select { + case err := <-errCh: + return err + case <-s.shutdownCh: + return ErrSessionShutdown + case <-timer.C: + return ErrConnectionWriteTimeout + } +} + +// sendNoWait does a send without waiting. Since there's the expectation that +// the send happens right here, we enforce the connection write timeout if we +// can't queue the header to be sent. +func (s *Session) sendNoWait(hdr header) error { + timer := time.NewTimer(s.config.ConnectionWriteTimeout) + defer timer.Stop() + + select { + case s.sendCh <- sendReady{Hdr: hdr}: + return nil + case <-s.shutdownCh: + return ErrSessionShutdown + case <-timer.C: + return ErrConnectionWriteTimeout + } +} + +// send is a long running goroutine that sends data +func (s *Session) send() { + for { + select { + case ready := <-s.sendCh: + // Send a header if ready + if ready.Hdr != nil { + sent := 0 + for sent < len(ready.Hdr) { + n, err := s.conn.Write(ready.Hdr[sent:]) + if err != nil { + s.logger.Printf("[ERR] yamux: Failed to write header: %v", err) + asyncSendErr(ready.Err, err) + s.exitErr(err) + return + } + sent += n + } + } + + // Send data from a body if given + if ready.Body != nil { + _, err := io.Copy(s.conn, ready.Body) + if err != nil { + s.logger.Printf("[ERR] yamux: Failed to write body: %v", err) + asyncSendErr(ready.Err, err) + s.exitErr(err) + return + } + } + + // No error, successful send + asyncSendErr(ready.Err, nil) + case <-s.shutdownCh: + return + } + } +} + +// recv is a long running goroutine that accepts new data +func (s *Session) recv() { + if err := s.recvLoop(); err != nil { + s.exitErr(err) + } +} + +// recvLoop continues to receive data until a fatal error is encountered +func (s *Session) recvLoop() error { + defer close(s.recvDoneCh) + hdr := header(make([]byte, headerSize)) + var handler func(header) error + for { + // Read the header + if _, err := io.ReadFull(s.bufRead, hdr); err != nil { + if err != io.EOF && !strings.Contains(err.Error(), "closed") && !strings.Contains(err.Error(), "reset by peer") { + s.logger.Printf("[ERR] yamux: Failed to read header: %v", err) + } + return err + } + + // Verify the version + if hdr.Version() != protoVersion { + s.logger.Printf("[ERR] yamux: Invalid protocol version: %d", hdr.Version()) + return ErrInvalidVersion + } + + // Switch on the type + switch hdr.MsgType() { + case typeData: + handler = s.handleStreamMessage + case typeWindowUpdate: + handler = s.handleStreamMessage + case typeGoAway: + handler = s.handleGoAway + case typePing: + handler = s.handlePing + default: + return ErrInvalidMsgType + } + + // Invoke the handler + if err := handler(hdr); err != nil { + return err + } + } +} + +// handleStreamMessage handles either a data or window update frame +func (s *Session) handleStreamMessage(hdr header) error { + // Check for a new stream creation + id := hdr.StreamID() + flags := hdr.Flags() + if flags&flagSYN == flagSYN { + if err := s.incomingStream(id); err != nil { + return err + } + } + + // Get the stream + s.streamLock.Lock() + stream := s.streams[id] + s.streamLock.Unlock() + + // If we do not have a stream, likely we sent a RST + if stream == nil { + // Drain any data on the wire + if hdr.MsgType() == typeData && hdr.Length() > 0 { + s.logger.Printf("[WARN] yamux: Discarding data for stream: %d", id) + if _, err := io.CopyN(ioutil.Discard, s.bufRead, int64(hdr.Length())); err != nil { + s.logger.Printf("[ERR] yamux: Failed to discard data: %v", err) + return nil + } + } else { + s.logger.Printf("[WARN] yamux: frame for missing stream: %v", hdr) + } + return nil + } + + // Check if this is a window update + if hdr.MsgType() == typeWindowUpdate { + if err := stream.incrSendWindow(hdr, flags); err != nil { + if sendErr := s.sendNoWait(s.goAway(goAwayProtoErr)); sendErr != nil { + s.logger.Printf("[WARN] yamux: failed to send go away: %v", sendErr) + } + return err + } + return nil + } + + // Read the new data + if err := stream.readData(hdr, flags, s.bufRead); err != nil { + if sendErr := s.sendNoWait(s.goAway(goAwayProtoErr)); sendErr != nil { + s.logger.Printf("[WARN] yamux: failed to send go away: %v", sendErr) + } + return err + } + return nil +} + +// handlePing is invokde for a typePing frame +func (s *Session) handlePing(hdr header) error { + flags := hdr.Flags() + pingID := hdr.Length() + + // Check if this is a query, respond back in a separate context so we + // don't interfere with the receiving thread blocking for the write. + if flags&flagSYN == flagSYN { + go func() { + hdr := header(make([]byte, headerSize)) + hdr.encode(typePing, flagACK, 0, pingID) + if err := s.sendNoWait(hdr); err != nil { + s.logger.Printf("[WARN] yamux: failed to send ping reply: %v", err) + } + }() + return nil + } + + // Handle a response + s.pingLock.Lock() + ch := s.pings[pingID] + if ch != nil { + delete(s.pings, pingID) + close(ch) + } + s.pingLock.Unlock() + return nil +} + +// handleGoAway is invokde for a typeGoAway frame +func (s *Session) handleGoAway(hdr header) error { + code := hdr.Length() + switch code { + case goAwayNormal: + atomic.SwapInt32(&s.remoteGoAway, 1) + case goAwayProtoErr: + s.logger.Printf("[ERR] yamux: received protocol error go away") + return fmt.Errorf("yamux protocol error") + case goAwayInternalErr: + s.logger.Printf("[ERR] yamux: received internal error go away") + return fmt.Errorf("remote yamux internal error") + default: + s.logger.Printf("[ERR] yamux: received unexpected go away") + return fmt.Errorf("unexpected go away received") + } + return nil +} + +// incomingStream is used to create a new incoming stream +func (s *Session) incomingStream(id uint32) error { + // Reject immediately if we are doing a go away + if atomic.LoadInt32(&s.localGoAway) == 1 { + hdr := header(make([]byte, headerSize)) + hdr.encode(typeWindowUpdate, flagRST, id, 0) + return s.sendNoWait(hdr) + } + + // Allocate a new stream + stream := newStream(s, id, streamSYNReceived) + + s.streamLock.Lock() + defer s.streamLock.Unlock() + + // Check if stream already exists + if _, ok := s.streams[id]; ok { + s.logger.Printf("[ERR] yamux: duplicate stream declared") + if sendErr := s.sendNoWait(s.goAway(goAwayProtoErr)); sendErr != nil { + s.logger.Printf("[WARN] yamux: failed to send go away: %v", sendErr) + } + return ErrDuplicateStream + } + + // Register the stream + s.streams[id] = stream + + // Check if we've exceeded the backlog + select { + case s.acceptCh <- stream: + return nil + default: + // Backlog exceeded! RST the stream + s.logger.Printf("[WARN] yamux: backlog exceeded, forcing connection reset") + delete(s.streams, id) + stream.sendHdr.encode(typeWindowUpdate, flagRST, id, 0) + return s.sendNoWait(stream.sendHdr) + } +} + +// closeStream is used to close a stream once both sides have +// issued a close. If there was an in-flight SYN and the stream +// was not yet established, then this will give the credit back. +func (s *Session) closeStream(id uint32) { + s.streamLock.Lock() + if _, ok := s.inflight[id]; ok { + select { + case <-s.synCh: + default: + s.logger.Printf("[ERR] yamux: SYN tracking out of sync") + } + } + delete(s.streams, id) + s.streamLock.Unlock() +} + +// establishStream is used to mark a stream that was in the +// SYN Sent state as established. +func (s *Session) establishStream(id uint32) { + s.streamLock.Lock() + if _, ok := s.inflight[id]; ok { + delete(s.inflight, id) + } else { + s.logger.Printf("[ERR] yamux: established stream without inflight SYN (no tracking entry)") + } + select { + case <-s.synCh: + default: + s.logger.Printf("[ERR] yamux: established stream without inflight SYN (didn't have semaphore)") + } + s.streamLock.Unlock() +} diff --git a/vendor/github.com/hashicorp/yamux/spec.md b/vendor/github.com/hashicorp/yamux/spec.md new file mode 100644 index 000000000..183d797bd --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/spec.md @@ -0,0 +1,140 @@ +# Specification + +We use this document to detail the internal specification of Yamux. +This is used both as a guide for implementing Yamux, but also for +alternative interoperable libraries to be built. + +# Framing + +Yamux uses a streaming connection underneath, but imposes a message +framing so that it can be shared between many logical streams. Each +frame contains a header like: + +* Version (8 bits) +* Type (8 bits) +* Flags (16 bits) +* StreamID (32 bits) +* Length (32 bits) + +This means that each header has a 12 byte overhead. +All fields are encoded in network order (big endian). +Each field is described below: + +## Version Field + +The version field is used for future backward compatibility. At the +current time, the field is always set to 0, to indicate the initial +version. + +## Type Field + +The type field is used to switch the frame message type. The following +message types are supported: + +* 0x0 Data - Used to transmit data. May transmit zero length payloads + depending on the flags. + +* 0x1 Window Update - Used to updated the senders receive window size. + This is used to implement per-session flow control. + +* 0x2 Ping - Used to measure RTT. It can also be used to heart-beat + and do keep-alives over TCP. + +* 0x3 Go Away - Used to close a session. + +## Flag Field + +The flags field is used to provide additional information related +to the message type. The following flags are supported: + +* 0x1 SYN - Signals the start of a new stream. May be sent with a data or + window update message. Also sent with a ping to indicate outbound. + +* 0x2 ACK - Acknowledges the start of a new stream. May be sent with a data + or window update message. Also sent with a ping to indicate response. + +* 0x4 FIN - Performs a half-close of a stream. May be sent with a data + message or window update. + +* 0x8 RST - Reset a stream immediately. May be sent with a data or + window update message. + +## StreamID Field + +The StreamID field is used to identify the logical stream the frame +is addressing. The client side should use odd ID's, and the server even. +This prevents any collisions. Additionally, the 0 ID is reserved to represent +the session. + +Both Ping and Go Away messages should always use the 0 StreamID. + +## Length Field + +The meaning of the length field depends on the message type: + +* Data - provides the length of bytes following the header +* Window update - provides a delta update to the window size +* Ping - Contains an opaque value, echoed back +* Go Away - Contains an error code + +# Message Flow + +There is no explicit connection setup, as Yamux relies on an underlying +transport to be provided. However, there is a distinction between client +and server side of the connection. + +## Opening a stream + +To open a stream, an initial data or window update frame is sent +with a new StreamID. The SYN flag should be set to signal a new stream. + +The receiver must then reply with either a data or window update frame +with the StreamID along with the ACK flag to accept the stream or with +the RST flag to reject the stream. + +Because we are relying on the reliable stream underneath, a connection +can begin sending data once the SYN flag is sent. The corresponding +ACK does not need to be received. This is particularly well suited +for an RPC system where a client wants to open a stream and immediately +fire a request without waiting for the RTT of the ACK. + +This does introduce the possibility of a connection being rejected +after data has been sent already. This is a slight semantic difference +from TCP, where the conection cannot be refused after it is opened. +Clients should be prepared to handle this by checking for an error +that indicates a RST was received. + +## Closing a stream + +To close a stream, either side sends a data or window update frame +along with the FIN flag. This does a half-close indicating the sender +will send no further data. + +Once both sides have closed the connection, the stream is closed. + +Alternatively, if an error occurs, the RST flag can be used to +hard close a stream immediately. + +## Flow Control + +When Yamux is initially starts each stream with a 256KB window size. +There is no window size for the session. + +To prevent the streams from stalling, window update frames should be +sent regularly. Yamux can be configured to provide a larger limit for +windows sizes. Both sides assume the initial 256KB window, but can +immediately send a window update as part of the SYN/ACK indicating a +larger window. + +Both sides should track the number of bytes sent in Data frames +only, as only they are tracked as part of the window size. + +## Session termination + +When a session is being terminated, the Go Away message should +be sent. The Length should be set to one of the following to +provide an error code: + +* 0x0 Normal termination +* 0x1 Protocol error +* 0x2 Internal error diff --git a/vendor/github.com/hashicorp/yamux/stream.go b/vendor/github.com/hashicorp/yamux/stream.go new file mode 100644 index 000000000..d216e281c --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/stream.go @@ -0,0 +1,457 @@ +package yamux + +import ( + "bytes" + "io" + "sync" + "sync/atomic" + "time" +) + +type streamState int + +const ( + streamInit streamState = iota + streamSYNSent + streamSYNReceived + streamEstablished + streamLocalClose + streamRemoteClose + streamClosed + streamReset +) + +// Stream is used to represent a logical stream +// within a session. +type Stream struct { + recvWindow uint32 + sendWindow uint32 + + id uint32 + session *Session + + state streamState + stateLock sync.Mutex + + recvBuf *bytes.Buffer + recvLock sync.Mutex + + controlHdr header + controlErr chan error + controlHdrLock sync.Mutex + + sendHdr header + sendErr chan error + sendLock sync.Mutex + + recvNotifyCh chan struct{} + sendNotifyCh chan struct{} + + readDeadline time.Time + writeDeadline time.Time +} + +// newStream is used to construct a new stream within +// a given session for an ID +func newStream(session *Session, id uint32, state streamState) *Stream { + s := &Stream{ + id: id, + session: session, + state: state, + controlHdr: header(make([]byte, headerSize)), + controlErr: make(chan error, 1), + sendHdr: header(make([]byte, headerSize)), + sendErr: make(chan error, 1), + recvWindow: initialStreamWindow, + sendWindow: initialStreamWindow, + recvNotifyCh: make(chan struct{}, 1), + sendNotifyCh: make(chan struct{}, 1), + } + return s +} + +// Session returns the associated stream session +func (s *Stream) Session() *Session { + return s.session +} + +// StreamID returns the ID of this stream +func (s *Stream) StreamID() uint32 { + return s.id +} + +// Read is used to read from the stream +func (s *Stream) Read(b []byte) (n int, err error) { + defer asyncNotify(s.recvNotifyCh) +START: + s.stateLock.Lock() + switch s.state { + case streamLocalClose: + fallthrough + case streamRemoteClose: + fallthrough + case streamClosed: + s.recvLock.Lock() + if s.recvBuf == nil || s.recvBuf.Len() == 0 { + s.recvLock.Unlock() + s.stateLock.Unlock() + return 0, io.EOF + } + s.recvLock.Unlock() + case streamReset: + s.stateLock.Unlock() + return 0, ErrConnectionReset + } + s.stateLock.Unlock() + + // If there is no data available, block + s.recvLock.Lock() + if s.recvBuf == nil || s.recvBuf.Len() == 0 { + s.recvLock.Unlock() + goto WAIT + } + + // Read any bytes + n, _ = s.recvBuf.Read(b) + s.recvLock.Unlock() + + // Send a window update potentially + err = s.sendWindowUpdate() + return n, err + +WAIT: + var timeout <-chan time.Time + var timer *time.Timer + if !s.readDeadline.IsZero() { + delay := s.readDeadline.Sub(time.Now()) + timer = time.NewTimer(delay) + timeout = timer.C + } + select { + case <-s.recvNotifyCh: + if timer != nil { + timer.Stop() + } + goto START + case <-timeout: + return 0, ErrTimeout + } +} + +// Write is used to write to the stream +func (s *Stream) Write(b []byte) (n int, err error) { + s.sendLock.Lock() + defer s.sendLock.Unlock() + total := 0 + for total < len(b) { + n, err := s.write(b[total:]) + total += n + if err != nil { + return total, err + } + } + return total, nil +} + +// write is used to write to the stream, may return on +// a short write. +func (s *Stream) write(b []byte) (n int, err error) { + var flags uint16 + var max uint32 + var body io.Reader +START: + s.stateLock.Lock() + switch s.state { + case streamLocalClose: + fallthrough + case streamClosed: + s.stateLock.Unlock() + return 0, ErrStreamClosed + case streamReset: + s.stateLock.Unlock() + return 0, ErrConnectionReset + } + s.stateLock.Unlock() + + // If there is no data available, block + window := atomic.LoadUint32(&s.sendWindow) + if window == 0 { + goto WAIT + } + + // Determine the flags if any + flags = s.sendFlags() + + // Send up to our send window + max = min(window, uint32(len(b))) + body = bytes.NewReader(b[:max]) + + // Send the header + s.sendHdr.encode(typeData, flags, s.id, max) + if err := s.session.waitForSendErr(s.sendHdr, body, s.sendErr); err != nil { + return 0, err + } + + // Reduce our send window + atomic.AddUint32(&s.sendWindow, ^uint32(max-1)) + + // Unlock + return int(max), err + +WAIT: + var timeout <-chan time.Time + if !s.writeDeadline.IsZero() { + delay := s.writeDeadline.Sub(time.Now()) + timeout = time.After(delay) + } + select { + case <-s.sendNotifyCh: + goto START + case <-timeout: + return 0, ErrTimeout + } + return 0, nil +} + +// sendFlags determines any flags that are appropriate +// based on the current stream state +func (s *Stream) sendFlags() uint16 { + s.stateLock.Lock() + defer s.stateLock.Unlock() + var flags uint16 + switch s.state { + case streamInit: + flags |= flagSYN + s.state = streamSYNSent + case streamSYNReceived: + flags |= flagACK + s.state = streamEstablished + } + return flags +} + +// sendWindowUpdate potentially sends a window update enabling +// further writes to take place. Must be invoked with the lock. +func (s *Stream) sendWindowUpdate() error { + s.controlHdrLock.Lock() + defer s.controlHdrLock.Unlock() + + // Determine the delta update + max := s.session.config.MaxStreamWindowSize + delta := max - atomic.LoadUint32(&s.recvWindow) + + // Determine the flags if any + flags := s.sendFlags() + + // Check if we can omit the update + if delta < (max/2) && flags == 0 { + return nil + } + + // Update our window + atomic.AddUint32(&s.recvWindow, delta) + + // Send the header + s.controlHdr.encode(typeWindowUpdate, flags, s.id, delta) + if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil { + return err + } + return nil +} + +// sendClose is used to send a FIN +func (s *Stream) sendClose() error { + s.controlHdrLock.Lock() + defer s.controlHdrLock.Unlock() + + flags := s.sendFlags() + flags |= flagFIN + s.controlHdr.encode(typeWindowUpdate, flags, s.id, 0) + if err := s.session.waitForSendErr(s.controlHdr, nil, s.controlErr); err != nil { + return err + } + return nil +} + +// Close is used to close the stream +func (s *Stream) Close() error { + closeStream := false + s.stateLock.Lock() + switch s.state { + // Opened means we need to signal a close + case streamSYNSent: + fallthrough + case streamSYNReceived: + fallthrough + case streamEstablished: + s.state = streamLocalClose + goto SEND_CLOSE + + case streamLocalClose: + case streamRemoteClose: + s.state = streamClosed + closeStream = true + goto SEND_CLOSE + + case streamClosed: + case streamReset: + default: + panic("unhandled state") + } + s.stateLock.Unlock() + return nil +SEND_CLOSE: + s.stateLock.Unlock() + s.sendClose() + s.notifyWaiting() + if closeStream { + s.session.closeStream(s.id) + } + return nil +} + +// forceClose is used for when the session is exiting +func (s *Stream) forceClose() { + s.stateLock.Lock() + s.state = streamClosed + s.stateLock.Unlock() + s.notifyWaiting() +} + +// processFlags is used to update the state of the stream +// based on set flags, if any. Lock must be held +func (s *Stream) processFlags(flags uint16) error { + // Close the stream without holding the state lock + closeStream := false + defer func() { + if closeStream { + s.session.closeStream(s.id) + } + }() + + s.stateLock.Lock() + defer s.stateLock.Unlock() + if flags&flagACK == flagACK { + if s.state == streamSYNSent { + s.state = streamEstablished + } + s.session.establishStream(s.id) + } + if flags&flagFIN == flagFIN { + switch s.state { + case streamSYNSent: + fallthrough + case streamSYNReceived: + fallthrough + case streamEstablished: + s.state = streamRemoteClose + s.notifyWaiting() + case streamLocalClose: + s.state = streamClosed + closeStream = true + s.notifyWaiting() + default: + s.session.logger.Printf("[ERR] yamux: unexpected FIN flag in state %d", s.state) + return ErrUnexpectedFlag + } + } + if flags&flagRST == flagRST { + s.state = streamReset + closeStream = true + s.notifyWaiting() + } + return nil +} + +// notifyWaiting notifies all the waiting channels +func (s *Stream) notifyWaiting() { + asyncNotify(s.recvNotifyCh) + asyncNotify(s.sendNotifyCh) +} + +// incrSendWindow updates the size of our send window +func (s *Stream) incrSendWindow(hdr header, flags uint16) error { + if err := s.processFlags(flags); err != nil { + return err + } + + // Increase window, unblock a sender + atomic.AddUint32(&s.sendWindow, hdr.Length()) + asyncNotify(s.sendNotifyCh) + return nil +} + +// readData is used to handle a data frame +func (s *Stream) readData(hdr header, flags uint16, conn io.Reader) error { + if err := s.processFlags(flags); err != nil { + return err + } + + // Check that our recv window is not exceeded + length := hdr.Length() + if length == 0 { + return nil + } + if remain := atomic.LoadUint32(&s.recvWindow); length > remain { + s.session.logger.Printf("[ERR] yamux: receive window exceeded (stream: %d, remain: %d, recv: %d)", s.id, remain, length) + return ErrRecvWindowExceeded + } + + // Wrap in a limited reader + conn = &io.LimitedReader{R: conn, N: int64(length)} + + // Copy into buffer + s.recvLock.Lock() + if s.recvBuf == nil { + // Allocate the receive buffer just-in-time to fit the full data frame. + // This way we can read in the whole packet without further allocations. + s.recvBuf = bytes.NewBuffer(make([]byte, 0, length)) + } + if _, err := io.Copy(s.recvBuf, conn); err != nil { + s.session.logger.Printf("[ERR] yamux: Failed to read stream data: %v", err) + s.recvLock.Unlock() + return err + } + + // Decrement the receive window + atomic.AddUint32(&s.recvWindow, ^uint32(length-1)) + s.recvLock.Unlock() + + // Unblock any readers + asyncNotify(s.recvNotifyCh) + return nil +} + +// SetDeadline sets the read and write deadlines +func (s *Stream) SetDeadline(t time.Time) error { + if err := s.SetReadDeadline(t); err != nil { + return err + } + if err := s.SetWriteDeadline(t); err != nil { + return err + } + return nil +} + +// SetReadDeadline sets the deadline for future Read calls. +func (s *Stream) SetReadDeadline(t time.Time) error { + s.readDeadline = t + return nil +} + +// SetWriteDeadline sets the deadline for future Write calls +func (s *Stream) SetWriteDeadline(t time.Time) error { + s.writeDeadline = t + return nil +} + +// Shrink is used to compact the amount of buffers utilized +// This is useful when using Yamux in a connection pool to reduce +// the idle memory utilization. +func (s *Stream) Shrink() { + s.recvLock.Lock() + if s.recvBuf != nil && s.recvBuf.Len() == 0 { + s.recvBuf = nil + } + s.recvLock.Unlock() +} diff --git a/vendor/github.com/hashicorp/yamux/util.go b/vendor/github.com/hashicorp/yamux/util.go new file mode 100644 index 000000000..5fe45afcd --- /dev/null +++ b/vendor/github.com/hashicorp/yamux/util.go @@ -0,0 +1,28 @@ +package yamux + +// asyncSendErr is used to try an async send of an error +func asyncSendErr(ch chan error, err error) { + if ch == nil { + return + } + select { + case ch <- err: + default: + } +} + +// asyncNotify is used to signal a waiting goroutine +func asyncNotify(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} + +// min computes the minimum of two values +func min(a, b uint32) uint32 { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/mitchellh/go-testing-interface/LICENSE b/vendor/github.com/mitchellh/go-testing-interface/LICENSE new file mode 100644 index 000000000..a3866a291 --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/go-testing-interface/README.md b/vendor/github.com/mitchellh/go-testing-interface/README.md new file mode 100644 index 000000000..26781bbae --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/README.md @@ -0,0 +1,52 @@ +# go-testing-interface + +go-testing-interface is a Go library that exports an interface that +`*testing.T` implements as well as a runtime version you can use in its +place. + +The purpose of this library is so that you can export test helpers as a +public API without depending on the "testing" package, since you can't +create a `*testing.T` struct manually. This lets you, for example, use the +public testing APIs to generate mock data at runtime, rather than just at +test time. + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/go-testing-interface). + +Given a test helper written using `go-testing-interface` like this: + + import "github.com/mitchellh/go-testing-interface" + + func TestHelper(t testing.T) { + t.Fatal("I failed") + } + +You can call the test helper in a real test easily: + + import "testing" + + func TestThing(t *testing.T) { + TestHelper(t) + } + +You can also call the test helper at runtime if needed: + + import "github.com/mitchellh/go-testing-interface" + + func main() { + TestHelper(&testing.RuntimeT{}) + } + +## Why?! + +**Why would I call a test helper that takes a *testing.T at runtime?** + +You probably shouldn't. The only use case I've seen (and I've had) for this +is to implement a "dev mode" for a service where the test helpers are used +to populate mock data, create a mock DB, perhaps run service dependencies +in-memory, etc. + +Outside of a "dev mode", I've never seen a use case for this and I think +there shouldn't be one since the point of the `testing.T` interface is that +you can fail immediately. diff --git a/vendor/github.com/mitchellh/go-testing-interface/testing.go b/vendor/github.com/mitchellh/go-testing-interface/testing.go new file mode 100644 index 000000000..204afb420 --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/testing.go @@ -0,0 +1,84 @@ +// +build !go1.9 + +package testing + +import ( + "fmt" + "log" +) + +// T is the interface that mimics the standard library *testing.T. +// +// In unit tests you can just pass a *testing.T struct. At runtime, outside +// of tests, you can pass in a RuntimeT struct from this package. +type T interface { + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Fail() + FailNow() + Failed() bool + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Log(args ...interface{}) + Logf(format string, args ...interface{}) + Name() string + Skip(args ...interface{}) + SkipNow() + Skipf(format string, args ...interface{}) + Skipped() bool +} + +// RuntimeT implements T and can be instantiated and run at runtime to +// mimic *testing.T behavior. Unlike *testing.T, this will simply panic +// for calls to Fatal. For calls to Error, you'll have to check the errors +// list to determine whether to exit yourself. Name and Skip methods are +// unimplemented noops. +type RuntimeT struct { + failed bool +} + +func (t *RuntimeT) Error(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) + t.Fail() +} + +func (t *RuntimeT) Errorf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) + t.Fail() +} + +func (t *RuntimeT) Fatal(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) + t.FailNow() +} + +func (t *RuntimeT) Fatalf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) + t.FailNow() +} + +func (t *RuntimeT) Fail() { + t.failed = true +} + +func (t *RuntimeT) FailNow() { + panic("testing.T failed, see logs for output (if any)") +} + +func (t *RuntimeT) Failed() bool { + return t.failed +} + +func (t *RuntimeT) Log(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) +} + +func (t *RuntimeT) Logf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) +} + +func (t *RuntimeT) Name() string { return "" } +func (t *RuntimeT) Skip(args ...interface{}) {} +func (t *RuntimeT) SkipNow() {} +func (t *RuntimeT) Skipf(format string, args ...interface{}) {} +func (t *RuntimeT) Skipped() bool { return false } diff --git a/vendor/github.com/mitchellh/go-testing-interface/testing_go19.go b/vendor/github.com/mitchellh/go-testing-interface/testing_go19.go new file mode 100644 index 000000000..31b42cadf --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/testing_go19.go @@ -0,0 +1,108 @@ +// +build go1.9 + +// NOTE: This is a temporary copy of testing.go for Go 1.9 with the addition +// of "Helper" to the T interface. Go 1.9 at the time of typing is in RC +// and is set for release shortly. We'll support this on master as the default +// as soon as 1.9 is released. + +package testing + +import ( + "fmt" + "log" +) + +// T is the interface that mimics the standard library *testing.T. +// +// In unit tests you can just pass a *testing.T struct. At runtime, outside +// of tests, you can pass in a RuntimeT struct from this package. +type T interface { + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Fail() + FailNow() + Failed() bool + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Log(args ...interface{}) + Logf(format string, args ...interface{}) + Name() string + Skip(args ...interface{}) + SkipNow() + Skipf(format string, args ...interface{}) + Skipped() bool + Helper() +} + +// RuntimeT implements T and can be instantiated and run at runtime to +// mimic *testing.T behavior. Unlike *testing.T, this will simply panic +// for calls to Fatal. For calls to Error, you'll have to check the errors +// list to determine whether to exit yourself. +type RuntimeT struct { + skipped bool + failed bool +} + +func (t *RuntimeT) Error(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) + t.Fail() +} + +func (t *RuntimeT) Errorf(format string, args ...interface{}) { + log.Printf(format, args...) + t.Fail() +} + +func (t *RuntimeT) Fail() { + t.failed = true +} + +func (t *RuntimeT) FailNow() { + panic("testing.T failed, see logs for output (if any)") +} + +func (t *RuntimeT) Failed() bool { + return t.failed +} + +func (t *RuntimeT) Fatal(args ...interface{}) { + log.Print(args...) + t.FailNow() +} + +func (t *RuntimeT) Fatalf(format string, args ...interface{}) { + log.Printf(format, args...) + t.FailNow() +} + +func (t *RuntimeT) Log(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) +} + +func (t *RuntimeT) Logf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) +} + +func (t *RuntimeT) Name() string { + return "" +} + +func (t *RuntimeT) Skip(args ...interface{}) { + log.Print(args...) + t.SkipNow() +} + +func (t *RuntimeT) SkipNow() { + t.skipped = true +} + +func (t *RuntimeT) Skipf(format string, args ...interface{}) { + log.Printf(format, args...) + t.SkipNow() +} + +func (t *RuntimeT) Skipped() bool { + return t.skipped +} + +func (t *RuntimeT) Helper() {} diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go new file mode 100644 index 000000000..89c4d459f --- /dev/null +++ b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go @@ -0,0 +1,176 @@ +// Code generated by protoc-gen-go. +// source: health.proto +// DO NOT EDIT! + +/* +Package grpc_health_v1 is a generated protocol buffer package. + +It is generated from these files: + health.proto + +It has these top-level messages: + HealthCheckRequest + HealthCheckResponse +*/ +package grpc_health_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type HealthCheckResponse_ServingStatus int32 + +const ( + HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 + HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 + HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 +) + +var HealthCheckResponse_ServingStatus_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SERVING", + 2: "NOT_SERVING", +} +var HealthCheckResponse_ServingStatus_value = map[string]int32{ + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, +} + +func (x HealthCheckResponse_ServingStatus) String() string { + return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) +} +func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptor0, []int{1, 0} +} + +type HealthCheckRequest struct { + Service string `protobuf:"bytes,1,opt,name=service" json:"service,omitempty"` +} + +func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } +func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } +func (*HealthCheckRequest) ProtoMessage() {} +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +type HealthCheckResponse struct { + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` +} + +func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } +func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } +func (*HealthCheckResponse) ProtoMessage() {} +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func init() { + proto.RegisterType((*HealthCheckRequest)(nil), "grpc.health.v1.HealthCheckRequest") + proto.RegisterType((*HealthCheckResponse)(nil), "grpc.health.v1.HealthCheckResponse") + proto.RegisterEnum("grpc.health.v1.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Health service + +type HealthClient interface { + Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) +} + +type healthClient struct { + cc *grpc.ClientConn +} + +func NewHealthClient(cc *grpc.ClientConn) HealthClient { + return &healthClient{cc} +} + +func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { + out := new(HealthCheckResponse) + err := grpc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Health service + +type HealthServer interface { + Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) +} + +func RegisterHealthServer(s *grpc.Server, srv HealthServer) { + s.RegisterService(&_Health_serviceDesc, srv) +} + +func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).Check(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.health.v1.Health/Check", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Health_serviceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.health.v1.Health", + HandlerType: (*HealthServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Check", + Handler: _Health_Check_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "health.proto", +} + +func init() { proto.RegisterFile("health.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 204 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x48, 0x4d, 0xcc, + 0x29, 0xc9, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4b, 0x2f, 0x2a, 0x48, 0xd6, 0x83, + 0x0a, 0x95, 0x19, 0x2a, 0xe9, 0x71, 0x09, 0x79, 0x80, 0x39, 0xce, 0x19, 0xa9, 0xc9, 0xd9, 0x41, + 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x12, 0x5c, 0xec, 0xc5, 0xa9, 0x45, 0x65, 0x99, 0xc9, + 0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x30, 0xae, 0xd2, 0x1c, 0x46, 0x2e, 0x61, 0x14, + 0x0d, 0xc5, 0x05, 0xf9, 0x79, 0xc5, 0xa9, 0x42, 0x9e, 0x5c, 0x6c, 0xc5, 0x25, 0x89, 0x25, 0xa5, + 0xc5, 0x60, 0x0d, 0x7c, 0x46, 0x86, 0x7a, 0xa8, 0x16, 0xe9, 0x61, 0xd1, 0xa4, 0x17, 0x0c, 0x32, + 0x34, 0x2f, 0x3d, 0x18, 0xac, 0x31, 0x08, 0x6a, 0x80, 0x92, 0x15, 0x17, 0x2f, 0x8a, 0x84, 0x10, + 0x37, 0x17, 0x7b, 0xa8, 0x9f, 0xb7, 0x9f, 0x7f, 0xb8, 0x9f, 0x00, 0x03, 0x88, 0x13, 0xec, 0x1a, + 0x14, 0xe6, 0xe9, 0xe7, 0x2e, 0xc0, 0x28, 0xc4, 0xcf, 0xc5, 0xed, 0xe7, 0x1f, 0x12, 0x0f, 0x13, + 0x60, 0x32, 0x8a, 0xe2, 0x62, 0x83, 0x58, 0x24, 0x14, 0xc0, 0xc5, 0x0a, 0xb6, 0x4c, 0x48, 0x09, + 0xaf, 0x4b, 0xc0, 0xfe, 0x95, 0x52, 0x26, 0xc2, 0xb5, 0x49, 0x6c, 0xe0, 0x10, 0x34, 0x06, 0x04, + 0x00, 0x00, 0xff, 0xff, 0xac, 0x56, 0x2a, 0xcb, 0x51, 0x01, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto new file mode 100644 index 000000000..6072fdc3b --- /dev/null +++ b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.proto @@ -0,0 +1,34 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package grpc.health.v1; + +message HealthCheckRequest { + string service = 1; +} + +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + } + ServingStatus status = 1; +} + +service Health{ + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); +} diff --git a/vendor/google.golang.org/grpc/health/health.go b/vendor/google.golang.org/grpc/health/health.go new file mode 100644 index 000000000..4dccbc76b --- /dev/null +++ b/vendor/google.golang.org/grpc/health/health.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package health provides some utility functions to health-check a server. The implementation +// is based on protobuf. Users need to write their own implementations if other IDLs are used. +package health + +import ( + "sync" + + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +// Server implements `service Health`. +type Server struct { + mu sync.Mutex + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: make(map[string]healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if in.Service == "" { + // check the server overall health status. + return &healthpb.HealthCheckResponse{ + Status: healthpb.HealthCheckResponse_SERVING, + }, nil + } + if status, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: status, + }, nil + } + return nil, grpc.Errorf(codes.NotFound, "unknown service") +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, status healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + s.statusMap[service] = status + s.mu.Unlock() +}