mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-19 21:56:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #40 from ncdc/backup-logs

Browse Source 
Save logs per backup
This commit is contained in:
Steve Kriss
2017-09-11 13:31:17 -07:00
committed by GitHub
parent b7265a59f2 ab99fcc2e2
commit f438c226e3
1145 changed files with 209338 additions and 67511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Godeps/Godeps.json
Generated
-1254
View File
File diff suppressed because it is too large Load Diff
Godeps/Readme
Generated
-5
View File
@@ -1,5 +0,0 @@
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.
Gopkg.lock
Generated
+396
View File
@@ -0,0 +1,396 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "cloud.google.com/go"
packages = ["compute/metadata","iam","internal","internal/optional","internal/version","storage"]
revision = "44bcd0b2078ba5e7fedbeb36808d1ed893534750"
version = "v0.11.0"
[[projects]]
name = "github.com/Azure/azure-sdk-for-go"
packages = ["arm/disk","arm/examples/helpers","arm/resources/subscriptions","storage"]
revision = "2d49bb8f2cee530cc16f1f1a9f0aae763dee257d"
version = "v10.2.1-beta"
[[projects]]
name = "github.com/Azure/go-autorest"
packages = ["autorest","autorest/adal","autorest/azure","autorest/date","autorest/to","autorest/validation"]
revision = "f6e08fe5e4d45c9a66e40196d3fed5f37331d224"
version = "v8.1.1"
[[projects]]
name = "github.com/PuerkitoBio/purell"
packages = ["."]
revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/PuerkitoBio/urlesc"
packages = ["."]
revision = "de5bf2ad457846296e2031421a34e2568e304e35"
[[projects]]
name = "github.com/asaskevich/govalidator"
packages = ["."]
revision = "4918b99a7cb949bb295f3c7bbaf24b577d806e35"
version = "v6"
[[projects]]
name = "github.com/aws/aws-sdk-go"
packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/ec2query","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/ec2","service/s3","service/sts"]
revision = "1850f427c33c2558a2118dc55c1cf95a633d7432"
version = "v1.10.27"
[[projects]]
name = "github.com/cpuguy83/go-md2man"
packages = ["md2man"]
revision = "a65d4d2de4d5f7c74868dfa9b202a3c8be315aaa"
version = "v1.0.6"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
name = "github.com/dgrijalva/jwt-go"
packages = ["."]
revision = "d2709f9f1f31ebcda9651b03077758c1f3a0018c"
version = "v3.0.0"
[[projects]]
name = "github.com/docker/distribution"
packages = ["digest","reference"]
revision = "48294d928ced5dd9b378f7fd7c6f5da3ff3f2c89"
version = "v2.6.2"
[[projects]]
name = "github.com/emicklei/go-restful"
packages = [".","log"]
revision = "68c9750c36bb8cb433f1b88c807b4b30df4acc40"
version = "v2.2.1"
[[projects]]
name = "github.com/emicklei/go-restful-swagger12"
packages = ["."]
revision = "dcef7f55730566d41eae5db10e7d6981829720f6"
version = "1.0.1"
[[projects]]
branch = "master"
name = "github.com/fatih/camelcase"
packages = ["."]
revision = "f6a740d52f961c60348ebb109adde9f4635d7540"
[[projects]]
name = "github.com/ghodss/yaml"
packages = ["."]
revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
version = "v1.0.0"
[[projects]]
name = "github.com/go-ini/ini"
packages = ["."]
revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd"
version = "v1.28.2"
[[projects]]
branch = "master"
name = "github.com/go-openapi/analysis"
packages = ["."]
revision = "8ed83f2ea9f00f945516462951a288eaa68bf0d6"
[[projects]]
branch = "master"
name = "github.com/go-openapi/errors"
packages = ["."]
revision = "03cfca65330da08a5a440053faf994a3c682b5bf"
[[projects]]
branch = "master"
name = "github.com/go-openapi/jsonpointer"
packages = ["."]
revision = "779f45308c19820f1a69e9a4cd965f496e0da10f"
[[projects]]
branch = "master"
name = "github.com/go-openapi/jsonreference"
packages = ["."]
revision = "36d33bfe519efae5632669801b180bf1a245da3b"
[[projects]]
branch = "master"
name = "github.com/go-openapi/loads"
packages = ["."]
revision = "a80dea3052f00e5f032e860dd7355cd0cc67e24d"
[[projects]]
branch = "master"
name = "github.com/go-openapi/spec"
packages = ["."]
revision = "3faa0055dbbf2110abc1f3b4e3adbb22721e96e7"
[[projects]]
branch = "master"
name = "github.com/go-openapi/strfmt"
packages = ["."]
revision = "93a31ef21ac23f317792fff78f9539219dd74619"
[[projects]]
branch = "master"
name = "github.com/go-openapi/swag"
packages = ["."]
revision = "f3f9494671f93fcff853e3c6e9e948b3eb71e590"
[[projects]]
name = "github.com/gogo/protobuf"
packages = ["proto","sortkeys"]
revision = "100ba4e885062801d56799d78530b73b178a78f3"
version = "v0.4"
[[projects]]
branch = "master"
name = "github.com/golang/glog"
packages = ["."]
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["proto","protoc-gen-go/descriptor","ptypes/any"]
revision = "ab9f9a6dab164b7d1246e0e688b0ab7b94d8553e"
[[projects]]
branch = "master"
name = "github.com/google/gofuzz"
packages = ["."]
revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
[[projects]]
branch = "master"
name = "github.com/googleapis/gax-go"
packages = ["."]
revision = "84ed26760e7f6f80887a2fbfb50db3cc415d2cea"
[[projects]]
branch = "master"
name = "github.com/hashicorp/golang-lru"
packages = [".","simplelru"]
revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6"
[[projects]]
branch = "master"
name = "github.com/howeyc/gopass"
packages = ["."]
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
[[projects]]
name = "github.com/imdario/mergo"
packages = ["."]
revision = "3e95a51e0639b4cf372f2ccf74c86749d747fbdc"
version = "0.2.2"
[[projects]]
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
name = "github.com/jmespath/go-jmespath"
packages = ["."]
revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9"
version = "0.2.2"
[[projects]]
branch = "master"
name = "github.com/juju/ratelimit"
packages = ["."]
revision = "5b9ff866471762aa2ab2dced63c9fb6f53921342"
[[projects]]
branch = "master"
name = "github.com/mailru/easyjson"
packages = ["buffer","jlexer","jwriter"]
revision = "2f5df55504ebc322e4d52d34df6a1f5b503bf26d"
[[projects]]
branch = "master"
name = "github.com/mitchellh/mapstructure"
packages = ["."]
revision = "d0303fe809921458f417bcf828397a65db30a7e4"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
name = "github.com/robfig/cron"
packages = ["."]
revision = "df38d32658d8788cd446ba74db4bb5375c4b0cb3"
[[projects]]
name = "github.com/russross/blackfriday"
packages = ["."]
revision = "93622da34e54fb6529bfb7c57e710f37a8d9cbd8"
[[projects]]
name = "github.com/satori/go.uuid"
packages = ["."]
revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
version = "v1.1.0"
[[projects]]
name = "github.com/satori/uuid"
packages = ["."]
revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/shurcooL/sanitized_anchor_name"
packages = ["."]
revision = "541ff5ee47f1dddf6a5281af78307d921524bcb5"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
packages = [".","mem"]
revision = "9be650865eab0c12963d8753212f4f9c66cdcf12"
[[projects]]
branch = "master"
name = "github.com/spf13/cobra"
packages = [".","doc"]
revision = "cb731b898346822cc0c225c28550a8a29d93c732"
[[projects]]
name = "github.com/spf13/pflag"
packages = ["."]
revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/stretchr/objx"
packages = ["."]
revision = "1a9d0bb9f541897e62256577b352fdbc1fb4fd94"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert","mock","require"]
revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
version = "v1.1.4"
[[projects]]
branch = "master"
name = "github.com/ugorji/go"
packages = ["codec"]
revision = "5efa3251c7f7d05e5d9704a69a984ec9f1386a40"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
revision = "1c05540f6879653db88113bc4a2b70aec4bd491f"
[[projects]]
branch = "master"
name = "golang.org/x/oauth2"
packages = [".","google","internal","jws","jwt"]
revision = "9a379c6b3e95a790ffc43293c2a78dee0d7b6e20"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
revision = "43e60d72a8e2bd92ee98319ba9a384a0e9837c08"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["internal/gen","internal/triegen","internal/ucd","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable","width"]
revision = "e56139fd9c5bc7244c76116c68e500765bb6db6b"
[[projects]]
branch = "master"
name = "google.golang.org/api"
packages = ["compute/v0.beta","gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","storage/v1","transport/http"]
revision = "ed10e890a8366167a7ce33fac2b12447987bcb1c"
[[projects]]
name = "google.golang.org/appengine"
packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/rpc/status"]
revision = "ee236bd376b077c7a89f260c026c4735b195e459"
[[projects]]
name = "google.golang.org/grpc"
packages = [".","codes","connectivity","credentials","grpclb/grpc_lb_v1","grpclog","internal","keepalive","metadata","naming","peer","stats","status","tap","transport"]
revision = "b3ddf786825de56a4178401b7e174ee332173b66"
version = "v1.5.2"
[[projects]]
name = "gopkg.in/inf.v0"
packages = ["."]
revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4"
version = "v0.9.0"
[[projects]]
branch = "v2"
name = "gopkg.in/mgo.v2"
packages = ["bson","internal/json"]
revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655"
[[projects]]
branch = "v2"
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
[[projects]]
name = "k8s.io/apimachinery"
packages = ["pkg/api/equality","pkg/api/errors","pkg/api/meta","pkg/api/resource","pkg/apimachinery","pkg/apimachinery/announced","pkg/apimachinery/registered","pkg/apis/meta/v1","pkg/apis/meta/v1/unstructured","pkg/apis/meta/v1alpha1","pkg/conversion","pkg/conversion/queryparams","pkg/conversion/unstructured","pkg/fields","pkg/labels","pkg/openapi","pkg/runtime","pkg/runtime/schema","pkg/runtime/serializer","pkg/runtime/serializer/json","pkg/runtime/serializer/protobuf","pkg/runtime/serializer/recognizer","pkg/runtime/serializer/streaming","pkg/runtime/serializer/versioning","pkg/selection","pkg/types","pkg/util/cache","pkg/util/clock","pkg/util/diff","pkg/util/errors","pkg/util/framer","pkg/util/intstr","pkg/util/json","pkg/util/net","pkg/util/rand","pkg/util/runtime","pkg/util/sets","pkg/util/validation","pkg/util/validation/field","pkg/util/wait","pkg/util/yaml","pkg/version","pkg/watch","third_party/forked/golang/reflect"]
revision = "1fd2e63a9a370677308a42f24fd40c86438afddf"
[[projects]]
name = "k8s.io/client-go"
packages = ["discovery","discovery/fake","dynamic","kubernetes","kubernetes/scheme","kubernetes/typed/admissionregistration/v1alpha1","kubernetes/typed/apps/v1beta1","kubernetes/typed/authentication/v1","kubernetes/typed/authentication/v1beta1","kubernetes/typed/authorization/v1","kubernetes/typed/authorization/v1beta1","kubernetes/typed/autoscaling/v1","kubernetes/typed/autoscaling/v2alpha1","kubernetes/typed/batch/v1","kubernetes/typed/batch/v2alpha1","kubernetes/typed/certificates/v1beta1","kubernetes/typed/core/v1","kubernetes/typed/extensions/v1beta1","kubernetes/typed/networking/v1","kubernetes/typed/policy/v1beta1","kubernetes/typed/rbac/v1alpha1","kubernetes/typed/rbac/v1beta1","kubernetes/typed/settings/v1alpha1","kubernetes/typed/storage/v1","kubernetes/typed/storage/v1beta1","pkg/api","pkg/api/v1","pkg/api/v1/ref","pkg/apis/admissionregistration","pkg/apis/admissionregistration/v1alpha1","pkg/apis/apps","pkg/apis/apps/v1beta1","pkg/apis/authentication","pkg/apis/authentication/v1","pkg/apis/authentication/v1beta1","pkg/apis/authorization","pkg/apis/authorization/v1","pkg/apis/authorization/v1beta1","pkg/apis/autoscaling","pkg/apis/autoscaling/v1","pkg/apis/autoscaling/v2alpha1","pkg/apis/batch","pkg/apis/batch/v1","pkg/apis/batch/v2alpha1","pkg/apis/certificates","pkg/apis/certificates/v1beta1","pkg/apis/extensions","pkg/apis/extensions/v1beta1","pkg/apis/networking","pkg/apis/networking/v1","pkg/apis/policy","pkg/apis/policy/v1beta1","pkg/apis/rbac","pkg/apis/rbac/v1alpha1","pkg/apis/rbac/v1beta1","pkg/apis/settings","pkg/apis/settings/v1alpha1","pkg/apis/storage","pkg/apis/storage/v1","pkg/apis/storage/v1beta1","pkg/util","pkg/util/parsers","pkg/version","plugin/pkg/client/auth/azure","plugin/pkg/client/auth/gcp","plugin/pkg/client/auth/oidc","rest","rest/watch","testing","third_party/forked/golang/template","tools/auth","tools/cache","tools/clientcmd","tools/clientcmd/api","tools/clientcmd/api/latest","tools/clientcmd/api/v1","tools/metrics","transport","util/cert","util/flowcontrol","util/homedir","util/integer","util/jsonpath","util/workqueue"]
revision = "d92e8497f71b7b4e0494e5bd204b48d34bd6f254"
version = "v4.0.0"
[[projects]]
branch = "master"
name = "k8s.io/gengo"
packages = ["args","generator","namer","parser","types"]
revision = "2ef5ef33e269934e14149598f5a85d1f561a7219"
[[projects]]
name = "k8s.io/kubernetes"
packages = ["cmd/libs/go2idl/client-gen","cmd/libs/go2idl/client-gen/args","cmd/libs/go2idl/client-gen/generators","cmd/libs/go2idl/client-gen/generators/fake","cmd/libs/go2idl/client-gen/generators/scheme","cmd/libs/go2idl/client-gen/path","cmd/libs/go2idl/client-gen/types","cmd/libs/go2idl/informer-gen","cmd/libs/go2idl/informer-gen/generators","cmd/libs/go2idl/lister-gen","cmd/libs/go2idl/lister-gen/generators","pkg/printers","pkg/util/slice"]
revision = "793658f2d7ca7f064d2bdf606519f9fe1229c381"
version = "v1.7.4"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "1739acba4589d0e177479f7cad479483b6951d15483af6b8c72028a8ec4b275e"
solver-name = "gps-cdcl"
solver-version = 1
Gopkg.toml
+100
View File
@@ -0,0 +1,100 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
required = [
"k8s.io/kubernetes/cmd/libs/go2idl/client-gen",
"k8s.io/kubernetes/cmd/libs/go2idl/lister-gen",
"k8s.io/kubernetes/cmd/libs/go2idl/informer-gen"
]
[[constraint]]
name = "cloud.google.com/go"
version = "0.11.0"
[[constraint]]
name = "github.com/Azure/azure-sdk-for-go"
version = "~10.2.1-beta"
[[constraint]]
name = "github.com/Azure/go-autorest"
version = "~8.1.x"
[[constraint]]
name = "github.com/aws/aws-sdk-go"
version = "1.10.26"
[[constraint]]
branch = "master"
name = "github.com/golang/glog"
[[constraint]]
name = "github.com/robfig/cron"
revision = "df38d32658d8788cd446ba74db4bb5375c4b0cb3"
[[constraint]]
name = "github.com/satori/go.uuid"
version = "1.1.0"
[[constraint]]
name = "github.com/satori/uuid"
version = "1.1.0"
[[constraint]]
branch = "master"
name = "github.com/spf13/afero"
[[constraint]]
branch = "master"
name = "github.com/spf13/cobra"
[[constraint]]
name = "github.com/spf13/pflag"
version = "1.0.0"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.1.4"
[[constraint]]
branch = "master"
name = "golang.org/x/oauth2"
[[constraint]]
branch = "master"
name = "google.golang.org/api"
[[constraint]]
name = "k8s.io/apimachinery"
revision = "1fd2e63a9a370677308a42f24fd40c86438afddf"
[[constraint]]
name = "k8s.io/client-go"
version = "~4.0"
[[constraint]]
name = "k8s.io/kubernetes"
version = "~1.7"
[[override]]
name = "github.com/russross/blackfriday"
revision = "93622da34e54fb6529bfb7c57e710f37a8d9cbd8"
Makefile
+1 -1
View File
@@ -61,10 +61,10 @@ ifneq ($(SKIP_TESTS), 1)
endif
update:
${ROOT_DIR}/hack/update-generated-docs.sh
${ROOT_DIR}/hack/update-generated-clientsets.sh
${ROOT_DIR}/hack/update-generated-listers.sh
${ROOT_DIR}/hack/update-generated-informers.sh
${ROOT_DIR}/hack/update-generated-docs.sh
all: cbuild container
docs/build-from-scratch.md
+6
View File
@@ -6,6 +6,7 @@ While the [README][0] pulls from the Heptio image registry, you can also build y
* [1. Download][2]
* [2. Build][3]
* [3. Run][7]
* [4. Vendoring dependencies][10]
## 0. Prerequisites
@@ -57,6 +58,9 @@ kubectl set image deployment/ark ark=$REGISTRY/$PROJECT:$VERSION
```
where `$REGISTRY`, `$PROJECT`, and `$VERSION` match what you used in the [build step][3].
## 4. Vendoring dependencies
If you need to add or update the vendored dependencies, please see [Vendoring dependencies][11].
[0]: ../README.md
[1]: #0-prerequisites
[2]: #1-download
@@ -67,3 +71,5 @@ where `$REGISTRY`, `$PROJECT`, and `$VERSION` match what you used in the [build
[7]: #3-run
[8]: reference.md#ark-config-definition
[9]: cloud-provider-specifics.md
[10]: #4-vendoring-dependencies
[11]: vendoring-dependencies.md
docs/cli-reference/ark.md
+1
View File
@@ -14,6 +14,7 @@ associated data.
```
--alsologtostderr log to standard error as well as files
-h, --help help for ark
--kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration
--log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0)
--log_dir string If non-empty, write log files in this directory
docs/cli-reference/ark_backup.md
+7
View File
@@ -7,6 +7,12 @@ Work with backups
Work with backups
### Options
```
-h, --help help for backup
```
### Options inherited from parent commands
```
@@ -24,4 +30,5 @@ Work with backups
* [ark](ark.md) - Back up and restore Kubernetes cluster resources.
* [ark backup create](ark_backup_create.md) - Create a backup
* [ark backup get](ark_backup_get.md) - Get backups
* [ark backup logs](ark_backup_logs.md) - Get backup logs
docs/cli-reference/ark_backup_create.md
+2 -1
View File
@@ -8,7 +8,7 @@ Create a backup
Create a backup
```
ark backup create NAME
ark backup create NAME [flags]
```
### Options
@@ -16,6 +16,7 @@ ark backup create NAME
```
--exclude-namespaces stringArray namespaces to exclude from the backup
--exclude-resources stringArray resources to exclude from the backup, formatted as resource.group, such as storageclasses.storage.k8s.io
-h, --help help for create
--include-namespaces stringArray namespaces to include in the backup (use '*' for all namespaces) (default *)
--include-resources stringArray resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)
--label-columns stringArray a comma-separated list of labels to be displayed as columns
docs/cli-reference/ark_backup_get.md
+2 -1
View File
@@ -8,12 +8,13 @@ Get backups
Get backups
```
ark backup get
ark backup get [flags]
```
### Options
```
-h, --help help for get
--label-columns stringArray a comma-separated list of labels to be displayed as columns
-o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. (default "table")
-l, --selector string only show items matching this label selector
docs/cli-reference/ark_backup_logs.md
+36
View File
@@ -0,0 +1,36 @@
## ark backup logs
Get backup logs
### Synopsis
Get backup logs
```
ark backup logs BACKUP [flags]
```
### Options
```
-h, --help help for logs
--timeout duration how long to wait to receive logs (default 1m0s)
```
### Options inherited from parent commands
```
--alsologtostderr log to standard error as well as files
--kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration
--log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0)
--log_dir string If non-empty, write log files in this directory
--logtostderr log to standard error instead of files
--stderrthreshold severity logs at or above this threshold go to stderr (default 2)
-v, --v Level log level for V logs
--vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
```
### SEE ALSO
* [ark backup](ark_backup.md) - Work with backups
docs/cli-reference/ark_restore.md
+6
View File
@@ -7,6 +7,12 @@ Work with restores
Work with restores
### Options
```
-h, --help help for restore
```
### Options inherited from parent commands
```
docs/cli-reference/ark_restore_create.md
+2 -1
View File
@@ -8,13 +8,14 @@ Create a restore
Create a restore
```
ark restore create BACKUP
ark restore create BACKUP [flags]
```
### Options
```
--exclude-namespaces stringArray namespaces to exclude from the restore
-h, --help help for create
--include-namespaces stringArray namespaces to include in the restore (use '*' for all namespaces) (default *)
--label-columns stringArray a comma-separated list of labels to be displayed as columns
--labels mapStringString labels to apply to the restore
docs/cli-reference/ark_restore_delete.md
+7 -1
View File
@@ -8,7 +8,13 @@ Delete a restore
Delete a restore
```
ark restore delete NAME
ark restore delete NAME [flags]
```
### Options
```
-h, --help help for delete
```
### Options inherited from parent commands
docs/cli-reference/ark_restore_get.md
+2 -1
View File
@@ -8,12 +8,13 @@ get restores
get restores
```
ark restore get
ark restore get [flags]
```
### Options
```
-h, --help help for get
--label-columns stringArray a comma-separated list of labels to be displayed as columns
-o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. (default "table")
-l, --selector string only show items matching this label selector
docs/cli-reference/ark_schedule.md
+6
View File
@@ -7,6 +7,12 @@ Work with schedules
Work with schedules
### Options
```
-h, --help help for schedule
```
### Options inherited from parent commands
```
docs/cli-reference/ark_schedule_create.md
+2 -1
View File
@@ -8,7 +8,7 @@ Create a schedule
Create a schedule
```
ark schedule create NAME
ark schedule create NAME [flags]
```
### Options
@@ -16,6 +16,7 @@ ark schedule create NAME
```
--exclude-namespaces stringArray namespaces to exclude from the backup
--exclude-resources stringArray resources to exclude from the backup, formatted as resource.group, such as storageclasses.storage.k8s.io
-h, --help help for create
--include-namespaces stringArray namespaces to include in the backup (use '*' for all namespaces) (default *)
--include-resources stringArray resources to include in the backup, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)
--label-columns stringArray a comma-separated list of labels to be displayed as columns
docs/cli-reference/ark_schedule_delete.md
+7 -1
View File
@@ -8,7 +8,13 @@ Delete a schedule
Delete a schedule
```
ark schedule delete NAME
ark schedule delete NAME [flags]
```
### Options
```
-h, --help help for delete
```
### Options inherited from parent commands
docs/cli-reference/ark_schedule_get.md
+2 -1
View File
@@ -8,12 +8,13 @@ Get schedules
Get schedules
```
ark schedule get
ark schedule get [flags]
```
### Options
```
-h, --help help for get
--label-columns stringArray a comma-separated list of labels to be displayed as columns
-o, --output string Output display format. For create commands, display the object but do not send it to the server. Valid formats are 'table', 'json', and 'yaml'. (default "table")
-l, --selector string only show items matching this label selector
docs/cli-reference/ark_server.md
+3 -2
View File
@@ -8,19 +8,20 @@ Run the ark server
Run the ark server
```
ark server
ark server [flags]
```
### Options
```
--kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration
-h, --help help for server
```
### Options inherited from parent commands
```
--alsologtostderr log to standard error as well as files
--kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration
--log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0)
--log_dir string If non-empty, write log files in this directory
--logtostderr log to standard error instead of files
docs/cli-reference/ark_version.md
+7 -1
View File
@@ -8,7 +8,13 @@ Print the ark version and associated image
Print the ark version and associated image
```
ark version
ark version [flags]
```
### Options
```
-h, --help help for version
```
### Options inherited from parent commands
docs/vendoring-dependencies.md
+23
View File
@@ -0,0 +1,23 @@
# Vendoring dependencies
## Overview
We are using [dep][0] to manage dependencies. You can install it by running
```
go get -u github.com/golang/dep/cmd/dep
```
Dep currently pulls in a bit more than we'd like, so
we have created a script to remove these extra files: `hack/dep-save.sh`.
## Adding a new dependency
Run `hack/dep-save.sh`. If you want to see verbose output, you can append `-v` as in
`hack/dep-save.sh -v`.
## Updating an existing dependency
Run `hack/dep-save.sh -update <pkg> [<pkg> ...]` to update one or more dependencies.
[0]: https://github.com/golang/dep
examples/common/00-prereqs.yaml
+15
View File
@@ -72,6 +72,21 @@ spec:
plural: configs
kind: Config
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: downloadrequests.ark.heptio.com
labels:
component: ark
spec:
group: ark.heptio.com
version: v1
scope: Namespaced
names:
plural: downloadrequests
kind: DownloadRequest
---
apiVersion: v1
kind: Namespace
hack/godep-save.sh → hack/dep-save.sh
+3 -8
View File
@@ -14,13 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
SAVES=(
k8s.io/kubernetes/cmd/libs/go2idl/client-gen
k8s.io/kubernetes/cmd/libs/go2idl/lister-gen
k8s.io/kubernetes/cmd/libs/go2idl/informer-gen
)
godep save ./... "${SAVES[@]}"
dep ensure $@
dep prune
# remove files we don't want
find vendor \( -name BUILD -o -name .travis.yml \) -exec rm {} \;
find vendor \( -name BUILD -o -name .travis.yml -o -name '*_test.go' \) -exec rm {} \;
pkg/apis/ark/v1/download_request.go
+82
View File
@@ -0,0 +1,82 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// DownloadRequestSpec is the specification for a download request.
type DownloadRequestSpec struct {
// Target is what to download (e.g. logs for a backup).
Target DownloadTarget `json:"target"`
}
// DownloadTargetKind represents what type of file to download.
type DownloadTargetKind string
const (
DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog"
)
// DownloadTarget is the specification for what kind of file to download, and the name of the
// resource with which it's associated.
type DownloadTarget struct {
// Kind is the type of file to download.
Kind DownloadTargetKind `json:"kind"`
// Name is the name of the kubernetes resource with which the file is associated.
Name string `json:"name"`
}
// DownloadRequestPhase represents the lifecycle phase of a DownloadRequest.
type DownloadRequestPhase string
const (
// DownloadRequestPhaseNew means the DownloadRequest has not been processed by the
// DownloadRequestController yet.
DownloadRequestPhaseNew DownloadRequestPhase = "New"
// DownloadRequestPhaseProcessed means the DownloadRequest has been processed by the
// DownloadRequestController.
DownloadRequestPhaseProcessed DownloadRequestPhase = "Processed"
)
// DownloadRequestStatus is the current status of a DownloadRequest.
type DownloadRequestStatus struct {
// Phase is the current state of the DownloadRequest.
Phase DownloadRequestPhase `json:"phase"`
// DownloadURL contains the pre-signed URL for the target file.
DownloadURL string `json:"downloadURL"`
// Expiration is when this DownloadRequest expires and can be deleted by the system.
Expiration metav1.Time `json:"expiration"`
}
// +genclient=true
// DownloadRequest is a request to download an artifact from backup object storage, such as a backup
// log file.
type DownloadRequest struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata"`
Spec DownloadRequestSpec `json:"spec"`
Status DownloadRequestStatus `json:"status,omitempty"`
}
// DownloadRequestList is a list of DownloadRequests.
type DownloadRequestList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata"`
Items []DownloadRequest `json:"items"`
}
pkg/apis/ark/v1/register.go
+2
View File
@@ -51,6 +51,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&RestoreList{},
&Config{},
&ConfigList{},
&DownloadRequest{},
&DownloadRequestList{},
)
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil
pkg/backup/backup.go
+85 -43
View File
@@ -25,8 +25,6 @@ import (
"strings"
"time"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -41,9 +39,9 @@ import (
// Backupper performs backups.
type Backupper interface {
// Backup takes a backup using the specification in the api.Backup and writes backup data to the
// given writers.
Backup(backup *api.Backup, data io.Writer) error
// Backup takes a backup using the specification in the api.Backup and writes backup and log data
// to the given writers.
Backup(backup *api.Backup, backupFile, logFile io.Writer) error
}
// kubernetesBackupper implements Backupper.
@@ -56,11 +54,20 @@ type kubernetesBackupper struct {
var _ Backupper = &kubernetesBackupper{}
// ActionContext contains contextual information for actions.
type ActionContext struct {
logger *logger
}
func (ac ActionContext) log(msg string, args ...interface{}) {
ac.logger.log(msg, args...)
}
// Action is an actor that performs an operation on an individual item being backed up.
type Action interface {
// Execute is invoked on an item being backed up. If an error is returned, the Backup is marked as
// failed.
Execute(item map[string]interface{}, backup *api.Backup) error
Execute(ctx ActionContext, item map[string]interface{}, backup *api.Backup) error
}
// NewKubernetesBackupper creates a new kubernetesBackupper.
@@ -98,29 +105,34 @@ func resolveActions(mapper meta.RESTMapper, actions map[string]Action) (map[sche
return ret, nil
}
// getResourceIncludesExcludes takes the lists of resources to include and exclude from the
// backup, uses the RESTMapper to resolve them to fully-qualified group-resource names, and returns
// an IncludesExcludes list.
func getResourceIncludesExcludes(mapper meta.RESTMapper, backup *api.Backup) *collections.IncludesExcludes {
// resolveResources uses the RESTMapper to resolve resources to their fully-qualified group-resource
// names. fn is invoked for each resolved resource. resolveResources returns a list of any resources that failed to resolve.
func (ctx *backupContext) resolveResources(mapper meta.RESTMapper, resources []string, allowAll bool, fn func(string)) {
for _, resource := range resources {
if allowAll && resource == "*" {
fn("*")
return
}
gr, err := resolveGroupResource(mapper, resource)
if err != nil {
ctx.log("Unable to resolve resource %q: %v", resource, err)
continue
}
fn(gr.String())
}
}
// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// RESTMapper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func (ctx *backupContext) getResourceIncludesExcludes(mapper meta.RESTMapper, includes, excludes []string) *collections.IncludesExcludes {
resources := collections.NewIncludesExcludes()
resolve := func(list []string, allowAll bool, f func(string)) {
for _, resource := range list {
if allowAll && resource == "*" {
f("*")
return
}
gr, err := resolveGroupResource(mapper, resource)
if err != nil {
glog.Errorf("unable to include resource %q in backup: %v", resource, err)
continue
}
f(gr.String())
}
}
ctx.resolveResources(mapper, includes, true, func(s string) { resources.Includes(s) })
ctx.resolveResources(mapper, excludes, false, func(s string) { resources.Excludes(s) })
resolve(backup.Spec.IncludedResources, true, func(s string) { resources.Includes(s) })
resolve(backup.Spec.ExcludedResources, false, func(s string) { resources.Excludes(s) })
ctx.log("Including resources: %v", strings.Join(resources.GetIncludes(), ", "))
ctx.log("Excluding resources: %v", strings.Join(resources.GetExcludes(), ", "))
return resources
}
@@ -141,9 +153,20 @@ func getNamespaceIncludesExcludes(backup *api.Backup) *collections.IncludesExclu
return collections.NewIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...)
}
type logger struct {
w io.Writer
}
func (l *logger) log(msg string, args ...interface{}) {
// TODO use a real logger that supports writing to files
now := time.Now().Format(time.RFC3339)
fmt.Fprintf(l.w, now+" "+msg+"\n", args...)
}
type backupContext struct {
backup *api.Backup
w tarWriter
logger *logger
namespaceIncludesExcludes *collections.IncludesExcludes
resourceIncludesExcludes *collections.IncludesExcludes
// deploymentsBackedUp marks whether we've seen and are backing up the deployments resource, from
@@ -156,32 +179,50 @@ type backupContext struct {
networkPoliciesBackedUp bool
}
// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to data. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(backup *api.Backup, data io.Writer) error {
gzw := gzip.NewWriter(data)
defer gzw.Close()
func (ctx *backupContext) log(msg string, args ...interface{}) {
ctx.logger.log(msg, args...)
}
tw := tar.NewWriter(gzw)
// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to backupFile. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(backup *api.Backup, backupFile, logFile io.Writer) error {
gzippedData := gzip.NewWriter(backupFile)
defer gzippedData.Close()
tw := tar.NewWriter(gzippedData)
defer tw.Close()
gzippedLog := gzip.NewWriter(logFile)
defer gzippedLog.Close()
var errs []error
ctx := &backupContext{
backup: backup,
w: tw,
logger: &logger{w: gzippedLog},
namespaceIncludesExcludes: getNamespaceIncludesExcludes(backup),
resourceIncludesExcludes: getResourceIncludesExcludes(kb.discoveryHelper.Mapper(), backup),
}
ctx.log("Starting backup")
ctx.resourceIncludesExcludes = ctx.getResourceIncludesExcludes(kb.discoveryHelper.Mapper(), backup.Spec.IncludedResources, backup.Spec.ExcludedResources)
for _, group := range kb.discoveryHelper.Resources() {
glog.V(2).Infof("Backing up group %q\n", group.GroupVersion)
ctx.log("Processing group %s", group.GroupVersion)
if err := kb.backupGroup(ctx, group); err != nil {
errs = append(errs, err)
}
}
return kuberrs.NewAggregate(errs)
err := kuberrs.NewAggregate(errs)
if err == nil {
ctx.log("Backup completed successfully")
} else {
ctx.log("Backup completed with errors: %v", err)
}
return err
}
type tarWriter interface {
@@ -195,7 +236,7 @@ func (kb *kubernetesBackupper) backupGroup(ctx *backupContext, group *metav1.API
var errs []error
for _, resource := range group.APIResources {
glog.V(2).Infof("Backing up resource %s/%s\n", group.GroupVersion, resource.Name)
ctx.log("Processing resource %s/%s", group.GroupVersion, resource.Name)
if err := kb.backupResource(ctx, group, resource); err != nil {
errs = append(errs, err)
}
@@ -228,7 +269,7 @@ func (kb *kubernetesBackupper) backupResource(
grString := gr.String()
if !ctx.resourceIncludesExcludes.ShouldInclude(grString) {
glog.V(2).Infof("Not including resource %s\n", grString)
ctx.log("Resource %s is excluded", grString)
return nil
}
@@ -240,7 +281,7 @@ func (kb *kubernetesBackupper) backupResource(
} else {
other = appsDeploymentsResource
}
glog.V(4).Infof("Skipping resource %q because it's a duplicate of %q", grString, other)
ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
return nil
}
@@ -255,7 +296,7 @@ func (kb *kubernetesBackupper) backupResource(
} else {
other = networkingNetworkPoliciesResource
}
glog.V(4).Infof("Skipping resource %q because it's a duplicate of %q", grString, other)
ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
return nil
}
@@ -357,20 +398,21 @@ func (*realItemBackupper) backupItem(ctx *backupContext, item map[string]interfa
namespace, err := collections.GetString(metadata, "namespace")
if err == nil {
if !ctx.namespaceIncludesExcludes.ShouldInclude(namespace) {
glog.V(2).Infof("Excluding item %s because namespace %s is excluded\n", name, namespace)
ctx.log("Excluding item %s because namespace %s is excluded", name, namespace)
return nil
}
}
if action != nil {
glog.V(4).Infof("Executing action on %s, ns=%s, name=%s", groupResource, namespace, name)
ctx.log("Executing action on %s, ns=%s, name=%s", groupResource, namespace, name)
if err := action.Execute(item, ctx.backup); err != nil {
actionCtx := ActionContext{logger: ctx.logger}
if err := action.Execute(actionCtx, item, ctx.backup); err != nil {
return err
}
}
glog.V(2).Infof("Backing up resource=%s, ns=%s, name=%s", groupResource, namespace, name)
ctx.log("Backing up resource=%s, ns=%s, name=%s", groupResource, namespace, name)
var filePath string
if namespace != "" {
pkg/backup/backup_test.go
+17 -9
View File
@@ -23,6 +23,7 @@ import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"reflect"
"sort"
"testing"
@@ -50,7 +51,7 @@ type fakeAction struct {
var _ Action = &fakeAction{}
func (a *fakeAction) Execute(item map[string]interface{}, backup *v1.Backup) error {
func (a *fakeAction) Execute(ctx ActionContext, item map[string]interface{}, backup *v1.Backup) error {
metadata, err := collections.GetMap(item, "metadata")
if err != nil {
return err
@@ -164,6 +165,13 @@ func TestGetResourceIncludesExcludes(t *testing.T) {
expectedIncludes: []string{"foodies.somegroup", "fields.somegroup"},
expectedExcludes: []string{"barnacles.anothergroup", "bazaars.anothergroup"},
},
{
name: "some unresolvable",
includes: []string{"foo", "fie", "bad1"},
excludes: []string{"bar", "baz", "bad2"},
expectedIncludes: []string{"foodies.somegroup", "fields.somegroup"},
expectedExcludes: []string{"barnacles.anothergroup", "bazaars.anothergroup"},
},
}
for _, test := range tests {
@@ -177,14 +185,12 @@ func TestGetResourceIncludesExcludes(t *testing.T) {
},
}
backup := &v1.Backup{
Spec: v1.BackupSpec{
IncludedResources: test.includes,
ExcludedResources: test.excludes,
},
b := new(bytes.Buffer)
ctx := &backupContext{
logger: &logger{w: b},
}
actual := getResourceIncludesExcludes(mapper, backup)
actual := ctx.getResourceIncludesExcludes(mapper, test.includes, test.excludes)
sort.Strings(test.expectedIncludes)
actualIncludes := actual.GetIncludes()
@@ -439,7 +445,7 @@ func TestBackupMethod(t *testing.T) {
require.NoError(t, err)
output := new(bytes.Buffer)
err = backupper.Backup(backup, output)
err = backupper.Backup(backup, output, ioutil.Discard)
require.NoError(t, err)
expectedFiles := sets.NewString(
@@ -775,6 +781,7 @@ func TestBackupResource(t *testing.T) {
namespaceIncludesExcludes: test.namespaceIncludesExcludes,
deploymentsBackedUp: test.deploymentsBackedUp,
networkPoliciesBackedUp: test.networkPoliciesBackedUp,
logger: &logger{w: new(bytes.Buffer)},
}
group := &metav1.APIResourceList{
@@ -1001,7 +1008,8 @@ func TestBackupItem(t *testing.T) {
ctx := &backupContext{
backup: backup,
namespaceIncludesExcludes: namespaces,
w: w,
w: w,
logger: &logger{w: new(bytes.Buffer)},
}
b := &realItemBackupper{}
err = b.backupItem(ctx, item, "resource.group", actionParam)
pkg/backup/volume_snapshot_action.go
+6 -8
View File
@@ -20,8 +20,6 @@ import (
"errors"
"fmt"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/clock"
api "github.com/heptio/ark/pkg/apis/ark/v1"
@@ -52,10 +50,10 @@ func NewVolumeSnapshotAction(snapshotService cloudprovider.SnapshotService) (Act
// Execute triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided
// backup has volume snapshots enabled and the PV is of a compatible type. Also records cloud
// disk type and IOPS (if applicable) to be able to restore to current state later.
func (a *volumeSnapshotAction) Execute(volume map[string]interface{}, backup *api.Backup) error {
func (a *volumeSnapshotAction) Execute(ctx ActionContext, volume map[string]interface{}, backup *api.Backup) error {
backupName := fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)
if backup.Spec.SnapshotVolumes != nil && !*backup.Spec.SnapshotVolumes {
glog.V(2).Infof("Backup %q has volume snapshots disabled; skipping volume snapshot action.", backupName)
ctx.log("Backup %q has volume snapshots disabled; skipping volume snapshot action.", backupName)
return nil
}
@@ -69,23 +67,23 @@ func (a *volumeSnapshotAction) Execute(volume map[string]interface{}, backup *ap
}
// no volumeID / nil error means unsupported PV source
if volumeID == "" {
glog.V(2).Infof("Backup %q: PersistentVolume %q is not a supported volume type for snapshots, skipping.", backupName, name)
ctx.log("Backup %q: PersistentVolume %q is not a supported volume type for snapshots, skipping.", backupName, name)
return nil
}
expiration := a.clock.Now().Add(backup.Spec.TTL.Duration)
glog.Infof("Backup %q: snapshotting PersistentVolume %q, volume-id %q, expiration %v", backupName, name, volumeID, expiration)
ctx.log("Backup %q: snapshotting PersistentVolume %q, volume-id %q, expiration %v", backupName, name, volumeID, expiration)
snapshotID, err := a.snapshotService.CreateSnapshot(volumeID)
if err != nil {
glog.V(4).Infof("error creating snapshot for backup %q, volume %q, volume-id %q: %v", backupName, name, volumeID, err)
ctx.log("error creating snapshot for backup %q, volume %q, volume-id %q: %v", backupName, name, volumeID, err)
return err
}
volumeType, iops, err := a.snapshotService.GetVolumeInfo(volumeID)
if err != nil {
glog.V(4).Infof("error getting volume info for backup %q, volume %q, volume-id %q: %v", backupName, name, volumeID, err)
ctx.log("error getting volume info for backup %q, volume %q, volume-id %q: %v", backupName, name, volumeID, err)
return err
}
pkg/backup/volume_snapshot_action_test.go
+3 -1
View File
@@ -17,6 +17,7 @@ limitations under the License.
package backup
import (
"bytes"
"reflect"
"testing"
"time"
@@ -185,7 +186,8 @@ func TestVolumeSnapshotAction(t *testing.T) {
t.Fatal(err)
}
err = action.Execute(pv, backup)
actionCtx := ActionContext{logger: &logger{w: new(bytes.Buffer)}}
err = action.Execute(actionCtx, pv, backup)
gotErr := err != nil
if e, a := test.expectError, gotErr; e != a {
pkg/cloudprovider/aws/object_storage_adapter.go
+35 -2
View File
@@ -19,10 +19,12 @@ package aws
import (
"errors"
"io"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/golang/glog"
"github.com/heptio/ark/pkg/cloudprovider"
)
@@ -107,8 +109,8 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
}
var ret []string
err := op.s3.ListObjectsV2Pages(req, func(res *s3.ListObjectsV2Output, lastPage bool) bool {
for _, prefix := range res.CommonPrefixes {
err := op.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, prefix := range page.CommonPrefixes {
ret = append(ret, *prefix.Prefix)
}
return !lastPage
@@ -121,6 +123,27 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
return ret, nil
}
func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) {
req := &s3.ListObjectsV2Input{
Bucket: &bucket,
Prefix: &prefix,
}
var ret []string
err := op.s3.ListObjectsV2Pages(req, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, obj := range page.Contents {
ret = append(ret, *obj.Key)
}
return !lastPage
})
if err != nil {
return nil, err
}
return ret, nil
}
func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
req := &s3.DeleteObjectInput{
Bucket: &bucket,
@@ -131,3 +154,13 @@ func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
return err
}
func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
glog.V(4).Infof("CreateSignedURL: bucket=%s, key=%s, ttl=%d", bucket, key, ttl)
req, _ := op.s3.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
return req.Presign(ttl)
}
pkg/cloudprovider/azure/block_storage_adapter.go
+5 -3
View File
@@ -26,6 +26,7 @@ import (
"github.com/Azure/azure-sdk-for-go/arm/disk"
"github.com/Azure/azure-sdk-for-go/arm/examples/helpers"
"github.com/Azure/azure-sdk-for-go/arm/resources/subscriptions"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/satori/uuid"
@@ -90,12 +91,13 @@ func NewBlockStorageAdapter(location string, apiTimeout time.Duration) (cloudpro
disksClient := disk.NewDisksClient(cfg[azureSubscriptionIDKey])
snapsClient := disk.NewSnapshotsClient(cfg[azureSubscriptionIDKey])
disksClient.Authorizer = spt
snapsClient.Authorizer = spt
authorizer := autorest.NewBearerAuthorizer(spt)
disksClient.Authorizer = authorizer
snapsClient.Authorizer = authorizer
// validate the location
groupClient := subscriptions.NewGroupClient()
groupClient.Authorizer = spt
groupClient.Authorizer = authorizer
locs, err := groupClient.ListLocations(cfg[azureSubscriptionIDKey])
if err != nil {
pkg/cloudprovider/azure/object_storage_adapter.go
+40
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"io"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/storage"
@@ -120,6 +121,29 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
return ret, nil
}
func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) {
container, err := getContainerReference(op.blobClient, bucket)
if err != nil {
return nil, err
}
params := storage.ListBlobsParameters{
Prefix: prefix,
}
res, err := container.ListBlobs(params)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(res.Blobs))
for _, blob := range res.Blobs {
ret = append(ret, blob.Name)
}
return ret, nil
}
func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
container, err := getContainerReference(op.blobClient, bucket)
if err != nil {
@@ -134,6 +158,22 @@ func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
return blob.Delete(nil)
}
const sasURIReadPermission = "r"
func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
container, err := getContainerReference(op.blobClient, bucket)
if err != nil {
return "", err
}
blob, err := getBlobReference(container, key)
if err != nil {
return "", err
}
return blob.GetSASURI(time.Now().Add(ttl), sasURIReadPermission)
}
func getContainerReference(blobClient *storage.BlobStorageClient, bucket string) (*storage.Container, error) {
container := blobClient.GetContainerReference(bucket)
if container == nil {
pkg/cloudprovider/backup_service.go
+51 -20
View File
@@ -38,21 +38,22 @@ type BackupService interface {
// UploadBackup uploads the specified Ark backup of a set of Kubernetes API objects, whose manifests are
// stored in the specified file, into object storage in an Ark bucket, tagged with Ark metadata. Returns
// an error if a problem is encountered accessing the file or performing the upload via the cloud API.
UploadBackup(bucket, name string, metadata, backup io.ReadSeeker) error
UploadBackup(bucket, name string, metadata, backup, log io.ReadSeeker) error
// DownloadBackup downloads an Ark backup with the specified object key from object storage via the cloud API.
// It returns the snapshot metadata and data (separately), or an error if a problem is encountered
// downloading or reading the file from the cloud API.
DownloadBackup(bucket, name string) (io.ReadCloser, error)
// DeleteBackup deletes the backup content in object storage for the given backup.
DeleteBackupFile(bucket, backupName string) error
// DeleteBackup deletes the backup metadata file in object storage for the given backup.
DeleteBackupMetadataFile(bucket, backupName string) error
// DeleteBackupDir deletes all files in object storage for the given backup.
DeleteBackupDir(bucket, backupName string) error
// GetBackup gets the specified api.Backup from the given bucket in object storage.
GetBackup(bucket, name string) (*api.Backup, error)
// CreateBackupLogSignedURL creates a pre-signed URL that can be used to download a backup's log
// file from object storage. The URL expires after ttl.
CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error)
}
// BackupGetter knows how to list backups in object storage.
@@ -62,10 +63,23 @@ type BackupGetter interface {
}
const (
metadataFileFormatString string = "%s/ark-backup.json"
backupFileFormatString string = "%s/%s.tar.gz"
metadataFileFormatString = "%s/ark-backup.json"
backupFileFormatString = "%s/%s-data.tar.gz"
logFileFormatString = "%s/%s-logs.gz"
)
func getMetadataKey(backup string) string {
return fmt.Sprintf(metadataFileFormatString, backup)
}
func getBackupKey(backup string) string {
return fmt.Sprintf(backupFileFormatString, backup, backup)
}
func getBackupLogKey(backup string) string {
return fmt.Sprintf(logFileFormatString, backup, backup)
}
type backupService struct {
objectStorage ObjectStorageAdapter
decoder runtime.Decoder
@@ -82,26 +96,34 @@ func NewBackupService(objectStorage ObjectStorageAdapter) BackupService {
}
}
func (br *backupService) UploadBackup(bucket, backupName string, metadata, backup io.ReadSeeker) error {
func (br *backupService) UploadBackup(bucket, backupName string, metadata, backup, log io.ReadSeeker) error {
// upload metadata file
metadataKey := fmt.Sprintf(metadataFileFormatString, backupName)
metadataKey := getMetadataKey(backupName)
if err := br.objectStorage.PutObject(bucket, metadataKey, metadata); err != nil {
// failure to upload metadata file is a hard-stop
return err
}
// upload tar file
if err := br.objectStorage.PutObject(bucket, fmt.Sprintf(backupFileFormatString, backupName, backupName), backup); err != nil {
if err := br.objectStorage.PutObject(bucket, getBackupKey(backupName), backup); err != nil {
// try to delete the metadata file since the data upload failed
deleteErr := br.objectStorage.DeleteObject(bucket, metadataKey)
return errors.NewAggregate([]error{err, deleteErr})
}
// uploading log file is best-effort; if it fails, we log the error but call the overall upload a
// success
logKey := getBackupLogKey(backupName)
if err := br.objectStorage.PutObject(bucket, logKey, log); err != nil {
glog.Errorf("error uploading %s/%s: %v", bucket, logKey, err)
}
return nil
}
func (br *backupService) DownloadBackup(bucket, backupName string) (io.ReadCloser, error) {
return br.objectStorage.GetObject(bucket, fmt.Sprintf(backupFileFormatString, backupName, backupName))
return br.objectStorage.GetObject(bucket, getBackupKey(backupName))
}
func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) {
@@ -155,16 +177,25 @@ func (br *backupService) GetBackup(bucket, name string) (*api.Backup, error) {
return backup, nil
}
func (br *backupService) DeleteBackupFile(bucket, backupName string) error {
key := fmt.Sprintf(backupFileFormatString, backupName, backupName)
glog.V(4).Infof("Trying to delete bucket=%s, key=%s", bucket, key)
return br.objectStorage.DeleteObject(bucket, key)
func (br *backupService) DeleteBackupDir(bucket, backupName string) error {
objects, err := br.objectStorage.ListObjects(bucket, backupName+"/")
if err != nil {
return err
}
var errs []error
for _, key := range objects {
glog.V(4).Infof("Trying to delete bucket=%s, key=%s", bucket, key)
if err := br.objectStorage.DeleteObject(bucket, key); err != nil {
errs = append(errs, err)
}
}
return errors.NewAggregate(errs)
}
func (br *backupService) DeleteBackupMetadataFile(bucket, backupName string) error {
key := fmt.Sprintf(metadataFileFormatString, backupName)
glog.V(4).Infof("Trying to delete bucket=%s, key=%s", bucket, key)
return br.objectStorage.DeleteObject(bucket, key)
func (br *backupService) CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error) {
return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(backupName), ttl)
}
// cachedBackupService wraps a real backup service with a cache for getting cloud backups.
pkg/cloudprovider/backup_service_test.go
+118 -289
View File
@@ -25,11 +25,12 @@ import (
"strings"
"testing"
testutil "github.com/heptio/ark/pkg/util/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/util/encode"
@@ -37,237 +38,140 @@ import (
func TestUploadBackup(t *testing.T) {
tests := []struct {
name string
bucket string
bucketExists bool
backupName string
metadata io.ReadSeeker
backup io.ReadSeeker
objectStoreErrs map[string]map[string]interface{}
expectedErr bool
expectedRes map[string][]byte
name string
metadata io.ReadSeeker
metadataError error
expectMetadataDelete bool
backup io.ReadSeeker
backupError error
log io.ReadSeeker
logError error
expectedErr string
}{
{
name: "normal case",
bucket: "test-bucket",
bucketExists: true,
backupName: "test-backup",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
expectedErr: false,
expectedRes: map[string][]byte{
"test-backup/ark-backup.json": []byte("foo"),
"test-backup/test-backup.tar.gz": []byte("bar"),
},
name: "normal case",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
log: newStringReadSeeker("baz"),
},
{
name: "no such bucket causes error",
bucket: "test-bucket",
bucketExists: false,
backupName: "test-backup",
expectedErr: true,
name: "error on metadata upload does not upload data or log",
metadata: newStringReadSeeker("foo"),
metadataError: errors.New("md"),
expectedErr: "md",
},
{
name: "error on metadata upload does not upload data",
bucket: "test-bucket",
bucketExists: true,
backupName: "test-backup",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
objectStoreErrs: map[string]map[string]interface{}{
"putobject": map[string]interface{}{
"test-bucket||test-backup/ark-backup.json": true,
},
},
expectedErr: true,
expectedRes: make(map[string][]byte),
name: "error on data upload deletes metadata",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
backupError: errors.New("backup"),
expectMetadataDelete: true,
expectedErr: "backup",
},
{
name: "error on data upload deletes metadata",
bucket: "test-bucket",
bucketExists: true,
backupName: "test-backup",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
objectStoreErrs: map[string]map[string]interface{}{
"putobject": map[string]interface{}{
"test-bucket||test-backup/test-backup.tar.gz": true,
},
},
expectedErr: true,
expectedRes: make(map[string][]byte),
name: "error on log upload is ok",
metadata: newStringReadSeeker("foo"),
backup: newStringReadSeeker("bar"),
log: newStringReadSeeker("baz"),
logError: errors.New("log"),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objStore := &fakeObjectStorage{
returnErrors: test.objectStoreErrs,
storage: make(map[string]map[string][]byte),
objStore := &testutil.ObjectStorageAdapter{}
bucket := "test-bucket"
backupName := "test-backup"
if test.metadata != nil {
objStore.On("PutObject", bucket, backupName+"/ark-backup.json", test.metadata).Return(test.metadataError)
}
if test.bucketExists {
objStore.storage[test.bucket] = make(map[string][]byte)
if test.backup != nil {
objStore.On("PutObject", bucket, backupName+"/"+backupName+"-data.tar.gz", test.backup).Return(test.backupError)
}
if test.log != nil {
objStore.On("PutObject", bucket, backupName+"/"+backupName+"-logs.gz", test.log).Return(test.logError)
}
if test.expectMetadataDelete {
objStore.On("DeleteObject", bucket, backupName+"/ark-backup.json").Return(nil)
}
backupService := NewBackupService(objStore)
err := backupService.UploadBackup(test.bucket, test.backupName, test.metadata, test.backup)
err := backupService.UploadBackup(bucket, backupName, test.metadata, test.backup, test.log)
assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
assert.Equal(t, test.expectedRes, objStore.storage[test.bucket])
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
}
objStore.AssertExpectations(t)
})
}
}
func TestDownloadBackup(t *testing.T) {
o := &testutil.ObjectStorageAdapter{}
bucket := "b"
backup := "bak"
o.On("GetObject", bucket, backup+"/"+backup+"-data.tar.gz").Return(ioutil.NopCloser(strings.NewReader("foo")), nil)
s := NewBackupService(o)
rc, err := s.DownloadBackup(bucket, backup)
require.NoError(t, err)
require.NotNil(t, rc)
data, err := ioutil.ReadAll(rc)
require.NoError(t, err)
assert.Equal(t, "foo", string(data))
o.AssertExpectations(t)
}
func TestDeleteBackup(t *testing.T) {
tests := []struct {
name string
bucket string
backupName string
storage map[string]map[string][]byte
expectedErr bool
expectedRes []byte
name string
listObjectsError error
deleteErrors []error
expectedErr string
}{
{
name: "normal case",
bucket: "test-bucket",
backupName: "test-backup",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{
"test-backup/test-backup.tar.gz": []byte("foo"),
},
},
expectedErr: false,
expectedRes: []byte("foo"),
name: "normal case",
},
{
name: "no such bucket causes error",
bucket: "test-bucket",
backupName: "test-backup",
storage: map[string]map[string][]byte{},
expectedErr: true,
},
{
name: "no such key causes error",
bucket: "test-bucket",
backupName: "test-backup",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{},
},
expectedErr: true,
name: "some delete errors, do as much as we can",
deleteErrors: []error{errors.New("a"), nil, errors.New("c")},
expectedErr: "[a, c]",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objStore := &fakeObjectStorage{storage: test.storage}
backupService := NewBackupService(objStore)
bucket := "bucket"
backup := "bak"
objects := []string{"bak/ark-backup.json", "bak/bak.tar.gz", "bak/bak.log.gz"}
rdr, err := backupService.DownloadBackup(test.bucket, test.backupName)
objStore := &testutil.ObjectStorageAdapter{}
objStore.On("ListObjects", bucket, backup+"/").Return(objects, test.listObjectsError)
for i, o := range objects {
var err error
if i < len(test.deleteErrors) {
err = test.deleteErrors[i]
}
assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
if err == nil {
res, err := ioutil.ReadAll(rdr)
assert.Nil(t, err)
assert.Equal(t, test.expectedRes, res)
objStore.On("DeleteObject", bucket, o).Return(err)
}
})
}
}
func TestDeleteBackupFile(t *testing.T) {
tests := []struct {
name string
bucket string
backupName string
storage map[string]map[string][]byte
expectedErr bool
expectedRes map[string][]byte
}{
{
name: "normal case",
bucket: "test-bucket",
backupName: "bak",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{
"bak/bak.tar.gz": nil,
},
},
expectedErr: false,
expectedRes: make(map[string][]byte),
},
{
name: "failed delete of backup returns error",
bucket: "test-bucket",
backupName: "bak",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{},
},
expectedErr: true,
expectedRes: make(map[string][]byte),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objStore := &fakeObjectStorage{storage: test.storage}
backupService := NewBackupService(objStore)
res := backupService.DeleteBackupFile(test.bucket, test.backupName)
err := backupService.DeleteBackupDir(bucket, backup)
assert.Equal(t, test.expectedErr, res != nil, "got error %v", res)
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
}
assert.Equal(t, test.expectedRes, objStore.storage[test.bucket])
})
}
}
func TestDeleteBackupMetadataFile(t *testing.T) {
tests := []struct {
name string
bucket string
backupName string
storage map[string]map[string][]byte
expectedErr bool
expectedRes map[string][]byte
}{
{
name: "normal case",
bucket: "test-bucket",
backupName: "bak",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{
"bak/ark-backup.json": nil,
},
},
expectedErr: false,
expectedRes: make(map[string][]byte),
},
{
name: "failed delete of file returns error",
bucket: "test-bucket",
backupName: "bak",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{},
},
expectedErr: true,
expectedRes: make(map[string][]byte),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objStore := &fakeObjectStorage{storage: test.storage}
backupService := NewBackupService(objStore)
res := backupService.DeleteBackupMetadataFile(test.bucket, test.backupName)
assert.Equal(t, test.expectedErr, res != nil, "got error %v", res)
assert.Equal(t, test.expectedRes, objStore.storage[test.bucket])
objStore.AssertExpectations(t)
})
}
}
@@ -275,21 +179,16 @@ func TestDeleteBackupMetadataFile(t *testing.T) {
func TestGetAllBackups(t *testing.T) {
tests := []struct {
name string
bucket string
storage map[string]map[string][]byte
storageData map[string][]byte
expectedRes []*api.Backup
expectedErr bool
expectedErr string
}{
{
name: "normal case",
bucket: "test-bucket",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
"backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
},
name: "normal case",
storageData: map[string][]byte{
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
"backup-2/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-2"}}),
},
expectedErr: false,
expectedRes: []*api.Backup{
&api.Backup{
TypeMeta: metav1.TypeMeta{Kind: "Backup", APIVersion: "ark.heptio.com/v1"},
@@ -302,13 +201,10 @@ func TestGetAllBackups(t *testing.T) {
},
},
{
name: "backup that can't be decoded is ignored",
bucket: "test-bucket",
storage: map[string]map[string][]byte{
"test-bucket": map[string][]byte{
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
"backup-2/ark-backup.json": []byte("this is not valid backup JSON"),
},
name: "backup that can't be decoded is ignored",
storageData: map[string][]byte{
"backup-1/ark-backup.json": encodeToBytes(&api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup-1"}}),
"backup-2/ark-backup.json": []byte("this is not valid backup JSON"),
},
expectedRes: []*api.Backup{
&api.Backup{
@@ -321,14 +217,26 @@ func TestGetAllBackups(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objStore := &fakeObjectStorage{storage: test.storage}
bucket := "bucket"
objStore := &testutil.ObjectStorageAdapter{}
objStore.On("ListCommonPrefixes", bucket, "/").Return([]string{"backup-1", "backup-2"}, nil)
objStore.On("GetObject", bucket, "backup-1/ark-backup.json").Return(ioutil.NopCloser(bytes.NewReader(test.storageData["backup-1/ark-backup.json"])), nil)
objStore.On("GetObject", bucket, "backup-2/ark-backup.json").Return(ioutil.NopCloser(bytes.NewReader(test.storageData["backup-2/ark-backup.json"])), nil)
backupService := NewBackupService(objStore)
res, err := backupService.GetAllBackups(test.bucket)
res, err := backupService.GetAllBackups(bucket)
assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
if test.expectedErr != "" {
assert.EqualError(t, err, test.expectedErr)
} else {
assert.NoError(t, err)
}
assert.Equal(t, test.expectedRes, res)
objStore.AssertExpectations(t)
})
}
}
@@ -362,82 +270,3 @@ func newStringReadSeeker(s string) *stringReadSeeker {
func (srs *stringReadSeeker) Seek(offset int64, whence int) (int64, error) {
panic("not implemented")
}
type fakeObjectStorage struct {
storage map[string]map[string][]byte
returnErrors map[string]map[string]interface{}
}
func (os *fakeObjectStorage) PutObject(bucket string, key string, body io.ReadSeeker) error {
if os.returnErrors["putobject"] != nil && os.returnErrors["putobject"][bucket+"||"+key] != nil {
return errors.New("error")
}
if os.storage[bucket] == nil {
return errors.New("bucket not found")
}
data, err := ioutil.ReadAll(body)
if err != nil {
return err
}
os.storage[bucket][key] = data
return nil
}
func (os *fakeObjectStorage) GetObject(bucket string, key string) (io.ReadCloser, error) {
if os.storage == nil {
return nil, errors.New("storage not initialized")
}
if os.storage[bucket] == nil {
return nil, errors.New("bucket not found")
}
if os.storage[bucket][key] == nil {
return nil, errors.New("key not found")
}
return ioutil.NopCloser(bytes.NewReader(os.storage[bucket][key])), nil
}
func (os *fakeObjectStorage) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
if os.storage == nil {
return nil, errors.New("storage not initialized")
}
if os.storage[bucket] == nil {
return nil, errors.New("bucket not found")
}
prefixes := sets.NewString()
for key := range os.storage[bucket] {
delimIdx := strings.LastIndex(key, delimiter)
if delimIdx == -1 {
prefixes.Insert(key)
}
prefixes.Insert(key[0:delimIdx])
}
return prefixes.List(), nil
}
func (os *fakeObjectStorage) DeleteObject(bucket string, key string) error {
if os.storage == nil {
return errors.New("storage not initialized")
}
if os.storage[bucket] == nil {
return errors.New("bucket not found")
}
if _, exists := os.storage[bucket][key]; !exists {
return errors.New("key not found")
}
delete(os.storage[bucket], key)
return nil
}
pkg/cloudprovider/gcp/object_storage_adapter.go
+41 -3
View File
@@ -17,23 +17,29 @@ limitations under the License.
package gcp
import (
"errors"
"io"
"strings"
"time"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
// TODO switch to using newstorage
newstorage "cloud.google.com/go/storage"
storage "google.golang.org/api/storage/v1"
"github.com/heptio/ark/pkg/cloudprovider"
)
type objectStorageAdapter struct {
gcs *storage.Service
gcs *storage.Service
googleAccessID string
privateKey []byte
}
var _ cloudprovider.ObjectStorageAdapter = &objectStorageAdapter{}
func NewObjectStorageAdapter() (cloudprovider.ObjectStorageAdapter, error) {
func NewObjectStorageAdapter(googleAccessID string, privateKey []byte) (cloudprovider.ObjectStorageAdapter, error) {
client, err := google.DefaultClient(oauth2.NoContext, storage.DevstorageReadWriteScope)
if err != nil {
return nil, err
@@ -45,7 +51,9 @@ func NewObjectStorageAdapter() (cloudprovider.ObjectStorageAdapter, error) {
}
return &objectStorageAdapter{
gcs: gcs,
gcs: gcs,
googleAccessID: googleAccessID,
privateKey: privateKey,
}, nil
}
@@ -84,6 +92,36 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
return ret, nil
}
func (op *objectStorageAdapter) ListObjects(bucket, prefix string) ([]string, error) {
res, err := op.gcs.Objects.List(bucket).Prefix(prefix).Do()
if err != nil {
return nil, err
}
ret := make([]string, 0, len(res.Items))
for _, item := range res.Items {
ret = append(ret, item.Name)
}
return ret, nil
}
func (op *objectStorageAdapter) DeleteObject(bucket string, key string) error {
return op.gcs.Objects.Delete(bucket, key).Do()
}
func (op *objectStorageAdapter) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
if op.googleAccessID == "" {
return "", errors.New("unable to create a pre-signed URL - make sure GOOGLE_APPLICATION_CREDENTIALS points to a valid GCE service account file (missing email address)")
}
if len(op.privateKey) == 0 {
return "", errors.New("unable to create a pre-signed URL - make sure GOOGLE_APPLICATION_CREDENTIALS points to a valid GCE service account file (missing private key)")
}
return newstorage.SignedURL(bucket, key, &newstorage.SignedURLOptions{
GoogleAccessID: op.googleAccessID,
PrivateKey: op.privateKey,
Method: "GET",
Expires: time.Now().Add(ttl),
})
}
pkg/cloudprovider/storage_interfaces.go
+10 -1
View File
@@ -16,7 +16,10 @@ limitations under the License.
package cloudprovider
import "io"
import (
"io"
"time"
)
// ObjectStorageAdapter exposes basic object-storage operations required
// by Ark.
@@ -34,9 +37,15 @@ type ObjectStorageAdapter interface {
// hierarchy in object storage).
ListCommonPrefixes(bucket string, delimiter string) ([]string, error)
// ListObjects gets a list of all objects in bucket that have the same prefix.
ListObjects(bucket, prefix string) ([]string, error)
// DeleteObject removes object with the specified key from the given
// bucket.
DeleteObject(bucket string, key string) error
// CreateSignedURL creates a pre-signed URL for the given bucket and key that expires after ttl.
CreateSignedURL(bucket, key string, ttl time.Duration) (string, error)
}
// BlockStorageAdapter exposes basic block-storage operations required
pkg/cmd/ark/ark.go
+6 -1
View File
@@ -31,7 +31,7 @@ import (
func NewCommand(name string) *cobra.Command {
c := &cobra.Command{
Use: name,
Use: name,
Short: "Back up and restore Kubernetes cluster resources.",
Long: `Heptio Ark is a tool for managing disaster recovery, specifically for
Kubernetes cluster resources. It provides a simple, configurable,
@@ -53,5 +53,10 @@ associated data.`,
// add the glog flags
c.PersistentFlags().AddGoFlagSet(flag.CommandLine)
// TODO: switch to a different logging library.
// Work around https://github.com/golang/glog/pull/13.
// See also https://github.com/kubernetes/kubernetes/issues/17162
flag.CommandLine.Parse([]string{})
return c
}
pkg/cmd/cli/backup/backup.go
+1
View File
@@ -32,6 +32,7 @@ func NewCommand(f client.Factory) *cobra.Command {
c.AddCommand(
NewCreateCommand(f),
NewGetCommand(f),
NewLogsCommand(f),
// Will implement describe later
// NewDescribeCommand(f),
pkg/cmd/cli/backup/logs.go
+55
View File
@@ -0,0 +1,55 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package backup
import (
"errors"
"os"
"time"
"github.com/spf13/cobra"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/client"
"github.com/heptio/ark/pkg/cmd"
"github.com/heptio/ark/pkg/cmd/util/downloadrequest"
)
func NewLogsCommand(f client.Factory) *cobra.Command {
timeout := time.Minute
c := &cobra.Command{
Use: "logs BACKUP",
Short: "Get backup logs",
Run: func(c *cobra.Command, args []string) {
if len(args) != 1 {
err := errors.New("backup name is required")
cmd.CheckError(err)
}
arkClient, err := f.Client()
cmd.CheckError(err)
err = downloadrequest.Stream(arkClient.ArkV1(), args[0], v1.DownloadTargetKindBackupLog, os.Stdout, timeout)
cmd.CheckError(err)
},
}
c.Flags().DurationVar(&timeout, "timeout", timeout, "how long to wait to receive logs")
return c
}
pkg/cmd/server/server.go
+38 -1
View File
@@ -19,10 +19,14 @@ package server
import (
"context"
"fmt"
"io/ioutil"
"os"
"reflect"
"sync"
"time"
"golang.org/x/oauth2/google"
"github.com/golang/glog"
"github.com/spf13/cobra"
@@ -305,7 +309,27 @@ func getObjectStorageProvider(cloudConfig api.CloudProviderConfig, field string)
cloudConfig.AWS.KMSKeyID,
cloudConfig.AWS.S3ForcePathStyle)
case cloudConfig.GCP != nil:
objectStorage, err = gcp.NewObjectStorageAdapter()
var email string
var privateKey []byte
credentialsFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
if credentialsFile != "" {
// Get the email and private key from the credentials file so we can pre-sign download URLs
creds, err := ioutil.ReadFile(credentialsFile)
if err != nil {
return nil, err
}
jwtConfig, err := google.JWTConfigFromJSON(creds)
if err != nil {
return nil, err
}
email = jwtConfig.Email
privateKey = jwtConfig.PrivateKey
} else {
glog.Warning("GOOGLE_APPLICATION_CREDENTIALS is undefined; some features such as downloading log files will not work")
}
objectStorage, err = gcp.NewObjectStorageAdapter(email, privateKey)
case cloudConfig.Azure != nil:
objectStorage, err = azure.NewObjectStorageAdapter()
}
@@ -465,6 +489,19 @@ func (s *server) runControllers(config *api.Config) error {
wg.Done()
}()
downloadRequestController := controller.NewDownloadRequestController(
s.arkClient.ArkV1(),
s.sharedInformerFactory.Ark().V1().DownloadRequests(),
s.sharedInformerFactory.Ark().V1().Backups(),
s.backupService,
config.BackupStorageProvider.Bucket,
)
wg.Add(1)
go func() {
downloadRequestController.Run(ctx, 1)
wg.Done()
}()
// SHARED INFORMERS HAVE TO BE STARTED AFTER ALL CONTROLLERS
go s.sharedInformerFactory.Start(ctx.Done())
pkg/cmd/util/downloadrequest/downloadrequest.go
+135
View File
@@ -0,0 +1,135 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package downloadrequest
import (
"compress/gzip"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"github.com/heptio/ark/pkg/apis/ark/v1"
arkclientv1 "github.com/heptio/ark/pkg/generated/clientset/typed/ark/v1"
)
func Stream(client arkclientv1.DownloadRequestsGetter, name string, kind v1.DownloadTargetKind, w io.Writer, timeout time.Duration) error {
req := &v1.DownloadRequest{
ObjectMeta: metav1.ObjectMeta{
Namespace: v1.DefaultNamespace,
Name: fmt.Sprintf("%s-%s", name, time.Now().Format("20060102150405")),
},
Spec: v1.DownloadRequestSpec{
Target: v1.DownloadTarget{
Kind: kind,
Name: name,
},
},
}
req, err := client.DownloadRequests(v1.DefaultNamespace).Create(req)
if err != nil {
return err
}
defer client.DownloadRequests(v1.DefaultNamespace).Delete(req.Name, nil)
listOptions := metav1.ListOptions{
//TODO: once kube-apiserver http://issue.k8s.io/51046 is fixed, uncomment
//FieldSelector: "metadata.name=" + req.Name
ResourceVersion: req.ResourceVersion,
}
watcher, err := client.DownloadRequests(v1.DefaultNamespace).Watch(listOptions)
if err != nil {
return err
}
defer watcher.Stop()
expired := time.NewTimer(timeout)
defer expired.Stop()
Loop:
for {
select {
case <-expired.C:
return errors.New("timed out waiting for download URL")
case e := <-watcher.ResultChan():
updated, ok := e.Object.(*v1.DownloadRequest)
if !ok {
return fmt.Errorf("unexpected type %T", e.Object)
}
if updated.Name != req.Name {
continue
}
switch e.Type {
case watch.Deleted:
errors.New("download request was unexpectedly deleted")
case watch.Modified:
if updated.Status.DownloadURL != "" {
req = updated
break Loop
}
}
}
}
if req.Status.DownloadURL == "" {
return errors.New("file not found")
}
httpClient := new(http.Client)
httpReq, err := http.NewRequest("GET", req.Status.DownloadURL, nil)
if err != nil {
return err
}
// Manually set this header so the net/http library does not automatically try to decompress. We
// need to handle this manually because it's not currently possible to set the MIME type for the
// pre-signed URLs for GCP or Azure.
httpReq.Header.Set("Accept-Encoding", "gzip")
resp, err := httpClient.Do(httpReq)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("request failed; unable to decode response body: %v", err)
}
return fmt.Errorf("request failed: %v", string(body))
}
gzipReader, err := gzip.NewReader(resp.Body)
if err != nil {
return err
}
defer gzipReader.Close()
_, err = io.Copy(w, gzipReader)
return err
}
pkg/cmd/util/downloadrequest/downloadrequest_test.go
+224
View File
@@ -0,0 +1,224 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package downloadrequest
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/generated/clientset/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
core "k8s.io/client-go/testing"
)
func TestStream(t *testing.T) {
tests := []struct {
name string
kind v1.DownloadTargetKind
timeout time.Duration
createError error
watchError error
watchAdds []runtime.Object
watchModifies []runtime.Object
watchDeletes []runtime.Object
updateWithURL bool
statusCode int
body string
deleteError error
expectedError string
}{
{
name: "error creating req",
createError: errors.New("foo"),
kind: v1.DownloadTargetKindBackupLog,
expectedError: "foo",
},
{
name: "error creating watch",
watchError: errors.New("bar"),
kind: v1.DownloadTargetKindBackupLog,
expectedError: "bar",
},
{
name: "timed out",
kind: v1.DownloadTargetKindBackupLog,
timeout: time.Millisecond,
expectedError: "timed out waiting for download URL",
},
{
name: "unexpected watch type",
kind: v1.DownloadTargetKindBackupLog,
watchAdds: []runtime.Object{&v1.Backup{}},
expectedError: "unexpected type *v1.Backup",
},
{
name: "other requests added/updated/deleted first",
kind: v1.DownloadTargetKindBackupLog,
watchAdds: []runtime.Object{
newDownloadRequest("foo").DownloadRequest,
},
watchModifies: []runtime.Object{
newDownloadRequest("foo").DownloadRequest,
},
watchDeletes: []runtime.Object{
newDownloadRequest("foo").DownloadRequest,
},
updateWithURL: true,
statusCode: http.StatusOK,
body: "download body",
},
{
name: "http error",
kind: v1.DownloadTargetKindBackupLog,
updateWithURL: true,
statusCode: http.StatusInternalServerError,
body: "some error",
expectedError: "request failed: some error",
},
}
const testTimeout = 30 * time.Second
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client := fake.NewSimpleClientset()
created := make(chan *v1.DownloadRequest, 1)
client.PrependReactor("create", "downloadrequests", func(action core.Action) (bool, runtime.Object, error) {
createAction := action.(core.CreateAction)
created <- createAction.GetObject().(*v1.DownloadRequest)
return true, createAction.GetObject(), test.createError
})
fakeWatch := watch.NewFake()
client.PrependWatchReactor("downloadrequests", core.DefaultWatchReactor(fakeWatch, test.watchError))
deleted := make(chan string, 1)
client.PrependReactor("delete", "downloadrequests", func(action core.Action) (bool, runtime.Object, error) {
deleteAction := action.(core.DeleteAction)
deleted <- deleteAction.GetName()
return true, nil, test.deleteError
})
timeout := test.timeout
if timeout == 0 {
timeout = testTimeout
}
var server *httptest.Server
var url string
if test.updateWithURL {
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(test.statusCode)
if test.statusCode == http.StatusOK {
gzipWriter := gzip.NewWriter(w)
fmt.Fprintf(gzipWriter, test.body)
gzipWriter.Close()
return
}
fmt.Fprintf(w, test.body)
}))
defer server.Close()
url = server.URL
}
output := new(bytes.Buffer)
errCh := make(chan error)
go func() {
err := Stream(client.ArkV1(), "name", test.kind, output, timeout)
errCh <- err
}()
for i := range test.watchAdds {
fakeWatch.Add(test.watchAdds[i])
}
for i := range test.watchModifies {
fakeWatch.Modify(test.watchModifies[i])
}
for i := range test.watchDeletes {
fakeWatch.Delete(test.watchDeletes[i])
}
var createdName string
if test.updateWithURL {
select {
case r := <-created:
createdName = r.Name
r.Status.DownloadURL = url
fakeWatch.Modify(r)
case <-time.After(testTimeout):
t.Fatalf("created object not received")
}
}
var err error
select {
case err = <-errCh:
case <-time.After(testTimeout):
t.Fatal("test timed out")
}
if test.expectedError != "" {
require.EqualError(t, err, test.expectedError)
return
}
require.NoError(t, err)
if test.statusCode != http.StatusOK {
assert.EqualError(t, err, "request failed: "+test.body)
return
}
assert.Equal(t, test.body, output.String())
select {
case name := <-deleted:
assert.Equal(t, createdName, name)
default:
t.Fatal("download request was not deleted")
}
})
}
}
type downloadRequest struct {
*v1.DownloadRequest
}
func newDownloadRequest(name string) *downloadRequest {
return &downloadRequest{
DownloadRequest: &v1.DownloadRequest{
ObjectMeta: metav1.ObjectMeta{
Namespace: v1.DefaultNamespace,
Name: name,
},
},
}
}
pkg/controller/backup_controller.go
+26 -9
View File
@@ -312,21 +312,36 @@ func (controller *backupController) runBackup(backup *api.Backup, bucket string)
if err != nil {
return err
}
logFile, err := ioutil.TempFile("", "")
if err != nil {
return err
}
defer func() {
var errs []error
errs = append(errs, err)
if closeErr := backupFile.Close(); closeErr != nil {
errs = append(errs, closeErr)
if err := backupFile.Close(); err != nil {
errs = append(errs, err)
}
if removeErr := os.Remove(backupFile.Name()); removeErr != nil {
errs = append(errs, removeErr)
if err := os.Remove(backupFile.Name()); err != nil {
errs = append(errs, err)
}
if err := logFile.Close(); err != nil {
errs = append(errs, err)
}
if err := os.Remove(logFile.Name()); err != nil {
errs = append(errs, err)
}
err = kuberrs.NewAggregate(errs)
}()
if err := controller.backupper.Backup(backup, backupFile); err != nil {
if err := controller.backupper.Backup(backup, backupFile, logFile); err != nil {
return err
}
@@ -340,11 +355,13 @@ func (controller *backupController) runBackup(backup *api.Backup, bucket string)
return err
}
// re-set the file offset to 0 for reading
_, err = backupFile.Seek(0, 0)
if err != nil {
// re-set the files' offset to 0 for reading
if _, err = backupFile.Seek(0, 0); err != nil {
return err
}
if _, err = logFile.Seek(0, 0); err != nil {
return err
}
return controller.backupService.UploadBackup(bucket, backup.Name, bytes.NewReader(buf.Bytes()), backupFile)
return controller.backupService.UploadBackup(bucket, backup.Name, bytes.NewReader(buf.Bytes()), backupFile, logFile)
}
pkg/controller/backup_controller_test.go
+5 -5
View File
@@ -40,8 +40,8 @@ type fakeBackupper struct {
mock.Mock
}
func (b *fakeBackupper) Backup(backup *v1.Backup, data io.Writer) error {
args := b.Called(backup, data)
func (b *fakeBackupper) Backup(backup *v1.Backup, data, log io.Writer) error {
args := b.Called(backup, data, log)
return args.Error(0)
}
@@ -155,7 +155,7 @@ func TestProcessBackup(t *testing.T) {
backupper := &fakeBackupper{}
cloudBackups := &fakeBackupService{}
cloudBackups := &BackupService{}
sharedInformers := informers.NewSharedInformerFactory(client, 0)
@@ -200,9 +200,9 @@ func TestProcessBackup(t *testing.T) {
backup.Status.Phase = v1.BackupPhaseInProgress
backup.Status.Expiration.Time = expiration
backup.Status.Version = 1
backupper.On("Backup", backup, mock.Anything).Return(nil)
backupper.On("Backup", backup, mock.Anything, mock.Anything).Return(nil)
cloudBackups.On("UploadBackup", "bucket", backup.Name, mock.Anything, mock.Anything).Return(nil)
cloudBackups.On("UploadBackup", "bucket", backup.Name, mock.Anything, mock.Anything, mock.Anything).Return(nil)
}
// this is necessary so the Update() call returns the appropriate object
pkg/controller/backup_sync_controller_test.go
+19 -23
View File
@@ -17,6 +17,7 @@ limitations under the License.
package controller
import (
"errors"
"testing"
"time"
@@ -24,46 +25,38 @@ import (
core "k8s.io/client-go/testing"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/generated/clientset/fake"
. "github.com/heptio/ark/pkg/util/test"
)
func TestRun(t *testing.T) {
func TestBackupSyncControllerRun(t *testing.T) {
tests := []struct {
name string
cloudBackups map[string][]*api.Backup
backupSvcErr error
name string
getAllBackupsError error
cloudBackups []*v1.Backup
}{
{
name: "no cloud backups",
},
{
name: "backup service returns error on GetAllBackups",
cloudBackups: map[string][]*api.Backup{
"nonexistent-bucket": []*api.Backup{
NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
},
},
name: "backup service returns error on GetAllBackups",
getAllBackupsError: errors.New("getAllBackups"),
},
{
name: "normal case",
cloudBackups: map[string][]*api.Backup{
"bucket": []*api.Backup{
NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
NewTestBackup().WithNamespace("ns-2").WithName("backup-3").Backup,
},
cloudBackups: []*v1.Backup{
NewTestBackup().WithNamespace("ns-1").WithName("backup-1").Backup,
NewTestBackup().WithNamespace("ns-1").WithName("backup-2").Backup,
NewTestBackup().WithNamespace("ns-2").WithName("backup-3").Backup,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var (
bs = &fakeBackupService{backupsByBucket: test.cloudBackups}
client = fake.NewSimpleClientset()
)
bs := &BackupService{}
client := fake.NewSimpleClientset()
c := NewBackupSyncController(
client.ArkV1(),
@@ -72,14 +65,16 @@ func TestRun(t *testing.T) {
time.Duration(0),
).(*backupSyncController)
bs.On("GetAllBackups", "bucket").Return(test.cloudBackups, test.getAllBackupsError)
c.run()
expectedActions := make([]core.Action, 0)
// we only expect creates for items within the target bucket
for _, cloudBackup := range test.cloudBackups["bucket"] {
for _, cloudBackup := range test.cloudBackups {
action := core.NewCreateAction(
api.SchemeGroupVersion.WithResource("backups"),
v1.SchemeGroupVersion.WithResource("backups"),
cloudBackup.Namespace,
cloudBackup,
)
@@ -88,6 +83,7 @@ func TestRun(t *testing.T) {
}
assert.Equal(t, expectedActions, client.Actions())
bs.AssertExpectations(t)
})
}
}
pkg/controller/download_request_controller.go
+285
View File
@@ -0,0 +1,285 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/golang/glog"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
"github.com/heptio/ark/pkg/generated/clientset/scheme"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
)
type downloadRequestController struct {
downloadRequestClient arkv1client.DownloadRequestsGetter
downloadRequestLister listers.DownloadRequestLister
downloadRequestListerSynced cache.InformerSynced
backupLister listers.BackupLister
backupListerSynced cache.InformerSynced
backupService cloudprovider.BackupService
bucket string
syncHandler func(key string) error
queue workqueue.RateLimitingInterface
clock clock.Clock
}
// NewDownloadRequestController creates a new DownloadRequestController.
func NewDownloadRequestController(
downloadRequestClient arkv1client.DownloadRequestsGetter,
downloadRequestInformer informers.DownloadRequestInformer,
backupInformer informers.BackupInformer,
backupService cloudprovider.BackupService,
bucket string,
) Interface {
c := &downloadRequestController{
downloadRequestClient: downloadRequestClient,
downloadRequestLister: downloadRequestInformer.Lister(),
downloadRequestListerSynced: downloadRequestInformer.Informer().HasSynced,
backupLister: backupInformer.Lister(),
backupListerSynced: backupInformer.Informer().HasSynced,
backupService: backupService,
bucket: bucket,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "backup"),
clock: &clock.RealClock{},
}
c.syncHandler = c.processDownloadRequest
downloadRequestInformer.Informer().AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
runtime.HandleError(fmt.Errorf("error creating queue key for %#v: %v", obj, err))
return
}
c.queue.Add(key)
},
},
)
return c
}
// Run is a blocking function that runs the specified number of worker goroutines
// to process items in the work queue. It will return when it receives on the
// ctx.Done() channel.
func (c *downloadRequestController) Run(ctx context.Context, numWorkers int) error {
var wg sync.WaitGroup
defer func() {
glog.Infof("Waiting for workers to finish their work")
c.queue.ShutDown()
// We have to wait here in the deferred function instead of at the bottom of the function body
// because we have to shut down the queue in order for the workers to shut down gracefully, and
// we want to shut down the queue via defer and not at the end of the body.
wg.Wait()
glog.Infof("All workers have finished")
}()
glog.Info("Starting DownloadRequestController")
defer glog.Infof("Shutting down DownloadRequestController")
glog.Info("Waiting for caches to sync")
if !cache.WaitForCacheSync(ctx.Done(), c.downloadRequestListerSynced, c.backupListerSynced) {
return errors.New("timed out waiting for caches to sync")
}
glog.Info("Caches are synced")
wg.Add(numWorkers)
for i := 0; i < numWorkers; i++ {
go func() {
wait.Until(c.runWorker, time.Second, ctx.Done())
wg.Done()
}()
}
wg.Add(1)
go func() {
wait.Until(c.resync, time.Minute, ctx.Done())
wg.Done()
}()
<-ctx.Done()
return nil
}
// runWorker runs a worker until the controller's queue indicates it's time to shut down.
func (c *downloadRequestController) runWorker() {
// continually take items off the queue (waits if it's
// empty) until we get a shutdown signal from the queue
for c.processNextWorkItem() {
}
}
// processNextWorkItem processes a single item from the queue.
func (c *downloadRequestController) processNextWorkItem() bool {
key, quit := c.queue.Get()
if quit {
return false
}
// always call done on this item, since if it fails we'll add
// it back with rate-limiting below
defer c.queue.Done(key)
err := c.syncHandler(key.(string))
if err == nil {
// If you had no error, tell the queue to stop tracking history for your key. This will reset
// things like failure counts for per-item rate limiting.
c.queue.Forget(key)
return true
}
glog.Errorf("syncHandler error: %v", err)
// we had an error processing the item so add it back
// into the queue for re-processing with rate-limiting
c.queue.AddRateLimited(key)
return true
}
// processDownloadRequest is the default per-item sync handler. It generates a pre-signed URL for
// a new DownloadRequest or deletes the DownloadRequest if it has expired.
func (c *downloadRequestController) processDownloadRequest(key string) error {
glog.V(4).Infof("processDownloadRequest for key %q", key)
ns, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
glog.V(4).Infof("error splitting key %q: %v", key, err)
return err
}
downloadRequest, err := c.downloadRequestLister.DownloadRequests(ns).Get(name)
if apierrors.IsNotFound(err) {
glog.V(4).Infof("unable to find DownloadRequest %q", key)
return nil
}
if err != nil {
glog.V(4).Infof("error getting DownloadRequest %q: %v", key, err)
return err
}
switch downloadRequest.Status.Phase {
case "", v1.DownloadRequestPhaseNew:
return c.generatePreSignedURL(downloadRequest)
case v1.DownloadRequestPhaseProcessed:
return c.deleteIfExpired(downloadRequest)
}
return nil
}
const signedURLTTL = 10 * time.Minute
// generatePreSignedURL generates a pre-signed URL for downloadRequest, changes the phase to
// Processed, and persists the changes to storage.
func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest) error {
switch downloadRequest.Spec.Target.Kind {
case v1.DownloadTargetKindBackupLog:
update, err := cloneDownloadRequest(downloadRequest)
if err != nil {
return err
}
update.Status.DownloadURL, err = c.backupService.CreateBackupLogSignedURL(c.bucket, update.Spec.Target.Name, signedURLTTL)
if err != nil {
return err
}
update.Status.Phase = v1.DownloadRequestPhaseProcessed
update.Status.Expiration = metav1.NewTime(c.clock.Now().Add(signedURLTTL))
_, err = c.downloadRequestClient.DownloadRequests(update.Namespace).Update(update)
return err
}
return fmt.Errorf("unsupported download target kind %q", downloadRequest.Spec.Target.Kind)
}
// deleteIfExpired deletes downloadRequest if it has expired.
func (c *downloadRequestController) deleteIfExpired(downloadRequest *v1.DownloadRequest) error {
glog.V(4).Infof("checking for expiration of %s/%s", downloadRequest.Namespace, downloadRequest.Name)
if downloadRequest.Status.Expiration.Time.Before(c.clock.Now()) {
glog.V(4).Infof("%s/%s has not expired", downloadRequest.Namespace, downloadRequest.Name)
return nil
}
glog.V(4).Infof("%s/%s has expired - deleting", downloadRequest.Namespace, downloadRequest.Name)
return c.downloadRequestClient.DownloadRequests(downloadRequest.Namespace).Delete(downloadRequest.Name, nil)
}
// resync requeues all the DownloadRequests in the lister's cache. This is mostly to handle deleting
// any expired requests that were not deleted as part of the normal client flow for whatever reason.
func (c *downloadRequestController) resync() {
list, err := c.downloadRequestLister.List(labels.Everything())
if err != nil {
runtime.HandleError(fmt.Errorf("error listing download requests: %v", err))
return
}
for _, dr := range list {
key, err := cache.MetaNamespaceKeyFunc(dr)
if err != nil {
runtime.HandleError(fmt.Errorf("error generating key for download request %#v: %v", dr, err))
continue
}
c.queue.Add(key)
}
}
// cloneDownloadRequest makes a deep copy of in.
func cloneDownloadRequest(in *v1.DownloadRequest) (*v1.DownloadRequest, error) {
clone, err := scheme.Scheme.DeepCopy(in)
if err != nil {
return nil, err
}
out, ok := clone.(*v1.DownloadRequest)
if !ok {
return nil, fmt.Errorf("unexpected type: %T", clone)
}
return out, nil
}
pkg/controller/gc_controller.go
+6 -15
View File
@@ -104,9 +104,9 @@ func (c *gcController) run() {
}
// garbageCollectBackup removes an expired backup by deleting any associated backup files (if
// deleteBackupFile = true), volume snapshots, restore API objects, and the backup API object
// deleteBackupFiles = true), volume snapshots, restore API objects, and the backup API object
// itself.
func (c *gcController) garbageCollectBackup(backup *api.Backup, deleteBackupFile bool) {
func (c *gcController) garbageCollectBackup(backup *api.Backup, deleteBackupFiles bool) {
// if the backup includes snapshots but we don't currently have a PVProvider, we don't
// want to orphan the snapshots so skip garbage-collection entirely.
if c.snapshotService == nil && len(backup.Status.VolumeBackups) > 0 {
@@ -128,23 +128,14 @@ func (c *gcController) garbageCollectBackup(backup *api.Backup, deleteBackupFile
}
}
// If applicable, delete backup & metadata file from object storage *before* deleting the API object
// If applicable, delete everything in the backup dir in object storage *before* deleting the API object
// because otherwise the backup sync controller could re-sync the backup from object storage.
if deleteBackupFile {
glog.Infof("Removing backup %s", kube.NamespaceAndName(backup))
if err := c.backupService.DeleteBackupFile(c.bucket, backup.Name); err != nil {
if deleteBackupFiles {
glog.Infof("Removing backup %s from object storage", kube.NamespaceAndName(backup))
if err := c.backupService.DeleteBackupDir(c.bucket, backup.Name); err != nil {
glog.Errorf("error deleting backup %s: %v", kube.NamespaceAndName(backup), err)
deletionFailure = true
}
if deletionFailure {
glog.Warningf("Backup %s will not be deleted due to errors deleting related object storage files(s) and/or volume snapshots", kube.NamespaceAndName(backup))
} else {
if err := c.backupService.DeleteBackupMetadataFile(c.bucket, backup.Name); err != nil {
glog.Errorf("error deleting backup metadata file for %s: %v", kube.NamespaceAndName(backup), err)
deletionFailure = true
}
}
}
glog.Infof("Getting restore API objects referencing backup %s", kube.NamespaceAndName(backup))
pkg/controller/gc_controller_test.go
+118 -370
View File
@@ -17,17 +17,11 @@ limitations under the License.
package controller
import (
"bytes"
"errors"
"io"
"io/ioutil"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/sets"
core "k8s.io/client-go/testing"
@@ -41,12 +35,11 @@ import (
type gcTest struct {
name string
bucket string
backups map[string][]*api.Backup
backups []*api.Backup
snapshots sets.String
nilSnapshotService bool
expectedBackupsRemaining map[string]sets.String
expectedDeletions sets.String
expectedSnapshotsRemaining sets.String
}
@@ -55,130 +48,96 @@ func TestGarbageCollect(t *testing.T) {
tests := []gcTest{
gcTest{
name: "basic-expired",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
name: "basic-expired",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
expectedBackupsRemaining: make(map[string]sets.String),
expectedDeletions: sets.NewString("backup-1"),
expectedSnapshotsRemaining: sets.NewString(),
},
gcTest{
name: "basic-unexpired",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
},
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
expectedBackupsRemaining: map[string]sets.String{
"bucket-1": sets.NewString("backup-1"),
name: "basic-unexpired",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
expectedDeletions: sets.NewString(),
expectedSnapshotsRemaining: sets.NewString("snapshot-1", "snapshot-2"),
},
gcTest{
name: "one expired, one unexpired",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-3", "snapshot-3").
WithSnapshot("pv-4", "snapshot-4").
Backup,
},
},
snapshots: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
expectedBackupsRemaining: map[string]sets.String{
"bucket-1": sets.NewString("backup-2"),
name: "one expired, one unexpired",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-3", "snapshot-3").
WithSnapshot("pv-4", "snapshot-4").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
expectedDeletions: sets.NewString("backup-1"),
expectedSnapshotsRemaining: sets.NewString("snapshot-3", "snapshot-4"),
},
gcTest{
name: "none expired in target bucket",
bucket: "bucket-2",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
"bucket-2": []*api.Backup{
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-3", "snapshot-3").
WithSnapshot("pv-4", "snapshot-4").
Backup,
},
},
snapshots: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
expectedBackupsRemaining: map[string]sets.String{
"bucket-1": sets.NewString("backup-1"),
"bucket-2": sets.NewString("backup-2"),
name: "none expired in target bucket",
backups: []*api.Backup{
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(1*time.Minute)).
WithSnapshot("pv-3", "snapshot-3").
WithSnapshot("pv-4", "snapshot-4").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
expectedDeletions: sets.NewString(),
expectedSnapshotsRemaining: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
},
gcTest{
name: "orphan snapshots",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
name: "orphan snapshots",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Minute)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2", "snapshot-3", "snapshot-4"),
expectedBackupsRemaining: make(map[string]sets.String),
expectedDeletions: sets.NewString("backup-1"),
expectedSnapshotsRemaining: sets.NewString("snapshot-3", "snapshot-4"),
},
gcTest{
name: "no snapshot service only GC's backups without snapshots",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(-1 * time.Second)).
Backup,
},
name: "no snapshot service only GC's backups without snapshots",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(-1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
NewTestBackup().WithName("backup-2").
WithExpiration(fakeClock.Now().Add(-1 * time.Second)).
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
nilSnapshotService: true,
expectedBackupsRemaining: map[string]sets.String{
"bucket-1": sets.NewString("backup-1"),
},
expectedDeletions: sets.NewString("backup-2"),
},
}
for _, test := range tests {
var (
backupService = &fakeBackupService{}
backupService = &BackupService{}
snapshotService *FakeSnapshotService
)
@@ -187,23 +146,11 @@ func TestGarbageCollect(t *testing.T) {
}
t.Run(test.name, func(t *testing.T) {
backupService.backupsByBucket = make(map[string][]*api.Backup)
backupService.backupMetadataByBucket = make(map[string][]*api.Backup)
for bucket, backups := range test.backups {
data := make([]*api.Backup, 0, len(backups))
for _, backup := range backups {
data = append(data, backup)
}
backupService.backupsByBucket[bucket] = data
backupService.backupMetadataByBucket[bucket] = data
}
var (
client = fake.NewSimpleClientset()
sharedInformers = informers.NewSharedInformerFactory(client, 0)
snapSvc cloudprovider.SnapshotService
bucket = "bucket"
)
if snapshotService != nil {
@@ -213,7 +160,7 @@ func TestGarbageCollect(t *testing.T) {
controller := NewGCController(
backupService,
snapSvc,
test.bucket,
bucket,
1*time.Millisecond,
sharedInformers.Ark().V1().Backups(),
client.ArkV1(),
@@ -222,161 +169,75 @@ func TestGarbageCollect(t *testing.T) {
).(*gcController)
controller.clock = fakeClock
controller.processBackups()
// verify every bucket has the backups we expect
for bucket, backups := range backupService.backupsByBucket {
// if actual and expected are both empty, no further verification needed
if len(backups) == 0 && len(test.expectedBackupsRemaining[bucket]) == 0 {
continue
}
// get all the actual backups remaining in this bucket
backupNames := sets.NewString()
for _, backup := range backupService.backupsByBucket[bucket] {
backupNames.Insert(backup.Name)
}
assert.Equal(t, test.expectedBackupsRemaining[bucket], backupNames)
backupService.On("GetAllBackups", bucket).Return(test.backups, nil)
for _, b := range test.expectedDeletions.List() {
backupService.On("DeleteBackupDir", bucket, b).Return(nil)
}
controller.processBackups()
if !test.nilSnapshotService {
assert.Equal(t, test.expectedSnapshotsRemaining, snapshotService.SnapshotsTaken)
}
backupService.AssertExpectations(t)
})
}
}
func TestGarbageCollectBackup(t *testing.T) {
tests := []struct {
name string
backup *api.Backup
deleteBackupFile bool
snapshots sets.String
backupFiles sets.String
backupMetadataFiles sets.String
restores []*api.Restore
expectedRestoreDeletes []string
expectedBackupDelete string
expectedSnapshots sets.String
expectedBackupFiles sets.String
expectedMetadataFiles sets.String
name string
backup *api.Backup
deleteBackupFile bool
snapshots sets.String
backupFiles sets.String
backupMetadataFiles sets.String
restores []*api.Restore
expectedRestoreDeletes []string
expectedBackupDelete string
expectedSnapshots sets.String
expectedObjectStorageDeletions sets.String
}{
{
name: "failed snapshot deletion shouldn't delete backup metadata file",
name: "deleteBackupFile=false, snapshot deletion fails, don't delete kube backup",
backup: NewTestBackup().WithName("backup-1").
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
deleteBackupFile: true,
snapshots: sets.NewString("snapshot-1"),
backupFiles: sets.NewString("backup-1"),
backupMetadataFiles: sets.NewString("backup-1"),
restores: nil,
expectedBackupDelete: "",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString(),
expectedMetadataFiles: sets.NewString("backup-1"),
deleteBackupFile: false,
snapshots: sets.NewString("snapshot-1"),
expectedSnapshots: sets.NewString(),
expectedObjectStorageDeletions: sets.NewString(),
},
{
name: "failed backup file deletion shouldn't delete backup metadata file",
backup: NewTestBackup().WithName("backup-1").
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
deleteBackupFile: true,
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
backupFiles: sets.NewString("doesn't-match-backup-name"),
backupMetadataFiles: sets.NewString("backup-1"),
restores: nil,
expectedBackupDelete: "",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString("doesn't-match-backup-name"),
expectedMetadataFiles: sets.NewString("backup-1"),
},
{
name: "missing backup metadata file still deletes snapshots & backup file",
backup: NewTestBackup().WithName("backup-1").
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
deleteBackupFile: true,
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
backupFiles: sets.NewString("backup-1"),
backupMetadataFiles: sets.NewString("doesn't-match-backup-name"),
restores: nil,
expectedBackupDelete: "",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString(),
expectedMetadataFiles: sets.NewString("doesn't-match-backup-name"),
},
{
name: "deleteBackupFile=false shouldn't error if no backup file exists",
backup: NewTestBackup().WithName("backup-1").
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
deleteBackupFile: false,
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
backupFiles: sets.NewString("non-matching-backup"),
backupMetadataFiles: sets.NewString("non-matching-backup"),
restores: nil,
expectedBackupDelete: "backup-1",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString("non-matching-backup"),
expectedMetadataFiles: sets.NewString("non-matching-backup"),
},
{
name: "deleteBackupFile=false should error if snapshot delete fails",
backup: NewTestBackup().WithName("backup-1").
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
deleteBackupFile: false,
snapshots: sets.NewString("snapshot-1"),
backupFiles: sets.NewString("non-matching-backup"),
backupMetadataFiles: sets.NewString("non-matching-backup"),
restores: nil,
expectedBackupDelete: "",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString("non-matching-backup"),
expectedMetadataFiles: sets.NewString("non-matching-backup"),
},
{
name: "related restores should be deleted",
backup: NewTestBackup().WithName("backup-1").Backup,
deleteBackupFile: false,
snapshots: sets.NewString(),
backupFiles: sets.NewString("non-matching-backup"),
backupMetadataFiles: sets.NewString("non-matching-backup"),
name: "related restores should be deleted",
backup: NewTestBackup().WithName("backup-1").Backup,
deleteBackupFile: true,
snapshots: sets.NewString(),
restores: []*api.Restore{
NewTestRestore(api.DefaultNamespace, "restore-1", api.RestorePhaseCompleted).WithBackup("backup-1").Restore,
NewTestRestore(api.DefaultNamespace, "restore-2", api.RestorePhaseCompleted).WithBackup("backup-2").Restore,
},
expectedRestoreDeletes: []string{"restore-1"},
expectedBackupDelete: "backup-1",
expectedSnapshots: sets.NewString(),
expectedBackupFiles: sets.NewString("non-matching-backup"),
expectedMetadataFiles: sets.NewString("non-matching-backup"),
expectedRestoreDeletes: []string{"restore-1"},
expectedBackupDelete: "backup-1",
expectedSnapshots: sets.NewString(),
expectedObjectStorageDeletions: sets.NewString("backup-1"),
},
}
for _, test := range tests {
var ()
t.Run(test.name, func(t *testing.T) {
var (
backupService = &fakeBackupService{
backupsByBucket: make(map[string][]*api.Backup),
backupMetadataByBucket: make(map[string][]*api.Backup),
}
backupService = &BackupService{}
snapshotService = &FakeSnapshotService{SnapshotsTaken: test.snapshots}
client = fake.NewSimpleClientset()
sharedInformers = informers.NewSharedInformerFactory(client, 0)
bucket = "bucket-1"
controller = NewGCController(
backupService,
snapshotService,
"bucket-1",
bucket,
1*time.Millisecond,
sharedInformers.Ark().V1().Backups(),
client.ArkV1(),
@@ -385,20 +246,15 @@ func TestGarbageCollectBackup(t *testing.T) {
).(*gcController)
)
for file := range test.backupFiles {
backup := &api.Backup{ObjectMeta: metav1.ObjectMeta{Name: file}}
backupService.backupsByBucket["bucket-1"] = append(backupService.backupsByBucket["bucket-1"], backup)
}
for file := range test.backupMetadataFiles {
backup := &api.Backup{ObjectMeta: metav1.ObjectMeta{Name: file}}
backupService.backupMetadataByBucket["bucket-1"] = append(backupService.backupMetadataByBucket["bucket-1"], backup)
}
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup)
for _, restore := range test.restores {
sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(restore)
}
for _, b := range test.expectedObjectStorageDeletions.List() {
backupService.On("DeleteBackupDir", bucket, b).Return(nil)
}
// METHOD UNDER TEST
controller.garbageCollectBackup(test.backup, test.deleteBackupFile)
@@ -407,22 +263,6 @@ func TestGarbageCollectBackup(t *testing.T) {
// remaining snapshots
assert.Equal(t, test.expectedSnapshots, snapshotService.SnapshotsTaken)
// remaining object storage backup files
expectedBackups := make([]*api.Backup, 0)
for file := range test.expectedBackupFiles {
backup := &api.Backup{ObjectMeta: metav1.ObjectMeta{Name: file}}
expectedBackups = append(expectedBackups, backup)
}
assert.Equal(t, expectedBackups, backupService.backupsByBucket["bucket-1"])
// remaining object storage backup metadata files
expectedBackups = make([]*api.Backup, 0)
for file := range test.expectedMetadataFiles {
backup := &api.Backup{ObjectMeta: metav1.ObjectMeta{Name: file}}
expectedBackups = append(expectedBackups, backup)
}
assert.Equal(t, expectedBackups, backupService.backupMetadataByBucket["bucket-1"])
expectedActions := make([]core.Action, 0)
// Restore client deletes
for _, restore := range test.expectedRestoreDeletes {
@@ -445,44 +285,32 @@ func TestGarbageCollectBackup(t *testing.T) {
}
assert.Equal(t, expectedActions, client.Actions())
backupService.AssertExpectations(t)
})
}
}
func TestGarbageCollectPicksUpBackupUponExpiration(t *testing.T) {
var (
backupService = &fakeBackupService{}
backupService = &BackupService{}
snapshotService = &FakeSnapshotService{}
fakeClock = clock.NewFakeClock(time.Now())
assert = assert.New(t)
)
scenario := gcTest{
name: "basic-expired",
bucket: "bucket-1",
backups: map[string][]*api.Backup{
"bucket-1": []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
name: "basic-expired",
backups: []*api.Backup{
NewTestBackup().WithName("backup-1").
WithExpiration(fakeClock.Now().Add(1*time.Second)).
WithSnapshot("pv-1", "snapshot-1").
WithSnapshot("pv-2", "snapshot-2").
Backup,
},
snapshots: sets.NewString("snapshot-1", "snapshot-2"),
}
backupService.backupsByBucket = make(map[string][]*api.Backup)
for bucket, backups := range scenario.backups {
data := make([]*api.Backup, 0, len(backups))
for _, backup := range backups {
data = append(data, backup)
}
backupService.backupsByBucket[bucket] = data
}
snapshotService.SnapshotsTaken = scenario.snapshots
var (
@@ -493,7 +321,7 @@ func TestGarbageCollectPicksUpBackupUponExpiration(t *testing.T) {
controller := NewGCController(
backupService,
snapshotService,
scenario.bucket,
"bucket",
1*time.Millisecond,
sharedInformers.Ark().V1().Backups(),
client.ArkV1(),
@@ -502,100 +330,20 @@ func TestGarbageCollectPicksUpBackupUponExpiration(t *testing.T) {
).(*gcController)
controller.clock = fakeClock
backupService.On("GetAllBackups", "bucket").Return(scenario.backups, nil)
// PASS 1
controller.processBackups()
assert.Equal(scenario.backups, backupService.backupsByBucket, "backups should not be garbage-collected yet.")
backupService.AssertExpectations(t)
assert.Equal(scenario.snapshots, snapshotService.SnapshotsTaken, "snapshots should not be garbage-collected yet.")
// PASS 2
fakeClock.Step(1 * time.Minute)
backupService.On("DeleteBackupDir", "bucket", "backup-1").Return(nil)
controller.processBackups()
assert.Equal(0, len(backupService.backupsByBucket[scenario.bucket]), "backups should have been garbage-collected.")
assert.Equal(0, len(snapshotService.SnapshotsTaken), "snapshots should have been garbage-collected.")
}
type fakeBackupService struct {
backupMetadataByBucket map[string][]*api.Backup
backupsByBucket map[string][]*api.Backup
mock.Mock
}
func (s *fakeBackupService) GetAllBackups(bucket string) ([]*api.Backup, error) {
backups, found := s.backupsByBucket[bucket]
if !found {
return nil, errors.New("bucket not found")
}
return backups, nil
}
func (s *fakeBackupService) GetBackup(bucket, name string) (*api.Backup, error) {
backups, err := s.GetAllBackups(bucket)
if err != nil {
return nil, err
}
for _, itm := range backups {
if itm.Name == name {
return itm, nil
}
}
return nil, errors.New("backup not found")
}
func (bs *fakeBackupService) UploadBackup(bucket, name string, metadata, backup io.ReadSeeker) error {
args := bs.Called(bucket, name, metadata, backup)
return args.Error(0)
}
func (s *fakeBackupService) DownloadBackup(bucket, name string) (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil
}
func (s *fakeBackupService) DeleteBackupMetadataFile(bucket, backupName string) error {
backups, found := s.backupMetadataByBucket[bucket]
if !found {
return errors.New("bucket not found")
}
deleteIdx := -1
for i, backup := range backups {
if backup.Name == backupName {
deleteIdx = i
break
}
}
if deleteIdx == -1 {
return errors.New("backup not found")
}
s.backupMetadataByBucket[bucket] = append(s.backupMetadataByBucket[bucket][0:deleteIdx], s.backupMetadataByBucket[bucket][deleteIdx+1:]...)
return nil
}
func (s *fakeBackupService) DeleteBackupFile(bucket, backupName string) error {
backups, err := s.GetAllBackups(bucket)
if err != nil {
return err
}
deleteIdx := -1
for i, backup := range backups {
if backup.Name == backupName {
deleteIdx = i
break
}
}
if deleteIdx == -1 {
return errors.New("backup not found")
}
s.backupsByBucket[bucket] = append(s.backupsByBucket[bucket][0:deleteIdx], s.backupsByBucket[bucket][deleteIdx+1:]...)
return nil
backupService.AssertExpectations(t)
}
pkg/controller/restore_controller_test.go
+66 -47
View File
@@ -17,8 +17,11 @@ limitations under the License.
package controller
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"testing"
"github.com/stretchr/testify/assert"
@@ -35,14 +38,15 @@ import (
. "github.com/heptio/ark/pkg/util/test"
)
func TestFetchRestore(t *testing.T) {
func TestFetchBackup(t *testing.T) {
tests := []struct {
name string
backupName string
informerBackups []*api.Backup
backupSvcBackups map[string][]*api.Backup
expectedRes *api.Backup
expectedErr bool
name string
backupName string
informerBackups []*api.Backup
backupServiceBackup *api.Backup
backupServiceError error
expectedRes *api.Backup
expectedErr bool
}{
{
name: "lister has backup",
@@ -51,17 +55,16 @@ func TestFetchRestore(t *testing.T) {
expectedRes: NewTestBackup().WithName("backup-1").Backup,
},
{
name: "backupSvc has backup",
backupName: "backup-1",
backupSvcBackups: map[string][]*api.Backup{
"bucket": []*api.Backup{NewTestBackup().WithName("backup-1").Backup},
},
expectedRes: NewTestBackup().WithName("backup-1").Backup,
name: "backupSvc has backup",
backupName: "backup-1",
backupServiceBackup: NewTestBackup().WithName("backup-1").Backup,
expectedRes: NewTestBackup().WithName("backup-1").Backup,
},
{
name: "no backup",
backupName: "backup-1",
expectedErr: true,
name: "no backup",
backupName: "backup-1",
backupServiceError: errors.New("no backup here"),
expectedErr: true,
},
}
@@ -71,7 +74,7 @@ func TestFetchRestore(t *testing.T) {
client = fake.NewSimpleClientset()
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
backupSvc = &fakeBackupService{}
backupSvc = &BackupService{}
)
c := NewRestoreController(
@@ -89,28 +92,34 @@ func TestFetchRestore(t *testing.T) {
sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm)
}
backupSvc.backupsByBucket = test.backupSvcBackups
if test.backupServiceBackup != nil || test.backupServiceError != nil {
backupSvc.On("GetBackup", "bucket", test.backupName).Return(test.backupServiceBackup, test.backupServiceError)
}
backup, err := c.fetchBackup("bucket", test.backupName)
if assert.Equal(t, test.expectedErr, err != nil) {
assert.Equal(t, test.expectedRes, backup)
}
backupSvc.AssertExpectations(t)
})
}
}
func TestProcessRestore(t *testing.T) {
tests := []struct {
name string
restoreKey string
restore *api.Restore
backup *api.Backup
restorerError error
allowRestoreSnapshots bool
expectedErr bool
expectedRestoreUpdates []*api.Restore
expectedRestorerCall *api.Restore
name string
restoreKey string
restore *api.Restore
backup *api.Backup
restorerError error
allowRestoreSnapshots bool
expectedErr bool
expectedRestoreUpdates []*api.Restore
expectedRestorerCall *api.Restore
backupServiceGetBackupError error
expectRestore bool
}{
{
name: "invalid key returns error",
@@ -161,18 +170,17 @@ func TestProcessRestore(t *testing.T) {
},
},
{
name: "restore with non-existent backup name fails",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedErr: false,
name: "restore with non-existent backup name fails",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
expectedErr: false,
backupServiceGetBackupError: errors.New("no backup here"),
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).
WithBackup("backup-1").
WithIncludedNamespace("ns-1").
WithErrors(api.RestoreResult{
// TODO this is the error msg returned by the fakeBackupService. When we switch to a mock obj,
// this will likely need to change.
Cluster: []string{"bucket not found"},
Cluster: []string{"no backup here"},
}).
Restore,
},
@@ -181,6 +189,7 @@ func TestProcessRestore(t *testing.T) {
name: "restorer throwing an error causes the restore to fail",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
restorerError: errors.New("blarg"),
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
@@ -197,10 +206,11 @@ func TestProcessRestore(t *testing.T) {
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
},
{
name: "valid restore gets executed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
name: "valid restore gets executed",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
@@ -208,10 +218,11 @@ func TestProcessRestore(t *testing.T) {
expectedRestorerCall: NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("ns-1").Restore,
},
{
name: "restore with no restorable namespaces gets defaulted to *",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectedErr: false,
name: "restore with no restorable namespaces gets defaulted to *",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
NewTestRestore("foo", "bar", api.RestorePhaseInProgress).WithBackup("backup-1").WithIncludedNamespace("*").Restore,
NewTestRestore("foo", "bar", api.RestorePhaseCompleted).WithBackup("backup-1").WithIncludedNamespace("*").Restore,
@@ -222,6 +233,7 @@ func TestProcessRestore(t *testing.T) {
name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true",
restore: NewTestRestore("foo", "bar", api.RestorePhaseNew).WithBackup("backup-1").WithIncludedNamespace("ns-1").WithRestorePVs(true).Restore,
backup: NewTestBackup().WithName("backup-1").Backup,
expectRestore: true,
allowRestoreSnapshots: true,
expectedErr: false,
expectedRestoreUpdates: []*api.Restore{
@@ -242,17 +254,14 @@ func TestProcessRestore(t *testing.T) {
},
}
// flag.Set("logtostderr", "true")
// flag.Set("v", "4")
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fmt.Println(test.name)
var (
client = fake.NewSimpleClientset()
restorer = &fakeRestorer{}
sharedInformers = informers.NewSharedInformerFactory(client, 0)
backupSvc = &fakeBackupService{}
backupSvc = &BackupService{}
)
c := NewRestoreController(
@@ -290,7 +299,11 @@ func TestProcessRestore(t *testing.T) {
if test.restorerError != nil {
errors.Namespaces = map[string][]string{"ns-1": {test.restorerError.Error()}}
}
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
if test.expectRestore {
downloadedBackup := ioutil.NopCloser(bytes.NewReader([]byte("hello world")))
backupSvc.On("DownloadBackup", mock.Anything, mock.Anything).Return(downloadedBackup, nil)
restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
}
var (
key = test.restoreKey
@@ -303,7 +316,13 @@ func TestProcessRestore(t *testing.T) {
}
}
if test.backupServiceGetBackupError != nil {
backupSvc.On("GetBackup", "bucket", test.restore.Spec.BackupName).Return(nil, test.backupServiceGetBackupError)
}
err = c.processRestore(key)
backupSvc.AssertExpectations(t)
restorer.AssertExpectations(t)
assert.Equal(t, test.expectedErr, err != nil, "got error %v", err)
pkg/generated/clientset/typed/ark/v1/ark_client.go
+5
View File
@@ -11,6 +11,7 @@ type ArkV1Interface interface {
RESTClient() rest.Interface
BackupsGetter
ConfigsGetter
DownloadRequestsGetter
RestoresGetter
SchedulesGetter
}
@@ -28,6 +29,10 @@ func (c *ArkV1Client) Configs(namespace string) ConfigInterface {
return newConfigs(c, namespace)
}
func (c *ArkV1Client) DownloadRequests(namespace string) DownloadRequestInterface {
return newDownloadRequests(c, namespace)
}
func (c *ArkV1Client) Restores(namespace string) RestoreInterface {
return newRestores(c, namespace)
}
pkg/generated/clientset/typed/ark/v1/downloadrequest.go
+156
View File
@@ -0,0 +1,156 @@
package v1
import (
v1 "github.com/heptio/ark/pkg/apis/ark/v1"
scheme "github.com/heptio/ark/pkg/generated/clientset/scheme"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
types "k8s.io/apimachinery/pkg/types"
watch "k8s.io/apimachinery/pkg/watch"
rest "k8s.io/client-go/rest"
)
// DownloadRequestsGetter has a method to return a DownloadRequestInterface.
// A group's client should implement this interface.
type DownloadRequestsGetter interface {
DownloadRequests(namespace string) DownloadRequestInterface
}
// DownloadRequestInterface has methods to work with DownloadRequest resources.
type DownloadRequestInterface interface {
Create(*v1.DownloadRequest) (*v1.DownloadRequest, error)
Update(*v1.DownloadRequest) (*v1.DownloadRequest, error)
UpdateStatus(*v1.DownloadRequest) (*v1.DownloadRequest, error)
Delete(name string, options *meta_v1.DeleteOptions) error
DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error
Get(name string, options meta_v1.GetOptions) (*v1.DownloadRequest, error)
List(opts meta_v1.ListOptions) (*v1.DownloadRequestList, error)
Watch(opts meta_v1.ListOptions) (watch.Interface, error)
Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.DownloadRequest, err error)
DownloadRequestExpansion
}
// downloadRequests implements DownloadRequestInterface
type downloadRequests struct {
client rest.Interface
ns string
}
// newDownloadRequests returns a DownloadRequests
func newDownloadRequests(c *ArkV1Client, namespace string) *downloadRequests {
return &downloadRequests{
client: c.RESTClient(),
ns: namespace,
}
}
// Create takes the representation of a downloadRequest and creates it. Returns the server's representation of the downloadRequest, and an error, if there is any.
func (c *downloadRequests) Create(downloadRequest *v1.DownloadRequest) (result *v1.DownloadRequest, err error) {
result = &v1.DownloadRequest{}
err = c.client.Post().
Namespace(c.ns).
Resource("downloadrequests").
Body(downloadRequest).
Do().
Into(result)
return
}
// Update takes the representation of a downloadRequest and updates it. Returns the server's representation of the downloadRequest, and an error, if there is any.
func (c *downloadRequests) Update(downloadRequest *v1.DownloadRequest) (result *v1.DownloadRequest, err error) {
result = &v1.DownloadRequest{}
err = c.client.Put().
Namespace(c.ns).
Resource("downloadrequests").
Name(downloadRequest.Name).
Body(downloadRequest).
Do().
Into(result)
return
}
// UpdateStatus was generated because the type contains a Status member.
// Add a +genclientstatus=false comment above the type to avoid generating UpdateStatus().
func (c *downloadRequests) UpdateStatus(downloadRequest *v1.DownloadRequest) (result *v1.DownloadRequest, err error) {
result = &v1.DownloadRequest{}
err = c.client.Put().
Namespace(c.ns).
Resource("downloadrequests").
Name(downloadRequest.Name).
SubResource("status").
Body(downloadRequest).
Do().
Into(result)
return
}
// Delete takes name of the downloadRequest and deletes it. Returns an error if one occurs.
func (c *downloadRequests) Delete(name string, options *meta_v1.DeleteOptions) error {
return c.client.Delete().
Namespace(c.ns).
Resource("downloadrequests").
Name(name).
Body(options).
Do().
Error()
}
// DeleteCollection deletes a collection of objects.
func (c *downloadRequests) DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error {
return c.client.Delete().
Namespace(c.ns).
Resource("downloadrequests").
VersionedParams(&listOptions, scheme.ParameterCodec).
Body(options).
Do().
Error()
}
// Get takes name of the downloadRequest, and returns the corresponding downloadRequest object, and an error if there is any.
func (c *downloadRequests) Get(name string, options meta_v1.GetOptions) (result *v1.DownloadRequest, err error) {
result = &v1.DownloadRequest{}
err = c.client.Get().
Namespace(c.ns).
Resource("downloadrequests").
Name(name).
VersionedParams(&options, scheme.ParameterCodec).
Do().
Into(result)
return
}
// List takes label and field selectors, and returns the list of DownloadRequests that match those selectors.
func (c *downloadRequests) List(opts meta_v1.ListOptions) (result *v1.DownloadRequestList, err error) {
result = &v1.DownloadRequestList{}
err = c.client.Get().
Namespace(c.ns).
Resource("downloadrequests").
VersionedParams(&opts, scheme.ParameterCodec).
Do().
Into(result)
return
}
// Watch returns a watch.Interface that watches the requested downloadRequests.
func (c *downloadRequests) Watch(opts meta_v1.ListOptions) (watch.Interface, error) {
opts.Watch = true
return c.client.Get().
Namespace(c.ns).
Resource("downloadrequests").
VersionedParams(&opts, scheme.ParameterCodec).
Watch()
}
// Patch applies the patch and returns the patched downloadRequest.
func (c *downloadRequests) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.DownloadRequest, err error) {
result = &v1.DownloadRequest{}
err = c.client.Patch(pt).
Namespace(c.ns).
Resource("downloadrequests").
SubResource(subresources...).
Name(name).
Body(data).
Do().
Into(result)
return
}
pkg/generated/clientset/typed/ark/v1/fake/fake_ark_client.go
+4
View File
@@ -18,6 +18,10 @@ func (c *FakeArkV1) Configs(namespace string) v1.ConfigInterface {
return &FakeConfigs{c, namespace}
}
func (c *FakeArkV1) DownloadRequests(namespace string) v1.DownloadRequestInterface {
return &FakeDownloadRequests{c, namespace}
}
func (c *FakeArkV1) Restores(namespace string) v1.RestoreInterface {
return &FakeRestores{c, namespace}
}
pkg/generated/clientset/typed/ark/v1/fake/fake_downloadrequest.go
+114
View File
@@ -0,0 +1,114 @@
package fake
import (
v1 "github.com/heptio/ark/pkg/apis/ark/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
labels "k8s.io/apimachinery/pkg/labels"
schema "k8s.io/apimachinery/pkg/runtime/schema"
types "k8s.io/apimachinery/pkg/types"
watch "k8s.io/apimachinery/pkg/watch"
testing "k8s.io/client-go/testing"
)
// FakeDownloadRequests implements DownloadRequestInterface
type FakeDownloadRequests struct {
Fake *FakeArkV1
ns string
}
var downloadrequestsResource = schema.GroupVersionResource{Group: "ark.heptio.com", Version: "v1", Resource: "downloadrequests"}
var downloadrequestsKind = schema.GroupVersionKind{Group: "ark.heptio.com", Version: "v1", Kind: "DownloadRequest"}
func (c *FakeDownloadRequests) Create(downloadRequest *v1.DownloadRequest) (result *v1.DownloadRequest, err error) {
obj, err := c.Fake.
Invokes(testing.NewCreateAction(downloadrequestsResource, c.ns, downloadRequest), &v1.DownloadRequest{})
if obj == nil {
return nil, err
}
return obj.(*v1.DownloadRequest), err
}
func (c *FakeDownloadRequests) Update(downloadRequest *v1.DownloadRequest) (result *v1.DownloadRequest, err error) {
obj, err := c.Fake.
Invokes(testing.NewUpdateAction(downloadrequestsResource, c.ns, downloadRequest), &v1.DownloadRequest{})
if obj == nil {
return nil, err
}
return obj.(*v1.DownloadRequest), err
}
func (c *FakeDownloadRequests) UpdateStatus(downloadRequest *v1.DownloadRequest) (*v1.DownloadRequest, error) {
obj, err := c.Fake.
Invokes(testing.NewUpdateSubresourceAction(downloadrequestsResource, "status", c.ns, downloadRequest), &v1.DownloadRequest{})
if obj == nil {
return nil, err
}
return obj.(*v1.DownloadRequest), err
}
func (c *FakeDownloadRequests) Delete(name string, options *meta_v1.DeleteOptions) error {
_, err := c.Fake.
Invokes(testing.NewDeleteAction(downloadrequestsResource, c.ns, name), &v1.DownloadRequest{})
return err
}
func (c *FakeDownloadRequests) DeleteCollection(options *meta_v1.DeleteOptions, listOptions meta_v1.ListOptions) error {
action := testing.NewDeleteCollectionAction(downloadrequestsResource, c.ns, listOptions)
_, err := c.Fake.Invokes(action, &v1.DownloadRequestList{})
return err
}
func (c *FakeDownloadRequests) Get(name string, options meta_v1.GetOptions) (result *v1.DownloadRequest, err error) {
obj, err := c.Fake.
Invokes(testing.NewGetAction(downloadrequestsResource, c.ns, name), &v1.DownloadRequest{})
if obj == nil {
return nil, err
}
return obj.(*v1.DownloadRequest), err
}
func (c *FakeDownloadRequests) List(opts meta_v1.ListOptions) (result *v1.DownloadRequestList, err error) {
obj, err := c.Fake.
Invokes(testing.NewListAction(downloadrequestsResource, downloadrequestsKind, c.ns, opts), &v1.DownloadRequestList{})
if obj == nil {
return nil, err
}
label, _, _ := testing.ExtractFromListOptions(opts)
if label == nil {
label = labels.Everything()
}
list := &v1.DownloadRequestList{}
for _, item := range obj.(*v1.DownloadRequestList).Items {
if label.Matches(labels.Set(item.Labels)) {
list.Items = append(list.Items, item)
}
}
return list, err
}
// Watch returns a watch.Interface that watches the requested downloadRequests.
func (c *FakeDownloadRequests) Watch(opts meta_v1.ListOptions) (watch.Interface, error) {
return c.Fake.
InvokesWatch(testing.NewWatchAction(downloadrequestsResource, c.ns, opts))
}
// Patch applies the patch and returns the patched downloadRequest.
func (c *FakeDownloadRequests) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.DownloadRequest, err error) {
obj, err := c.Fake.
Invokes(testing.NewPatchSubresourceAction(downloadrequestsResource, c.ns, name, data, subresources...), &v1.DownloadRequest{})
if obj == nil {
return nil, err
}
return obj.(*v1.DownloadRequest), err
}
pkg/generated/clientset/typed/ark/v1/generated_expansion.go
+2
View File
@@ -4,6 +4,8 @@ type BackupExpansion interface{}
type ConfigExpansion interface{}
type DownloadRequestExpansion interface{}
type RestoreExpansion interface{}
type ScheduleExpansion interface{}
pkg/generated/informers/externalversions/ark/v1/downloadrequest.go
+52
View File
@@ -0,0 +1,52 @@
// This file was automatically generated by informer-gen
package v1
import (
ark_v1 "github.com/heptio/ark/pkg/apis/ark/v1"
clientset "github.com/heptio/ark/pkg/generated/clientset"
internalinterfaces "github.com/heptio/ark/pkg/generated/informers/externalversions/internalinterfaces"
v1 "github.com/heptio/ark/pkg/generated/listers/ark/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
watch "k8s.io/apimachinery/pkg/watch"
cache "k8s.io/client-go/tools/cache"
time "time"
)
// DownloadRequestInformer provides access to a shared informer and lister for
// DownloadRequests.
type DownloadRequestInformer interface {
Informer() cache.SharedIndexInformer
Lister() v1.DownloadRequestLister
}
type downloadRequestInformer struct {
factory internalinterfaces.SharedInformerFactory
}
func newDownloadRequestInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
sharedIndexInformer := cache.NewSharedIndexInformer(
&cache.ListWatch{
ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {
return client.ArkV1().DownloadRequests(meta_v1.NamespaceAll).List(options)
},
WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) {
return client.ArkV1().DownloadRequests(meta_v1.NamespaceAll).Watch(options)
},
},
&ark_v1.DownloadRequest{},
resyncPeriod,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
)
return sharedIndexInformer
}
func (f *downloadRequestInformer) Informer() cache.SharedIndexInformer {
return f.factory.InformerFor(&ark_v1.DownloadRequest{}, newDownloadRequestInformer)
}
func (f *downloadRequestInformer) Lister() v1.DownloadRequestLister {
return v1.NewDownloadRequestLister(f.Informer().GetIndexer())
}
pkg/generated/informers/externalversions/ark/v1/interface.go
+7
View File
@@ -12,6 +12,8 @@ type Interface interface {
Backups() BackupInformer
// Configs returns a ConfigInformer.
Configs() ConfigInformer
// DownloadRequests returns a DownloadRequestInformer.
DownloadRequests() DownloadRequestInformer
// Restores returns a RestoreInformer.
Restores() RestoreInformer
// Schedules returns a ScheduleInformer.
@@ -37,6 +39,11 @@ func (v *version) Configs() ConfigInformer {
return &configInformer{factory: v.SharedInformerFactory}
}
// DownloadRequests returns a DownloadRequestInformer.
func (v *version) DownloadRequests() DownloadRequestInformer {
return &downloadRequestInformer{factory: v.SharedInformerFactory}
}
// Restores returns a RestoreInformer.
func (v *version) Restores() RestoreInformer {
return &restoreInformer{factory: v.SharedInformerFactory}
pkg/generated/informers/externalversions/generic.go
+2
View File
@@ -40,6 +40,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().Backups().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("configs"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().Configs().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("downloadrequests"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().DownloadRequests().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("restores"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ark().V1().Restores().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("schedules"):
pkg/generated/listers/ark/v1/downloadrequest.go
+78
View File
@@ -0,0 +1,78 @@
// This file was automatically generated by lister-gen
package v1
import (
v1 "github.com/heptio/ark/pkg/apis/ark/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/cache"
)
// DownloadRequestLister helps list DownloadRequests.
type DownloadRequestLister interface {
// List lists all DownloadRequests in the indexer.
List(selector labels.Selector) (ret []*v1.DownloadRequest, err error)
// DownloadRequests returns an object that can list and get DownloadRequests.
DownloadRequests(namespace string) DownloadRequestNamespaceLister
DownloadRequestListerExpansion
}
// downloadRequestLister implements the DownloadRequestLister interface.
type downloadRequestLister struct {
indexer cache.Indexer
}
// NewDownloadRequestLister returns a new DownloadRequestLister.
func NewDownloadRequestLister(indexer cache.Indexer) DownloadRequestLister {
return &downloadRequestLister{indexer: indexer}
}
// List lists all DownloadRequests in the indexer.
func (s *downloadRequestLister) List(selector labels.Selector) (ret []*v1.DownloadRequest, err error) {
err = cache.ListAll(s.indexer, selector, func(m interface{}) {
ret = append(ret, m.(*v1.DownloadRequest))
})
return ret, err
}
// DownloadRequests returns an object that can list and get DownloadRequests.
func (s *downloadRequestLister) DownloadRequests(namespace string) DownloadRequestNamespaceLister {
return downloadRequestNamespaceLister{indexer: s.indexer, namespace: namespace}
}
// DownloadRequestNamespaceLister helps list and get DownloadRequests.
type DownloadRequestNamespaceLister interface {
// List lists all DownloadRequests in the indexer for a given namespace.
List(selector labels.Selector) (ret []*v1.DownloadRequest, err error)
// Get retrieves the DownloadRequest from the indexer for a given namespace and name.
Get(name string) (*v1.DownloadRequest, error)
DownloadRequestNamespaceListerExpansion
}
// downloadRequestNamespaceLister implements the DownloadRequestNamespaceLister
// interface.
type downloadRequestNamespaceLister struct {
indexer cache.Indexer
namespace string
}
// List lists all DownloadRequests in the indexer for a given namespace.
func (s downloadRequestNamespaceLister) List(selector labels.Selector) (ret []*v1.DownloadRequest, err error) {
err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
ret = append(ret, m.(*v1.DownloadRequest))
})
return ret, err
}
// Get retrieves the DownloadRequest from the indexer for a given namespace and name.
func (s downloadRequestNamespaceLister) Get(name string) (*v1.DownloadRequest, error) {
obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
if err != nil {
return nil, err
}
if !exists {
return nil, errors.NewNotFound(v1.Resource("downloadrequest"), name)
}
return obj.(*v1.DownloadRequest), nil
}
pkg/generated/listers/ark/v1/expansion_generated.go
+8
View File
@@ -18,6 +18,14 @@ type ConfigListerExpansion interface{}
// ConfigNamespaceLister.
type ConfigNamespaceListerExpansion interface{}
// DownloadRequestListerExpansion allows custom methods to be added to
// DownloadRequestLister.
type DownloadRequestListerExpansion interface{}
// DownloadRequestNamespaceListerExpansion allows custom methods to be added to
// DownloadRequestNamespaceLister.
type DownloadRequestNamespaceListerExpansion interface{}
// RestoreListerExpansion allows custom methods to be added to
// RestoreLister.
type RestoreListerExpansion interface{}
pkg/util/test/backup_service.go
+146
View File
@@ -0,0 +1,146 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by mockery v1.0.0
package test
import io "io"
import mock "github.com/stretchr/testify/mock"
import time "time"
import v1 "github.com/heptio/ark/pkg/apis/ark/v1"
// BackupService is an autogenerated mock type for the BackupService type
type BackupService struct {
mock.Mock
}
// CreateBackupLogSignedURL provides a mock function with given fields: bucket, backupName, ttl
func (_m *BackupService) CreateBackupLogSignedURL(bucket string, backupName string, ttl time.Duration) (string, error) {
ret := _m.Called(bucket, backupName, ttl)
var r0 string
if rf, ok := ret.Get(0).(func(string, string, time.Duration) string); ok {
r0 = rf(bucket, backupName, ttl)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, time.Duration) error); ok {
r1 = rf(bucket, backupName, ttl)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// DeleteBackupDir provides a mock function with given fields: bucket, backupName
func (_m *BackupService) DeleteBackupDir(bucket string, backupName string) error {
ret := _m.Called(bucket, backupName)
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(bucket, backupName)
} else {
r0 = ret.Error(0)
}
return r0
}
// DownloadBackup provides a mock function with given fields: bucket, name
func (_m *BackupService) DownloadBackup(bucket string, name string) (io.ReadCloser, error) {
ret := _m.Called(bucket, name)
var r0 io.ReadCloser
if rf, ok := ret.Get(0).(func(string, string) io.ReadCloser); ok {
r0 = rf(bucket, name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(io.ReadCloser)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(bucket, name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetAllBackups provides a mock function with given fields: bucket
func (_m *BackupService) GetAllBackups(bucket string) ([]*v1.Backup, error) {
ret := _m.Called(bucket)
var r0 []*v1.Backup
if rf, ok := ret.Get(0).(func(string) []*v1.Backup); ok {
r0 = rf(bucket)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*v1.Backup)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(bucket)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetBackup provides a mock function with given fields: bucket, name
func (_m *BackupService) GetBackup(bucket string, name string) (*v1.Backup, error) {
ret := _m.Called(bucket, name)
var r0 *v1.Backup
if rf, ok := ret.Get(0).(func(string, string) *v1.Backup); ok {
r0 = rf(bucket, name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v1.Backup)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(bucket, name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// UploadBackup provides a mock function with given fields: bucket, name, metadata, backup, log
func (_m *BackupService) UploadBackup(bucket string, name string, metadata io.ReadSeeker, backup io.ReadSeeker, log io.ReadSeeker) error {
ret := _m.Called(bucket, name, metadata, backup, log)
var r0 error
if rf, ok := ret.Get(0).(func(string, string, io.ReadSeeker, io.ReadSeeker, io.ReadSeeker) error); ok {
r0 = rf(bucket, name, metadata, backup, log)
} else {
r0 = ret.Error(0)
}
return r0
}
pkg/util/test/object_storage_adapter.go
+145
View File
@@ -0,0 +1,145 @@
/*
Copyright 2017 Heptio Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by mockery v1.0.0
package test
import io "io"
import mock "github.com/stretchr/testify/mock"
import time "time"
// ObjectStorageAdapter is an autogenerated mock type for the ObjectStorageAdapter type
type ObjectStorageAdapter struct {
mock.Mock
}
// CreateSignedURL provides a mock function with given fields: bucket, key, ttl
func (_m *ObjectStorageAdapter) CreateSignedURL(bucket string, key string, ttl time.Duration) (string, error) {
ret := _m.Called(bucket, key, ttl)
var r0 string
if rf, ok := ret.Get(0).(func(string, string, time.Duration) string); ok {
r0 = rf(bucket, key, ttl)
} else {
r0 = ret.Get(0).(string)
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, time.Duration) error); ok {
r1 = rf(bucket, key, ttl)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// DeleteObject provides a mock function with given fields: bucket, key
func (_m *ObjectStorageAdapter) DeleteObject(bucket string, key string) error {
ret := _m.Called(bucket, key)
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(bucket, key)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetObject provides a mock function with given fields: bucket, key
func (_m *ObjectStorageAdapter) GetObject(bucket string, key string) (io.ReadCloser, error) {
ret := _m.Called(bucket, key)
var r0 io.ReadCloser
if rf, ok := ret.Get(0).(func(string, string) io.ReadCloser); ok {
r0 = rf(bucket, key)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(io.ReadCloser)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(bucket, key)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListCommonPrefixes provides a mock function with given fields: bucket, delimiter
func (_m *ObjectStorageAdapter) ListCommonPrefixes(bucket string, delimiter string) ([]string, error) {
ret := _m.Called(bucket, delimiter)
var r0 []string
if rf, ok := ret.Get(0).(func(string, string) []string); ok {
r0 = rf(bucket, delimiter)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(bucket, delimiter)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListObjects provides a mock function with given fields: bucket, prefix
func (_m *ObjectStorageAdapter) ListObjects(bucket string, prefix string) ([]string, error) {
ret := _m.Called(bucket, prefix)
var r0 []string
if rf, ok := ret.Get(0).(func(string, string) []string); ok {
r0 = rf(bucket, prefix)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string) error); ok {
r1 = rf(bucket, prefix)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// PutObject provides a mock function with given fields: bucket, key, body
func (_m *ObjectStorageAdapter) PutObject(bucket string, key string, body io.ReadSeeker) error {
ret := _m.Called(bucket, key, body)
var r0 error
if rf, ok := ret.Get(0).(func(string, string, io.ReadSeeker) error); ok {
r0 = rf(bucket, key, body)
} else {
r0 = ret.Error(0)
}
return r0
}
vendor/cloud.google.com/go/CONTRIBUTING.md
Generated Vendored
+138
View File
@@ -0,0 +1,138 @@
# Contributing
1. Sign one of the contributor license agreements below.
1. `go get golang.org/x/review/git-codereview` to install the code reviewing tool.
1. You will need to ensure that your `GOBIN` directory (by default
`$GOPATH/bin`) is in your `PATH` so that git can find the command.
1. If you would like, you may want to set up aliases for git-codereview,
such that `git codereview change` becomes `git change`. See the
[godoc](https://godoc.org/golang.org/x/review/git-codereview) for details.
1. Should you run into issues with the git-codereview tool, please note
that all error messages will assume that you have set up these
aliases.
1. Get the cloud package by running `go get -d cloud.google.com/go`.
1. If you have already checked out the source, make sure that the remote git
origin is https://code.googlesource.com/gocloud:
git remote set-url origin https://code.googlesource.com/gocloud
1. Make sure your auth is configured correctly by visiting
https://code.googlesource.com, clicking "Generate Password", and following
the directions.
1. Make changes and create a change by running `git codereview change <name>`,
provide a commit message, and use `git codereview mail` to create a Gerrit CL.
1. Keep amending to the change with `git codereview change` and mail as your receive
feedback. Each new mailed amendment will create a new patch set for your change in Gerrit.
## Integration Tests
In addition to the unit tests, you may run the integration test suite.
To run the integrations tests, creating and configuration of a project in the
Google Developers Console is required.
After creating a project, you must [create a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount).
Ensure the project-level **Owner** [IAM role](console.cloud.google.com/iam-admin/iam/project)
(or **Editor** and **Logs Configuration Writer** roles) are added to the
service account.
Once you create a project, set the following environment variables to be able to
run the against the actual APIs.
- **GCLOUD_TESTS_GOLANG_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455)
- **GCLOUD_TESTS_GOLANG_KEY**: The path to the JSON key file.
- **GCLOUD_TESTS_API_KEY**: Your API key.
Install the [gcloud command-line tool][gcloudcli] to your machine and use it
to create some resources used in integration tests.
From the project's root directory:
``` sh
# Set the default project in your env.
$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID
# Authenticate the gcloud tool with your account.
$ gcloud auth login
# Create the indexes used in the datastore integration tests.
$ gcloud preview datastore create-indexes datastore/testdata/index.yaml
# Create a Google Cloud storage bucket with the same name as your test project,
# and with the Stackdriver Logging service account as owner, for the sink
# integration tests in logging.
$ gsutil mb gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
$ gsutil acl ch -g cloud-logs@google.com:O gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
# Create a Spanner instance for the spanner integration tests.
$ gcloud beta spanner instances create go-integration-test --config regional-us-central1 --nodes 1 --description 'Instance for go client test'
# NOTE: Spanner instances are priced by the node-hour, so you may want to delete
# the instance after testing with 'gcloud beta spanner instances delete'.
```
Once you've set the environment variables, you can run the integration tests by
running:
``` sh
$ go test -v cloud.google.com/go/...
```
## Contributor License Agreements
Before we can accept your pull requests you'll need to sign a Contributor
License Agreement (CLA):
- **If you are an individual writing original source code** and **you own the
- intellectual property**, then you'll need to sign an [individual CLA][indvcla].
- **If you work for a company that wants to allow you to contribute your work**,
then you'll need to sign a [corporate CLA][corpcla].
You can sign these electronically (just scroll to the bottom). After that,
we'll be able to accept your pull requests.
## Contributor Code of Conduct
As contributors and maintainers of this project,
and in the interest of fostering an open and welcoming community,
we pledge to respect all people who contribute through reporting issues,
posting feature requests, updating documentation,
submitting pull requests or patches, and other activities.
We are committed to making participation in this project
a harassment-free experience for everyone,
regardless of level of experience, gender, gender identity and expression,
sexual orientation, disability, personal appearance,
body size, race, ethnicity, age, religion, or nationality.
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* Public or private harassment
* Publishing other's private information,
such as physical or electronic
addresses, without explicit permission
* Other unethical or unprofessional conduct.
Project maintainers have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct.
By adopting this Code of Conduct,
project maintainers commit themselves to fairly and consistently
applying these principles to every aspect of managing this project.
Project maintainers who do not follow or enforce the Code of Conduct
may be permanently removed from the project team.
This code of conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community.
Instances of abusive, harassing, or otherwise unacceptable behavior
may be reported by opening an issue
or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/
[indvcla]: https://developers.google.com/open-source/cla/individual
[corpcla]: https://developers.google.com/open-source/cla/corporate
vendor/cloud.google.com/go/CONTRIBUTORS
Generated Vendored
+3
View File
@@ -11,6 +11,7 @@
# Keep the list alphabetically sorted.
Alexis Hunt <lexer@google.com>
Andreas Litt <andreas.litt@gmail.com>
Andrew Gerrand <adg@golang.org>
Brad Fitzpatrick <bradfitz@golang.org>
@@ -24,11 +25,13 @@ Ingo Oeser <nightlyone@googlemail.com>
Johan Euphrosine <proppy@google.com>
Jonathan Amsterdam <jba@google.com>
Luna Duclos <luna.duclos@palmstonegames.com>
Magnus Hiie <magnus.hiie@gmail.com>
Michael McGreevy <mcgreevy@golang.org>
Omar Jarjur <ojarjur@google.com>
Paweł Knap <pawelknap88@gmail.com>
Péter Szilágyi <peterke@gmail.com>
Sarah Adams <shadams@google.com>
Thanatat Tamtan <acoshift@gmail.com>
Toby Burress <kurin@google.com>
Tuo Shan <shantuo@google.com>
Tyler Treat <ttreat31@gmail.com>
vendor/cloud.google.com/go/MIGRATION.md
Generated Vendored
+54
View File
@@ -0,0 +1,54 @@
# Code Changes
## v0.10.0
- pubsub: Replace
```
sub.ModifyPushConfig(ctx, pubsub.PushConfig{Endpoint: "https://example.com/push"})
```
with
```
sub.Update(ctx, pubsub.SubscriptionConfigToUpdate{
PushConfig: &pubsub.PushConfig{Endpoint: "https://example.com/push"},
})
```
- trace: traceGRPCServerInterceptor will be provided from *trace.Client.
Given an initialized `*trace.Client` named `tc`, instead of
```
s := grpc.NewServer(grpc.UnaryInterceptor(trace.GRPCServerInterceptor(tc)))
```
write
```
s := grpc.NewServer(grpc.UnaryInterceptor(tc.GRPCServerInterceptor()))
```
- trace trace.GRPCClientInterceptor will also provided from *trace.Client.
Instead of
```
conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(trace.GRPCClientInterceptor()))
```
write
```
conn, err := grpc.Dial(srv.Addr, grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor()))
```
- trace: We removed the deprecated `trace.EnableGRPCTracing`. Use the gRPC
interceptor as a dial option as shown below when initializing Cloud package
clients:
```
c, err := pubsub.NewClient(ctx, "project-id", option.WithGRPCDialOption(grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor())))
if err != nil {
...
}
```
vendor/cloud.google.com/go/README.md
Generated Vendored
+460
View File
@@ -0,0 +1,460 @@
# Google Cloud Client Libraries for Go
[![GoDoc](https://godoc.org/cloud.google.com/go?status.svg)](https://godoc.org/cloud.google.com/go)
Go packages for [Google Cloud Platform](https://cloud.google.com) services.
``` go
import "cloud.google.com/go"
```
To install the packages on your system,
```
$ go get -u cloud.google.com/go/...
```
**NOTE:** Some of these packages are under development, and may occasionally
make backwards-incompatible changes.
**NOTE:** Github repo is a mirror of [https://code.googlesource.com/gocloud](https://code.googlesource.com/gocloud).
* [News](#news)
* [Supported APIs](#supported-apis)
* [Go Versions Supported](#go-versions-supported)
* [Authorization](#authorization)
* [Cloud Datastore](#cloud-datastore-)
* [Cloud Storage](#cloud-storage-)
* [Cloud Pub/Sub](#cloud-pub-sub-)
* [Cloud BigQuery](#cloud-bigquery-)
* [Stackdriver Logging](#stackdriver-logging-)
* [Cloud Spanner](#cloud-spanner-)
## News
_July 31, 2017_
*v0.11.0*
- Clients for spanner, pubsub and video are now in beta.
- New client for DLP.
- spanner: performance and testing improvements.
- storage: requester-pays buckets are supported.
- storage, profiler, bigtable, bigquery: bug fixes and other minor improvements.
- pubsub: bug fixes and other minor improvements
_June 17, 2017_
*v0.10.0*
- pubsub: Subscription.ModifyPushConfig replaced with Subscription.Update.
- pubsub: Subscription.Receive now runs concurrently for higher throughput.
- vision: cloud.google.com/go/vision is deprecated. Use
cloud.google.com/go/vision/apiv1 instead.
- translation: now stable.
- trace: several changes to the surface. See the link below.
[Code changes required from v0.9.0.](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/MIGRATION.md)
[Older news](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/old-news.md)
## Supported APIs
Google API | Status | Package
---------------------------------|--------------|-----------------------------------------------------------
[Datastore][cloud-datastore] | stable | [`cloud.google.com/go/datastore`][cloud-datastore-ref]
[Storage][cloud-storage] | stable | [`cloud.google.com/go/storage`][cloud-storage-ref]
[Bigtable][cloud-bigtable] | beta | [`cloud.google.com/go/bigtable`][cloud-bigtable-ref]
[BigQuery][cloud-bigquery] | beta | [`cloud.google.com/go/bigquery`][cloud-bigquery-ref]
[Logging][cloud-logging] | stable | [`cloud.google.com/go/logging`][cloud-logging-ref]
[Monitoring][cloud-monitoring] | alpha | [`cloud.google.com/go/monitoring/apiv3`][cloud-monitoring-ref]
[Pub/Sub][cloud-pubsub] | beta | [`cloud.google.com/go/pubsub`][cloud-pubsub-ref]
[Vision][cloud-vision] | beta | [`cloud.google.com/go/vision/apiv1`][cloud-vision-ref]
[Language][cloud-language] | beta | [`cloud.google.com/go/language/apiv1`][cloud-language-ref]
[Speech][cloud-speech] | beta | [`cloud.google.com/go/speech/apiv1`][cloud-speech-ref]
[Spanner][cloud-spanner] | beta | [`cloud.google.com/go/spanner`][cloud-spanner-ref]
[Translation][cloud-translation] | stable | [`cloud.google.com/go/translate`][cloud-translation-ref]
[Trace][cloud-trace] | alpha | [`cloud.google.com/go/trace`][cloud-trace-ref]
[Video Intelligence][cloud-video]| beta | [`cloud.google.com/go/videointelligence/apiv1beta1`][cloud-video-ref]
[ErrorReporting][cloud-errors] | alpha | [`cloud.google.com/go/errors`][cloud-errors-ref]
> **Alpha status**: the API is still being actively developed. As a
> result, it might change in backward-incompatible ways and is not recommended
> for production use.
>
> **Beta status**: the API is largely complete, but still has outstanding
> features and bugs to be addressed. There may be minor backwards-incompatible
> changes where necessary.
>
> **Stable status**: the API is mature and ready for production use. We will
> continue addressing bugs and feature requests.
Documentation and examples are available at
https://godoc.org/cloud.google.com/go
Visit or join the
[google-api-go-announce group](https://groups.google.com/forum/#!forum/google-api-go-announce)
for updates on these packages.
## Go Versions Supported
We support the two most recent major versions of Go. If Google App Engine uses
an older version, we support that as well. You can see which versions are
currently supported by looking at the lines following `go:` in
[`.travis.yml`](.travis.yml).
## Authorization
By default, each API will use [Google Application Default Credentials][default-creds]
for authorization credentials used in calling the API endpoints. This will allow your
application to run in many environments without requiring explicit configuration.
[snip]:# (auth)
```go
client, err := storage.NewClient(ctx)
```
To authorize using a
[JSON key file](https://cloud.google.com/iam/docs/managing-service-account-keys),
pass
[`option.WithServiceAccountFile`](https://godoc.org/google.golang.org/api/option#WithServiceAccountFile)
to the `NewClient` function of the desired package. For example:
[snip]:# (auth-JSON)
```go
client, err := storage.NewClient(ctx, option.WithServiceAccountFile("path/to/keyfile.json"))
```
You can exert more control over authorization by using the
[`golang.org/x/oauth2`](https://godoc.org/golang.org/x/oauth2) package to
create an `oauth2.TokenSource`. Then pass
[`option.WithTokenSource`](https://godoc.org/google.golang.org/api/option#WithTokenSource)
to the `NewClient` function:
[snip]:# (auth-ts)
```go
tokenSource := ...
client, err := storage.NewClient(ctx, option.WithTokenSource(tokenSource))
```
## Cloud Datastore [![GoDoc](https://godoc.org/cloud.google.com/go/datastore?status.svg)](https://godoc.org/cloud.google.com/go/datastore)
- [About Cloud Datastore][cloud-datastore]
- [Activating the API for your project][cloud-datastore-activation]
- [API documentation][cloud-datastore-docs]
- [Go client documentation](https://godoc.org/cloud.google.com/go/datastore)
- [Complete sample program](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/datastore/tasks)
### Example Usage
First create a `datastore.Client` to use throughout your application:
[snip]:# (datastore-1)
```go
client, err := datastore.NewClient(ctx, "my-project-id")
if err != nil {
log.Fatal(err)
}
```
Then use that client to interact with the API:
[snip]:# (datastore-2)
```go
type Post struct {
Title string
Body string `datastore:",noindex"`
PublishedAt time.Time
}
keys := []*datastore.Key{
datastore.NameKey("Post", "post1", nil),
datastore.NameKey("Post", "post2", nil),
}
posts := []*Post{
{Title: "Post 1", Body: "...", PublishedAt: time.Now()},
{Title: "Post 2", Body: "...", PublishedAt: time.Now()},
}
if _, err := client.PutMulti(ctx, keys, posts); err != nil {
log.Fatal(err)
}
```
## Cloud Storage [![GoDoc](https://godoc.org/cloud.google.com/go/storage?status.svg)](https://godoc.org/cloud.google.com/go/storage)
- [About Cloud Storage][cloud-storage]
- [API documentation][cloud-storage-docs]
- [Go client documentation](https://godoc.org/cloud.google.com/go/storage)
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage)
### Example Usage
First create a `storage.Client` to use throughout your application:
[snip]:# (storage-1)
```go
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatal(err)
}
```
[snip]:# (storage-2)
```go
// Read the object1 from bucket.
rc, err := client.Bucket("bucket").Object("object1").NewReader(ctx)
if err != nil {
log.Fatal(err)
}
defer rc.Close()
body, err := ioutil.ReadAll(rc)
if err != nil {
log.Fatal(err)
}
```
## Cloud Pub/Sub [![GoDoc](https://godoc.org/cloud.google.com/go/pubsub?status.svg)](https://godoc.org/cloud.google.com/go/pubsub)
- [About Cloud Pubsub][cloud-pubsub]
- [API documentation][cloud-pubsub-docs]
- [Go client documentation](https://godoc.org/cloud.google.com/go/pubsub)
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/pubsub)
### Example Usage
First create a `pubsub.Client` to use throughout your application:
[snip]:# (pubsub-1)
```go
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
log.Fatal(err)
}
```
Then use the client to publish and subscribe:
[snip]:# (pubsub-2)
```go
// Publish "hello world" on topic1.
topic := client.Topic("topic1")
res := topic.Publish(ctx, &pubsub.Message{
Data: []byte("hello world"),
})
// The publish happens asynchronously.
// Later, you can get the result from res:
...
msgID, err := res.Get(ctx)
if err != nil {
log.Fatal(err)
}
// Use a callback to receive messages via subscription1.
sub := client.Subscription("subscription1")
err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
fmt.Println(m.Data)
m.Ack() // Acknowledge that we've consumed the message.
})
if err != nil {
log.Println(err)
}
```
## Cloud BigQuery [![GoDoc](https://godoc.org/cloud.google.com/go/bigquery?status.svg)](https://godoc.org/cloud.google.com/go/bigquery)
- [About Cloud BigQuery][cloud-bigquery]
- [API documentation][cloud-bigquery-docs]
- [Go client documentation][cloud-bigquery-ref]
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/bigquery)
### Example Usage
First create a `bigquery.Client` to use throughout your application:
[snip]:# (bq-1)
```go
c, err := bigquery.NewClient(ctx, "my-project-ID")
if err != nil {
// TODO: Handle error.
}
```
Then use that client to interact with the API:
[snip]:# (bq-2)
```go
// Construct a query.
q := c.Query(`
SELECT year, SUM(number)
FROM [bigquery-public-data:usa_names.usa_1910_2013]
WHERE name = "William"
GROUP BY year
ORDER BY year
`)
// Execute the query.
it, err := q.Read(ctx)
if err != nil {
// TODO: Handle error.
}
// Iterate through the results.
for {
var values []bigquery.Value
err := it.Next(&values)
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(values)
}
```
## Stackdriver Logging [![GoDoc](https://godoc.org/cloud.google.com/go/logging?status.svg)](https://godoc.org/cloud.google.com/go/logging)
- [About Stackdriver Logging][cloud-logging]
- [API documentation][cloud-logging-docs]
- [Go client documentation][cloud-logging-ref]
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/logging)
### Example Usage
First create a `logging.Client` to use throughout your application:
[snip]:# (logging-1)
```go
ctx := context.Background()
client, err := logging.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
```
Usually, you'll want to add log entries to a buffer to be periodically flushed
(automatically and asynchronously) to the Stackdriver Logging service.
[snip]:# (logging-2)
```go
logger := client.Logger("my-log")
logger.Log(logging.Entry{Payload: "something happened!"})
```
Close your client before your program exits, to flush any buffered log entries.
[snip]:# (logging-3)
```go
err = client.Close()
if err != nil {
// TODO: Handle error.
}
```
## Cloud Spanner [![GoDoc](https://godoc.org/cloud.google.com/go/spanner?status.svg)](https://godoc.org/cloud.google.com/go/spanner)
- [About Cloud Spanner][cloud-spanner]
- [API documentation][cloud-spanner-docs]
- [Go client documentation](https://godoc.org/cloud.google.com/go/spanner)
### Example Usage
First create a `spanner.Client` to use throughout your application:
[snip]:# (spanner-1)
```go
client, err := spanner.NewClient(ctx, "projects/P/instances/I/databases/D")
if err != nil {
log.Fatal(err)
}
```
[snip]:# (spanner-2)
```go
// Simple Reads And Writes
_, err = client.Apply(ctx, []*spanner.Mutation{
spanner.Insert("Users",
[]string{"name", "email"},
[]interface{}{"alice", "a@example.com"})})
if err != nil {
log.Fatal(err)
}
row, err := client.Single().ReadRow(ctx, "Users",
spanner.Key{"alice"}, []string{"email"})
if err != nil {
log.Fatal(err)
}
```
## Contributing
Contributions are welcome. Please, see the
[CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md)
document for details. We're using Gerrit for our code reviews. Please don't open pull
requests against this repo, new pull requests will be automatically closed.
Please note that this project is released with a Contributor Code of Conduct.
By participating in this project you agree to abide by its terms.
See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md#contributor-code-of-conduct)
for more information.
[cloud-datastore]: https://cloud.google.com/datastore/
[cloud-datastore-ref]: https://godoc.org/cloud.google.com/go/datastore
[cloud-datastore-docs]: https://cloud.google.com/datastore/docs
[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
[cloud-pubsub]: https://cloud.google.com/pubsub/
[cloud-pubsub-ref]: https://godoc.org/cloud.google.com/go/pubsub
[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs
[cloud-storage]: https://cloud.google.com/storage/
[cloud-storage-ref]: https://godoc.org/cloud.google.com/go/storage
[cloud-storage-docs]: https://cloud.google.com/storage/docs
[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
[cloud-bigtable]: https://cloud.google.com/bigtable/
[cloud-bigtable-ref]: https://godoc.org/cloud.google.com/go/bigtable
[cloud-bigquery]: https://cloud.google.com/bigquery/
[cloud-bigquery-docs]: https://cloud.google.com/bigquery/docs
[cloud-bigquery-ref]: https://godoc.org/cloud.google.com/go/bigquery
[cloud-logging]: https://cloud.google.com/logging/
[cloud-logging-docs]: https://cloud.google.com/logging/docs
[cloud-logging-ref]: https://godoc.org/cloud.google.com/go/logging
[cloud-monitoring]: https://cloud.google.com/monitoring/
[cloud-monitoring-ref]: https://godoc.org/cloud.google.com/go/monitoring/apiv3
[cloud-vision]: https://cloud.google.com/vision
[cloud-vision-ref]: https://godoc.org/cloud.google.com/go/vision/apiv1
[cloud-language]: https://cloud.google.com/natural-language
[cloud-language-ref]: https://godoc.org/cloud.google.com/go/language/apiv1
[cloud-speech]: https://cloud.google.com/speech
[cloud-speech-ref]: https://godoc.org/cloud.google.com/go/speech/apiv1
[cloud-spanner]: https://cloud.google.com/spanner/
[cloud-spanner-ref]: https://godoc.org/cloud.google.com/go/spanner
[cloud-spanner-docs]: https://cloud.google.com/spanner/docs
[cloud-translation]: https://cloud.google.com/translation
[cloud-translation-ref]: https://godoc.org/cloud.google.com/go/translation
[cloud-trace]: https://cloud.google.com/trace/
[cloud-trace-ref]: https://godoc.org/cloud.google.com/go/trace
[cloud-video]: https://cloud.google.com/video-intelligence/
[cloud-video-ref]: https://godoc.org/cloud.google.com/go/videointelligence/apiv1beta1
[cloud-errors]: https://cloud.google.com/error-reporting/
[cloud-errors-ref]: https://godoc.org/cloud.google.com/go/errors
[default-creds]: https://developers.google.com/identity/protocols/application-default-credentials
vendor/cloud.google.com/go/appveyor.yml
Generated Vendored
+32
View File
@@ -0,0 +1,32 @@
# This file configures AppVeyor (http://www.appveyor.com),
# a Windows-based CI service similar to Travis.
# Identifier for this run
version: "{build}"
# Clone the repo into this path, which conforms to the standard
# Go workspace structure.
clone_folder: c:\gopath\src\cloud.google.com\go
environment:
GOPATH: c:\gopath
GCLOUD_TESTS_GOLANG_PROJECT_ID: dulcet-port-762
GCLOUD_TESTS_GOLANG_KEY: c:\gopath\src\cloud.google.com\go\key.json
KEYFILE_CONTENTS:
secure: IvRbDAhM2PIQqzVkjzJ4FjizUvoQ+c3vG/qhJQG+HlZ/L5KEkqLu+x6WjLrExrNMyGku4znB2jmbTrUW3Ob4sGG+R5vvqeQ3YMHCVIkw5CxY+/bUDkW5RZWsVbuCnNa/vKsWmCP+/sZW6ICe29yKJ2ZOb6QaauI4s9R6j+cqBbU9pumMGYFRb0Rw3uUU7DKmVFCy+NjTENZIlDP9rmjANgAzigowJJEb2Tg9sLlQKmQeKiBSRN8lKc5Nq60a+fIzHGKvql4eIitDDDpOpyHv15/Xr1BzFw2yDoiR4X1lng0u7q0X9RgX4VIYa6gT16NXBEmQgbuX8gh7SfPMp9RhiZD9sVUaV+yogEabYpyPnmUURo0hXwkctKaBkQlEmKvjHwF5dvbg8+yqGhwtjAgFNimXG3INrwQsfQsZskkQWanutbJf9xy50GyWWFZZdi0uT4oXP/b5P7aklPXKXsvrJKBh7RjEaqBrhi86IJwOjBspvoR4l2WmcQyxb2xzQS1pjbBJFQfYJJ8+JgsstTL8PBO9d4ybJC0li1Om1qnWxkaewvPxxuoHJ9LpRKof19yRYWBmhTXb2tTASKG/zslvl4fgG4DmQBS93WC7dsiGOhAraGw2eCTgd0lYZOhk1FjWl9TS80aktXxzH/7nTvem5ohm+eDl6O0wnTL4KXjQVNSQ1PyLn4lGRJ5MNGzBTRFWIr2API2rca4Fysyfh/UdmazPGlNbY9JPGqb9+F04QzLfqm+Zz/cHy59E7lOSMBlUI4KD6d6ZNNKNRH+/g9i+fSiyiXKugTfda8KBnWGyPwprxuWGYaiQUGUYOwJY5R6x5c4mjImAB310V+Wo33UbWFJiwxEDsiCNqW1meVkBzt2er26vh4qbgCUIQ3iM3gFPfHgy+QxkmIhic7Q1HYacQElt8AAP41M7cCKWCuZidegP37MBB//mjjiNt047ZSQEvB4tqsX/OvfbByVef+cbtVw9T0yjHvmCdPW1XrhyrCCgclu6oYYdbmc5D7BBDRbjjMWGv6YvceAbfGf6ukdB5PuV+TGEN/FoQ1QTRA6Aqf+3fLMg4mS4oyTfw5xyYNbv3qoyLPrp+BnxI53WB9p0hfMg4n9FD6NntBxjDq+Q3Lk/bjC/Y4MaRWdzbMzF9a0lgGfcw9DURlK5p7uGJC9vg34feNoQprxVEZRQ01cHLeob6eGkYm4HxSRx8JY39Mh+9wzJo+k/aIvFleNC3e35NOrkXr6wb5e42n2DwBdPqdNolTLtLFRglAL1LTpp27UjvjieWJAKfoDTR5CKl01sZqt0wPdLLcvsMj6CiPFmccUIOYeZMe86kLBD61Qa5F1EwkgO3Om2qSjW96FzL4skRc+BmU5RrHlAFSldR1wpUgtkUMv9vH5Cy+UJdcvpZ8KbmhZ2PsjF7ddJ1ve9RAw3cP325AyIMwZ77Ef1mgTM0NJze6eSW1qKlEsgt1FADPyeUu1NQTA2H2dueMPGlArWTSUgyWR9AdfpqouT7eg0JWI5w+yUZZC+/rPglYbt84oLmYpwuli0z8FyEQRPIc3EtkfWIv/yYgDr2TZ0N2KvGfpi/MAUWgxI1gleC2uKgEOEtuJthd3XZjF2NoE7IBqjQOINybcJOjyeB5vRLDY1FLuxYzdg1y1etkV4XQig/vje
install:
# Info for debugging.
- echo %PATH%
- go version
- go env
- go get -v -d -t ./...
# Provide a build script, or AppVeyor will call msbuild.
build_script:
- go install -v ./...
- echo %KEYFILE_CONTENTS% > %GCLOUD_TESTS_GOLANG_KEY%
test_script:
- go test -v ./...
vendor/cloud.google.com/go/cloud.go
Generated Vendored
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package cloud is the root of the packages used to access Google Cloud
// Services. See https://godoc.org/cloud.google.com/go for a full list
// of sub-packages.
//
// This package documents how to authorize and authenticate the sub packages.
package cloud // import "cloud.google.com/go"
vendor/cloud.google.com/go/compute/metadata/metadata.go
Generated Vendored
+18 -19
View File
@@ -17,7 +17,7 @@
//
// This package is a wrapper around the GCE metadata service,
// as documented at https://developers.google.com/compute/docs/metadata.
package metadata
package metadata // import "cloud.google.com/go/compute/metadata"
import (
"encoding/json"
@@ -34,8 +34,6 @@ import (
"golang.org/x/net/context"
"golang.org/x/net/context/ctxhttp"
"cloud.google.com/go/internal"
)
const (
@@ -48,6 +46,8 @@ const (
// This is variable name is not defined by any spec, as far as
// I know; it was made up for the Go package.
metadataHostEnv = "GCE_METADATA_HOST"
userAgent = "gcloud-golang/0.1"
)
type cachedValue struct {
@@ -65,24 +65,20 @@ var (
var (
metaClient = &http.Client{
Transport: &internal.Transport{
Base: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
ResponseHeaderTimeout: 2 * time.Second,
},
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
ResponseHeaderTimeout: 2 * time.Second,
},
}
subscribeClient = &http.Client{
Transport: &internal.Transport{
Base: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
},
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
},
}
)
@@ -132,6 +128,7 @@ func getETag(client *http.Client, suffix string) (value, etag string, err error)
url := "http://" + host + "/computeMetadata/v1/" + suffix
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("Metadata-Flavor", "Google")
req.Header.Set("User-Agent", userAgent)
res, err := client.Do(req)
if err != nil {
return "", "", err
@@ -202,7 +199,9 @@ func testOnGCE() bool {
// Try two strategies in parallel.
// See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194
go func() {
res, err := ctxhttp.Get(ctx, metaClient, "http://"+metadataIP)
req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
req.Header.Set("User-Agent", userAgent)
res, err := ctxhttp.Do(ctx, metaClient, req)
if err != nil {
resc <- false
return
vendor/cloud.google.com/go/iam/iam.go
Generated Vendored
+256
View File
@@ -0,0 +1,256 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package iam supports the resource-specific operations of Google Cloud
// IAM (Identity and Access Management) for the Google Cloud Libraries.
// See https://cloud.google.com/iam for more about IAM.
//
// Users of the Google Cloud Libraries will typically not use this package
// directly. Instead they will begin with some resource that supports IAM, like
// a pubsub topic, and call its IAM method to get a Handle for that resource.
package iam
import (
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/iam/v1"
"google.golang.org/grpc"
)
// client abstracts the IAMPolicy API to allow multiple implementations.
type client interface {
Get(ctx context.Context, resource string) (*pb.Policy, error)
Set(ctx context.Context, resource string, p *pb.Policy) error
Test(ctx context.Context, resource string, perms []string) ([]string, error)
}
// grpcClient implements client for the standard gRPC-based IAMPolicy service.
type grpcClient struct {
c pb.IAMPolicyClient
}
func (g *grpcClient) Get(ctx context.Context, resource string) (*pb.Policy, error) {
proto, err := g.c.GetIamPolicy(ctx, &pb.GetIamPolicyRequest{Resource: resource})
if err != nil {
return nil, err
}
return proto, nil
}
func (g *grpcClient) Set(ctx context.Context, resource string, p *pb.Policy) error {
_, err := g.c.SetIamPolicy(ctx, &pb.SetIamPolicyRequest{
Resource: resource,
Policy: p,
})
return err
}
func (g *grpcClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
res, err := g.c.TestIamPermissions(ctx, &pb.TestIamPermissionsRequest{
Resource: resource,
Permissions: perms,
})
if err != nil {
return nil, err
}
return res.Permissions, nil
}
// A Handle provides IAM operations for a resource.
type Handle struct {
c client
resource string
}
// InternalNewHandle is for use by the Google Cloud Libraries only.
//
// InternalNewHandle returns a Handle for resource.
// The conn parameter refers to a server that must support the IAMPolicy service.
func InternalNewHandle(conn *grpc.ClientConn, resource string) *Handle {
return InternalNewHandleClient(&grpcClient{c: pb.NewIAMPolicyClient(conn)}, resource)
}
// InternalNewHandleClient is for use by the Google Cloud Libraries only.
//
// InternalNewHandleClient returns a Handle for resource using the given
// client implementation.
func InternalNewHandleClient(c client, resource string) *Handle {
return &Handle{
c: c,
resource: resource,
}
}
// Policy retrieves the IAM policy for the resource.
func (h *Handle) Policy(ctx context.Context) (*Policy, error) {
proto, err := h.c.Get(ctx, h.resource)
if err != nil {
return nil, err
}
return &Policy{InternalProto: proto}, nil
}
// SetPolicy replaces the resource's current policy with the supplied Policy.
//
// If policy was created from a prior call to Get, then the modification will
// only succeed if the policy has not changed since the Get.
func (h *Handle) SetPolicy(ctx context.Context, policy *Policy) error {
return h.c.Set(ctx, h.resource, policy.InternalProto)
}
// TestPermissions returns the subset of permissions that the caller has on the resource.
func (h *Handle) TestPermissions(ctx context.Context, permissions []string) ([]string, error) {
return h.c.Test(ctx, h.resource, permissions)
}
// A RoleName is a name representing a collection of permissions.
type RoleName string
// Common role names.
const (
Owner RoleName = "roles/owner"
Editor RoleName = "roles/editor"
Viewer RoleName = "roles/viewer"
)
const (
// AllUsers is a special member that denotes all users, even unauthenticated ones.
AllUsers = "allUsers"
// AllAuthenticatedUsers is a special member that denotes all authenticated users.
AllAuthenticatedUsers = "allAuthenticatedUsers"
)
// A Policy is a list of Bindings representing roles
// granted to members.
//
// The zero Policy is a valid policy with no bindings.
type Policy struct {
// TODO(jba): when type aliases are available, put Policy into an internal package
// and provide an exported alias here.
// This field is exported for use by the Google Cloud Libraries only.
// It may become unexported in a future release.
InternalProto *pb.Policy
}
// Members returns the list of members with the supplied role.
// The return value should not be modified. Use Add and Remove
// to modify the members of a role.
func (p *Policy) Members(r RoleName) []string {
b := p.binding(r)
if b == nil {
return nil
}
return b.Members
}
// HasRole reports whether member has role r.
func (p *Policy) HasRole(member string, r RoleName) bool {
return memberIndex(member, p.binding(r)) >= 0
}
// Add adds member member to role r if it is not already present.
// A new binding is created if there is no binding for the role.
func (p *Policy) Add(member string, r RoleName) {
b := p.binding(r)
if b == nil {
if p.InternalProto == nil {
p.InternalProto = &pb.Policy{}
}
p.InternalProto.Bindings = append(p.InternalProto.Bindings, &pb.Binding{
Role: string(r),
Members: []string{member},
})
return
}
if memberIndex(member, b) < 0 {
b.Members = append(b.Members, member)
return
}
}
// Remove removes member from role r if it is present.
func (p *Policy) Remove(member string, r RoleName) {
bi := p.bindingIndex(r)
if bi < 0 {
return
}
bindings := p.InternalProto.Bindings
b := bindings[bi]
mi := memberIndex(member, b)
if mi < 0 {
return
}
// Order doesn't matter for bindings or members, so to remove, move the last item
// into the removed spot and shrink the slice.
if len(b.Members) == 1 {
// Remove binding.
last := len(bindings) - 1
bindings[bi] = bindings[last]
bindings[last] = nil
p.InternalProto.Bindings = bindings[:last]
return
}
// Remove member.
// TODO(jba): worry about multiple copies of m?
last := len(b.Members) - 1
b.Members[mi] = b.Members[last]
b.Members[last] = ""
b.Members = b.Members[:last]
}
// Roles returns the names of all the roles that appear in the Policy.
func (p *Policy) Roles() []RoleName {
if p.InternalProto == nil {
return nil
}
var rns []RoleName
for _, b := range p.InternalProto.Bindings {
rns = append(rns, RoleName(b.Role))
}
return rns
}
// binding returns the Binding for the suppied role, or nil if there isn't one.
func (p *Policy) binding(r RoleName) *pb.Binding {
i := p.bindingIndex(r)
if i < 0 {
return nil
}
return p.InternalProto.Bindings[i]
}
func (p *Policy) bindingIndex(r RoleName) int {
if p.InternalProto == nil {
return -1
}
for i, b := range p.InternalProto.Bindings {
if b.Role == string(r) {
return i
}
}
return -1
}
// memberIndex returns the index of m in b's Members, or -1 if not found.
func memberIndex(m string, b *pb.Binding) int {
if b == nil {
return -1
}
for i, mm := range b.Members {
if mm == m {
return i
}
}
return -1
}
vendor/cloud.google.com/go/internal/cloud.go
Generated Vendored
-64
View File
@@ -1,64 +0,0 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package internal provides support for the cloud packages.
//
// Users should not import this package directly.
package internal
import (
"fmt"
"net/http"
)
const userAgent = "gcloud-golang/0.1"
// Transport is an http.RoundTripper that appends Google Cloud client's
// user-agent to the original request's user-agent header.
type Transport struct {
// TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does.
// Do User-Agent some other way.
// Base is the actual http.RoundTripper
// requests will use. It must not be nil.
Base http.RoundTripper
}
// RoundTrip appends a user-agent to the existing user-agent
// header and delegates the request to the base http.RoundTripper.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
req = cloneRequest(req)
ua := req.Header.Get("User-Agent")
if ua == "" {
ua = userAgent
} else {
ua = fmt.Sprintf("%s %s", ua, userAgent)
}
req.Header.Set("User-Agent", ua)
return t.Base.RoundTrip(req)
}
// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
// shallow copy of the struct
r2 := new(http.Request)
*r2 = *r
// deep copy of the Header
r2.Header = make(http.Header)
for k, s := range r.Header {
r2.Header[k] = s
}
return r2
}
vendor/cloud.google.com/go/internal/optional/optional.go
Generated Vendored
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package optional provides versions of primitive types that can
// be nil. These are useful in methods that update some of an API object's
// fields.
package optional
import (
"fmt"
"strings"
)
type (
// Bool is either a bool or nil.
Bool interface{}
// String is either a string or nil.
String interface{}
// Int is either an int or nil.
Int interface{}
// Uint is either a uint or nil.
Uint interface{}
// Float64 is either a float64 or nil.
Float64 interface{}
)
// ToBool returns its argument as a bool.
// It panics if its argument is nil or not a bool.
func ToBool(v Bool) bool {
x, ok := v.(bool)
if !ok {
doPanic("Bool", v)
}
return x
}
// ToString returns its argument as a string.
// It panics if its argument is nil or not a string.
func ToString(v String) string {
x, ok := v.(string)
if !ok {
doPanic("String", v)
}
return x
}
// ToInt returns its argument as an int.
// It panics if its argument is nil or not an int.
func ToInt(v Int) int {
x, ok := v.(int)
if !ok {
doPanic("Int", v)
}
return x
}
// ToUint returns its argument as a uint.
// It panics if its argument is nil or not a uint.
func ToUint(v Uint) uint {
x, ok := v.(uint)
if !ok {
doPanic("Uint", v)
}
return x
}
// ToFloat64 returns its argument as a float64.
// It panics if its argument is nil or not a float64.
func ToFloat64(v Float64) float64 {
x, ok := v.(float64)
if !ok {
doPanic("Float64", v)
}
return x
}
func doPanic(capType string, v interface{}) {
panic(fmt.Sprintf("optional.%s value should be %s, got %T", capType, strings.ToLower(capType), v))
}
vendor/cloud.google.com/go/internal/retry.go
Generated Vendored
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"fmt"
"time"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
)
// Retry calls the supplied function f repeatedly according to the provided
// backoff parameters. It returns when one of the following occurs:
// When f's first return value is true, Retry immediately returns with f's second
// return value.
// When the provided context is done, Retry returns with an error that
// includes both ctx.Error() and the last error returned by f.
func Retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error)) error {
return retry(ctx, bo, f, gax.Sleep)
}
func retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error),
sleep func(context.Context, time.Duration) error) error {
var lastErr error
for {
stop, err := f()
if stop {
return err
}
// Remember the last "real" error from f.
if err != nil && err != context.Canceled && err != context.DeadlineExceeded {
lastErr = err
}
p := bo.Pause()
if cerr := sleep(ctx, p); cerr != nil {
if lastErr != nil {
return fmt.Errorf("%v; last function err: %v", cerr, lastErr)
}
return cerr
}
}
}
vendor/cloud.google.com/go/internal/version/update_version.sh
Generated Vendored Executable
+6
View File
@@ -0,0 +1,6 @@
#!/bin/bash
today=$(date +%Y%m%d)
sed -i -r -e 's/const Repo = "([0-9]{8})"/const Repo = "'$today'"/' $GOFILE
vendor/cloud.google.com/go/internal/version/version.go
Generated Vendored
+71
View File
@@ -0,0 +1,71 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:generate ./update_version.sh
// Package version contains version information for Google Cloud Client
// Libraries for Go, as reported in request headers.
package version
import (
"runtime"
"strings"
"unicode"
)
// Repo is the current version of the client libraries in this
// repo. It should be a date in YYYYMMDD format.
const Repo = "20170621"
// Go returns the Go runtime version. The returned string
// has no whitespace.
func Go() string {
return goVersion
}
var goVersion = goVer(runtime.Version())
const develPrefix = "devel +"
func goVer(s string) string {
if strings.HasPrefix(s, develPrefix) {
s = s[len(develPrefix):]
if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 {
s = s[:p]
}
return s
}
if strings.HasPrefix(s, "go1") {
s = s[2:]
var prerelease string
if p := strings.IndexFunc(s, notSemverRune); p >= 0 {
s, prerelease = s[:p], s[p:]
}
if strings.HasSuffix(s, ".") {
s += "0"
} else if strings.Count(s, ".") < 2 {
s += ".0"
}
if prerelease != "" {
s += "-" + prerelease
}
return s
}
return ""
}
func notSemverRune(r rune) bool {
return strings.IndexRune("0123456789.", r) < 0
}
vendor/cloud.google.com/go/key.json.enc
Generated Vendored
BIN
View File
Binary file not shown.
vendor/cloud.google.com/go/old-news.md
Generated Vendored
+451
View File
@@ -0,0 +1,451 @@
_March 17, 2017_
Breaking Pubsub changes.
* Publish is now asynchronous
([announcement](https://groups.google.com/d/topic/google-api-go-announce/aaqRDIQ3rvU/discussion)).
* Subscription.Pull replaced by Subscription.Receive, which takes a callback ([announcement](https://groups.google.com/d/topic/google-api-go-announce/8pt6oetAdKc/discussion)).
* Message.Done replaced with Message.Ack and Message.Nack.
_February 14, 2017_
Release of a client library for Spanner. See
the
[blog post](https://cloudplatform.googleblog.com/2017/02/introducing-Cloud-Spanner-a-global-database-service-for-mission-critical-applications.html).
Note that although the Spanner service is beta, the Go client library is alpha.
_December 12, 2016_
Beta release of BigQuery, DataStore, Logging and Storage. See the
[blog post](https://cloudplatform.googleblog.com/2016/12/announcing-new-google-cloud-client.html).
Also, BigQuery now supports structs. Read a row directly into a struct with
`RowIterator.Next`, and upload a row directly from a struct with `Uploader.Put`.
You can also use field tags. See the [package documentation][cloud-bigquery-ref]
for details.
_December 5, 2016_
More changes to BigQuery:
* The `ValueList` type was removed. It is no longer necessary. Instead of
```go
var v ValueList
... it.Next(&v) ..
```
use
```go
var v []Value
... it.Next(&v) ...
```
* Previously, repeatedly calling `RowIterator.Next` on the same `[]Value` or
`ValueList` would append to the slice. Now each call resets the size to zero first.
* Schema inference will infer the SQL type BYTES for a struct field of
type []byte. Previously it inferred STRING.
* The types `uint`, `uint64` and `uintptr` are no longer supported in schema
inference. BigQuery's integer type is INT64, and those types may hold values
that are not correctly represented in a 64-bit signed integer.
* The SQL types DATE, TIME and DATETIME are now supported. They correspond to
the `Date`, `Time` and `DateTime` types in the new `cloud.google.com/go/civil`
package.
_November 17, 2016_
Change to BigQuery: values from INTEGER columns will now be returned as int64,
not int. This will avoid errors arising from large values on 32-bit systems.
_November 8, 2016_
New datastore feature: datastore now encodes your nested Go structs as Entity values,
instead of a flattened list of the embedded struct's fields.
This means that you may now have twice-nested slices, eg.
```go
type State struct {
Cities []struct{
Populations []int
}
}
```
See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/79jtrdeuJAg) for
more details.
_November 8, 2016_
Breaking changes to datastore: contexts no longer hold namespaces; instead you
must set a key's namespace explicitly. Also, key functions have been changed
and renamed.
* The WithNamespace function has been removed. To specify a namespace in a Query, use the Query.Namespace method:
```go
q := datastore.NewQuery("Kind").Namespace("ns")
```
* All the fields of Key are exported. That means you can construct any Key with a struct literal:
```go
k := &Key{Kind: "Kind", ID: 37, Namespace: "ns"}
```
* As a result of the above, the Key methods Kind, ID, d.Name, Parent, SetParent and Namespace have been removed.
* `NewIncompleteKey` has been removed, replaced by `IncompleteKey`. Replace
```go
NewIncompleteKey(ctx, kind, parent)
```
with
```go
IncompleteKey(kind, parent)
```
and if you do use namespaces, make sure you set the namespace on the returned key.
* `NewKey` has been removed, replaced by `NameKey` and `IDKey`. Replace
```go
NewKey(ctx, kind, name, 0, parent)
NewKey(ctx, kind, "", id, parent)
```
with
```go
NameKey(kind, name, parent)
IDKey(kind, id, parent)
```
and if you do use namespaces, make sure you set the namespace on the returned key.
* The `Done` variable has been removed. Replace `datastore.Done` with `iterator.Done`, from the package `google.golang.org/api/iterator`.
* The `Client.Close` method will have a return type of error. It will return the result of closing the underlying gRPC connection.
See [the announcement](https://groups.google.com/forum/#!topic/google-api-go-announce/hqXtM_4Ix-0) for
more details.
_October 27, 2016_
Breaking change to bigquery: `NewGCSReference` is now a function,
not a method on `Client`.
New bigquery feature: `Table.LoaderFrom` now accepts a `ReaderSource`, enabling
loading data into a table from a file or any `io.Reader`.
_October 21, 2016_
Breaking change to pubsub: removed `pubsub.Done`.
Use `iterator.Done` instead, where `iterator` is the package
`google.golang.org/api/iterator`.
_October 19, 2016_
Breaking changes to cloud.google.com/go/bigquery:
* Client.Table and Client.OpenTable have been removed.
Replace
```go
client.OpenTable("project", "dataset", "table")
```
with
```go
client.DatasetInProject("project", "dataset").Table("table")
```
* Client.CreateTable has been removed.
Replace
```go
client.CreateTable(ctx, "project", "dataset", "table")
```
with
```go
client.DatasetInProject("project", "dataset").Table("table").Create(ctx)
```
* Dataset.ListTables have been replaced with Dataset.Tables.
Replace
```go
tables, err := ds.ListTables(ctx)
```
with
```go
it := ds.Tables(ctx)
for {
table, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
// TODO: use table.
}
```
* Client.Read has been replaced with Job.Read, Table.Read and Query.Read.
Replace
```go
it, err := client.Read(ctx, job)
```
with
```go
it, err := job.Read(ctx)
```
and similarly for reading from tables or queries.
* The iterator returned from the Read methods is now named RowIterator. Its
behavior is closer to the other iterators in these libraries. It no longer
supports the Schema method; see the next item.
Replace
```go
for it.Next(ctx) {
var vals ValueList
if err := it.Get(&vals); err != nil {
// TODO: Handle error.
}
// TODO: use vals.
}
if err := it.Err(); err != nil {
// TODO: Handle error.
}
```
with
```
for {
var vals ValueList
err := it.Next(&vals)
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
// TODO: use vals.
}
```
Instead of the `RecordsPerRequest(n)` option, write
```go
it.PageInfo().MaxSize = n
```
Instead of the `StartIndex(i)` option, write
```go
it.StartIndex = i
```
* ValueLoader.Load now takes a Schema in addition to a slice of Values.
Replace
```go
func (vl *myValueLoader) Load(v []bigquery.Value)
```
with
```go
func (vl *myValueLoader) Load(v []bigquery.Value, s bigquery.Schema)
```
* Table.Patch is replace by Table.Update.
Replace
```go
p := table.Patch()
p.Description("new description")
metadata, err := p.Apply(ctx)
```
with
```go
metadata, err := table.Update(ctx, bigquery.TableMetadataToUpdate{
Description: "new description",
})
```
* Client.Copy is replaced by separate methods for each of its four functions.
All options have been replaced by struct fields.
* To load data from Google Cloud Storage into a table, use Table.LoaderFrom.
Replace
```go
client.Copy(ctx, table, gcsRef)
```
with
```go
table.LoaderFrom(gcsRef).Run(ctx)
```
Instead of passing options to Copy, set fields on the Loader:
```go
loader := table.LoaderFrom(gcsRef)
loader.WriteDisposition = bigquery.WriteTruncate
```
* To extract data from a table into Google Cloud Storage, use
Table.ExtractorTo. Set fields on the returned Extractor instead of
passing options.
Replace
```go
client.Copy(ctx, gcsRef, table)
```
with
```go
table.ExtractorTo(gcsRef).Run(ctx)
```
* To copy data into a table from one or more other tables, use
Table.CopierFrom. Set fields on the returned Copier instead of passing options.
Replace
```go
client.Copy(ctx, dstTable, srcTable)
```
with
```go
dst.Table.CopierFrom(srcTable).Run(ctx)
```
* To start a query job, create a Query and call its Run method. Set fields
on the query instead of passing options.
Replace
```go
client.Copy(ctx, table, query)
```
with
```go
query.Run(ctx)
```
* Table.NewUploader has been renamed to Table.Uploader. Instead of options,
configure an Uploader by setting its fields.
Replace
```go
u := table.NewUploader(bigquery.UploadIgnoreUnknownValues())
```
with
```go
u := table.NewUploader(bigquery.UploadIgnoreUnknownValues())
u.IgnoreUnknownValues = true
```
_October 10, 2016_
Breaking changes to cloud.google.com/go/storage:
* AdminClient replaced by methods on Client.
Replace
```go
adminClient.CreateBucket(ctx, bucketName, attrs)
```
with
```go
client.Bucket(bucketName).Create(ctx, projectID, attrs)
```
* BucketHandle.List replaced by BucketHandle.Objects.
Replace
```go
for query != nil {
objs, err := bucket.List(d.ctx, query)
if err != nil { ... }
query = objs.Next
for _, obj := range objs.Results {
fmt.Println(obj)
}
}
```
with
```go
iter := bucket.Objects(d.ctx, query)
for {
obj, err := iter.Next()
if err == iterator.Done {
break
}
if err != nil { ... }
fmt.Println(obj)
}
```
(The `iterator` package is at `google.golang.org/api/iterator`.)
Replace `Query.Cursor` with `ObjectIterator.PageInfo().Token`.
Replace `Query.MaxResults` with `ObjectIterator.PageInfo().MaxSize`.
* ObjectHandle.CopyTo replaced by ObjectHandle.CopierFrom.
Replace
```go
attrs, err := src.CopyTo(ctx, dst, nil)
```
with
```go
attrs, err := dst.CopierFrom(src).Run(ctx)
```
Replace
```go
attrs, err := src.CopyTo(ctx, dst, &storage.ObjectAttrs{ContextType: "text/html"})
```
with
```go
c := dst.CopierFrom(src)
c.ContextType = "text/html"
attrs, err := c.Run(ctx)
```
* ObjectHandle.ComposeFrom replaced by ObjectHandle.ComposerFrom.
Replace
```go
attrs, err := dst.ComposeFrom(ctx, []*storage.ObjectHandle{src1, src2}, nil)
```
with
```go
attrs, err := dst.ComposerFrom(src1, src2).Run(ctx)
```
* ObjectHandle.Update's ObjectAttrs argument replaced by ObjectAttrsToUpdate.
Replace
```go
attrs, err := obj.Update(ctx, &storage.ObjectAttrs{ContextType: "text/html"})
```
with
```go
attrs, err := obj.Update(ctx, storage.ObjectAttrsToUpdate{ContextType: "text/html"})
```
* ObjectHandle.WithConditions replaced by ObjectHandle.If.
Replace
```go
obj.WithConditions(storage.Generation(gen), storage.IfMetaGenerationMatch(mgen))
```
with
```go
obj.Generation(gen).If(storage.Conditions{MetagenerationMatch: mgen})
```
Replace
```go
obj.WithConditions(storage.IfGenerationMatch(0))
```
with
```go
obj.If(storage.Conditions{DoesNotExist: true})
```
* `storage.Done` replaced by `iterator.Done` (from package `google.golang.org/api/iterator`).
_October 6, 2016_
Package preview/logging deleted. Use logging instead.
_September 27, 2016_
Logging client replaced with preview version (see below).
_September 8, 2016_
* New clients for some of Google's Machine Learning APIs: Vision, Speech, and
Natural Language.
* Preview version of a new [Stackdriver Logging][cloud-logging] client in
[`cloud.google.com/go/preview/logging`](https://godoc.org/cloud.google.com/go/preview/logging).
This client uses gRPC as its transport layer, and supports log reading, sinks
and metrics. It will replace the current client at `cloud.google.com/go/logging` shortly.
vendor/cloud.google.com/go/run-tests.sh
Generated Vendored Executable
+86
View File
@@ -0,0 +1,86 @@
#!/bin/bash
# Selectively run tests for this repo, based on what has changed
# in a commit. Runs short tests for the whole repo, and full tests
# for changed directories.
set -e
prefix=cloud.google.com/go
dryrun=false
if [[ $1 == "-n" ]]; then
dryrun=true
shift
fi
if [[ $1 == "" ]]; then
echo >&2 "usage: $0 [-n] COMMIT"
exit 1
fi
# Files or directories that cause all tests to run if modified.
declare -A run_all
run_all=([.travis.yml]=1 [run-tests.sh]=1)
function run {
if $dryrun; then
echo $*
else
(set -x; $*)
fi
}
# Find all the packages that have changed in this commit.
declare -A changed_packages
for f in $(git diff-tree --no-commit-id --name-only -r $1); do
if [[ ${run_all[$f]} == 1 ]]; then
# This change requires a full test. Do it and exit.
run go test -race -v $prefix/...
exit
fi
# Map, e.g., "spanner/client.go" to "$prefix/spanner".
d=$(dirname $f)
if [[ $d == "." ]]; then
pkg=$prefix
else
pkg=$prefix/$d
fi
changed_packages[$pkg]=1
done
echo "changed packages: ${!changed_packages[*]}"
# Reports whether its argument, a package name, depends (recursively)
# on a changed package.
function depends_on_changed_package {
# According to go list, a package does not depend on itself, so
# we test that separately.
if [[ ${changed_packages[$1]} == 1 ]]; then
return 0
fi
for dep in $(go list -f '{{range .Deps}}{{.}} {{end}}' $1); do
if [[ ${changed_packages[$dep]} == 1 ]]; then
return 0
fi
done
return 1
}
# Collect the packages into two separate lists. (It is faster go test a list of
# packages than to individually go test each one.)
shorts=
fulls=
for pkg in $(go list $prefix/...); do # for each package in the repo
if depends_on_changed_package $pkg; then # if it depends on a changed package
fulls="$fulls $pkg" # run the full test
else # otherwise
shorts="$shorts $pkg" # run the short test
fi
done
run go test -race -v -short $shorts
run go test -race -v $fulls
vendor/cloud.google.com/go/storage/acl.go
Generated Vendored
+252
View File
@@ -0,0 +1,252 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"net/http"
"reflect"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
// ACLRole is the level of access to grant.
type ACLRole string
const (
RoleOwner ACLRole = "OWNER"
RoleReader ACLRole = "READER"
RoleWriter ACLRole = "WRITER"
)
// ACLEntity refers to a user or group.
// They are sometimes referred to as grantees.
//
// It could be in the form of:
// "user-<userId>", "user-<email>", "group-<groupId>", "group-<email>",
// "domain-<domain>" and "project-team-<projectId>".
//
// Or one of the predefined constants: AllUsers, AllAuthenticatedUsers.
type ACLEntity string
const (
AllUsers ACLEntity = "allUsers"
AllAuthenticatedUsers ACLEntity = "allAuthenticatedUsers"
)
// ACLRule represents a grant for a role to an entity (user, group or team) for a Google Cloud Storage object or bucket.
type ACLRule struct {
Entity ACLEntity
Role ACLRole
}
// ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object.
type ACLHandle struct {
c *Client
bucket string
object string
isDefault bool
userProject string // for requester-pays buckets
}
// Delete permanently deletes the ACL entry for the given entity.
func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error {
if a.object != "" {
return a.objectDelete(ctx, entity)
}
if a.isDefault {
return a.bucketDefaultDelete(ctx, entity)
}
return a.bucketDelete(ctx, entity)
}
// Set sets the permission level for the given entity.
func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error {
if a.object != "" {
return a.objectSet(ctx, entity, role, false)
}
if a.isDefault {
return a.objectSet(ctx, entity, role, true)
}
return a.bucketSet(ctx, entity, role)
}
// List retrieves ACL entries.
func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) {
if a.object != "" {
return a.objectList(ctx)
}
if a.isDefault {
return a.bucketDefaultList(ctx)
}
return a.bucketList(ctx)
}
func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err)
}
return toACLRules(acls.Items), nil
}
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.BucketAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.List(a.bucket)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err)
}
r := make([]ACLRule, len(acls.Items))
for i, v := range acls.Items {
r[i].Entity = ACLEntity(v.Entity)
r[i].Role = ACLRole(v.Role)
}
return r, nil
}
func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
acl := &raw.BucketAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
err := runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
a.configureCall(req, ctx)
_, err := req.Do()
return err
})
if err != nil {
return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err)
}
return toACLRules(acls.Items), nil
}
func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error {
type setRequest interface {
Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error)
Header() http.Header
}
acl := &raw.ObjectAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
var req setRequest
if isBucketDefault {
req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl)
} else {
req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
}
a.configureCall(req, ctx)
err := runWithRetry(ctx, func() error {
_, err := req.Do()
return err
})
if err != nil {
if isBucketDefault {
return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
} else {
return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err)
}
}
return nil
}
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err)
}
return nil
}
func (a *ACLHandle) configureCall(call interface {
Header() http.Header
}, ctx context.Context) {
vc := reflect.ValueOf(call)
vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)})
if a.userProject != "" {
vc.MethodByName("UserProject").Call([]reflect.Value{reflect.ValueOf(a.userProject)})
}
setClientHeader(call.Header())
}
func toACLRules(items []*raw.ObjectAccessControl) []ACLRule {
r := make([]ACLRule, 0, len(items))
for _, item := range items {
r = append(r, ACLRule{Entity: ACLEntity(item.Entity), Role: ACLRole(item.Role)})
}
return r
}
vendor/cloud.google.com/go/storage/bucket.go
Generated Vendored
+590
View File
@@ -0,0 +1,590 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"net/http"
"reflect"
"time"
"cloud.google.com/go/internal/optional"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
raw "google.golang.org/api/storage/v1"
)
// BucketHandle provides operations on a Google Cloud Storage bucket.
// Use Client.Bucket to get a handle.
type BucketHandle struct {
c *Client
name string
acl ACLHandle
defaultObjectACL ACLHandle
conds *BucketConditions
userProject string // project for requester-pays buckets
}
// Bucket returns a BucketHandle, which provides operations on the named bucket.
// This call does not perform any network operations.
//
// The supplied name must contain only lowercase letters, numbers, dashes,
// underscores, and dots. The full specification for valid bucket names can be
// found at:
// https://cloud.google.com/storage/docs/bucket-naming
func (c *Client) Bucket(name string) *BucketHandle {
return &BucketHandle{
c: c,
name: name,
acl: ACLHandle{
c: c,
bucket: name,
},
defaultObjectACL: ACLHandle{
c: c,
bucket: name,
isDefault: true,
},
}
}
// Create creates the Bucket in the project.
// If attrs is nil the API defaults will be used.
func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error {
var bkt *raw.Bucket
if attrs != nil {
bkt = attrs.toRawBucket()
} else {
bkt = &raw.Bucket{}
}
bkt.Name = b.name
req := b.c.raw.Buckets.Insert(projectID, bkt)
setClientHeader(req.Header())
return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err })
}
// Delete deletes the Bucket.
func (b *BucketHandle) Delete(ctx context.Context) error {
req, err := b.newDeleteCall()
if err != nil {
return err
}
return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
}
func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
req := b.c.raw.Buckets.Delete(b.name)
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// ACL returns an ACLHandle, which provides access to the bucket's access control list.
// This controls who can list, create or overwrite the objects in a bucket.
// This call does not perform any network operations.
func (b *BucketHandle) ACL() *ACLHandle {
return &b.acl
}
// DefaultObjectACL returns an ACLHandle, which provides access to the bucket's default object ACLs.
// These ACLs are applied to newly created objects in this bucket that do not have a defined ACL.
// This call does not perform any network operations.
func (b *BucketHandle) DefaultObjectACL() *ACLHandle {
return &b.defaultObjectACL
}
// Object returns an ObjectHandle, which provides operations on the named object.
// This call does not perform any network operations.
//
// name must consist entirely of valid UTF-8-encoded runes. The full specification
// for valid object names can be found at:
// https://cloud.google.com/storage/docs/bucket-naming
func (b *BucketHandle) Object(name string) *ObjectHandle {
return &ObjectHandle{
c: b.c,
bucket: b.name,
object: name,
acl: ACLHandle{
c: b.c,
bucket: b.name,
object: name,
userProject: b.userProject,
},
gen: -1,
userProject: b.userProject,
}
}
// Attrs returns the metadata for the bucket.
func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) {
req, err := b.newGetCall()
if err != nil {
return nil, err
}
var resp *raw.Bucket
err = runWithRetry(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
})
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
return nil, ErrBucketNotExist
}
if err != nil {
return nil, err
}
return newBucket(resp), nil
}
func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) {
req := b.c.raw.Buckets.Get(b.name).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) {
req, err := b.newPatchCall(&uattrs)
if err != nil {
return nil, err
}
// TODO(jba): retry iff metagen is set?
rb, err := req.Context(ctx).Do()
if err != nil {
return nil, err
}
return newBucket(rb), nil
}
func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
rb := uattrs.toRawBucket()
req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
type BucketAttrs struct {
// Name is the name of the bucket.
Name string
// ACL is the list of access control rules on the bucket.
ACL []ACLRule
// DefaultObjectACL is the list of access controls to
// apply to new objects when no object ACL is provided.
DefaultObjectACL []ACLRule
// Location is the location of the bucket. It defaults to "US".
Location string
// MetaGeneration is the metadata generation of the bucket.
MetaGeneration int64
// StorageClass is the default storage class of the bucket. This defines
// how objects in the bucket are stored and determines the SLA
// and the cost of storage. Typical values are "MULTI_REGIONAL",
// "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" and
// "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD", which
// is equivalent to "MULTI_REGIONAL" or "REGIONAL" depending on
// the bucket's location settings.
StorageClass string
// Created is the creation time of the bucket.
Created time.Time
// VersioningEnabled reports whether this bucket has versioning enabled.
// This field is read-only.
VersioningEnabled bool
// Labels are the bucket's labels.
Labels map[string]string
// RequesterPays reports whether the bucket is a Requester Pays bucket.
RequesterPays bool
}
func newBucket(b *raw.Bucket) *BucketAttrs {
if b == nil {
return nil
}
bucket := &BucketAttrs{
Name: b.Name,
Location: b.Location,
MetaGeneration: b.Metageneration,
StorageClass: b.StorageClass,
Created: convertTime(b.TimeCreated),
VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled,
Labels: b.Labels,
RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
}
acl := make([]ACLRule, len(b.Acl))
for i, rule := range b.Acl {
acl[i] = ACLRule{
Entity: ACLEntity(rule.Entity),
Role: ACLRole(rule.Role),
}
}
bucket.ACL = acl
objACL := make([]ACLRule, len(b.DefaultObjectAcl))
for i, rule := range b.DefaultObjectAcl {
objACL[i] = ACLRule{
Entity: ACLEntity(rule.Entity),
Role: ACLRole(rule.Role),
}
}
bucket.DefaultObjectACL = objACL
return bucket
}
// toRawBucket copies the editable attribute from b to the raw library's Bucket type.
func (b *BucketAttrs) toRawBucket() *raw.Bucket {
var acl []*raw.BucketAccessControl
if len(b.ACL) > 0 {
acl = make([]*raw.BucketAccessControl, len(b.ACL))
for i, rule := range b.ACL {
acl[i] = &raw.BucketAccessControl{
Entity: string(rule.Entity),
Role: string(rule.Role),
}
}
}
dACL := toRawObjectACL(b.DefaultObjectACL)
// Copy label map.
var labels map[string]string
if len(b.Labels) > 0 {
labels = make(map[string]string, len(b.Labels))
for k, v := range b.Labels {
labels[k] = v
}
}
// Ignore VersioningEnabled if it is false. This is OK because
// we only call this method when creating a bucket, and by default
// new buckets have versioning off.
var v *raw.BucketVersioning
if b.VersioningEnabled {
v = &raw.BucketVersioning{Enabled: true}
}
var bb *raw.BucketBilling
if b.RequesterPays {
bb = &raw.BucketBilling{RequesterPays: true}
}
return &raw.Bucket{
Name: b.Name,
DefaultObjectAcl: dACL,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: acl,
Versioning: v,
Labels: labels,
Billing: bb,
}
}
type BucketAttrsToUpdate struct {
// VersioningEnabled, if set, updates whether the bucket uses versioning.
VersioningEnabled optional.Bool
// RequesterPays, if set, updates whether the bucket is a Requester Pays bucket.
RequesterPays optional.Bool
setLabels map[string]string
deleteLabels map[string]bool
}
// SetLabel causes a label to be added or modified when ua is used
// in a call to Bucket.Update.
func (ua *BucketAttrsToUpdate) SetLabel(name, value string) {
if ua.setLabels == nil {
ua.setLabels = map[string]string{}
}
ua.setLabels[name] = value
}
// DeleteLabel causes a label to be deleted when ua is used in a
// call to Bucket.Update.
func (ua *BucketAttrsToUpdate) DeleteLabel(name string) {
if ua.deleteLabels == nil {
ua.deleteLabels = map[string]bool{}
}
ua.deleteLabels[name] = true
}
func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
rb := &raw.Bucket{}
if ua.VersioningEnabled != nil {
rb.Versioning = &raw.BucketVersioning{
Enabled: optional.ToBool(ua.VersioningEnabled),
ForceSendFields: []string{"Enabled"},
}
}
if ua.RequesterPays != nil {
rb.Billing = &raw.BucketBilling{
RequesterPays: optional.ToBool(ua.RequesterPays),
ForceSendFields: []string{"RequesterPays"},
}
}
if ua.setLabels != nil || ua.deleteLabels != nil {
rb.Labels = map[string]string{}
for k, v := range ua.setLabels {
rb.Labels[k] = v
}
if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 {
rb.ForceSendFields = append(rb.ForceSendFields, "Labels")
}
for l := range ua.deleteLabels {
rb.NullFields = append(rb.NullFields, "Labels."+l)
}
}
return rb
}
// If returns a new BucketHandle that applies a set of preconditions.
// Preconditions already set on the BucketHandle are ignored.
// Operations on the new handle will only occur if the preconditions are
// satisfied. The only valid preconditions for buckets are MetagenerationMatch
// and MetagenerationNotMatch.
func (b *BucketHandle) If(conds BucketConditions) *BucketHandle {
b2 := *b
b2.conds = &conds
return &b2
}
// BucketConditions constrain bucket methods to act on specific metagenerations.
//
// The zero value is an empty set of constraints.
type BucketConditions struct {
// MetagenerationMatch specifies that the bucket must have the given
// metageneration for the operation to occur.
// If MetagenerationMatch is zero, it has no effect.
MetagenerationMatch int64
// MetagenerationNotMatch specifies that the bucket must not have the given
// metageneration for the operation to occur.
// If MetagenerationNotMatch is zero, it has no effect.
MetagenerationNotMatch int64
}
func (c *BucketConditions) validate(method string) error {
if *c == (BucketConditions{}) {
return fmt.Errorf("storage: %s: empty conditions", method)
}
if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 {
return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
}
return nil
}
// UserProject returns a new BucketHandle that passes the project ID as the user
// project for all subsequent calls. A user project is required for all operations
// on requester-pays buckets.
func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
b2 := *b
b2.userProject = projectID
b2.acl.userProject = projectID
b2.defaultObjectACL.userProject = projectID
return &b2
}
// applyBucketConds modifies the provided call using the conditions in conds.
// call is something that quacks like a *raw.WhateverCall.
func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
if conds == nil {
return nil
}
if err := conds.validate(method); err != nil {
return err
}
cval := reflect.ValueOf(call)
switch {
case conds.MetagenerationMatch != 0:
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
return nil
}
// Objects returns an iterator over the objects in the bucket that match the Query q.
// If q is nil, no filtering is done.
func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
it := &ObjectIterator{
ctx: ctx,
bucket: b,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
if q != nil {
it.query = *q
}
return it
}
// An ObjectIterator is an iterator over ObjectAttrs.
type ObjectIterator struct {
ctx context.Context
bucket *BucketHandle
query Query
pageInfo *iterator.PageInfo
nextFunc func() error
items []*ObjectAttrs
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
//
// If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will
// have a non-empty Prefix field, and a zero value for all other fields. These
// represent prefixes.
func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.bucket.c.raw.Objects.List(it.bucket.name)
setClientHeader(req.Header())
req.Projection("full")
req.Delimiter(it.query.Delimiter)
req.Prefix(it.query.Prefix)
req.Versions(it.query.Versions)
req.PageToken(pageToken)
if it.bucket.userProject != "" {
req.UserProject(it.bucket.userProject)
}
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Objects
var err error
err = runWithRetry(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
})
if err != nil {
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
err = ErrBucketNotExist
}
return "", err
}
for _, item := range resp.Items {
it.items = append(it.items, newObject(item))
}
for _, prefix := range resp.Prefixes {
it.items = append(it.items, &ObjectAttrs{Prefix: prefix})
}
return resp.NextPageToken, nil
}
// TODO(jbd): Add storage.buckets.update.
// Buckets returns an iterator over the buckets in the project. You may
// optionally set the iterator's Prefix field to restrict the list to buckets
// whose names begin with the prefix. By default, all buckets in the project
// are returned.
func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator {
it := &BucketIterator{
ctx: ctx,
client: c,
projectID: projectID,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.buckets) },
func() interface{} { b := it.buckets; it.buckets = nil; return b })
return it
}
// A BucketIterator is an iterator over BucketAttrs.
type BucketIterator struct {
// Prefix restricts the iterator to buckets whose names begin with it.
Prefix string
ctx context.Context
client *Client
projectID string
buckets []*BucketAttrs
pageInfo *iterator.PageInfo
nextFunc func() error
}
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
func (it *BucketIterator) Next() (*BucketAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
b := it.buckets[0]
it.buckets = it.buckets[1:]
return b, nil
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.client.raw.Buckets.List(it.projectID)
setClientHeader(req.Header())
req.Projection("full")
req.Prefix(it.Prefix)
req.PageToken(pageToken)
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Buckets
var err error
err = runWithRetry(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
})
if err != nil {
return "", err
}
for _, item := range resp.Items {
it.buckets = append(it.buckets, newBucket(item))
}
return resp.NextPageToken, nil
}
vendor/cloud.google.com/go/storage/copy.go
Generated Vendored
+201
View File
@@ -0,0 +1,201 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"errors"
"fmt"
"golang.org/x/net/context"
raw "google.golang.org/api/storage/v1"
)
// CopierFrom creates a Copier that can copy src to dst.
// You can immediately call Run on the returned Copier, or
// you can configure it first.
//
// For Requester Pays buckets, the user project of dst is billed, unless it is empty,
// in which case the user project of src is billed.
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
return &Copier{dst: dst, src: src}
}
// A Copier copies a source object to a destination.
type Copier struct {
// ObjectAttrs are optional attributes to set on the destination object.
// Any attributes must be initialized before any calls on the Copier. Nil
// or zero-valued attributes are ignored.
ObjectAttrs
// RewriteToken can be set before calling Run to resume a copy
// operation. After Run returns a non-nil error, RewriteToken will
// have been updated to contain the value needed to resume the copy.
RewriteToken string
// ProgressFunc can be used to monitor the progress of a multi-RPC copy
// operation. If ProgressFunc is not nil and copying requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
// ProgressFunc will be invoked after each call with the number of bytes of
// content copied so far and the total size in bytes of the source object.
//
// ProgressFunc is intended to make upload progress available to the
// application. For example, the implementation of ProgressFunc may update
// a progress bar in the application's UI, or log the result of
// float64(copiedBytes)/float64(totalBytes).
//
// ProgressFunc should return quickly without blocking.
ProgressFunc func(copiedBytes, totalBytes uint64)
dst, src *ObjectHandle
}
// Run performs the copy.
func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
if err := c.src.validate(); err != nil {
return nil, err
}
if err := c.dst.validate(); err != nil {
return nil, err
}
// Convert destination attributes to raw form, omitting the bucket.
// If the bucket is included but name or content-type aren't, the service
// returns a 400 with "Required" as the only message. Omitting the bucket
// does not cause any problems.
rawObject := c.ObjectAttrs.toRawObject("")
for {
res, err := c.callRewrite(ctx, rawObject)
if err != nil {
return nil, err
}
if c.ProgressFunc != nil {
c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize))
}
if res.Done { // Finished successfully.
return newObject(res.Resource), nil
}
}
}
func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) {
call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj)
call.Context(ctx).Projection("full")
if c.RewriteToken != "" {
call.RewriteToken(c.RewriteToken)
}
if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
} else if c.src.userProject != "" {
call.UserProject(c.src.userProject)
}
if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
return nil, err
}
var res *raw.RewriteResponse
var err error
setClientHeader(call.Header())
err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
if err != nil {
return nil, err
}
c.RewriteToken = res.RewriteToken
return res, nil
}
// ComposerFrom creates a Composer that can compose srcs into dst.
// You can immediately call Run on the returned Composer, or you can
// configure it first.
//
// The encryption key for the destination object will be used to decrypt all
// source objects and encrypt the destination object. It is an error
// to specify an encryption key for any of the source objects.
func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
return &Composer{dst: dst, srcs: srcs}
}
// A Composer composes source objects into a destination object.
//
// For Requester Pays buckets, the user project of dst is billed.
type Composer struct {
// ObjectAttrs are optional attributes to set on the destination object.
// Any attributes must be initialized before any calls on the Composer. Nil
// or zero-valued attributes are ignored.
ObjectAttrs
dst *ObjectHandle
srcs []*ObjectHandle
}
// Run performs the compose operation.
func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
if err := c.dst.validate(); err != nil {
return nil, err
}
if len(c.srcs) == 0 {
return nil, errors.New("storage: at least one source object must be specified")
}
req := &raw.ComposeRequest{}
// Compose requires a non-empty Destination, so we always set it,
// even if the caller-provided ObjectAttrs is the zero value.
req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
for _, src := range c.srcs {
if err := src.validate(); err != nil {
return nil, err
}
if src.bucket != c.dst.bucket {
return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket)
}
if src.encryptionKey != nil {
return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
}
srcObj := &raw.ComposeRequestSourceObjects{
Name: src.object,
}
if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
return nil, err
}
req.SourceObjects = append(req.SourceObjects, srcObj)
}
call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
var obj *raw.Object
var err error
setClientHeader(call.Header())
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
if err != nil {
return nil, err
}
return newObject(obj), nil
}
vendor/cloud.google.com/go/storage/doc.go
Generated Vendored
+161
View File
@@ -0,0 +1,161 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package storage provides an easy way to work with Google Cloud Storage.
Google Cloud Storage stores data in named objects, which are grouped into buckets.
More information about Google Cloud Storage is available at
https://cloud.google.com/storage/docs.
All of the methods of this package use exponential backoff to retry calls
that fail with certain errors, as described in
https://cloud.google.com/storage/docs/exponential-backoff.
Note: This package is in beta. Some backwards-incompatible changes may occur.
Creating a Client
To start working with this package, create a client:
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
Buckets
A Google Cloud Storage bucket is a collection of objects. To work with a
bucket, make a bucket handle:
bkt := client.Bucket(bucketName)
A handle is a reference to a bucket. You can have a handle even if the
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
call Create on the handle:
if err := bkt.Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
Note that although buckets are associated with projects, bucket names are
global across all projects.
Each bucket has associated metadata, represented in this package by
BucketAttrs. The third argument to BucketHandle.Create allows you to set
the intial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
Attrs:
attrs, err := bkt.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)
Objects
An object holds arbitrary data as a sequence of bytes, like a file. You
refer to objects using a handle, just as with buckets. You can use the
standard Go io.Reader and io.Writer interfaces to read and write
object data:
obj := bkt.Object("data")
// Write something to obj.
// w implements io.Writer.
w := obj.NewWriter(ctx)
// Write some text to obj. This will overwrite whatever is there.
if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
// TODO: Handle error.
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
// TODO: Handle error.
}
// Read it back.
r, err := obj.NewReader(ctx)
if err != nil {
// TODO: Handle error.
}
defer r.Close()
if _, err := io.Copy(os.Stdout, r); err != nil {
// TODO: Handle error.
}
// Prints "This object contains text."
Objects also have attributes, which you can fetch with Attrs:
objAttrs, err := obj.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("object %s has size %d and can be read using %s\n",
objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)
ACLs
Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
ACLRules, each of which specifies the role of a user, group or project. ACLs
are suitable for fine-grained control, but you may prefer using IAM to control
access at the project level (see
https://cloud.google.com/storage/docs/access-control/iam).
To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:
acls, err := obj.ACL().List(ctx)
if err != nil {
// TODO: Handle error.
}
for _, rule := range acls {
fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
}
You can also set and delete ACLs.
Conditions
Every object has a generation and a metageneration. The generation changes
whenever the content changes, and the metageneration changes whenever the
metadata changes. Conditions let you check these values before an operation;
the operation only executes if the conditions match. You can use conditions to
prevent race conditions in read-modify-write operations.
For example, say you've read an object's metadata into objAttrs. Now
you want to write to that object, but only if its contents haven't changed
since you read it. Here is how to express that:
w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
// Proceed with writing as above.
Signed URLs
You can obtain a URL that lets anyone read or write an object for a limited time.
You don't need to create a client to do this. See the documentation of
SignedURL for details.
url, err := storage.SignedURL(bucketName, "shared-object", opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
Authentication
See examples of authorization and authentication at
https://godoc.org/cloud.google.com/go#pkg-examples.
*/
package storage // import "cloud.google.com/go/storage"
vendor/cloud.google.com/go/storage/go17.go
Generated Vendored
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.7
package storage
import (
"context"
"net/http"
)
func withContext(r *http.Request, ctx context.Context) *http.Request {
return r.WithContext(ctx)
}
vendor/cloud.google.com/go/storage/iam.go
Generated Vendored
+108
View File
@@ -0,0 +1,108 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"cloud.google.com/go/iam"
"golang.org/x/net/context"
raw "google.golang.org/api/storage/v1"
iampb "google.golang.org/genproto/googleapis/iam/v1"
)
// IAM provides access to IAM access control for the bucket.
func (b *BucketHandle) IAM() *iam.Handle {
return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name)
}
// iamClient implements the iam.client interface.
type iamClient struct {
raw *raw.Service
}
func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) {
req := c.raw.Buckets.GetIamPolicy(resource)
setClientHeader(req.Header())
var rp *raw.Policy
var err error
err = runWithRetry(ctx, func() error {
rp, err = req.Context(ctx).Do()
return err
})
if err != nil {
return nil, err
}
return iamFromStoragePolicy(rp), nil
}
func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error {
rp := iamToStoragePolicy(p)
req := c.raw.Buckets.SetIamPolicy(resource, rp)
setClientHeader(req.Header())
return runWithRetry(ctx, func() error {
_, err := req.Context(ctx).Do()
return err
})
}
func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
req := c.raw.Buckets.TestIamPermissions(resource, perms)
setClientHeader(req.Header())
var res *raw.TestIamPermissionsResponse
var err error
err = runWithRetry(ctx, func() error {
res, err = req.Context(ctx).Do()
return err
})
if err != nil {
return nil, err
}
return res.Permissions, nil
}
func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy {
return &raw.Policy{
Bindings: iamToStorageBindings(ip.Bindings),
Etag: string(ip.Etag),
}
}
func iamToStorageBindings(ibs []*iampb.Binding) []*raw.PolicyBindings {
var rbs []*raw.PolicyBindings
for _, ib := range ibs {
rbs = append(rbs, &raw.PolicyBindings{
Role: ib.Role,
Members: ib.Members,
})
}
return rbs
}
func iamFromStoragePolicy(rp *raw.Policy) *iampb.Policy {
return &iampb.Policy{
Bindings: iamFromStorageBindings(rp.Bindings),
Etag: []byte(rp.Etag),
}
}
func iamFromStorageBindings(rbs []*raw.PolicyBindings) []*iampb.Binding {
var ibs []*iampb.Binding
for _, rb := range rbs {
ibs = append(ibs, &iampb.Binding{
Role: rb.Role,
Members: rb.Members,
})
}
return ibs
}
vendor/cloud.google.com/go/storage/invoke.go
Generated Vendored
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"cloud.google.com/go/internal"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
)
// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
func runWithRetry(ctx context.Context, call func() error) error {
return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
err = call()
if err == nil {
return true, nil
}
e, ok := err.(*googleapi.Error)
if !ok {
return true, err
}
// Retry on 429 and 5xx, according to
// https://cloud.google.com/storage/docs/exponential-backoff.
if e.Code == 429 || (e.Code >= 500 && e.Code < 600) {
return false, nil
}
return true, err
})
}
vendor/cloud.google.com/go/storage/not_go17.go
Generated Vendored
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.7
package storage
import (
"net/http"
)
func withContext(r *http.Request, _ interface{}) *http.Request {
// In Go 1.6 and below, ignore the context.
return r
}
vendor/cloud.google.com/go/storage/reader.go
Generated Vendored
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"hash/crc32"
"io"
)
var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
// Reader reads a Cloud Storage object.
// It implements io.Reader.
type Reader struct {
body io.ReadCloser
remain, size int64
contentType string
checkCRC bool // should we check the CRC?
wantCRC uint32 // the CRC32c value the server sent in the header
gotCRC uint32 // running crc
}
// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
return r.body.Close()
}
func (r *Reader) Read(p []byte) (int, error) {
n, err := r.body.Read(p)
if r.remain != -1 {
r.remain -= int64(n)
}
if r.checkCRC {
r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n])
// Check CRC here. It would be natural to check it in Close, but
// everybody defers Close on the assumption that it doesn't return
// anything worth looking at.
if r.remain == 0 && r.gotCRC != r.wantCRC {
return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d",
r.gotCRC, r.wantCRC)
}
}
return n, err
}
// Size returns the size of the object in bytes.
// The returned value is always the same and is not affected by
// calls to Read or Close.
func (r *Reader) Size() int64 {
return r.size
}
// Remain returns the number of bytes left to read, or -1 if unknown.
func (r *Reader) Remain() int64 {
return r.remain
}
// ContentType returns the content type of the object.
func (r *Reader) ContentType() string {
return r.contentType
}
vendor/cloud.google.com/go/storage/storage.go
Generated Vendored
+1118
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/storage/writer.go
Generated Vendored
+192
View File
@@ -0,0 +1,192 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"encoding/base64"
"errors"
"fmt"
"io"
"unicode/utf8"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
// A Writer writes a Cloud Storage object.
type Writer struct {
// ObjectAttrs are optional attributes to set on the object. Any attributes
// must be initialized before the first Write call. Nil or zero-valued
// attributes are ignored.
ObjectAttrs
// SendCRC specifies whether to transmit a CRC32C field. It should be set
// to true in addition to setting the Writer's CRC32C field, because zero
// is a valid CRC and normally a zero would not be transmitted.
SendCRC32C bool
// ChunkSize controls the maximum number of bytes of the object that the
// Writer will attempt to send to the server in a single request. Objects
// smaller than the size will be sent in a single request, while larger
// objects will be split over multiple requests. The size will be rounded up
// to the nearest multiple of 256K. If zero, chunking will be disabled and
// the object will be uploaded in a single request.
//
// ChunkSize will default to a reasonable value. Any custom configuration
// must be done before the first Write call.
ChunkSize int
// ProgressFunc can be used to monitor the progress of a large write.
// operation. If ProgressFunc is not nil and writing requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload),
// then ProgressFunc will be invoked after each call with the number of bytes of
// content copied so far.
//
// ProgressFunc should return quickly without blocking.
ProgressFunc func(int64)
ctx context.Context
o *ObjectHandle
opened bool
pw *io.PipeWriter
donec chan struct{} // closed after err and obj are set.
err error
obj *ObjectAttrs
}
func (w *Writer) open() error {
attrs := w.ObjectAttrs
// Check the developer didn't change the object Name (this is unfortunate, but
// we don't want to store an object under the wrong name).
if attrs.Name != w.o.object {
return fmt.Errorf("storage: Writer.Name %q does not match object name %q", attrs.Name, w.o.object)
}
if !utf8.ValidString(attrs.Name) {
return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name)
}
pr, pw := io.Pipe()
w.pw = pw
w.opened = true
if w.ChunkSize < 0 {
return errors.New("storage: Writer.ChunkSize must non-negative")
}
mediaOpts := []googleapi.MediaOption{
googleapi.ChunkSize(w.ChunkSize),
}
if c := attrs.ContentType; c != "" {
mediaOpts = append(mediaOpts, googleapi.ContentType(c))
}
go func() {
defer close(w.donec)
rawObj := attrs.toRawObject(w.o.bucket)
if w.SendCRC32C {
rawObj.Crc32c = encodeUint32(attrs.CRC32C)
}
if w.MD5 != nil {
rawObj.Md5Hash = base64.StdEncoding.EncodeToString(w.MD5)
}
call := w.o.c.raw.Objects.Insert(w.o.bucket, rawObj).
Media(pr, mediaOpts...).
Projection("full").
Context(w.ctx)
if w.ProgressFunc != nil {
call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) })
}
if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil {
w.err = err
pr.CloseWithError(w.err)
return
}
var resp *raw.Object
err := applyConds("NewWriter", w.o.gen, w.o.conds, call)
if err == nil {
if w.o.userProject != "" {
call.UserProject(w.o.userProject)
}
setClientHeader(call.Header())
// We will only retry here if the initial POST, which obtains a URI for
// the resumable upload, fails with a retryable error. The upload itself
// has its own retry logic.
err = runWithRetry(w.ctx, func() error {
var err2 error
resp, err2 = call.Do()
return err2
})
}
if err != nil {
w.err = err
pr.CloseWithError(w.err)
return
}
w.obj = newObject(resp)
}()
return nil
}
// Write appends to w. It implements the io.Writer interface.
//
// Since writes happen asynchronously, Write may return a nil
// error even though the write failed (or will fail). Always
// use the error returned from Writer.Close to determine if
// the upload was successful.
func (w *Writer) Write(p []byte) (n int, err error) {
if w.err != nil {
return 0, w.err
}
if !w.opened {
if err := w.open(); err != nil {
return 0, err
}
}
return w.pw.Write(p)
}
// Close completes the write operation and flushes any buffered data.
// If Close doesn't return an error, metadata about the written object
// can be retrieved by calling Attrs.
func (w *Writer) Close() error {
if !w.opened {
if err := w.open(); err != nil {
return err
}
}
if err := w.pw.Close(); err != nil {
return err
}
<-w.donec
return w.err
}
// CloseWithError aborts the write operation with the provided error.
// CloseWithError always returns nil.
func (w *Writer) CloseWithError(err error) error {
if !w.opened {
return nil
}
return w.pw.CloseWithError(err)
}
// Attrs returns metadata about a successfully-written object.
// It's only valid to call it after Close returns nil.
func (w *Writer) Attrs() *ObjectAttrs {
return w.obj
}
vendor/github.com/Azure/azure-sdk-for-go/.gitignore
Generated
+32
View File
@@ -0,0 +1,32 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
# Editor swap files
*.swp
*~
.DS_Store
# ignore vendor/
vendor/
vendor/github.com/Azure/azure-sdk-for-go/CHANGELOG.md
Generated Vendored
+520
View File
@@ -0,0 +1,520 @@
# CHANGELOG
## `v10.2.1-beta`
- Fixes polymorphic structs in `mysql` and `postgresql` packages.
## `v10.2.0-beta`
### ARM
| api | version | note |
|:------------------------------------|:-------------------|:------------------------------------|
| arm/cosmos-db | 2015-04-08 | new |
| arm/mysql | 2017-04-30-preview | new |
| arm/postgresql | 2017-04-30-preview | new |
### Storage
- Bug fixes.
### Generated code notes
- [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs) commit: 485ded7560c6309efb2f795ec6e46b7436dc6fdb
- [AutoRest](https://github.com/Azure/autorest) commit: c180952b850e677a8624655abeaded307d95cae3
## `v10.1.0-beta`
### ARM
| api | version | note |
|:------------------------------------|:-------------------|:------------------------------------|
| arm/recoveryservicessiterecovery | 2016-08-10 | new |
| arm/managedapplications | 2016-09-01-preview | new |
| arm/storsimple8000series | 2017-06-01 | new |
| arm/streamanalytics | multiple | new |
### Storage
- Bug fixes.
### Generated code notes
- [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs) commit: a2cdf005407b81edb161c1f7b5c49b5ce8e7f041
- [AutoRest](https://github.com/Azure/autorest) commit: 8e9c2d3704a04913a175ab76972b7d9597c77687
-----
## `v10.0.0-beta`
### ARM
In addition to the tabulated changes below, each package had the following updates:
- Long running operations now run inside a goroutine and return channels for the response and the errors.
- Some functions changed from returning `autorest.Response` to return the already unmarshaled struct.
- Uses go-autorest v8.0.0.
| api | version | note |
|:------------------------------------|:-------------------|:------------------------------------|
| arm/advisor | 2017-04-19 | new |
| arm/analysisservices | 2016-05-16 | refactor |
| arm/apimanagement | 2016-10-10 | update to latest swagger & refactor |
| arm/appinsights | 2015-05-01 | new |
| arm/automation | 2015-10-31 | new |
| arm/billing | 2017-04-24-preview | update to latest swagger & refactor |
| arm/cdn | 2016-10-02 | refactor |
| arm/commerce | 2015-06-01-preview | refactor |
| arm/compute | 2016-04-30-preview | refactor |
| arm/consumption | 2017-04-24-preview | new |
| arm/containerregistry | 2017-03-01 | update to latest swagger & refactor |
| arm/containerservice | 2017-01-31 | update to latest swagger & refactor |
| arm/customer-insights | 2017-01-01 | refactor |
| arm/datalake-analytics/account | 2016-11-01 | refactor |
| arm/datalake-store/account | 2016-11-01 | refactor |
| arm/devtestlabs | 2016-05-15 | refactor |
| arm/disk | 2016-04-30-preview | refactor |
| arm/dns | 2016-04-01 | refactor |
| arm/documentdb | 2015-04-08 | refactor |
| arm/eventhub | 2015-08-01 | refactor |
| arm/graphrbac | 1.6 | refactor |
| arm/hdinsight | 2015-03-01-preview | new |
| arm/insights | multiple | new |
| arm/intune | 2015-01-14-preview | refactor |
| arm/iothub | 2016-02-03 | refactor |
| arm/machinelearning/commitmentplans | 2016-05-01-preview | refactor |
| arm/machinelearning/webservices | 2017-01-01 | update to latest swagger & refactor |
| arm/monitor | multiple | new |
| arm/network | 2017-03-01 | update to latest swagger & refactor |
| arm/notificationhubs | 2017-04-01 | update to latest swagger & refactor |
| arm/operationalinsights | 2015-11-01-preview | update to latest swagger & refactor |
| arm/powerbiembedded | 2016-01-29 | refactor |
| arm/recoveryservices | 2016-12-01 | refactor |
| arm/recoveryservicesbackup | 2016-12-01 | new |
| arm/redis | 2016-04-01 | refactor |
| arm/relay | 2016-07-01 | new |
| arm/resourcehealth | 2015-01-01 | new |
| arm/resources/features | 2015-12-01 | refactor |
| arm/resources/links | 2016-09-01 | refactor |
| arm/resources/resources | 2016-09-01 | refactor |
| arm/resources/subscriptions | 2016-06-01 | refactor |
| arm/scheduler | 2016-03-01 | refactor |
| arm/servermanagement | 2016-07-01-preview | refactor |
| arm/servicebus | 2015-08-01 | refactor |
| arm/servicefabric | 2016-09-01 | new |
| arm/service-map | 2015-11-01-preview | refactor |
| arm/sql | multiple | update to latest swagger & refactor |
| arm/storage | 2016-12-01 | update to latest swagger & refactor |
| arm/storageimportexport | 2016-11-01 | refactor |
| arm/web | multiple | refactor |
### Data plane
| api | version | note |
|:------------------------------------|:-------------------|:------------------------------------|
| dataplane/keyvault | 2016-10-01 | refactor |
### Storage
Storage has returned to this repo.
It has also been refactored:
- Blobs, containers, tables, etc are now method receivers. These structs are the ones being
updated with each operation.
- When creating a client, the SDK checks if the storage account provided is valid.
- Added retry logic. It provides the flexibility for user to provide their own retry logic.
- Added operations:
- Get table
- Get entity
- Get and set queue ACL
- Table batch
- Page blob incremental copy
- All operations that previously had `extraHeaders` as parameter now recieve a struct with well
defined possible headers and other options. Some functions are easier to use.
- Storage tests now use HTTP recordings.
### Generated code notes
- [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs) commit: 519980465d9c195622d466dc4601b1999a448ed5
- [AutoRest](https://github.com/Azure/autorest) commit: ced950d64e39735b84d41876a56b54b27c227dc7
## `v9.0.0-beta`
### ARM
In addition to the tabulated changes below, each package had the following updates:
- API Version is now associated with individual methods, instead of the client. This was done to
support composite swaggers, which logically may contain more than one API Version.
- Version numbers are now calculated in the generator instead of at runtime. This keeps us from
adding new allocations, while removing the race-conditions that were added.
| api | version | note |
|:------------------------------------|:-------------------|:-----------------------------------|
| arm/analysisservices | 2016-05-16 | update to latest swagger |
| arm/authorization | 2015-07-01 | refactoring |
| arm/batch | 2017-01-01 | update to latest swagger &refactor |
| arm/cdn | 2016-10-02 | update to latest swagger |
| arm/compute | 2016-04-30-preview | update to latest swagger |
| arm/dns | 2016-04-01 | update to latest swagger &refactor |
| arm/eventhub | 2015-08-01 | refactoring |
| arm/logic | 2016-06-01 | update to latest swagger &refactor |
| arm/notificationshub | 2016-03-01 | update to latest swagger &refactor |
| arm/redis | 2016-04-01 | update to latest swagger &refactor |
| arm/resources/resources | 2016-09-01 | update to latest swagger |
| arm/servicebus | 2015-08-01 | update to latest swagger |
| arm/sql | 2014-04-01 | update to latest swagger |
| arm/web | multiple | generating from composite |
| datalake-analytics/account | 2016-11-01 | update to latest swagger |
| datalake-store/filesystem | 2016-11-01 | update to latest swagger |
### Storage
Storage has been moved to its own repository which can be found here:
https://github.com/Azure/azure-storage-go
For backwards compatibility, a submodule has been added to this repo. However, consuming storage
via this repository is deprecated and may be deleted in future versions.
## `v8.1.0-beta`
### ARM
| api | version | note |
|:------------------------------------|:-------------------|:-----------------------------------|
| arm/apimanagement | 2016-07-07 | new |
| arm/apideployment | 2016-07-07 | new |
| arm/billing | 2017-02-27-preview | new |
| arm/compute | 2016-04-30-preview | update to latest swagger |
| arm/containerservice | 2017-01-31 | update to latest swagger |
| arm/customer-insights | 2017-01-01 | new |
| arm/graphrbac | 1.6 | new |
| arm/networkwatcher | 2016-12-01 | new |
| arm/operationalinsights | 2015-11-01-preview | new |
| arm/service-map | 2015-11-01-preview | new |
| arm/storageimportexport | 2016-11-01 | new |
### Data plane
| api | version | note |
|:------------------------------------|:-------------------|:-----------------------------------|
| dataplane/keyvault | 2016-10-01 | new |
- Uses go-autorest v7.3.0
## `v8.0.0-beta`
### ARM
- In addition to the tablulated changes below, all updated packages received performance
improvements to their Version() method.
- Some validation that was taking place in the runtime was erroneously blocking calls.
all packages have been updated to take that bug fix.
| api | version | note |
|:------------------------------------|:-------------------|:-----------------------------------|
| arm/analysisservices | 2016-05-16 | update to latest swagger |
| arm/cdn | 2016-10-02 | update to latest swagger |
| arm/cognitiveservices | 2016-02-01-preview | update to latest swagger |
| arm/compute | 2016-03-30 | update to latest swagger, refactor |
| arm/containerregistry | 2016-06-27-preview | update to latest swagger |
| arm/containerservice | 2016-09-30 | update to latest swagger |
| arm/datalake-analytics | 2016-11-01 | update to latest swagger |
| arm/datalake-store | 2016-11-01 | update to latest swagger |
| arm/disk | 2016-04-30-preview | new |
| arm/documentdb | 2015-04-08 | update to latest swagger |
| arm/iothub | 2016-02-03 | update to latest swagger |
| arm/keyvault | 2015-06-01 | update to latest swagger |
| arm/logic | 2016-06-01 | update to latest swagger |
| arm/machinelearning | 2016-05-01-preview | update to latest swagger |
| arm/mobileengagement | 2014-12-01 | update to latest swagger, refactor |
| arm/redis | 2016-04-01 | update to latest swagger |
| arm/resources/locks | 2016-09-01 | refactor |
| arm/resources/policy | 2016-12-01 | previous version was deleted |
| arm/resources/resources | 2016-09-01 | update to latest swagger, refactor |
| arm/scheduler | 2016-03-01 | refactor |
| arm/search | 2015-08-19 | refactor |
| arm/web | 2015-08-01 | refactor |
## `v7.0.0-beta`
| api | version | note |
|:------------------------------------|:-------------------|:-----------------------------------|
| arm/analysisservices | 2016-05-16 | new |
| arm/cdn | 2016-10-02 | update to latest swagger |
| arm/commerce | 2015-06-01-preview | new |
| arm/containerservice | 2016-09-30 | update to latest swagger |
| arm/containerregistry | 2016-06-27-preview | new |
| arm/datalake-analytics/account | 2016-11-01 | update to latest swagger |
| arm/datalake-store/account | 2016-11-01 | update to latest swagger |
| arm/datalake-store/filesystem | 2016-11-01 | update to latest swagger |
| arm/documentdb | 2015-04-08 | new |
| arm/machinelearning/commitmentplans | 2016-05-01-preview | new |
| arm/recoveryservices | 2016-06-01 | new |
| arm/resources/subscriptions | 2016-06-01 | new |
| arm/search | 2015-08-19 | update to latest swagger |
| arm/sql | 2014-04-01 | previous version was deleted |
### Storage
- Can now update messages in storage queues.
- Added support for blob snapshots and aborting blob copy operations.
- Added support for getting and setting ACLs on containers.
- Added various APIs for file and directory manipulation.
### Support for the following swagger extensions was added to the Go generator which affected codegen.
- x-ms-client-flatten
- x-ms-paramater-location
## `v6.0.0-beta`
| api | version | note |
|:-------------------------------|:-------------------|:-----------------------------------|
| arm/authorization | no change | code refactoring |
| arm/batch | no change | code refactoring |
| arm/compute | no change | code refactoring |
| arm/containerservice | 2016-03-30 | return |
| arm/datalake-analytics/account | 2015-10-01-preview | new |
| arm/datalake-store/filesystem | no change | moved to datalake-store/filesystem |
| arm/eventhub | no change | code refactoring |
| arm/intune | no change | code refactoring |
| arm/iothub | no change | code refactoring |
| arm/keyvault | no change | code refactoring |
| arm/mediaservices | no change | code refactoring |
| arm/network | no change | code refactoring |
| arm/notificationhubs | no change | code refactoring |
| arm/redis | no change | code refactoring |
| arm/resources/resources | no change | code refactoring |
| arm/resources/links | 2016-09-01 | new |
| arm/resources/locks | 2016-09-01 | updated |
| arm/resources/policy | no change | code refactoring |
| arm/resources/resources | 2016-09-01 | updated |
| arm/servermanagement | 2016-07-01-preview | updated |
| arm/web | no change | code refactoring |
- storage: Added blob lease functionality and tests
## `v5.0.0-beta`
| api | version | note |
|:------------------------------|:--------------------|:-----------------|
| arm/network | 2016-09-01 | updated |
| arm/servermanagement | 2015-07-01-preview | new |
| arm/eventhub | 2015-08-01 | new |
| arm/containerservice | -- | removed |
| arm/resources/subscriptions | no change | code refactoring |
| arm/resources/features | no change | code refactoring |
| arm/resources/resources | no change | code refactoring |
| arm/datalake-store/accounts | no change | code refactoring |
| arm/datalake-store/filesystem | no change | code refactoring |
| arm/notificationhubs | no change | code refactoring |
| arm/redis | no change | code refactoring |
- storage: Add more file storage share operations.
- azure-rest-api-specs/commit/b8cdc2c50a0872fc0039f20c2b6b33aa0c2af4bf
- Uses go-autorest v7.2.1
## `v4.0.0-beta`
- arm/logic: breaking change in package logic.
- arm: parameter validation code added in all arm packages.
- Uses go-autorest v7.2.0.
## `v3.2.0-beta`
| api | version | note |
|:----------------------------|:--------------------|:----------|
| arm/mediaservices | 2015-10-01 | new |
| arm/keyvault | 2015-06-01 | new |
| arm/iothub | 2016-02-03 | new |
| arm/datalake-store | 2015-12-01 | new |
| arm/network | 2016-06-01 | updated |
| arm/resources/resources | 2016-07-01 | updated |
| arm/resources/policy | 2016-04-01 | updated |
| arm/servicebus | 2015-08-01 | updated |
- arm: uses go-autorest version v7.1.0.
- storage: fix for operating on blobs names containing special characters.
- storage: add SetBlobProperties(), update BlobProperties response fields.
- storage: make storage client work correctly with read-only secondary account.
- storage: add Azure Storage Emulator support.
## `v3.1.0-beta`
- Added a new arm/compute/containerservice (2016-03-30) package
- Reintroduced NewxxClientWithBaseURI method.
- Uses go-autorest version - v7.0.7.
## `v3.0.0-beta`
This release brings the Go SDK ARM packages up-to-date with Azure ARM Swagger files for most
services. Since the underlying [Swagger files](https://github.com/Azure/azure-rest-api-specs)
continue to change substantially, the ARM packages are still in *beta* status.
The ARM packages now align with the following API versions (*highlighted* packages are new or
updated in this release):
| api | version | note |
|:----------------------------|:--------------------|:----------|
| arm/authorization | 2015-07-01 | no change |
| arm/intune | 2015-01-14-preview | no change |
| arm/notificationhubs | 2014-09-01 | no change |
| arm/resources/features | 2015-12-01 | no change |
| arm/resources/subscriptions | 2015-11-01 | no change |
| arm/web | 2015-08-01 | no change |
| arm/cdn | 2016-04-02 | updated |
| arm/compute | 2016-03-30 | updated |
| arm/dns | 2016-04-01 | updated |
| arm/logic | 2015-08-01-preview | updated |
| arm/network | 2016-03-30 | updated |
| arm/redis | 2016-04-01 | updated |
| arm/resources/resources | 2016-02-01 | updated |
| arm/resources/policy | 2015-10-01-preview | updated |
| arm/resources/locks | 2015-01-01 | updated (resources/authorization earlier)|
| arm/scheduler | 2016-03-01 | updated |
| arm/storage | 2016-01-01 | updated |
| arm/search | 2015-02-28 | updated |
| arm/batch | 2015-12-01 | new |
| arm/cognitiveservices | 2016-02-01-preview | new |
| arm/devtestlabs | 2016-05-15 | new |
| arm/machinelearning | 2016-05-01-preview | new |
| arm/powerbiembedded | 2016-01-29 | new |
| arm/mobileengagement | 2014-12-01 | new |
| arm/servicebus | 2014-09-01 | new |
| arm/sql | 2015-05-01 | new |
| arm/trafficmanager | 2015-11-01 | new |
Below are some design changes.
- Removed Api version from method arguments.
- Removed New...ClientWithBaseURI() method in all clients. BaseURI value is set in client.go.
- Uses go-autorest version v7.0.6.
## `v2.2.0-beta`
- Uses go-autorest version v7.0.5.
- Update version of pacakges "jwt-go" and "crypto" in glide.lock.
## `v2.1.1-beta`
- arm: Better error messages for long running operation failures (Uses go-autorest version v7.0.4).
## `v2.1.0-beta`
- arm: Uses go-autorest v7.0.3 (polling related updates).
- arm: Cancel channel argument added in long-running calls.
- storage: Allow caller to provide headers for DeleteBlob methods.
- storage: Enables connection sharing with http keepalive.
- storage: Add BlobPrefixes and Delimiter to BlobListResponse
## `v2.0.0-beta`
- Uses go-autorest v6.0.0 (Polling and Asynchronous requests related changes).
## `v0.5.0-beta`
Updated following packages to new API versions:
- arm/resources/features 2015-12-01
- arm/resources/resources 2015-11-01
- arm/resources/subscriptions 2015-11-01
### Changes
- SDK now uses go-autorest v3.0.0.
## `v0.4.0-beta`
This release brings the Go SDK ARM packages up-to-date with Azure ARM Swagger files for most
services. Since the underlying [Swagger files](https://github.com/Azure/azure-rest-api-specs)
continue to change substantially, the ARM packages are still in *beta* status.
The ARM packages now align with the following API versions (*highlighted* packages are new or
updated in this release):
- *arm/authorization 2015-07-01*
- *arm/cdn 2015-06-01*
- arm/compute 2015-06-15
- arm/dns 2015-05-04-preview
- *arm/intune 2015-01-14-preview*
- arm/logic 2015-02-01-preview
- *arm/network 2015-06-15*
- *arm/notificationhubs 2014-09-01*
- arm/redis 2015-08-01
- *arm/resources/authorization 2015-01-01*
- *arm/resources/features 2014-08-01-preview*
- *arm/resources/resources 2014-04-01-preview*
- *arm/resources/subscriptions 2014-04-01-preview*
- *arm/scheduler 2016-01-01*
- arm/storage 2015-06-15
- arm/web 2015-08-01
### Changes
- Moved the arm/authorization, arm/features, arm/resources, and arm/subscriptions packages under a new, resources, package (to reflect the corresponding Swagger structure)
- Added a new arm/authoriation (2015-07-01) package
- Added a new arm/cdn (2015-06-01) package
- Added a new arm/intune (2015-01-14-preview) package
- Udated arm/network (2015-06-01)
- Added a new arm/notificationhubs (2014-09-01) package
- Updated arm/scheduler (2016-01-01) package
-----
## `v0.3.0-beta`
- Corrected unintentional struct field renaming and client renaming in v0.2.0-beta
-----
## `v0.2.0-beta`
- Added support for DNS, Redis, and Web site services
- Updated Storage service to API version 2015-06-15
- Updated Network to include routing table support
- Address https://github.com/Azure/azure-sdk-for-go/issues/232
- Address https://github.com/Azure/azure-sdk-for-go/issues/231
- Address https://github.com/Azure/azure-sdk-for-go/issues/230
- Address https://github.com/Azure/azure-sdk-for-go/issues/224
- Address https://github.com/Azure/azure-sdk-for-go/issues/184
- Address https://github.com/Azure/azure-sdk-for-go/issues/183
------
## `v0.1.1-beta`
- Improves the UserAgent string to disambiguate arm packages from others in the SDK
- Improves setting the http.Response into generated results (reduces likelihood of a nil reference)
- Adds gofmt, golint, and govet to Travis CI for the arm packages
##### Fixed Issues
- https://github.com/Azure/azure-sdk-for-go/issues/196
- https://github.com/Azure/azure-sdk-for-go/issues/213
------
## v0.1.0-beta
This release addresses the issues raised against the alpha release and adds more features. Most
notably, to address the challenges of encoding JSON
(see the [comments](https://github.com/Azure/go-autorest#handling-empty-values) in the
[go-autorest](https://github.com/Azure/go-autorest) package) by using pointers for *all* structure
fields (with the exception of enumerations). The
[go-autorest/autorest/to](https://github.com/Azure/go-autorest/tree/master/autorest/to) package
provides helpers to convert to / from pointers. The examples demonstrate their usage.
Additionally, the packages now align with Go coding standards and pass both `golint` and `govet`.
Accomplishing this required renaming various fields and parameters (such as changing Url to URL).
##### Changes
- Changed request / response structures to use pointer fields.
- Changed methods to return `error` instead of `autorest.Error`.
- Re-divided methods to ease asynchronous requests.
- Added paged results support.
- Added a UserAgent string.
- Added changes necessary to pass golint and govet.
- Updated README.md with details on asynchronous requests and paging.
- Saved package dependencies through Godep (for the entire SDK).
##### Fixed Issues:
- https://github.com/Azure/azure-sdk-for-go/issues/205
- https://github.com/Azure/azure-sdk-for-go/issues/206
- https://github.com/Azure/azure-sdk-for-go/issues/211
- https://github.com/Azure/azure-sdk-for-go/issues/212
-----
## v0.1.0-alpha
This release introduces the Azure Resource Manager packages generated from the corresponding
[Swagger API](http://swagger.io) [definitions](https://github.com/Azure/azure-rest-api-specs).
vendor/github.com/Azure/azure-sdk-for-go/README.md
Generated Vendored
+59
View File
@@ -0,0 +1,59 @@
# Microsoft Azure SDK for Go
[![GoDoc](https://godoc.org/github.com/Azure/azure-sdk-for-go?status.svg)](https://godoc.org/github.com/Azure/azure-sdk-for-go)
[![Build Status](https://travis-ci.org/Azure/azure-sdk-for-go.svg?branch=master)](https://travis-ci.org/Azure/azure-sdk-for-go)
[![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-sdk-for-go)](https://goreportcard.com/report/github.com/Azure/azure-sdk-for-go)
This is Microsoft Azure's core repository for hosting Go packages which offer a more convenient way of targeting Azure
REST endpoints. Here, you'll find a mix of code generated by [Autorest](https://github.com/Azure/autorest) and hand
maintained packages.
> **NOTE:** This repository is under heavy ongoing development and should be considered a preview. Vendoring your
dependencies is always a good idea, but it is doubly important if you're consuming this library.
# Installation
- If you don't already have it, install [the Go Programming Language](https://golang.org/dl/).
- Go get the SDK:
```
$ go get -u github.com/Azure/azure-sdk-for-go/...
```
> **IMPORTANT:** We highly suggest vendoring Azure SDK for Go as a dependency. For vendoring dependencies, Azure SDK
for Go uses [glide](https://github.com/Masterminds/glide).
# Versioning
## SDK Versions
The tags in this repository are based on, but do not conform to [SemVer.org's recommendations](http://semver.org/).
For now, the "-beta" tag is an indicator that we are still in preview and still are planning on releasing some breaking
changes.
## Azure Versions
Azure services _mostly_ do not use SemVer based versions. Rather, they use profiles identified by dates. One will often
see this casually referred to as an "APIVersion". At the moment, our SDK only supports the most recent profiles. In
order to lock to an API version, one must also lock to an SDK version. However, as discussed in
[#517](https://github.com/Azure/azure-sdk-for-go/issues/517), our objective is to reorganize and publish independent
packages for each profile. In that way, we'll be able to have parallel support in a single SDK version for all
APIVersions supported by Azure.
# Documentation
- Azure SDK for Go Documentation is available at [GoDoc.org](http://godoc.org/github.com/Azure/azure-sdk-for-go/).
- Azure REST APIs used by packages in this repository are documented at [Microsoft Docs, Azure REST](https://docs.microsoft.com/en-us/rest/api/).
- Azure Services are discussed in detail at [Microsoft Docs, Azure Services](https://docs.microsoft.com/en-us/azure/#pivot=services).
# Code samples
- [Getting Started with Azure Blob Service in Go](https://github.com/Azure-Samples/storage-blob-go-getting-started)
# License
This project is published under [Apache 2.0 License](LICENSE).
# Contribute
If you would like to become an active contributor to this project please follow the instructions provided in [Microsoft
Azure Projects Contribution Guidelines](http://azure.github.io/guidelines/).
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact
[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
vendor/github.com/Azure/azure-sdk-for-go/arm/README.md
Generated Vendored
+285
View File
@@ -0,0 +1,285 @@
# Introducing the Azure Resource Manager packages for Go
The `github.com/Azure/azure-sdk-for-go/arm` packages are used to perform operations using the Azure Resource Manager (ARM). Read more about [Azure Resource Manager vs. classic deployment](https://azure.microsoft.com/documentation/articles/resource-manager-deployment-model/). Packages for Azure Service Manager or classic deployment are in the [management](https://github.com/Azure/azure-sdk-for-go/tree/master/management) folder.
## How Did We Get Here?
Azure is growing rapidly, regularly adding new services and features. While rapid growth
is good for users, it is hard on SDKs. Each new service and each new feature requires someone to
learn the details and add the needed code to the SDK. As a result, the
[Azure SDK for Go](https://github.com/Azure/azure-sdk-for-go)
has lagged behind Azure. It is missing
entire services and has not kept current with features. There is simply too much change to maintain
a hand-written SDK.
For this reason, the
[Azure SDK for Go](https://github.com/Azure/azure-sdk-for-go),
with the release of the Azure Resource Manager (ARM)
packages, is transitioning to a generated-code model. Other Azure SDKs, notably the
[Azure SDK for .NET](https://github.com/Azure/azure-sdk-for-net), have successfully adopted a
generated-code strategy. Recently, Microsoft published the
[AutoRest](https://github.com/Azure/autorest) tool used to create these SDKs and we have been adding support for Go. The ARM packages are
the first set generated using this new toolchain. The input for AutoRest are the [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs), files in Swagger JSON format.
There are a couple of items to note. First, since both the tooling and the underlying support
packages are new, the code is not yet "production ready". Treat these packages as of
***beta*** quality.
That's not to say we don't believe in the code, but we want to see what others think and how well
they work in a variety of environments before settling down into an official, first release. If you
find problems or have suggestions, please submit a pull request to document what you find. However,
since the code is generated, we'll use your pull request to guide changes we make to the underlying
generator versus merging the pull request itself.
The second item of note is that, to keep the generated code clean and reliable, it depends on
another new package [go-autorest](https://github.com/Azure/go-autorest).
Though part of the SDK, we separated the code to better control versioning and maintain agility.
Since
[go-autorest](https://github.com/Azure/go-autorest)
is hand-crafted, we will take pull requests in the same manner as for our other repositories.
We intend to rapidly improve these packages until they are "production ready".
So, try them out and give us your thoughts.
## What Have We Done?
Creating new frameworks is hard and often leads to "cliffs": The code is easy to use until some
special case or tweak arises and then, well, then you're stuck. Often times small differences in
requirements can lead to forking the code and investing a lot of time. Cliffs occur even more
frequently in generated code. We wanted to avoid them and believe the new model does. Our initial
goals were:
* Easy-to-use out of the box. It should be "clone and go" for straight-forward use.
* Easy composition to handle the majority of complex cases.
* Easy to integrate with existing frameworks, fit nicely with channels, supporting fan-out /
fan-in set ups.
These are best shown in a series of examples, all of which are included in the
[examples](/arm/examples) sub-folder.
## How is the SDK tested?
Testing the SDK is currently a work in progress. It includes three different points:
* Test the [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs) against the APIs themselves. This way we can find if the specs are reflecting correctly the API behavior. All Azure SDKs can benefit from this tests.
* Add [acceptance tests](https://github.com/Azure/autorest/blob/master/docs/developer/guide/writing-tests.md) to AutoRest.
* Test the generated SDK with code samples. This would catch bugs that escaped the previous tests, and provide some documentation.
## First a Sidenote: Authentication and the Azure Resource Manager
Before using the Azure Resource Manager packages, you need to understand how it authenticates and
authorizes requests.
Azure Resource Manager requests can be authorized through [OAuth2](http://oauth.net). While OAuth2 provides many advantages over
certificates, programmatic use, such as for scripts on headless servers, requires understanding and
creating one or more *Service Principals.*
The Azure-SDK-for-Node has an excellent tutorial that includes instructions for how to create Service Principals in the Portal and using the Azure CLI, both of which are applicable to Go.
Find that documentation here: [Authenticaion, Azure/azure-sdk-for-node](https://github.com/Azure/azure-sdk-for-node/blob/master/Documentation/Authentication.md)
In addition, there are several good blog posts, such as
[Automating Azure on your CI server using a Service Principal](http://blog.davidebbo.com/2014/12/azure-service-principal.html)
and
[Microsoft Azure REST API + OAuth 2.0](https://ahmetalpbalkan.com/blog/azure-rest-api-with-oauth2/),
that describe what this means.
For details on creating and authorizing Service Principals, see the MSDN articles
[Azure API Management REST API Authentication](https://msdn.microsoft.com/library/azure/5b13010a-d202-4af5-aabf-7ebc26800b3d)
and
[Create a new Azure Service Principal using the Azure portal](https://azure.microsoft.com/documentation/articles/resource-group-create-service-principal-portal/).
Dushyant Gill, a Senior Program Manager for Azure Active Directory, has written an extensive blog
post,
[Developer's Guide to Auth with Azure Resource Manager API](http://www.dushyantgill.com/blog/2015/05/23/developers-guide-to-auth-with-azure-resource-manager-api/),
that is also quite helpful.
### Complete source code
Get code for a full example of [authenticating to Azure via certificate or device authorization](https://github.com/Azure/go-autorest/tree/master/autorest/azure/example).
## A Simple Example: Checking availability of name within Azure Storage
Each ARM provider, such as
[Azure Storage](http://azure.microsoft.com/documentation/services/storage/)
or
[Azure Compute](https://azure.microsoft.com/documentation/services/virtual-machines/),
has its own package. Start by importing
the packages for the providers you need. Next, most packages divide their APIs across multiple
clients to avoid name collision and improve usability. For example, the
[Azure Storage](http://azure.microsoft.com/documentation/services/storage/)
package has
two clients:
[storage.AccountsClient](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#AccountsClient)
and
[storage.UsageOperationsClient](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#UsageOperationsClient).
To check if a name is available, use the
[storage.AccountsClient](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#AccountsClient).
Each ARM client composes with [autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client).
[autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client)
enables altering the behavior of the API calls by leveraging the decorator pattern of
[go-autorest](https://github.com/Azure/go-autorest). For example, in the code above, the
[azure.ServicePrincipalToken](https://godoc.org/github.com/Azure/go-autorest/autorest/azure#ServicePrincipalToken)
includes a
[WithAuthorization](https://godoc.org/github.com/Azure/go-autorest/autorest#Client.WithAuthorization)
[autorest.PrepareDecorator](https://godoc.org/github.com/Azure/go-autorest/autorest#PrepareDecorator)
that applies the OAuth2 authorization token to the request. It will, as needed, refresh the token
using the supplied credentials.
Providing a decorated
[autorest.Sender](https://godoc.org/github.com/Azure/go-autorest/autorest#Sender) or populating
the [autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client)
with a custom
[autorest.PrepareDecorator](https://godoc.org/github.com/Azure/go-autorest/autorest#PrepareDecorator)
or
[autorest.RespondDecorator](https://godoc.org/github.com/Azure/go-autorest/autorest#RespondDecorator)
enables more control. See the included example file
[check.go](/arm/examples/check/check.go)
for more details. Through these you can modify the outgoing request, inspect the incoming response,
or even go so far as to provide a
[circuit breaker](https://msdn.microsoft.com/library/dn589784.aspx)
to protect your service from unexpected latencies.
Lastly, all Azure ARM API calls return an instance of
[autorest.DetailedError](https://godoc.org/github.com/Azure/go-autorest/autorest#DetailedError).
Not only DetailedError gives anonymous access to the original
[error](http://golang.org/ref/spec#Errors),
but provides the package type (e.g.,
[storage.AccountsClient](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#AccountsClient)),
the failing method (e.g.,
[CheckNameAvailability](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#AccountsClient.CheckNameAvailability)),
and a detailed error message.
### Complete source code
Complete source code for this example can be found in [check.go](/arm/examples/check/check.go).
1. Create a [service principal](https://azure.microsoft.com/documentation/articles/resource-group-authenticate-service-principal-cli/). You will need the Tenant ID, Client ID and Client Secret for [authentication](#first-a-sidenote-authentication-and-the-azure-resource-manager), so keep them as soon as you get them.
2. Get your Azure Subscription ID using either of the methods mentioned below:
- Get it through the [portal](portal.azure.com) in the subscriptions section.
- Get it using the [Azure CLI](https://azure.microsoft.com/documentation/articles/xplat-cli-install/) with command `azure account show`.
- Get it using [Azure Powershell](https://azure.microsoft.com/documentation/articles/powershell-install-configure/) with cmdlet `Get-AzureRmSubscription`.
3. Set environment variables `AZURE_TENANT_ID = <TENANT_ID>`, `AZURE_CLIENT_ID = <CLIENT_ID>`, `AZURE_CLIENT_SECRET = <CLIENT_SECRET>` and `AZURE_SUBSCRIPTION_ID = <SUBSCRIPTION_ID>`.
4. Run the sample with commands:
```
$ cd arm/examples/check
$ go run check.go
```
## Something a Bit More Complex: Creating a new Azure Storage account
Redundancy, both local and across regions, and service load affect service responsiveness. Some
API calls will return before having completed the request. An Azure ARM API call indicates the
request is incomplete (versus the request failed for some reason) by returning HTTP status code
'202 Accepted.' The
[autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client)
composed into
all of the Azure ARM clients, provides support for basic request polling. The default is to
poll until a specified duration has passed (with polling frequency determined by the
HTTP [Retry-After](http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.37)
header in the response). By changing the
[autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client)
settings, you can poll for a fixed number of attempts or elect to not poll at all.
Whether you elect to poll or not, all Azure ARM client responses compose with an instance of
[autorest.Response](https://godoc.org/github.com/Azure/go-autorest/autorest#Response).
At present,
[autorest.Response](https://godoc.org/github.com/Azure/go-autorest/autorest#Response)
only composes over the standard
[http.Response](https://golang.org/pkg/net/http/#Response)
object (that may change as we implement more features). When your code receives an error from an
Azure ARM API call, you may find it useful to inspect the HTTP status code contained in the returned
[autorest.Response](https://godoc.org/github.com/Azure/go-autorest/autorest#Response).
If, for example, it is an HTTP 202, then you can use the
[GetPollingLocation](https://godoc.org/github.com/Azure/go-autorest/autorest#Response.GetPollingLocation)
response method to extract the URL at which to continue polling. Similarly, the
[GetPollingDelay](https://godoc.org/github.com/Azure/go-autorest/autorest#Response.GetPollingDelay)
response method returns, as a
[time.Duration](http://golang.org/pkg/time/#Duration),
the service suggested minimum polling delay.
Creating a new Azure storage account is a straight-forward way to see these concepts.
[autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client)
portion of the
[storage.AccountsClient](https://godoc.org/github.com/Azure/azure-sdk-for-go/arm/storage#AccountsClient)
to poll for a fixed number of attempts versus polling for a set duration (which is the default).
If an error occurs creating the storage account, the code inspects the HTTP status code and
prints the URL the
[Azure Storage](http://azure.microsoft.com/documentation/services/storage/)
service returned for polling.
### Complete source for the example
More details, including deleting the created account, are in the example code file [create.go](/arm/examples/create/create.go)
1. Create a [service principal](https://azure.microsoft.com/documentation/articles/resource-group-authenticate-service-principal-cli/). You will need the Tenant ID, Client ID and Client Secret for [authentication](#first-a-sidenote-authentication-and-the-azure-resource-manager), so keep them as soon as you get them.
2. Get your Azure Subscription ID using either of the methods mentioned below:
- Get it through the [portal](portal.azure.com) in the subscriptions section.
- Get it using the [Azure CLI](https://azure.microsoft.com/documentation/articles/xplat-cli-install/) with command `azure account show`.
- Get it using [Azure Powershell](https://azure.microsoft.com/documentation/articles/powershell-install-configure/) with cmdlet `Get-AzureRmSubscription`.
3. Set environment variables `AZURE_TENANT_ID = <TENANT_ID>`, `AZURE_CLIENT_ID = <CLIENT_ID>`, `AZURE_CLIENT_SECRET = <CLIENT_SECRET>` and `AZURE_SUBSCRIPTION_ID = <SUBSCRIPTION_ID>`.
4. Create a resource group and add its name in the first line of the main function.
5. Run the example with commands:
```
$ cd arm/examples/create
$ go run create.go
```
## Making Asynchronous Requests
One of Go's many strong points is how natural it makes sending and managing asynchronous requests
by means of goroutines. We wanted the ARM packages to fit naturally in the variety of asynchronous
patterns used in Go code, but also be straight-forward for simple use cases. We accomplished both
by adopting a pattern for all APIs. Each package API includes (at least) four methods
(more if the API returns a paged result set). For example, for an API call named `Foo` the package
defines:
- `FooPreparer`: This method accepts the arguments for the API and returns a prepared
`http.Request`.
- `FooSender`: This method sends the prepared `http.Request`. It handles the possible status codes
and will, unless the disabled in the [autorest.Client](https://godoc.org/github.com/Azure/go-autorest/autorest#Client), handling polling.
- `FooResponder`: This method accepts and handles the `http.Response` returned by the sender
and unmarshals the JSON, if any, into the result.
- `Foo`: This method accepts the arguments for the API and returns the result. It is a wrapper
around the `FooPreparer`, `FooSender`, and `FooResponder`.
By using the preparer, sender, and responder methods, package users can spread request and
response handling across goroutines as needed. Further, adding a cancel channel to the
`http.Response` (most easily through a
[PrepareDecorator](https://godoc.org/github.com/Azure/go-autorest/autorest#PrepareDecorator)),
enables canceling sent requests (see the documentation on
[http.Request](https://golang.org/pkg/net/http/#Request)) for details.
## Paged Result Sets
Some API calls return partial results. Typically, when they do, the result structure will include
a `Value` array and a `NextLink` URL. The `NextLink` URL is used to retrieve the next page or
block of results.
The packages add two methods to make working with and retrieving paged results natural. First,
on paged result structures, the packages include a preparer method that returns an `http.Request`
for the next set of results. For a result set returned in a structure named `FooResults`, the
package will include a method named `FooResultsPreparer`. If the `NextLink` is `nil` or empty, the
method returns `nil`.
The corresponding API (which typically includes "List" in the name) has a method to ease retrieving
the next result set given a result set. For example, for an API named `FooList`, the package will
include `FooListNextResults` that accepts the results of the last call and returns the next set.
## Summing Up
The new Azure Resource Manager packages for the Azure SDK for Go are a big step toward keeping the
SDK current with Azure's rapid growth.
As mentioned, we intend to rapidly stabilize these packages for production use.
We'll also add more examples, including some highlighting the
[Azure Resource Manager Templates](https://msdn.microsoft.com/library/azure/dn790568.aspx)
and the other providers.
So, give the packages a try, explore the various ARM providers, and let us know what you think.
We look forward to hearing from you!
## License
See the Azure SDK for Go LICENSE file.
vendor/github.com/Azure/azure-sdk-for-go/arm/disk/version.go
Generated Vendored
+4 -5
View File
@@ -14,16 +14,15 @@ package disk
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Code generated by Microsoft (R) AutoRest Code Generator 1.0.1.0
// Changes may cause incorrect behavior and will be lost if the code is
// regenerated.
// Code generated by Microsoft (R) AutoRest Code Generator 1.1.0.0
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// UserAgent returns the UserAgent string to use when sending http.Requests.
func UserAgent() string {
return "Azure-SDK-For-Go/v10.0.2-beta arm-disk/2016-04-30-preview"
return "Azure-SDK-For-Go/v10.2.0-beta arm-disk/2016-04-30-preview"
}
// Version returns the semantic version (see http://semver.org) of the client.
func Version() string {
return "v10.0.2-beta"
return "v10.2.0-beta"
}
vendor/github.com/Azure/azure-sdk-for-go/arm/examples/helpers/helpers.go
Generated Vendored
+4 -4
View File
@@ -3,7 +3,7 @@ package helpers
import (
"encoding/json"
"fmt"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/Azure/go-autorest/autorest/azure"
)
@@ -20,12 +20,12 @@ func ToJSON(v interface{}) (string, error) {
// NewServicePrincipalTokenFromCredentials creates a new ServicePrincipalToken using values of the
// passed credentials map.
func NewServicePrincipalTokenFromCredentials(c map[string]string, scope string) (*azure.ServicePrincipalToken, error) {
oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(c["AZURE_TENANT_ID"])
func NewServicePrincipalTokenFromCredentials(c map[string]string, scope string) (*adal.ServicePrincipalToken, error) {
oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, c["AZURE_TENANT_ID"])
if err != nil {
panic(err)
}
return azure.NewServicePrincipalToken(*oauthConfig, c["AZURE_CLIENT_ID"], c["AZURE_CLIENT_SECRET"], scope)
return adal.NewServicePrincipalToken(*oauthConfig, c["AZURE_CLIENT_ID"], c["AZURE_CLIENT_SECRET"], scope)
}
func ensureValueStrings(mapOfInterface map[string]interface{}) map[string]string {
vendor/github.com/Azure/azure-sdk-for-go/arm/resources/subscriptions/version.go
Generated Vendored
+4 -5
View File
@@ -14,16 +14,15 @@ package subscriptions
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Code generated by Microsoft (R) AutoRest Code Generator 1.0.1.0
// Changes may cause incorrect behavior and will be lost if the code is
// regenerated.
// Code generated by Microsoft (R) AutoRest Code Generator 1.1.0.0
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// UserAgent returns the UserAgent string to use when sending http.Requests.
func UserAgent() string {
return "Azure-SDK-For-Go/v10.0.2-beta arm-subscriptions/2016-06-01"
return "Azure-SDK-For-Go/v10.2.0-beta arm-subscriptions/2016-06-01"
}
// Version returns the semantic version (see http://semver.org) of the client.
func Version() string {
return "v10.0.2-beta"
return "v10.2.0-beta"
}
vendor/github.com/Azure/azure-sdk-for-go/buildTerraform.sh
Generated Vendored
+36
View File
@@ -0,0 +1,36 @@
# This script tries to build Terraform related packages,
# and find possible breaking changes regarding the Azure
# SDK for Go
set -x
# This should only run on cronjobs
if [ "cron" != $TRAVIS_EVENT_TYPE ]; then
exit 0
fi
# Only meant to run on latest go version
if [ "go version go1.8 linux/amd64" != "$(go version)" ]; then
exit 0
fi
go get github.com/kardianos/govendor
REALEXITSTATUS=0
packages=(github.com/hashicorp/terraform
github.com/terraform-providers/terraform-provider-azurerm
github.com/terraform-providers/terraform-provider-azure)
for package in ${packages[*]}; do
go get $package
cd $GOPATH/src/$package
# update to latest SDK
govendor update github.com/Azure/azure-sdk-for-go/...
# try to build
make
REALEXITSTATUS=$(($REALEXITSTATUS+$?))
done
exit $REALEXITSTATUS
vendor/github.com/Azure/azure-sdk-for-go/glide.lock
Generated Vendored
+63
View File
@@ -0,0 +1,63 @@
hash: bdeb606eca05304f80e5577159227ae4f00a002213f353e304066c593419a31b
updated: 2017-07-31T15:32:36.5529621-07:00
imports:
- name: github.com/Azure/go-autorest
version: f6e08fe5e4d45c9a66e40196d3fed5f37331d224
subpackages:
- autorest
- autorest/adal
- autorest/azure
- autorest/date
- autorest/to
- autorest/validation
- name: github.com/dgrijalva/jwt-go
version: a539ee1a749a2b895533f979515ac7e6e0f5b650
- name: github.com/howeyc/gopass
version: bf9dde6d0d2c004a008c27aaee91170c786f6db8
- name: github.com/mattn/go-colorable
version: 040dd0b833b34b4182673409ada4df2853b26537
- name: github.com/mattn/go-isatty
version: fc9e8d8ef48496124e79ae0df75490096eccf6fe
- name: github.com/mgutz/ansi
version: 9520e82c474b0a04dd04f8a40959027271bab992
- name: github.com/mgutz/minimist
version: 39eb8cf573ca29344bd7d7e6ba4d7febdebd37a9
- name: github.com/mgutz/str
version: 968bf66e3da857419e4f6e71b2d5c9ae95682dc4
- name: github.com/mgutz/to
version: 00c06406c2dd2e011f153a6502a21473676db33f
- name: github.com/MichaelTJones/walk
version: 4748e29d5718c2df4028a6543edf86fd8cc0f881
- name: github.com/nozzle/throttler
version: d9b45f19996c645d38c9266d1f5cf1990e930119
- name: github.com/satori/uuid
version: 5bf94b69c6b68ee1b541973bb8e1144db23a194b
- name: github.com/shopspring/decimal
version: 3c692774ac4c47c7a296ec96e553719dba1a68fc
- name: golang.org/x/crypto
version: 558b6879de74bc843225cde5686419267ff707ca
subpackages:
- pkcs12
- pkcs12/internal/rc2
- ssh/terminal
- name: golang.org/x/sys
version: 0f826bdd13b500be0f1d4004938ad978fcc6031e
subpackages:
- unix
- name: gopkg.in/check.v1
version: 20d25e2804050c1cd24a7eea1e7a6447dd0e74ec
- name: gopkg.in/godo.v2
version: b5fd2f0bef1ebe832e628cfad18ab1cc707f65a1
subpackages:
- glob
- util
- watcher
- watcher/fswatch
testImports:
- name: github.com/dnaeon/go-vcr
version: 87d4990451a858cc210399285be976e63bc3c364
subpackages:
- cassette
- recorder
- name: gopkg.in/yaml.v2
version: 25c4ec802a7d637f88d584ab26798e94ad14c13b
vendor/github.com/Azure/azure-sdk-for-go/glide.yaml
Generated Vendored
+13
View File
@@ -0,0 +1,13 @@
package: github.com/Azure/azure-sdk-for-go
import:
- package: github.com/Azure/go-autorest
version: ~8.1.1
subpackages:
- /autorest
- autorest/azure
- autorest/date
- autorest/to
- package: golang.org/x/crypto
subpackages:
- /pkcs12
- package: gopkg.in/check.v1
vendor/github.com/Azure/azure-sdk-for-go/rungas.sh
Generated Vendored
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
GITBRANCH=`git rev-parse --abbrev-ref HEAD`
#We intend to only run gas on release branches.
if [ "master" != $GITBRANCH ]; then
exit 0
fi
REALEXITSTATUS=0
go get -u github.com/HewlettPackard/gas
gas -skip=*/arm/*/models.go -skip=*/management/examples/*.go -skip=*vendor* -skip=*/Gododir/* ./... | tee /dev/stderr
REALEXITSTATUS=$(($REALEXITSTATUS+$?))
gas -exclude=G101 ./arm/... ./management/examples/... | tee /dev/stderr
REALEXITSTATUS=$(($REALEXITSTATUS+$?))
gas -exclude=G204 ./Gododir/... | tee /dev/stderr
REALEXITSTATUS=$(($REALEXITSTATUS+$?))
exit $REALEXITSTATUS

Some files were not shown because too many files have changed in this diff Show More