diff --git a/.gitignore b/.gitignore index 6e9f9babb..fb6e4e241 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ _testmain.go debug /ark +.idea/ diff --git a/Makefile b/Makefile index 65cf51b6a..ac27ccf81 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,11 @@ VERSION ?= v0.3.3 GOTARGET = github.com/heptio/$(PROJECT) OUTPUT_DIR = $(ROOT_DIR)/_output BIN_DIR = $(OUTPUT_DIR)/bin +GIT_SHA=$(shell git rev-parse --short HEAD) +GIT_DIRTY=$(shell git status --porcelain $(ROOT_DIR) 2> /dev/null) +ifneq ($(GIT_DIRTY),) + GIT_SHA := $(GIT_SHA)-dirty +endif # docker related vars DOCKER ?= docker @@ -26,7 +31,7 @@ REGISTRY ?= gcr.io/heptio-images BUILD_IMAGE ?= gcr.io/heptio-images/golang:1.8-alpine3.6 # go build -i installs compiled packages so they can be reused later. # This speeds up recompiles. -BUILDCMD = go build -i -v -ldflags "-X $(GOTARGET)/pkg/buildinfo.Version=$(VERSION) -X $(GOTARGET)/pkg/buildinfo.DockerImage=$(REGISTRY)/$(PROJECT)" +BUILDCMD = go build -i -v -ldflags "-X $(GOTARGET)/pkg/buildinfo.Version=$(VERSION) -X $(GOTARGET)/pkg/buildinfo.DockerImage=$(REGISTRY)/$(PROJECT) -X $(GOTARGET)/pkg/buildinfo.GitSHA=$(GIT_SHA)" BUILDMNT = /go/src/$(GOTARGET) EXTRA_MNTS ?= @@ -44,6 +49,10 @@ $(BINARIES): mkdir -p $(BIN_DIR) $(BUILDCMD) -o $(BIN_DIR)/$@ $(GOTARGET)/cmd/$@ +fmt: + gofmt -w=true $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/generated/*") + goimports -w=true -d $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/generated/*") + test: ifneq ($(SKIP_TESTS), 1) # go test -i installs compiled packages so they can be reused later @@ -60,7 +69,7 @@ ifneq ($(SKIP_TESTS), 1) ${ROOT_DIR}/hack/verify-generated-informers.sh endif -update: +update: fmt ${ROOT_DIR}/hack/update-generated-clientsets.sh ${ROOT_DIR}/hack/update-generated-listers.sh ${ROOT_DIR}/hack/update-generated-informers.sh @@ -80,7 +89,7 @@ container-local: $(BINARIES) push: docker -- push $(REGISTRY)/$(PROJECT):$(VERSION) -.PHONY: all local container cbuild push test verify update $(BINARIES) +.PHONY: all local container cbuild push test verify update fmt $(BINARIES) clean: rm -rf $(OUTPUT_DIR) diff --git a/docs/cli-reference/ark_backup.md b/docs/cli-reference/ark_backup.md index aabdc0a06..35638f0d7 100644 --- a/docs/cli-reference/ark_backup.md +++ b/docs/cli-reference/ark_backup.md @@ -29,6 +29,7 @@ Work with backups ### SEE ALSO * [ark](ark.md) - Back up and restore Kubernetes cluster resources. * [ark backup create](ark_backup_create.md) - Create a backup +* [ark backup download](ark_backup_download.md) - Download a backup * [ark backup get](ark_backup_get.md) - Get backups * [ark backup logs](ark_backup_logs.md) - Get backup logs diff --git a/docs/cli-reference/ark_backup_download.md b/docs/cli-reference/ark_backup_download.md new file mode 100644 index 000000000..89621488a --- /dev/null +++ b/docs/cli-reference/ark_backup_download.md @@ -0,0 +1,38 @@ +## ark backup download + +Download a backup + +### Synopsis + + +Download a backup + +``` +ark backup download NAME [flags] +``` + +### Options + +``` + --force forces the download and will overwrite file if it exists already + -h, --help help for download + --output-dir string directory to download backup to. (Default cwd) + --timeout duration maximum time to wait to process download request (default 1m0s) +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + --kubeconfig string Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + +### SEE ALSO +* [ark backup](ark_backup.md) - Work with backups + diff --git a/docs/generate/ark.go b/docs/generate/ark.go index 3d716cdbd..d64cf8d2a 100644 --- a/docs/generate/ark.go +++ b/docs/generate/ark.go @@ -18,17 +18,17 @@ package main import ( "log" - "os" + "os" - "github.com/spf13/cobra/doc" "github.com/heptio/ark/pkg/cmd/ark" + "github.com/spf13/cobra/doc" ) func main() { - cmdName := os.Args[1] + cmdName := os.Args[1] outputDir := os.Args[2] - cmd := ark.NewCommand(cmdName) + cmd := ark.NewCommand(cmdName) // Remove auto-generated timestamps cmd.DisableAutoGenTag = true diff --git a/pkg/apis/ark/v1/download_request.go b/pkg/apis/ark/v1/download_request.go index feadaf22e..fcbb61dc0 100644 --- a/pkg/apis/ark/v1/download_request.go +++ b/pkg/apis/ark/v1/download_request.go @@ -28,7 +28,8 @@ type DownloadRequestSpec struct { type DownloadTargetKind string const ( - DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog" + DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog" + DownloadTargetKindBackupContents DownloadTargetKind = "BackupContents" ) // DownloadTarget is the specification for what kind of file to download, and the name of the diff --git a/pkg/buildinfo/version.go b/pkg/buildinfo/version.go index acf7ae83b..8d2989168 100644 --- a/pkg/buildinfo/version.go +++ b/pkg/buildinfo/version.go @@ -25,3 +25,6 @@ var Version string // DockerImage is the full path to the docker image for this build, for example // gcr.io/heptio-images/ark. var DockerImage string + +// GitSHA is the actual commit that is being built, set by the go linker's -X flag at build time. +var GitSHA string diff --git a/pkg/cloudprovider/backup_service.go b/pkg/cloudprovider/backup_service.go index ff6863584..befa8b1aa 100644 --- a/pkg/cloudprovider/backup_service.go +++ b/pkg/cloudprovider/backup_service.go @@ -53,7 +53,7 @@ type BackupService interface { // CreateBackupLogSignedURL creates a pre-signed URL that can be used to download a backup's log // file from object storage. The URL expires after ttl. - CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error) + CreateBackupSignedURL(backupType api.DownloadTargetKind, bucket, backupName string, ttl time.Duration) (string, error) } // BackupGetter knows how to list backups in object storage. @@ -72,7 +72,7 @@ func getMetadataKey(backup string) string { return fmt.Sprintf(metadataFileFormatString, backup) } -func getBackupKey(backup string) string { +func getBackupContentsKey(backup string) string { return fmt.Sprintf(backupFileFormatString, backup, backup) } @@ -105,7 +105,7 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu } // upload tar file - if err := br.objectStorage.PutObject(bucket, getBackupKey(backupName), backup); err != nil { + if err := br.objectStorage.PutObject(bucket, getBackupContentsKey(backupName), backup); err != nil { // try to delete the metadata file since the data upload failed deleteErr := br.objectStorage.DeleteObject(bucket, metadataKey) @@ -123,7 +123,7 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu } func (br *backupService) DownloadBackup(bucket, backupName string) (io.ReadCloser, error) { - return br.objectStorage.GetObject(bucket, getBackupKey(backupName)) + return br.objectStorage.GetObject(bucket, getBackupContentsKey(backupName)) } func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) { @@ -194,8 +194,15 @@ func (br *backupService) DeleteBackupDir(bucket, backupName string) error { return errors.NewAggregate(errs) } -func (br *backupService) CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error) { - return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(backupName), ttl) +func (br *backupService) CreateBackupSignedURL(backupType api.DownloadTargetKind, bucket, backupName string, ttl time.Duration) (string, error) { + switch backupType { + case api.DownloadTargetKindBackupContents: + return br.objectStorage.CreateSignedURL(bucket, getBackupContentsKey(backupName), ttl) + case api.DownloadTargetKindBackupLog: + return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(backupName), ttl) + default: + return "", fmt.Errorf("unsupported download target kind %q", backupType) + } } // cachedBackupService wraps a real backup service with a cache for getting cloud backups. diff --git a/pkg/cmd/cli/backup/backup.go b/pkg/cmd/cli/backup/backup.go index 96ece4f17..63db36a3e 100644 --- a/pkg/cmd/cli/backup/backup.go +++ b/pkg/cmd/cli/backup/backup.go @@ -33,6 +33,7 @@ func NewCommand(f client.Factory) *cobra.Command { NewCreateCommand(f), NewGetCommand(f), NewLogsCommand(f), + NewDownloadCommand(f), // Will implement describe later // NewDescribeCommand(f), diff --git a/pkg/cmd/cli/backup/download.go b/pkg/cmd/cli/backup/download.go new file mode 100644 index 000000000..f590bbb17 --- /dev/null +++ b/pkg/cmd/cli/backup/download.go @@ -0,0 +1,113 @@ +/* +Copyright 2017 Heptio Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup + +import ( + "errors" + "fmt" + "os" + "path" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/heptio/ark/pkg/apis/ark/v1" + "github.com/heptio/ark/pkg/client" + "github.com/heptio/ark/pkg/cmd" + "github.com/heptio/ark/pkg/cmd/util/downloadrequest" +) + +func NewDownloadCommand(f client.Factory) *cobra.Command { + o := NewDownloadOptions() + c := &cobra.Command{ + Use: "download NAME", + Short: "Download a backup", + Run: func(c *cobra.Command, args []string) { + cmd.CheckError(o.Validate(c, args)) + cmd.CheckError(o.Complete(args)) + cmd.CheckError(o.Run(c, f)) + }, + } + + o.BindFlags(c.Flags()) + + return c +} + +type DownloadOptions struct { + Name string + Path string + Force bool + Timeout time.Duration + writeOptions int +} + +func NewDownloadOptions() *DownloadOptions { + return &DownloadOptions{ + Timeout: time.Minute, + } +} + +func (o *DownloadOptions) BindFlags(flags *pflag.FlagSet) { + flags.StringVar(&o.Path, "output-dir", "", "directory to download backup to. (Default cwd)") + flags.BoolVar(&o.Force, "force", o.Force, "forces the download and will overwrite file if it exists already") + flags.DurationVar(&o.Timeout, "timeout", o.Timeout, "maximum time to wait to process download request") +} + +func (o *DownloadOptions) Validate(c *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("backup name is required") + } + + o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_EXCL + if o.Force { + o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_TRUNC + } + + if o.Path == "" { + path, err := os.Getwd() + if err != nil { + return errors.New("an issue occurred attempting to determine the current working directory.") + } + o.Path = path + } + + return nil +} + +func (o *DownloadOptions) Complete(args []string) error { + o.Name = args[0] + return nil +} + +func (o *DownloadOptions) Run(c *cobra.Command, f client.Factory) error { + arkClient, err := f.Client() + cmd.CheckError(err) + + backupDest, err := os.OpenFile(path.Join(o.Path, fmt.Sprintf("%s-data.tar.gz", o.Name)), o.writeOptions, 0600) + if err != nil { + return err + } + defer backupDest.Close() + + err = downloadrequest.Stream(arkClient.ArkV1(), o.Name, v1.DownloadTargetKindBackupContents, backupDest, o.Timeout) + cmd.CheckError(err) + + fmt.Printf("Backup %s has been successfully downloaded to %s\n", o.Name, backupDest.Name()) + return nil +} diff --git a/pkg/cmd/version/version.go b/pkg/cmd/version/version.go index f5d070c0e..8eafe4329 100644 --- a/pkg/cmd/version/version.go +++ b/pkg/cmd/version/version.go @@ -29,7 +29,7 @@ func NewCommand() *cobra.Command { Use: "version", Short: "Print the ark version and associated image", Run: func(cmd *cobra.Command, args []string) { - fmt.Println(buildinfo.Version) + fmt.Printf("Version: [%s] - [%s]\n", buildinfo.Version, buildinfo.GitSHA) fmt.Println("Configured docker image:", buildinfo.DockerImage) }, } diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 0f798ba58..707ed01b8 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -216,13 +216,13 @@ const signedURLTTL = 10 * time.Minute // Processed, and persists the changes to storage. func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest) error { switch downloadRequest.Spec.Target.Kind { - case v1.DownloadTargetKindBackupLog: + case v1.DownloadTargetKindBackupLog, v1.DownloadTargetKindBackupContents: update, err := cloneDownloadRequest(downloadRequest) if err != nil { return err } - update.Status.DownloadURL, err = c.backupService.CreateBackupLogSignedURL(c.bucket, update.Spec.Target.Name, signedURLTTL) + update.Status.DownloadURL, err = c.backupService.CreateBackupSignedURL(downloadRequest.Spec.Target.Kind, c.bucket, update.Spec.Target.Name, signedURLTTL) if err != nil { return err } @@ -233,8 +233,8 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow _, err = c.downloadRequestClient.DownloadRequests(update.Namespace).Update(update) return err } - return fmt.Errorf("unsupported download target kind %q", downloadRequest.Spec.Target.Kind) + } // deleteIfExpired deletes downloadRequest if it has expired. diff --git a/pkg/util/test/backup_service.go b/pkg/util/test/backup_service.go index 85352d369..fe1140a11 100644 --- a/pkg/util/test/backup_service.go +++ b/pkg/util/test/backup_service.go @@ -28,7 +28,7 @@ type BackupService struct { } // CreateBackupLogSignedURL provides a mock function with given fields: bucket, backupName, ttl -func (_m *BackupService) CreateBackupLogSignedURL(bucket string, backupName string, ttl time.Duration) (string, error) { +func (_m *BackupService) CreateBackupSignedURL(backupType v1.DownloadTargetKind, bucket string, backupName string, ttl time.Duration) (string, error) { ret := _m.Called(bucket, backupName, ttl) var r0 string