add restic integration for doing pod volume backups/restores

Signed-off-by: Steve Kriss <steve@heptio.com>

pkg/cmd/ark/ark.go

@@ -30,8 +30,10 @@ import (
 	"github.com/heptio/ark/pkg/cmd/cli/describe"
 	"github.com/heptio/ark/pkg/cmd/cli/get"
 	"github.com/heptio/ark/pkg/cmd/cli/plugin"
+	"github.com/heptio/ark/pkg/cmd/cli/restic"
 	"github.com/heptio/ark/pkg/cmd/cli/restore"
 	"github.com/heptio/ark/pkg/cmd/cli/schedule"
+	"github.com/heptio/ark/pkg/cmd/daemonset"
 	"github.com/heptio/ark/pkg/cmd/server"
 	runplugin "github.com/heptio/ark/pkg/cmd/server/plugin"
 	"github.com/heptio/ark/pkg/cmd/version"
@@ -67,6 +69,8 @@ operations can also be performed as 'ark backup get' and 'ark schedule create'.`
 		delete.NewCommand(f),
 		cliclient.NewCommand(),
 		completion.NewCommand(),
+		daemonset.NewCommand(f),
+		restic.NewCommand(f),
 	)
 
 	// add the glog flags

pkg/cmd/cli/restic/init_repository.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restic
+
+import (
+	"crypto/rand"
+	"io/ioutil"
+
+	"github.com/heptio/ark/pkg/client"
+	"github.com/heptio/ark/pkg/cmd"
+	"github.com/heptio/ark/pkg/restic"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kclientset "k8s.io/client-go/kubernetes"
+)
+
+func NewInitRepositoryCommand(f client.Factory) *cobra.Command {
+	o := NewInitRepositoryOptions()
+
+	c := &cobra.Command{
+		Use:   "init-repository",
+		Short: "create an encryption key for a restic repository",
+		Long:  "create an encryption key for a restic repository",
+		Run: func(c *cobra.Command, args []string) {
+			cmd.CheckError(o.Complete(f))
+			cmd.CheckError(o.Validate(f))
+			cmd.CheckError(o.Run(f))
+		},
+	}
+
+	o.BindFlags(c.Flags())
+
+	return c
+}
+
+type InitRepositoryOptions struct {
+	Namespace string
+	KeyFile   string
+	KeyData   string
+	KeySize   int
+
+	kubeClient kclientset.Interface
+	keyBytes   []byte
+}
+
+func NewInitRepositoryOptions() *InitRepositoryOptions {
+	return &InitRepositoryOptions{
+		KeySize: 1024,
+	}
+}
+
+func (o *InitRepositoryOptions) BindFlags(flags *pflag.FlagSet) {
+	flags.StringVar(&o.KeyFile, "key-file", o.KeyFile, "Path to file containing the encryption key for the restic repository. Optional; if unset, Ark will generate a random key for you.")
+	flags.StringVar(&o.KeyData, "key-data", o.KeyData, "Encryption key for the restic repository. Optional; if unset, Ark will generate a random key for you.")
+	flags.IntVar(&o.KeySize, "key-size", o.KeySize, "Size of the generated key for the restic repository")
+}
+
+func (o *InitRepositoryOptions) Complete(f client.Factory) error {
+	if o.KeyFile != "" && o.KeyData != "" {
+		return errors.Errorf("only one of --key-file and --key-data may be specified")
+	}
+
+	if o.KeyFile == "" && o.KeyData == "" && o.KeySize < 1 {
+		return errors.Errorf("--key-size must be at least 1")
+	}
+
+	o.Namespace = f.Namespace()
+
+	if o.KeyFile != "" {
+		data, err := ioutil.ReadFile(o.KeyFile)
+		if err != nil {
+			return err
+		}
+		o.keyBytes = data
+	}
+
+	if len(o.KeyData) == 0 {
+		o.keyBytes = make([]byte, o.KeySize)
+		// rand.Reader always returns a nil error
+		_, _ = rand.Read(o.keyBytes)
+	}
+
+	return nil
+}
+
+func (o *InitRepositoryOptions) Validate(f client.Factory) error {
+	if len(o.keyBytes) == 0 {
+		return errors.Errorf("keyBytes is required")
+	}
+
+	kubeClient, err := f.KubeClient()
+	if err != nil {
+		return err
+	}
+	o.kubeClient = kubeClient
+
+	if _, err := kubeClient.CoreV1().Namespaces().Get(o.Namespace, metav1.GetOptions{}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (o *InitRepositoryOptions) Run(f client.Factory) error {
+	return restic.NewRepositoryKey(o.kubeClient.CoreV1(), o.Namespace, o.keyBytes)
+}

pkg/cmd/cli/restic/restic.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restic
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/heptio/ark/pkg/client"
+)
+
+func NewCommand(f client.Factory) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "restic",
+		Short: "Work with restic repositories",
+		Long:  "Work with restic repositories",
+	}
+
+	c.AddCommand(
+		NewInitRepositoryCommand(f),
+	)
+
+	return c
+}

pkg/cmd/daemonset/daemonset.go

@@ -0,0 +1,156 @@
+package daemonset
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeinformers "k8s.io/client-go/informers"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+
+	"github.com/heptio/ark/pkg/buildinfo"
+	"github.com/heptio/ark/pkg/client"
+	"github.com/heptio/ark/pkg/cmd"
+	"github.com/heptio/ark/pkg/cmd/util/signals"
+	"github.com/heptio/ark/pkg/controller"
+	clientset "github.com/heptio/ark/pkg/generated/clientset/versioned"
+	informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
+	"github.com/heptio/ark/pkg/util/logging"
+)
+
+func NewCommand(f client.Factory) *cobra.Command {
+	var logLevelFlag = logging.LogLevelFlag(logrus.InfoLevel)
+
+	var command = &cobra.Command{
+		Use:   "daemonset",
+		Short: "Run the ark daemonset",
+		Long:  "Run the ark daemonset",
+		Run: func(c *cobra.Command, args []string) {
+			logLevel := logLevelFlag.Parse()
+			logrus.Infof("setting log-level to %s", strings.ToUpper(logLevel.String()))
+
+			logger := logging.DefaultLogger(logLevel)
+			logger.Infof("Starting Ark restic daemonset %s", buildinfo.FormattedGitSHA())
+
+			s, err := newDaemonServer(logger, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()))
+			cmd.CheckError(err)
+
+			s.run()
+		},
+	}
+
+	command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("the level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", ")))
+
+	return command
+}
+
+type daemonServer struct {
+	kubeClient          kubernetes.Interface
+	arkClient           clientset.Interface
+	arkInformerFactory  informers.SharedInformerFactory
+	kubeInformerFactory kubeinformers.SharedInformerFactory
+	podInformer         cache.SharedIndexInformer
+	logger              logrus.FieldLogger
+	ctx                 context.Context
+	cancelFunc          context.CancelFunc
+}
+
+func newDaemonServer(logger logrus.FieldLogger, baseName string) (*daemonServer, error) {
+	clientConfig, err := client.Config("", "", baseName)
+	if err != nil {
+		return nil, err
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	arkClient, err := clientset.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	// use a stand-alone pod informer because we want to use a field selector to
+	// filter to only pods scheduled on this node.
+	podInformer := corev1informers.NewFilteredPodInformer(
+		kubeClient,
+		"",
+		0,
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+		func(opts *metav1.ListOptions) {
+			opts.FieldSelector = fmt.Sprintf("spec.nodeName=%s", os.Getenv("NODE_NAME"))
+		},
+	)
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+
+	return &daemonServer{
+		kubeClient:          kubeClient,
+		arkClient:           arkClient,
+		arkInformerFactory:  informers.NewFilteredSharedInformerFactory(arkClient, 0, os.Getenv("HEPTIO_ARK_NAMESPACE"), nil),
+		kubeInformerFactory: kubeinformers.NewSharedInformerFactory(kubeClient, 0),
+		podInformer:         podInformer,
+		logger:              logger,
+		ctx:                 ctx,
+		cancelFunc:          cancelFunc,
+	}, nil
+}
+
+func (s *daemonServer) run() {
+	signals.CancelOnShutdown(s.cancelFunc, s.logger)
+
+	s.logger.Info("Starting controllers")
+
+	var wg sync.WaitGroup
+
+	backupController := controller.NewPodVolumeBackupController(
+		s.logger,
+		s.arkInformerFactory.Ark().V1().PodVolumeBackups(),
+		s.arkClient.ArkV1(),
+		s.podInformer,
+		s.kubeInformerFactory.Core().V1().Secrets(),
+		s.kubeInformerFactory.Core().V1().PersistentVolumeClaims(),
+		os.Getenv("NODE_NAME"),
+	)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		backupController.Run(s.ctx, 1)
+	}()
+
+	restoreController := controller.NewPodVolumeRestoreController(
+		s.logger,
+		s.arkInformerFactory.Ark().V1().PodVolumeRestores(),
+		s.arkClient.ArkV1(),
+		s.podInformer,
+		s.kubeInformerFactory.Core().V1().Secrets(),
+		s.kubeInformerFactory.Core().V1().PersistentVolumeClaims(),
+		os.Getenv("NODE_NAME"),
+	)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		restoreController.Run(s.ctx, 1)
+	}()
+
+	go s.arkInformerFactory.Start(s.ctx.Done())
+	go s.kubeInformerFactory.Start(s.ctx.Done())
+	go s.podInformer.Run(s.ctx.Done())
+
+	s.logger.Info("Controllers started successfully")
+
+	<-s.ctx.Done()
+
+	s.logger.Info("Waiting for all controllers to shut down gracefully")
+	wg.Wait()
+}

pkg/cmd/server/plugin/plugin.go

@@ -106,6 +106,8 @@ func NewCommand(f client.Factory) *cobra.Command {
 					action = restore.NewPodAction(logger)
 				case "svc":
 					action = restore.NewServiceAction(logger)
+				case "restic":
+					action = restore.NewResticRestoreAction(logger)
 				default:
 					logger.Fatal("Unrecognized plugin name")
 				}

pkg/cmd/server/server.go

@@ -22,15 +22,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/signal"
 	"reflect"
-	"sort"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
-	"github.com/heptio/ark/pkg/buildinfo"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -44,23 +40,25 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
+	corev1informers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
-	kcorev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 
 	api "github.com/heptio/ark/pkg/apis/ark/v1"
 	"github.com/heptio/ark/pkg/backup"
+	"github.com/heptio/ark/pkg/buildinfo"
 	"github.com/heptio/ark/pkg/client"
 	"github.com/heptio/ark/pkg/cloudprovider"
 	"github.com/heptio/ark/pkg/cmd"
-	"github.com/heptio/ark/pkg/cmd/util/flag"
+	"github.com/heptio/ark/pkg/cmd/util/signals"
 	"github.com/heptio/ark/pkg/controller"
 	arkdiscovery "github.com/heptio/ark/pkg/discovery"
 	clientset "github.com/heptio/ark/pkg/generated/clientset/versioned"
-	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
 	"github.com/heptio/ark/pkg/plugin"
+	"github.com/heptio/ark/pkg/podexec"
+	"github.com/heptio/ark/pkg/restic"
 	"github.com/heptio/ark/pkg/restore"
 	"github.com/heptio/ark/pkg/util/kube"
 	"github.com/heptio/ark/pkg/util/logging"
@@ -69,9 +67,8 @@ import (
 
 func NewCommand() *cobra.Command {
 	var (
-		sortedLogLevels = getSortedLogLevels()
-		logLevelFlag    = flag.NewEnum(logrus.InfoLevel.String(), sortedLogLevels...)
-		pluginDir       = "/plugins"
+		logLevelFlag = logging.LogLevelFlag(logrus.InfoLevel)
+		pluginDir    = "/plugins"
 	)
 
 	var command = &cobra.Command{
@@ -79,19 +76,10 @@ func NewCommand() *cobra.Command {
 		Short: "Run the ark server",
 		Long:  "Run the ark server",
 		Run: func(c *cobra.Command, args []string) {
-			logLevel := logrus.InfoLevel
-
-			if parsed, err := logrus.ParseLevel(logLevelFlag.String()); err == nil {
-				logLevel = parsed
-			} else {
-				// This should theoretically never happen assuming the enum flag
-				// is constructed correctly because the enum flag will not allow
-				//  an invalid value to be set.
-				logrus.Errorf("log-level flag has invalid value %s", strings.ToUpper(logLevelFlag.String()))
-			}
+			logLevel := logLevelFlag.Parse()
 			logrus.Infof("setting log-level to %s", strings.ToUpper(logLevel.String()))
 
-			logger := newLogger(logLevel, &logging.ErrorLocationHook{}, &logging.LogLocationHook{})
+			logger := logging.DefaultLogger(logLevel)
 			logger.Infof("Starting Ark server %s", buildinfo.FormattedGitSHA())
 
 			// NOTE: the namespace flag is bound to ark's persistent flags when the root ark command
@@ -109,14 +97,13 @@ func NewCommand() *cobra.Command {
 			namespace := getServerNamespace(namespaceFlag)
 
 			s, err := newServer(namespace, fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name()), pluginDir, logger)
-
 			cmd.CheckError(err)
 
 			cmd.CheckError(s.run())
 		},
 	}
 
-	command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("the level at which to log. Valid values are %s.", strings.Join(sortedLogLevels, ", ")))
+	command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("the level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", ")))
 	command.Flags().StringVar(&pluginDir, "plugin-dir", pluginDir, "directory containing Ark plugins")
 
 	return command
@@ -136,42 +123,12 @@ func getServerNamespace(namespaceFlag *pflag.Flag) string {
 	return api.DefaultNamespace
 }
 
-func newLogger(level logrus.Level, hooks ...logrus.Hook) *logrus.Logger {
-	logger := logrus.New()
-	logger.Level = level
-
-	for _, hook := range hooks {
-		logger.Hooks.Add(hook)
-	}
-
-	return logger
-}
-
-// getSortedLogLevels returns a string slice containing all of the valid logrus
-// log levels (based on logrus.AllLevels), sorted in ascending order of severity.
-func getSortedLogLevels() []string {
-	var (
-		sortedLogLevels  = make([]logrus.Level, len(logrus.AllLevels))
-		logLevelsStrings []string
-	)
-
-	copy(sortedLogLevels, logrus.AllLevels)
-
-	// logrus.Panic has the lowest value, so the compare function uses ">"
-	sort.Slice(sortedLogLevels, func(i, j int) bool { return sortedLogLevels[i] > sortedLogLevels[j] })
-
-	for _, level := range sortedLogLevels {
-		logLevelsStrings = append(logLevelsStrings, level.String())
-	}
-
-	return logLevelsStrings
-}
-
 type server struct {
 	namespace             string
 	kubeClientConfig      *rest.Config
 	kubeClient            kubernetes.Interface
 	arkClient             clientset.Interface
+	objectStore           cloudprovider.ObjectStore
 	backupService         cloudprovider.BackupService
 	snapshotService       cloudprovider.SnapshotService
 	discoveryClient       discovery.DiscoveryInterface
@@ -181,6 +138,7 @@ type server struct {
 	cancelFunc            context.CancelFunc
 	logger                logrus.FieldLogger
 	pluginManager         plugin.Manager
+	resticManager         restic.RepositoryManager
 }
 
 func newServer(namespace, baseName, pluginDir string, logger *logrus.Logger) (*server, error) {
@@ -225,7 +183,8 @@ func newServer(namespace, baseName, pluginDir string, logger *logrus.Logger) (*s
 
 func (s *server) run() error {
 	defer s.pluginManager.CleanupClients()
-	s.handleShutdownSignals()
+
+	signals.CancelOnShutdown(s.cancelFunc, s.logger)
 
 	if err := s.ensureArkNamespace(); err != nil {
 		return err
@@ -251,6 +210,21 @@ func (s *server) run() error {
 		return err
 	}
 
+	if config.BackupStorageProvider.ResticLocation != "" {
+		if err := s.initRestic(config.BackupStorageProvider); err != nil {
+			return err
+		}
+		s.runResticMaintenance()
+
+		// warn if restic daemonset does not exist
+		_, err := s.kubeClient.AppsV1().DaemonSets(s.namespace).Get("restic", metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			s.logger.Warn("Ark restic DaemonSet not found; restic backups will fail until it's created")
+		} else if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
 	if err := s.runControllers(config); err != nil {
 		return err
 	}
@@ -258,6 +232,20 @@ func (s *server) run() error {
 	return nil
 }
 
+func (s *server) runResticMaintenance() {
+	go func() {
+		interval := time.Hour
+
+		<-time.After(interval)
+
+		wait.Forever(func() {
+			if err := s.resticManager.PruneAllRepos(); err != nil {
+				s.logger.WithError(err).Error("error pruning repos")
+			}
+		}, interval)
+	}()
+}
+
 func (s *server) ensureArkNamespace() error {
 	logContext := s.logger.WithField("namespace", s.namespace)
 
@@ -301,11 +289,21 @@ func (s *server) loadConfig() (*api.Config, error) {
 }
 
 const (
-	defaultGCSyncPeriod       = 60 * time.Minute
-	defaultBackupSyncPeriod   = 60 * time.Minute
-	defaultScheduleSyncPeriod = time.Minute
+	defaultGCSyncPeriod              = 60 * time.Minute
+	defaultBackupSyncPeriod          = 60 * time.Minute
+	defaultScheduleSyncPeriod        = time.Minute
+	defaultPodVolumeOperationTimeout = 60 * time.Minute
 )
 
+// - Namespaces go first because all namespaced resources depend on them.
+// - PVs go before PVCs because PVCs depend on them.
+// - PVCs go before pods or controllers so they can be mounted as volumes.
+// - Secrets and config maps go before pods or controllers so they can be mounted
+// 	 as volumes.
+// - Service accounts go before pods or controllers so pods can use them.
+// - Limit ranges go before pods or controllers so pods can use them.
+// - Pods go before controllers so they can be explicitly restored and potentially
+//	 have restic restores run before controllers adopt the pods.
 var defaultResourcePriorities = []string{
 	"namespaces",
 	"persistentvolumes",
@@ -314,6 +312,7 @@ var defaultResourcePriorities = []string{
 	"configmaps",
 	"serviceaccounts",
 	"limitranges",
+	"pods",
 }
 
 func applyConfigDefaults(c *api.Config, logger logrus.FieldLogger) {
@@ -329,6 +328,10 @@ func applyConfigDefaults(c *api.Config, logger logrus.FieldLogger) {
 		c.ScheduleSyncPeriod.Duration = defaultScheduleSyncPeriod
 	}
 
+	if c.PodVolumeOperationTimeout.Duration == 0 {
+		c.PodVolumeOperationTimeout.Duration = defaultPodVolumeOperationTimeout
+	}
+
 	if len(c.ResourcePriorities) == 0 {
 		c.ResourcePriorities = defaultResourcePriorities
 		logger.WithField("priorities", c.ResourcePriorities).Info("Using default resource priorities")
@@ -379,17 +382,6 @@ func (s *server) watchConfig(config *api.Config) {
 	})
 }
 
-func (s *server) handleShutdownSignals() {
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-
-	go func() {
-		sig := <-sigs
-		s.logger.Infof("Received signal %s, gracefully shutting down", sig)
-		s.cancelFunc()
-	}()
-}
-
 func (s *server) initBackupService(config *api.Config) error {
 	s.logger.Info("Configuring cloud provider for backup service")
 	objectStore, err := getObjectStore(config.BackupStorageProvider.CloudProviderConfig, s.pluginManager)
@@ -397,6 +389,7 @@ func (s *server) initBackupService(config *api.Config) error {
 		return err
 	}
 
+	s.objectStore = objectStore
 	s.backupService = cloudprovider.NewBackupService(objectStore, s.logger)
 	return nil
 }
@@ -457,6 +450,42 @@ func durationMin(a, b time.Duration) time.Duration {
 	return b
 }
 
+func (s *server) initRestic(config api.ObjectStorageProviderConfig) error {
+	// set the env vars that restic uses for creds purposes
+	if config.Name == string(restic.AzureBackend) {
+		os.Setenv("AZURE_ACCOUNT_NAME", os.Getenv("AZURE_STORAGE_ACCOUNT_ID"))
+		os.Setenv("AZURE_ACCOUNT_KEY", os.Getenv("AZURE_STORAGE_KEY"))
+	}
+
+	secretsInformer := corev1informers.NewFilteredSecretInformer(
+		s.kubeClient,
+		"",
+		0,
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+		func(opts *metav1.ListOptions) {
+			opts.FieldSelector = fmt.Sprintf("metadata.name=%s", restic.CredentialsSecretName)
+		},
+	)
+	go secretsInformer.Run(s.ctx.Done())
+
+	res, err := restic.NewRepositoryManager(
+		s.ctx,
+		s.objectStore,
+		config,
+		s.arkClient,
+		secretsInformer,
+		s.kubeClient.CoreV1(),
+		s.logger,
+	)
+	if err != nil {
+		return err
+	}
+	s.resticManager = res
+
+	s.logger.Info("Checking restic repositories")
+	return s.resticManager.CheckAllRepos()
+}
+
 func (s *server) runControllers(config *api.Config) error {
 	s.logger.Info("Starting controllers")
 
@@ -505,8 +534,16 @@ func (s *server) runControllers(config *api.Config) error {
 	} else {
 		backupTracker := controller.NewBackupTracker()
 
-		backupper, err := newBackupper(discoveryHelper, s.clientPool, s.backupService, s.snapshotService, s.kubeClientConfig, s.kubeClient.CoreV1())
+		backupper, err := backup.NewKubernetesBackupper(
+			discoveryHelper,
+			client.NewDynamicFactory(s.clientPool),
+			podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()),
+			s.snapshotService,
+			s.resticManager,
+			config.PodVolumeOperationTimeout.Duration,
+		)
 		cmd.CheckError(err)
+
 		backupController := controller.NewBackupController(
 			s.sharedInformerFactory.Ark().V1().Backups(),
 			s.arkClient.ArkV1(),
@@ -561,6 +598,8 @@ func (s *server) runControllers(config *api.Config) error {
 			s.sharedInformerFactory.Ark().V1().Restores(),
 			s.arkClient.ArkV1(), // restoreClient
 			backupTracker,
+			s.resticManager,
+			s.sharedInformerFactory.Ark().V1().PodVolumeBackups(),
 		)
 		wg.Add(1)
 		go func() {
@@ -570,14 +609,16 @@ func (s *server) runControllers(config *api.Config) error {
 
 	}
 
-	restorer, err := newRestorer(
+	restorer, err := restore.NewKubernetesRestorer(
 		discoveryHelper,
-		s.clientPool,
+		client.NewDynamicFactory(s.clientPool),
 		s.backupService,
 		s.snapshotService,
 		config.ResourcePriorities,
 		s.arkClient.ArkV1(),
-		s.kubeClient,
+		s.kubeClient.CoreV1().Namespaces(),
+		s.resticManager,
+		config.PodVolumeOperationTimeout.Duration,
 		s.logger,
 	)
 	cmd.CheckError(err)
@@ -670,41 +711,3 @@ func (s *server) removeDeprecatedGCFinalizer() {
 		}
 	}
 }
-
-func newBackupper(
-	discoveryHelper arkdiscovery.Helper,
-	clientPool dynamic.ClientPool,
-	backupService cloudprovider.BackupService,
-	snapshotService cloudprovider.SnapshotService,
-	kubeClientConfig *rest.Config,
-	kubeCoreV1Client kcorev1client.CoreV1Interface,
-) (backup.Backupper, error) {
-	return backup.NewKubernetesBackupper(
-		discoveryHelper,
-		client.NewDynamicFactory(clientPool),
-		backup.NewPodCommandExecutor(kubeClientConfig, kubeCoreV1Client.RESTClient()),
-		snapshotService,
-	)
-}
-
-func newRestorer(
-	discoveryHelper arkdiscovery.Helper,
-	clientPool dynamic.ClientPool,
-	backupService cloudprovider.BackupService,
-	snapshotService cloudprovider.SnapshotService,
-	resourcePriorities []string,
-	backupClient arkv1client.BackupsGetter,
-	kubeClient kubernetes.Interface,
-	logger logrus.FieldLogger,
-) (restore.Restorer, error) {
-	return restore.NewKubernetesRestorer(
-		discoveryHelper,
-		client.NewDynamicFactory(clientPool),
-		backupService,
-		snapshotService,
-		resourcePriorities,
-		backupClient,
-		kubeClient.CoreV1().Namespaces(),
-		logger,
-	)
-}

pkg/cmd/util/flag/enum.go

@@ -18,15 +18,13 @@ package flag
 
 import (
 	"github.com/pkg/errors"
-
-	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Enum is a Cobra-compatible wrapper for defining
 // a string flag that can be one of a specified set
 // of values.
 type Enum struct {
-	allowedValues sets.String
+	allowedValues []string
 	value         string
 }
 
@@ -35,7 +33,7 @@ type Enum struct {
 // none is set.
 func NewEnum(defaultValue string, allowedValues ...string) *Enum {
 	return &Enum{
-		allowedValues: sets.NewString(allowedValues...),
+		allowedValues: allowedValues,
 		value:         defaultValue,
 	}
 }
@@ -50,12 +48,14 @@ func (e *Enum) String() string {
 // receiver. It returns an error if the string
 // is not an allowed value.
 func (e *Enum) Set(s string) error {
-	if !e.allowedValues.Has(s) {
-		return errors.Errorf("invalid value: %q", s)
+	for _, val := range e.allowedValues {
+		if val == s {
+			e.value = s
+			return nil
+		}
 	}
 
-	e.value = s
-	return nil
+	return errors.Errorf("invalid value: %q", s)
 }
 
 // Type returns a string representation of the
@@ -66,3 +66,9 @@ func (e *Enum) Type() string {
 	// the possible options.
 	return ""
 }
+
+// AllowedValues returns a slice of the flag's valid
+// values.
+func (e *Enum) AllowedValues() []string {
+	return e.allowedValues
+}

pkg/cmd/util/signals/signals.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package signals
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+// CancelOnShutdown starts a goroutine that will call cancelFunc when
+// either SIGINT or SIGTERM is received
+func CancelOnShutdown(cancelFunc context.CancelFunc, logger logrus.FieldLogger) {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		sig := <-sigs
+		logger.Infof("Received signal %s, shutting down", sig)
+		cancelFunc()
+	}()
+}