Merge pull request #4628 from shubham-pampattiwar/add-restore-policy

Add `existingResourcePolicy` to Restore API

changelogs/unreleased/4628-shubham-pampattiwar

@@ -0,0 +1 @@
+Add existingResourcePolicy to Restore API

config/crd/v1/bases/velero.io_restores.yaml

@@ -53,6 +53,11 @@ spec:
                   type: string
                 nullable: true
                 type: array
+              existingResourcePolicy:
+                description: ExistingResourcePolicy specifies the restore behaviour
+                  for the kubernetes resource to be restored
+                nullable: true
+                type: string
               hooks:
                 description: Hooks represent custom behaviors that should be executed
                   during or post restore.

pkg/apis/velero/v1/restore.go

@@ -92,6 +92,11 @@ type RestoreSpec struct {
 	// Hooks represent custom behaviors that should be executed during or post restore.
 	// +optional
 	Hooks RestoreHooks `json:"hooks,omitempty"`
+
+	// ExistingResourcePolicy specifies the restore behaviour for the kubernetes resource to be restored
+	// +optional
+	// +nullable
+	ExistingResourcePolicy PolicyType `json:"existingResourcePolicy,omitempty"`
 }
 
 // RestoreHooks contains custom behaviors that should be executed during or post restore.
@@ -212,6 +217,14 @@ const (
 	// RestorePhaseFailed means the restore was unable to execute.
 	// The failing error is recorded in status.FailureReason.
 	RestorePhaseFailed RestorePhase = "Failed"
+
+	// PolicyTypeNone means velero will not overwrite the resource
+	// in cluster with the one in backup whether changed/unchanged.
+	PolicyTypeNone PolicyType = "none"
+
+	// PolicyTypeUpdate means velero will try to attempt a patch on
+	// the changed resources.
+	PolicyTypeUpdate PolicyType = "update"
 )
 
 // RestoreStatus captures the current status of a Velero restore
@@ -302,3 +315,6 @@ type RestoreList struct {
 
 	Items []Restore `json:"items"`
 }
+
+// PolicyType helps specify the ExistingResourcePolicy
+type PolicyType string

pkg/builder/restore_builder.go

@@ -95,6 +95,12 @@ func (b *RestoreBuilder) ExcludedResources(resources ...string) *RestoreBuilder
 	return b
 }
 
+// ExistingResourcePolicy sets the Restore's resource policy.
+func (b *RestoreBuilder) ExistingResourcePolicy(policy string) *RestoreBuilder {
+	b.object.Spec.ExistingResourcePolicy = velerov1api.PolicyType(policy)
+	return b
+}
+
 // IncludeClusterResources sets the Restore's "include cluster resources" flag.
 func (b *RestoreBuilder) IncludeClusterResources(val bool) *RestoreBuilder {
 	b.object.Spec.IncludeClusterResources = &val

pkg/cmd/cli/restore/create.go

@@ -82,6 +82,7 @@ type CreateOptions struct {
 	Labels                  flag.Map
 	IncludeNamespaces       flag.StringArray
 	ExcludeNamespaces       flag.StringArray
+	ExistingResourcePolicy  string
 	IncludeResources        flag.StringArray
 	ExcludeResources        flag.StringArray
 	NamespaceMappings       flag.Map
@@ -113,6 +114,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.Var(&o.Labels, "labels", "Labels to apply to the restore.")
 	flags.Var(&o.IncludeResources, "include-resources", "Resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources).")
 	flags.Var(&o.ExcludeResources, "exclude-resources", "Resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io.")
+	flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow, can be - none or update")
 	flags.VarP(&o.Selector, "selector", "l", "Only restore resources matching this label selector.")
 	f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "Whether to restore volumes from snapshots.")
 	// this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true"
@@ -172,6 +174,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto
 		return errors.New("Velero client is not set; unable to proceed")
 	}
 
+	if len(o.ExistingResourcePolicy) > 0 && !isResourcePolicyValid(o.ExistingResourcePolicy) {
+		return errors.New("existing-resource-policy has invalid value, it accepts only none, update as value")
+	}
+
 	switch {
 	case o.BackupName != "":
 		if _, err := o.client.VeleroV1().Backups(f.Namespace()).Get(context.TODO(), o.BackupName, metav1.GetOptions{}); err != nil {
@@ -264,6 +270,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 			ExcludedNamespaces:      o.ExcludeNamespaces,
 			IncludedResources:       o.IncludeResources,
 			ExcludedResources:       o.ExcludeResources,
+			ExistingResourcePolicy:  api.PolicyType(o.ExistingResourcePolicy),
 			NamespaceMapping:        o.NamespaceMappings.Data(),
 			LabelSelector:           o.Selector.LabelSelector,
 			RestorePVs:              o.RestoreVolumes.Value,
@@ -351,3 +358,10 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 
 	return nil
 }
+
+func isResourcePolicyValid(resourcePolicy string) bool {
+	if resourcePolicy == string(api.PolicyTypeNone) || resourcePolicy == string(api.PolicyTypeUpdate) {
+		return true
+	}
+	return false
+}

pkg/cmd/util/output/restore_describer.go

@@ -148,6 +148,13 @@ func DescribeRestore(ctx context.Context, kbClient kbclient.Client, restore *vel
 			describePodVolumeRestores(d, podVolumeRestores, details)
 		}
 
+		d.Println()
+		s = "<none>"
+		if restore.Spec.ExistingResourcePolicy != "" {
+			s = string(restore.Spec.ExistingResourcePolicy)
+		}
+		d.Printf("Existing Resource Policy: \t%s\n", s)
+
 		d.Println()
 		d.Printf("Preserve Service NodePorts:\t%s\n", BoolPointerString(restore.Spec.PreserveNodePorts, "false", "true", "auto"))
 

pkg/restore/restore.go

@@ -1240,6 +1240,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		// labels, so copy them from the object we attempted to restore.
 		labels := obj.GetLabels()
 		addRestoreLabels(fromCluster, labels[velerov1api.RestoreNameLabel], labels[velerov1api.BackupNameLabel])
+		fromClusterWithLabels := fromCluster.DeepCopy() // saving the in-cluster object so that we can create label patch if overall patch fails
 
 		if !equality.Semantic.DeepEqual(fromCluster, obj) {
 			switch groupResource {
@@ -1267,17 +1268,58 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 				_, err = resourceClient.Patch(name, patchBytes)
 				if err != nil {
 					warnings.Add(namespace, err)
+					// check if there is existingResourcePolicy and if it is set to update policy
+					if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate {
+						// remove restore labels so that we apply the latest backup/restore names on the object via patch
+						removeRestoreLabels(fromCluster)
+						//try patching just the backup/restore labels
+						warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, fromClusterWithLabels, namespace, resourceClient)
+						warnings.Merge(&warningsFromUpdate)
+						errs.Merge(&errsFromUpdate)
+					}
 				} else {
 					ctx.log.Infof("ServiceAccount %s successfully updated", kube.NamespaceAndName(obj))
 				}
 			default:
-				e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version.",
-					obj.GetKind(), obj.GetName())
-				warnings.Add(namespace, e)
+				// check for the presence of existingResourcePolicy
+				if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 {
+					resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy
+					ctx.log.Infof("restore API has resource policy defined %s , executing restore workflow accordingly for changed resource %s %s", resourcePolicy, fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+
+					// existingResourcePolicy is set as none, add warning
+					if resourcePolicy == velerov1api.PolicyTypeNone {
+						e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version.",
+							obj.GetKind(), obj.GetName())
+						warnings.Add(namespace, e)
+						// existingResourcePolicy is set as update, attempt patch on the resource and add warning if it fails
+					} else if resourcePolicy == velerov1api.PolicyTypeUpdate {
+						// processing update as existingResourcePolicy
+						warningsFromUpdateRP, errsFromUpdateRP := ctx.processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj, namespace, resourceClient)
+						warnings.Merge(&warningsFromUpdateRP)
+						errs.Merge(&errsFromUpdateRP)
+					}
+				} else {
+					// Preserved Velero behavior when existingResourcePolicy is not specified by the user
+					e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version.",
+						obj.GetKind(), obj.GetName())
+					warnings.Add(namespace, e)
+				}
 			}
 			return warnings, errs
 		}
 
+		//update backup/restore labels on the unchanged resources if existingResourcePolicy is set as update
+		if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate {
+			resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy
+			ctx.log.Infof("restore API has resource policy defined %s , executing restore workflow accordingly for unchanged resource %s %s ", resourcePolicy, obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+			// remove restore labels so that we apply the latest backup/restore names on the object via patch
+			removeRestoreLabels(fromCluster)
+			// try updating the backup/restore labels for the in-cluster object
+			warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, obj, namespace, resourceClient)
+			warnings.Merge(&warningsFromUpdate)
+			errs.Merge(&errsFromUpdate)
+		}
+
 		ctx.log.Infof("Restore of %s, %v skipped: it already exists in the cluster and is the same as the backed up version", obj.GroupVersionKind().Kind, name)
 		return warnings, errs
 	}
@@ -1831,3 +1873,81 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource, targetNamespace
 	}
 	return restorable, warnings, errs
 }
+
+// removeRestoreLabels removes the restore name and the
+// restored backup's name.
+func removeRestoreLabels(obj metav1.Object) {
+	labels := obj.GetLabels()
+
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+
+	labels[velerov1api.BackupNameLabel] = ""
+	labels[velerov1api.RestoreNameLabel] = ""
+
+	obj.SetLabels(labels)
+}
+
+// updates the backup/restore labels
+func (ctx *restoreContext) updateBackupRestoreLabels(fromCluster, fromClusterWithLabels *unstructured.Unstructured, namespace string, resourceClient client.Dynamic) (warnings, errs Result) {
+	patchBytes, err := generatePatch(fromCluster, fromClusterWithLabels)
+	if err != nil {
+		ctx.log.Errorf("error generating patch for %s %s: %v", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster), err)
+		errs.Add(namespace, err)
+		return warnings, errs
+	}
+
+	if patchBytes == nil {
+		// In-cluster and desired state are the same, so move on to
+		// the next items
+		ctx.log.Errorf("skipped updating backup/restore labels for %s %s: in-cluster and desired state are the same along-with the labels", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+		return warnings, errs
+	}
+
+	// try patching the in-cluster resource (with only latest backup/restore labels)
+	_, err = resourceClient.Patch(fromCluster.GetName(), patchBytes)
+	if err != nil {
+		ctx.log.Errorf("backup/restore label patch attempt failed for %s %s: %v", fromCluster.GroupVersionKind(), kube.NamespaceAndName(fromCluster), err)
+		errs.Add(namespace, err)
+	} else {
+		ctx.log.Infof("backup/restore labels successfully updated for %s %s", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+	}
+	return warnings, errs
+}
+
+// function to process existingResourcePolicy as update, tries to patch the diff between in-cluster and restore obj first
+// if the patch fails then tries to update the backup/restore labels for the in-cluster version
+func (ctx *restoreContext) processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj *unstructured.Unstructured, namespace string, resourceClient client.Dynamic) (warnings, errs Result) {
+	ctx.log.Infof("restore API has existingResourcePolicy defined as update , executing restore workflow accordingly for changed resource %s %s ", obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+	ctx.log.Infof("attempting patch on %s %q", fromCluster.GetKind(), fromCluster.GetName())
+	// remove restore labels so that we apply the latest backup/restore names on the object via patch
+	removeRestoreLabels(fromCluster)
+	patchBytes, err := generatePatch(fromCluster, obj)
+	if err != nil {
+		ctx.log.Errorf("error generating patch for %s %s: %v", obj.GroupVersionKind().Kind, kube.NamespaceAndName(obj), err)
+		errs.Add(namespace, err)
+		return warnings, errs
+	}
+
+	if patchBytes == nil {
+		// In-cluster and desired state are the same, so move on to
+		// the next items
+		ctx.log.Errorf("skipped updating %s %s: in-cluster and desired state are the same", fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster))
+		return warnings, errs
+	}
+
+	// try patching the in-cluster resource (resource diff plus latest backup/restore labels)
+	_, err = resourceClient.Patch(obj.GetName(), patchBytes)
+	if err != nil {
+		ctx.log.Errorf("patch attempt failed for %s %s: %v", fromCluster.GroupVersionKind(), kube.NamespaceAndName(fromCluster), err)
+		warnings.Add(namespace, err)
+		// try just patching the labels
+		warningsFromUpdate, errsFromUpdate := ctx.updateBackupRestoreLabels(fromCluster, fromClusterWithLabels, namespace, resourceClient)
+		warnings.Merge(&warningsFromUpdate)
+		errs.Merge(&errsFromUpdate)
+	} else {
+		ctx.log.Infof("%s %s successfully updated", obj.GroupVersionKind().Kind, kube.NamespaceAndName(obj))
+	}
+	return warnings, errs
+}

pkg/restore/restore_test.go

@@ -967,6 +967,76 @@ func TestRestoreItems(t *testing.T) {
 				test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").Result()),
 			},
 		},
+		{
+			name:    "update secret data and labels when secret exists in cluster and is not identical to the backed up one, existing resource policy is update",
+			restore: defaultRestore().ExistingResourcePolicy("update").Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("secrets", builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"key-1": []byte("value-1")}).Result()).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.Secrets(builder.ForSecret("ns-1", "sa-1").Data(map[string][]byte{"foo": []byte("bar")}).Result()),
+			},
+			want: []*test.APIResource{
+				test.Secrets(builder.ForSecret("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Data(map[string][]byte{"key-1": []byte("value-1")}).Result()),
+			},
+		},
+		{
+			name:    "update service account labels when service account exists in cluster and is identical to the backed up one, existing resource policy is update",
+			restore: defaultRestore().ExistingResourcePolicy("update").Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("serviceaccounts", builder.ForServiceAccount("ns-1", "sa-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+			want: []*test.APIResource{
+				test.ServiceAccounts(builder.ForServiceAccount("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result()),
+			},
+		},
+		{
+			name:    "update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is update",
+			restore: defaultRestore().ExistingResourcePolicy("update").Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+			want: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1")).Result()),
+			},
+		},
+		{
+			name:    "do not update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is none",
+			restore: defaultRestore().ExistingResourcePolicy("none").Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+			want: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+		},
+		{
+			name:    "do not update pod labels when pod exists in cluster and is identical to the backed up one, existing resource policy is not specified, velero behavior is preserved",
+			restore: defaultRestore().Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "sa-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+			want: []*test.APIResource{
+				test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()),
+			},
+		},
 		{
 			name:    "service account secrets and image pull secrets are restored when service account already exists in cluster",
 			restore: defaultRestore().Result(),
@@ -2168,7 +2238,7 @@ func TestRestorePersistentVolumes(t *testing.T) {
 						ObjectMeta(
 							builder.WithAnnotations("velero.io/original-pv-name", "source-pv"),
 							builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"),
-						// the namespace for this PV's claimRef should be the one that the PVC was remapped into.
+							// the namespace for this PV's claimRef should be the one that the PVC was remapped into.
 						).ClaimRef("target-ns", "pvc-1").
 						AWSEBSVolumeID("new-volume").
 						Result(),

site/content/docs/main/api-types/restore.md

@@ -73,6 +73,9 @@ spec:
   # to restore from. If specified, and BackupName is empty, Velero will
   # restore from the most recent successful backup created from this schedule.
   scheduleName: my-scheduled-backup-name
+  # ExistingResourcePolicy specifies the restore behaviour
+  # for the kubernetes resource to be restored. Optional
+  existingResourcePolicy: none
   # Actions to perform during or post restore. The only hooks currently supported are
   # adding an init container to a pod before it can be restored and executing a command in a
   # restored pod's container. Optional.