Enhance Backup to backup resources in specific order. (#2724)

Signed-off-by: Phuong Hoang <phuong.n.hoang@dell.com>

Co-authored-by: Phuong Hoang <phuong.n.hoang@dell.com>

changelogs/unreleased/2724-phuong

@@ -0,0 +1 @@
+Enhance Backup to support backing up resources in specific orders and add --ordered-resources option to support this feature.

config/crd/bases/velero.io_backups.yaml

@@ -303,6 +303,15 @@ spec:
                     are ANDed.
                   type: object
               type: object
+            orderedResources:
+              additionalProperties:
+                type: string
+              description: OrderedResources specifies the backup order of resources
+                of specific Kind. The map key is the Kind name and value is a list
+                of resource names separeted by commas. Each resource name has format
+                "namespace/resourcename".  For cluster resources, simply use "resourcename".
+              nullable: true
+              type: object
             snapshotVolumes:
               description: SnapshotVolumes specifies whether to take cloud snapshots
                 of any PV's referenced in the set of objects included in the Backup.

config/crd/bases/velero.io_schedules.yaml

@@ -318,6 +318,16 @@ spec:
                         are ANDed.
                       type: object
                   type: object
+                orderedResources:
+                  additionalProperties:
+                    type: string
+                  description: OrderedResources specifies the backup order of resources
+                    of specific Kind. The map key is the Kind name and value is a
+                    list of resource names separeted by commas. Each resource name
+                    has format "namespace/resourcename".  For cluster resources, simply
+                    use "resourcename".
+                  nullable: true
+                  type: object
                 snapshotVolumes:
                   description: SnapshotVolumes specifies whether to take cloud snapshots
                     of any PV's referenced in the set of objects included in the Backup.

pkg/apis/velero/v1/backup.go

@@ -88,6 +88,13 @@ type BackupSpec struct {
 	// +optional
 	// + nullable
 	DefaultVolumesToRestic *bool `json:"defaultVolumesToRestic,omitempty"`
+
+	// OrderedResources specifies the backup order of resources of specific Kind.
+	// The map key is the Kind name and value is a list of resource names separeted by commas.
+	// Each resource name has format "namespace/resourcename".  For cluster resources, simply use "resourcename".
+	// +optional
+	// +nullable
+	OrderedResources map[string]string `json:"orderedResources,omitempty"`
 }
 
 // BackupHooks contains custom behaviors that should be executed at different phases of the backup.

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -252,6 +252,13 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.OrderedResources != nil {
+		in, out := &in.OrderedResources, &out.OrderedResources
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 

pkg/backup/item_collector.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017, 2020 the Velero contributors.
+Copyright 2020 the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package backup
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"sort"
 	"strings"
@@ -99,6 +100,64 @@ func (r *itemCollector) getGroupItems(log logrus.FieldLogger, group *metav1.APIR
 	return items, nil
 }
 
+// sortResourcesByOrder sorts items by the names specified in "order".  Items are not in order will be put at the end in original order.
+func sortResourcesByOrder(log logrus.FieldLogger, items []*kubernetesResource, order []string) []*kubernetesResource {
+	if len(order) == 0 {
+		return items
+	}
+	log.Debugf("Sorting resources using the following order %v...", order)
+	itemMap := make(map[string]*kubernetesResource)
+	for _, item := range items {
+		var fullname string
+		if item.namespace != "" {
+			fullname = fmt.Sprintf("%s/%s", item.namespace, item.name)
+		} else {
+			fullname = item.name
+		}
+		itemMap[fullname] = item
+	}
+	var sortedItems []*kubernetesResource
+	// First select items from the order
+	for _, name := range order {
+		if item, ok := itemMap[name]; ok {
+			sortedItems = append(sortedItems, item)
+			log.Debugf("%s added to sorted resource list.", item.name)
+			delete(itemMap, name)
+		} else {
+			log.Warnf("Cannot find resource '%s'.", name)
+		}
+	}
+	// Now append the rest in sortedGroupItems, maintain the original order
+	for _, item := range items {
+		var fullname string
+		if item.namespace != "" {
+			fullname = fmt.Sprintf("%s/%s", item.namespace, item.name)
+		} else {
+			fullname = item.name
+		}
+		if _, ok := itemMap[fullname]; !ok {
+			//This item has been inserted in the result
+			continue
+		}
+		sortedItems = append(sortedItems, item)
+		log.Debugf("%s added to sorted resource list.", item.name)
+	}
+	return sortedItems
+}
+
+// getOrderedResourcesForType gets order of resourceType from orderResources.
+func getOrderedResourcesForType(log logrus.FieldLogger, orderedResources map[string]string, resourceType string) []string {
+	if orderedResources == nil {
+		return nil
+	}
+	orderStr, ok := orderedResources[resourceType]
+	if !ok || len(orderStr) == 0 {
+		return nil
+	}
+	orders := strings.Split(orderStr, ",")
+	return orders
+}
+
 // getResourceItems collects all relevant items for a given group-version-resource.
 func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.GroupVersion, resource metav1.APIResource) ([]*kubernetesResource, error) {
 	log = log.WithField("resource", resource.Name)
@@ -111,6 +170,7 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
 		clusterScoped = !resource.Namespaced
 	)
 
+	orders := getOrderedResourcesForType(log, r.backupRequest.Backup.Spec.OrderedResources, resource.Name)
 	// Getting the preferred group version of this resource
 	preferredGVR, _, err := r.discoveryHelper.ResourceFor(gr.WithVersion(""))
 	if err != nil {
@@ -260,6 +320,9 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
 			})
 		}
 	}
+	if len(orders) > 0 {
+		items = sortResourcesByOrder(r.log, items, orders)
+	}
 
 	return items, nil
 }

pkg/backup/item_collector_test.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"testing"
 
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -48,3 +49,32 @@ func TestSortCoreGroup(t *testing.T) {
 		assert.Equal(t, expected[i], r.Name)
 	}
 }
+
+func TestSortOrderedResource(t *testing.T) {
+	log := logrus.StandardLogger()
+	podResources := []*kubernetesResource{
+		{namespace: "ns1", name: "pod1"},
+		{namespace: "ns1", name: "pod2"},
+	}
+	order := []string{"ns1/pod2", "ns1/pod1"}
+	expectedResources := []*kubernetesResource{
+		{namespace: "ns1", name: "pod2"},
+		{namespace: "ns1", name: "pod1"},
+	}
+	sortedResources := sortResourcesByOrder(log, podResources, order)
+	assert.Equal(t, sortedResources, expectedResources)
+
+	// Test cluster resources
+	pvResources := []*kubernetesResource{
+		{name: "pv1"},
+		{name: "pv2"},
+	}
+	pvOrder := []string{"pv5", "pv2", "pv1"}
+	expectedPvResources := []*kubernetesResource{
+		{name: "pv2"},
+		{name: "pv1"},
+	}
+	sortedPvResources := sortResourcesByOrder(log, pvResources, pvOrder)
+	assert.Equal(t, sortedPvResources, expectedPvResources)
+
+}

pkg/builder/backup_builder.go

@@ -181,3 +181,9 @@ func (b *BackupBuilder) Hooks(hooks velerov1api.BackupHooks) *BackupBuilder {
 	b.object.Spec.Hooks = hooks
 	return b
 }
+
+// OrderedResources sets the Backup's OrderedResources
+func (b *BackupBuilder) OrderedResources(orders map[string]string) *BackupBuilder {
+	b.object.Spec.OrderedResources = orders
+	return b
+}

pkg/cmd/cli/backup/create.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -96,6 +97,7 @@ type CreateOptions struct {
 	StorageLocation         string
 	SnapshotLocations       []string
 	FromSchedule            string
+	OrderedResources        string
 
 	client veleroclient.Interface
 }
@@ -120,6 +122,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.StorageLocation, "storage-location", "", "Location in which to store the backup.")
 	flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.")
 	flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.")
+	flags.StringVar(&o.OrderedResources, "ordered-resources", "", "mapping Kinds to an ordered list of specific resources of that Kind.  Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon.  Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'.  Optional.")
 	f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup.")
 	// this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true"
 	// like a normal bool flag
@@ -281,6 +284,28 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 	return nil
 }
 
+// parseOrderedResources converts to map of Kinds to an ordered list of specific resources of that Kind.
+// Resource names in the list are in format 'namespace/resourcename' and separated by commas.
+// Key-value pairs in the mapping are separated by semi-colon.
+// Ex: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'.
+func parseOrderedResources(orderMapStr string) (map[string]string, error) {
+	entries := strings.Split(orderMapStr, ";")
+	if len(entries) == 0 {
+		return nil, fmt.Errorf("Invalid OrderedResources '%s'.", orderMapStr)
+	}
+	orderedResources := make(map[string]string)
+	for _, entry := range entries {
+		kv := strings.Split(entry, "=")
+		if len(kv) != 2 {
+			return nil, fmt.Errorf("Invalid OrderedResources '%s'.", entry)
+		}
+		kind := strings.TrimSpace(kv[0])
+		order := strings.TrimSpace(kv[1])
+		orderedResources[kind] = order
+	}
+	return orderedResources, nil
+}
+
 func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, error) {
 	var backupBuilder *builder.BackupBuilder
 
@@ -304,6 +329,13 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 			TTL(o.TTL).
 			StorageLocation(o.StorageLocation).
 			VolumeSnapshotLocations(o.SnapshotLocations...)
+		if len(o.OrderedResources) > 0 {
+			orders, err := parseOrderedResources(o.OrderedResources)
+			if err != nil {
+				return nil, err
+			}
+			backupBuilder.OrderedResources(orders)
+		}
 
 		if o.SnapshotVolumes.Value != nil {
 			backupBuilder.SnapshotVolumes(*o.SnapshotVolumes.Value)

pkg/cmd/cli/backup/create_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 the Velero contributors.
+Copyright 2020 the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,6 +33,9 @@ const testNamespace = "velero"
 func TestCreateOptions_BuildBackup(t *testing.T) {
 	o := NewCreateOptions()
 	o.Labels.Set("velero.io/test=true")
+	o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2"
+	orders, err := parseOrderedResources(o.OrderedResources)
+	assert.NoError(t, err)
 
 	backup, err := o.BuildBackup(testNamespace)
 	assert.NoError(t, err)
@@ -42,11 +45,16 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 		IncludedNamespaces:      []string(o.IncludeNamespaces),
 		SnapshotVolumes:         o.SnapshotVolumes.Value,
 		IncludeClusterResources: o.IncludeClusterResources.Value,
+		OrderedResources:        orders,
 	}, backup.Spec)
 
 	assert.Equal(t, map[string]string{
 		"velero.io/test": "true",
 	}, backup.GetLabels())
+	assert.Equal(t, map[string]string{
+		"pods":                   "p1,p2",
+		"persistentvolumeclaims": "pvc1,pvc2",
+	}, backup.Spec.OrderedResources)
 }
 
 func TestCreateOptions_BuildBackupFromSchedule(t *testing.T) {
@@ -87,3 +95,27 @@ func TestCreateOptions_BuildBackupFromSchedule(t *testing.T) {
 		}, backup.GetLabels())
 	})
 }
+
+func TestCreateOptions_OrderedResources(t *testing.T) {
+	orderedResources, err := parseOrderedResources("pods= ns1/p1; ns1/p2; persistentvolumeclaims=ns2/pvc1, ns2/pvc2")
+	assert.NotNil(t, err)
+
+	orderedResources, err = parseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumeclaims=ns2/pvc1,ns2/pvc2")
+	assert.NoError(t, err)
+
+	expectedResources := map[string]string{
+		"pods":                   "ns1/p1,ns1/p2",
+		"persistentvolumeclaims": "ns2/pvc1,ns2/pvc2",
+	}
+	assert.Equal(t, orderedResources, expectedResources)
+
+	orderedResources, err = parseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumes=pv1,pv2")
+	assert.NoError(t, err)
+
+	expectedMixedResources := map[string]string{
+		"pods":              "ns1/p1,ns1/p2",
+		"persistentvolumes": "pv1,pv2",
+	}
+	assert.Equal(t, orderedResources, expectedMixedResources)
+
+}

pkg/cmd/util/output/backup_describer.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017, 2019 the Velero contributors.
+Copyright 2020 the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -218,6 +218,14 @@ func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) {
 		}
 	}
 
+	if spec.OrderedResources != nil {
+		d.Println()
+		d.Printf("OrderedResources:\n")
+		for key, value := range spec.OrderedResources {
+			d.Printf("\t%s: %s\n", key, value)
+		}
+	}
+
 }
 
 // DescribeBackupStatus describes a backup status in human-readable format.

site/docs/main/backup-reference.md

@@ -7,3 +7,12 @@ It is possible to exclude individual items from being backed up, even if they ma
 ```bash
 kubectl label -n <ITEM_NAMESPACE> <RESOURCE>/<NAME> velero.io/exclude-from-backup=true
 ```
+
+## Specify Backup Orders of Resources of Specific Kind
+
+To backup resources of specific Kind in a specific order, use option --ordered-resources to specify a mapping Kinds to an ordered list of specific resources of that Kind.  Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon.  Kind name is in plural form.
+
+```bash
+velero backup create backupName --include-cluster-resources=true --ordered-resources 'pods=ns1/pod1,ns1/pod2;persistentvolumes=pv4,pv8' --include-namespaces=ns1
+velero backup create backupName --ordered-resources 'statefulsets=ns1/sts1,ns1/sts0' --include-namespaces=ns1
+```