mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-20 23:14:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add multiple label selector support to backup API

Browse Source 
Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

remove backup CLI bits

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

labelselectors spec option for velero restore

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

add changelog file

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

update spec name to OrLabelSelectors

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

minor fixes

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

add validations for labelSelector and orLabelSelectors

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

update crds.gp after fixing conflicts

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

fix CI and add unit tests

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

updated OrLabelSelector spec description and added validation failure unit tests

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

add comments and change log level

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

update site docs

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

wrap pager client calls in a function

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

resolve confilcts and update crds.go

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

rebase and update crds.go

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>

combine listing items for a given label

Signed-off-by: Shubham Pampattiwar <shubhampampattiwar7@gmail.com>
This commit is contained in:
Shubham Pampattiwar
2022-05-11 09:57:47 -04:00
parent e51865eec1
commit bfdb68a35a
20 changed files with 441 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/backup/backup_test.go
+29
View File
@@ -331,6 +331,35 @@ func TestBackupResourceFiltering(t *testing.T) {
"resources/persistentvolumes/v1-preferredversion/cluster/bar.json",
},
},
{
name: "OrLabelSelector only backs up matching resources",
backup: defaultBackup().
OrLabelSelector([]*metav1.LabelSelector{{MatchLabels: map[string]string{"a1": "b1"}}, {MatchLabels: map[string]string{"a2": "b2"}},
{MatchLabels: map[string]string{"a3": "b3"}}, {MatchLabels: map[string]string{"a4": "b4"}}}).
Result(),
apiResources: []*test.APIResource{
test.Pods(
builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("a1", "b1")).Result(),
builder.ForPod("zoo", "raz").Result(),
),
test.Deployments(
builder.ForDeployment("foo", "bar").Result(),
builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("a2", "b2")).Result(),
),
test.PVs(
builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a4", "b4")).Result(),
builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a5", "b5")).Result(),
),
},
want: []string{
"resources/pods/namespaces/foo/bar.json",
"resources/deployments.apps/namespaces/zoo/raz.json",
"resources/persistentvolumes/cluster/bar.json",
"resources/pods/v1-preferredversion/namespaces/foo/bar.json",
"resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json",
"resources/persistentvolumes/v1-preferredversion/cluster/bar.json",
},
},
{
name: "resources with velero.io/exclude-from-backup=true label are not included",
backup: defaultBackup().
pkg/backup/item_collector.go
+87 -39
View File
@@ -291,53 +291,44 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
continue
}
var labelSelector string
if selector := r.backupRequest.Spec.LabelSelector; selector != nil {
labelSelector = metav1.FormatLabelSelector(selector)
var orLabelSelectors []string
if r.backupRequest.Spec.OrLabelSelectors != nil {
for _, s := range r.backupRequest.Spec.OrLabelSelectors {
orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(s))
}
} else {
orLabelSelectors = []string{}
}
log.Info("Listing items")
unstructuredItems := make([]unstructured.Unstructured, 0)
if r.pageSize > 0 {
// If limit is positive, use a pager to split list over multiple requests
// Use Velero's dynamic list function instead of the default
listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
return resourceClient.List(opts)
}))
// Use the page size defined in the server config
// TODO allow configuration of page buffer size
listPager.PageSize = int64(r.pageSize)
// Add each item to temporary slice
list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
// Listing items for orLabelSelectors
errListingForNS := false
for _, label := range orLabelSelectors {
unstructuredItems, err = r.listItemsForLabel(unstructuredItems, gr, label, resourceClient)
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing resources")
errListingForNS = true
}
}
if errListingForNS {
log.WithError(err).Error("Error listing items")
continue
}
var labelSelector string
if selector := r.backupRequest.Spec.LabelSelector; selector != nil {
labelSelector = metav1.FormatLabelSelector(selector)
}
// Listing items for labelSelector (singular)
if len(orLabelSelectors) == 0 {
unstructuredItems, err = r.listItemsForLabel(unstructuredItems, gr, labelSelector, resourceClient)
if err != nil {
log.WithError(err).Error("Error listing items")
continue
}
if !paginated {
log.Infof("list for groupResource %s was not paginated", gr)
}
err = meta.EachListItem(list, func(object runtime.Object) error {
u, ok := object.(*unstructured.Unstructured)
if !ok {
log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry in the list")
return fmt.Errorf("expected *unstructured.Unstructured but got %T", u)
}
unstructuredItems = append(unstructuredItems, *u)
return nil
})
if err != nil {
log.WithError(errors.WithStack(err)).Error("unable to understand paginated list")
continue
}
} else {
// If limit is not positive, do not use paging. Instead, request all items at once
unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector})
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing items")
continue
}
unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
}
log.Infof("Retrieved %d items", len(unstructuredItems))
@@ -467,3 +458,60 @@ func newCohabitatingResource(resource, group1, group2 string) *cohabitatingResou
seen: false,
}
}
// function to process pager client calls when the pageSize is specified
func (r *itemCollector) processPagerClientCalls(gr schema.GroupResource, label string, resourceClient client.Dynamic) (runtime.Object, error) {
// If limit is positive, use a pager to split list over multiple requests
// Use Velero's dynamic list function instead of the default
listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
return resourceClient.List(opts)
}))
// Use the page size defined in the server config
// TODO allow configuration of page buffer size
listPager.PageSize = int64(r.pageSize)
// Add each item to temporary slice
list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: label})
if err != nil {
r.log.WithError(errors.WithStack(err)).Error("Error listing resources")
return list, err
}
if !paginated {
r.log.Infof("list for groupResource %s was not paginated", gr)
}
return list, nil
}
func (r *itemCollector) listItemsForLabel(unstructuredItems []unstructured.Unstructured, gr schema.GroupResource, label string, resourceClient client.Dynamic) ([]unstructured.Unstructured, error) {
if r.pageSize > 0 {
// process pager client calls
list, err := r.processPagerClientCalls(gr, label, resourceClient)
if err != nil {
return unstructuredItems, err
}
err = meta.EachListItem(list, func(object runtime.Object) error {
u, ok := object.(*unstructured.Unstructured)
if !ok {
r.log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry in the list")
return fmt.Errorf("expected *unstructured.Unstructured but got %T", u)
}
unstructuredItems = append(unstructuredItems, *u)
return nil
})
if err != nil {
r.log.WithError(errors.WithStack(err)).Error("unable to understand paginated list")
return unstructuredItems, err
}
} else {
unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: label})
if err != nil {
r.log.WithError(errors.WithStack(err)).Error("Error listing items")
return unstructuredItems, err
}
unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
}
return unstructuredItems, nil
}