mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-03 11:45:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

- fix buggy pager func

Browse Source 
fix paging items in to use list options passed by the paging function

The client-go pager sets the Limit options for the list call
to paginate the request[1]. This PR fixes the paging function
to use the options passed by the pager instead of shadowed options
This is required for the pagination to work correctly.

- simplify the pager list implementation by using pager.List()
The List() function already implements a lot of the logic that was
needed for paging here, using it simplifies the code.

1. 3f40906dd8/staging/src/k8s.io/client-go/tools/pager/pager.go (L219)

Signed-off-by: Alay Patel <alay1431@gmail.com>
This commit is contained in:
Alay Patel
2021-11-01 18:21:55 -04:00
parent de0fe7ff67
commit b2fe7fe304
1 changed files with 24 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
pkg/backup/item_collector.go
View File

@@ -26,7 +26,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
@@ -293,7 +293,6 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
if selector := r.backupRequest.Spec.LabelSelector; selector != nil {
labelSelector = metav1.FormatLabelSelector(selector)
}
listOptions := metav1.ListOptions{LabelSelector: labelSelector}
log.Info("Listing items")
unstructuredItems := make([]unstructured.Unstructured, 0)
@@ -301,50 +300,42 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group
if r.pageSize > 0 {
// If limit is positive, use a pager to split list over multiple requests
// Use Velero's dynamic list function instead of the default
listFunc := pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
list, err := resourceClient.List(listOptions)
if err != nil {
return nil, err
}
return list, nil
})
listPager := pager.New(listFunc)
listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
return resourceClient.List(opts)
}))
// Use the page size defined in the server config
// TODO allow configuration of page buffer size
listPager.PageSize = int64(r.pageSize)
// Add each item to temporary slice
var items []unstructured.Unstructured
err := listPager.EachListItem(context.Background(), listOptions, func(object runtime.Object) error {
item, isUnstructured := object.(*unstructured.Unstructured)
if !isUnstructured {
// We should never hit this
log.Error("Got type other than Unstructured from pager func")
return nil
}
items = append(items, *item)
return nil
})
if statusError, isStatusError := err.(*apierrors.StatusError); isStatusError && statusError.Status().Reason == metav1.StatusReasonExpired {
log.WithError(errors.WithStack(err)).Error("Error paging item list. Falling back on unpaginated list")
unstructuredList, err := resourceClient.List(listOptions)
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing items")
continue
}
items = unstructuredList.Items
} else if err != nil {
log.WithError(errors.WithStack(err)).Error("Error paging item list")
list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing resources")
continue
}
if !paginated {
log.Infof("list for groupResource %s was not paginated", gr)
}
err = meta.EachListItem(list, func(object runtime.Object) error {
u, ok := object.(*unstructured.Unstructured)
if !ok {
log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry in the list")
return fmt.Errorf("expected *unstructured.Unstructured but got %T", u)
}
unstructuredItems = append(unstructuredItems, *u)
return nil
})
if err != nil {
log.WithError(errors.WithStack(err)).Error("unable to understand paginated list")
continue
}
unstructuredItems = append(unstructuredItems, items...)
} else {
// If limit is not positive, do not use paging. Instead, request all items at once
unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector})
unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
if err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing items")
continue
}
unstructuredItems = append(unstructuredItems, unstructuredList.Items...)
}
log.Infof("Retrieved %d items", len(unstructuredItems))