From f017a23d4104052ba5cdbe0f6558635f52c37ddb Mon Sep 17 00:00:00 2001 From: Justin Nauman Date: Sun, 27 Aug 2017 23:23:05 -0500 Subject: [PATCH] AWS Paging Support - Adding in paging support for the S3 and Snapshot AWS integration. As a testing note, you can add in a a MaxKeys to the S3 request as an easy way to ensure that paging is working properly without having to creation over 1k backups. Signed-off-by: Justin Nauman --- pkg/cloudprovider/aws/block_storage_adapter.go | 15 ++++++++------- pkg/cloudprovider/aws/object_storage_adapter.go | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/cloudprovider/aws/block_storage_adapter.go b/pkg/cloudprovider/aws/block_storage_adapter.go index f57618e2a..328ef91eb 100644 --- a/pkg/cloudprovider/aws/block_storage_adapter.go +++ b/pkg/cloudprovider/aws/block_storage_adapter.go @@ -166,17 +166,18 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st req.Filters = append(req.Filters, filter) } - res, err := op.ec2.DescribeSnapshots(req) + var ret []string + err := op.ec2.DescribeSnapshotsPages(req, func (res *ec2.DescribeSnapshotsOutput, lastPage bool) bool { + for _, snapshot := range res.Snapshots { + ret = append(ret, *snapshot.SnapshotId) + } + + return !lastPage + }) if err != nil { return nil, err } - var ret []string - - for _, snapshot := range res.Snapshots { - ret = append(ret, *snapshot.SnapshotId) - } - return ret, nil } diff --git a/pkg/cloudprovider/aws/object_storage_adapter.go b/pkg/cloudprovider/aws/object_storage_adapter.go index 856a32683..9b7d9ea17 100644 --- a/pkg/cloudprovider/aws/object_storage_adapter.go +++ b/pkg/cloudprovider/aws/object_storage_adapter.go @@ -106,17 +106,18 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri Delimiter: &delimiter, } - res, err := op.s3.ListObjectsV2(req) + var ret []string + err := op.s3.ListObjectsV2Pages(req, func(res *s3.ListObjectsV2Output, lastPage bool) bool { + for _, prefix := range res.CommonPrefixes { + ret = append(ret, *prefix.Prefix) + } + return !lastPage + }) + if err != nil { return nil, err } - ret := make([]string, 0, len(res.CommonPrefixes)) - - for _, prefix := range res.CommonPrefixes { - ret = append(ret, *prefix.Prefix) - } - return ret, nil }