mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-03 22:57:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add E2E test cases for backup VolumeInfo feature.

Browse Source 
Signed-off-by: Xun Jiang <blackpigletbruce@gmail.com>
This commit is contained in:
Xun Jiang
2024-02-22 16:18:55 +08:00
committed by Xun Jiang
parent 174c10fa8a
commit effbcba521
20 changed files with 677 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/util/k8s/common.go
+6
View File
@@ -204,6 +204,12 @@ func KubectlApplyByFile(ctx context.Context, file string) error {
return exec.CommandContext(ctx, "kubectl", args...).Run()
}
func KubectlDeleteByFile(ctx context.Context, file string) error {
args := []string{"delete", "-f", file, "--force=true"}
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}
func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error {
cmd := exec.CommandContext(ctx, "kubectl",
"config", "use-context", kubectlContext)
test/util/providers/aws_utils.go
+54
View File
@@ -20,6 +20,7 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net/http"
"net/url"
"os"
@@ -415,3 +416,56 @@ func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefi
}
return totalSize, nil
}
func (s AWSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
config := flag.NewMap()
config.Set(bslConfig)
objectsInput := s3.ListObjectsV2Input{}
objectsInput.Bucket = aws.String(bslBucket)
objectsInput.Delimiter = aws.String("/")
if bslPrefix != "" {
objectsInput.Prefix = aws.String(bslPrefix)
}
var err error
var s3Config aws.Config
var s3Client *s3.Client
region := config.Data()["region"]
s3url := ""
if region == "" {
region, err = GetBucketRegion(bslBucket)
if err != nil {
return nil, errors.Wrapf(err, "failed to get region for bucket %s", bslBucket)
}
}
if region == "minio" {
s3url = config.Data()["s3Url"]
s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, true, "")
if err != nil {
return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region)
}
s3Client, err = newS3Client(s3Config, s3url, true)
} else {
s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, false, "")
if err != nil {
return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region)
}
s3Client, err = newS3Client(s3Config, s3url, true)
}
if err != nil {
return nil, errors.Wrapf(err, "failed to create S3 client of region %s", region)
}
fullObjectKey := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(objectKey, "/")
result, err := s3Client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bslBucket),
Key: aws.String(fullObjectKey),
})
if err != nil {
return nil, errors.Wrapf(err, "failed to get object %s", fullObjectKey)
}
return result.Body, nil
}
test/util/providers/azure_utils.go
+30
View File
@@ -18,6 +18,7 @@ package providers
import (
"fmt"
"io"
"log"
"net/url"
"os"
@@ -364,3 +365,32 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN
return nil
}
}
func (s AzureStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
ctx := context.Background()
accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
if err != nil {
return nil, errors.Wrapf(err, "Fail to get storage account name and key of bucket %s", bslBucket)
}
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
log.Fatal("Invalid credentials with error: " + err.Error())
}
p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
URL, _ := url.Parse(
fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, bslBucket))
containerURL := azblob.NewContainerURL(*URL, p)
_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
handleErrors(err)
blobURL := containerURL.NewBlockBlobURL(strings.Join([]string{bslPrefix, objectKey}, "/"))
downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
if err != nil {
handleErrors(err)
}
return downloadResponse.Body(azblob.RetryReaderOptions{}), nil
}
test/util/providers/common.go
+61
View File
@@ -17,13 +17,17 @@ limitations under the License.
package providers
import (
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"strings"
"time"
"github.com/pkg/errors"
"github.com/vmware-tanzu/velero/internal/volume"
. "github.com/vmware-tanzu/velero/test"
velero "github.com/vmware-tanzu/velero/test/util/velero"
)
@@ -32,6 +36,7 @@ type ObjectsInStorage interface {
IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error)
}
func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
@@ -189,3 +194,59 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig
}
return nil
}
func GetVolumeInfoMetadataContent(
cloudProvider,
cloudCredentialsFile,
bslBucket,
bslPrefix,
bslConfig,
backupName,
subPrefix string,
) (io.Reader, error) {
bslPrefix = strings.Trim(getFullPrefix(bslPrefix, subPrefix), "/")
volumeFileName := backupName + "-volumeinfo.json.gz"
fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix)
s, err := getProvider(cloudProvider)
if err != nil {
return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
}
return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName)
}
func GetVolumeInfo(
cloudProvider,
cloudCredentialsFile,
bslBucket,
bslPrefix,
bslConfig,
backupName,
subPrefix string,
) ([]*volume.VolumeInfo, error) {
readCloser, err := GetVolumeInfoMetadataContent(cloudProvider,
cloudCredentialsFile,
bslBucket,
bslPrefix,
bslConfig,
backupName,
subPrefix,
)
if err != nil {
return nil, err
}
gzr, err := gzip.NewReader(readCloser)
if err != nil {
return nil, errors.WithStack(err)
}
defer gzr.Close()
volumeInfos := make([]*volume.VolumeInfo, 0)
if err := json.NewDecoder(gzr).Decode(&volumeInfos); err != nil {
return nil, errors.Wrap(err, "error decoding object data")
}
return volumeInfos, nil
}
test/util/providers/gcloud_utils.go
+12
View File
@@ -19,6 +19,7 @@ package providers
import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
@@ -65,6 +66,7 @@ func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix
}
}
}
func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error {
q := &storage.Query{
Prefix: bslPrefix,
@@ -141,3 +143,13 @@ func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj
return nil
}
}
func (s GCSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) {
ctx := context.Background()
client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile))
if err != nil {
return nil, errors.Wrapf(err, "fail to create GCloud client")
}
return client.Bucket(bslBucket).Object(strings.Join([]string{bslPrefix, objectKey}, "/")).NewReader(ctx)
}