mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-05-02 05:15:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #6684 from danfengliu/fix-storage-class-failure-issue

Browse Source 
Replace pod with deployment for E2E test
This commit is contained in:
danfengliu
2023-08-25 16:45:29 +08:00
committed by GitHub
parent 2e5aeb799a 6ef013d7c0
commit e6b510e32b
16 changed files with 302 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/e2e/basic/pvc-selected-node-changing.go
View File

@@ -98,7 +98,7 @@ func (p *PVCSelectedNodeChanging) CreateResources() error {
By("Prepare ConfigMap data", func() {
nodeNameList, err := GetWorkerNodes(context.Background())
Expect(err).To(Succeed())
Expect(len(nodeNameList) > 2).To(Equal(true))
Expect(len(nodeNameList) > 1).To(Equal(true))
for _, nodeName := range nodeNameList {
if nodeName != p.oldNodeName {
p.newNodeName = nodeName
@@ -142,7 +142,7 @@ func (p *PVCSelectedNodeChanging) Restore() error {
}
func (p *PVCSelectedNodeChanging) Verify() error {
By(fmt.Sprintf("PVC selected node should be %s", p.newNodeName), func() {
pvcNameList, err := GetPvcByPodName(context.Background(), p.mappedNS, p.pvcName)
pvcNameList, err := GetPvcByPVCName(context.Background(), p.mappedNS, p.pvcName)
Expect(err).To(Succeed())
Expect(len(pvcNameList)).Should(Equal(1))
pvc, err := GetPVC(context.Background(), p.Client, p.mappedNS, pvcNameList[0])

79
test/e2e/basic/storage-class-changing.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"time"
"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -24,44 +25,44 @@ type StorageClasssChanging struct {
namespace string
srcStorageClass string
desStorageClass string
pvcName string
volume string
podName string
mappedNS string
deploymentName string
CaseBaseName string
}
const SCCBaseName string = "scc-"
var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{
namespace: SCCBaseName + "1", TestCase: TestCase{NSBaseName: SCCBaseName}})
var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{})
func (s *StorageClasssChanging) Init() error {
s.TestCase.Init()
UUIDgen, err := uuid.NewRandom()
Expect(err).To(Succeed())
s.CaseBaseName = SCCBaseName + UUIDgen.String()
s.namespace = SCCBaseName + UUIDgen.String()
s.BackupName = "backup-" + s.CaseBaseName
s.RestoreName = "restore-" + s.CaseBaseName
s.mappedNS = s.namespace + "-mapped"
s.VeleroCfg = VeleroCfg
s.Client = *s.VeleroCfg.ClientToInstallVelero
s.NSBaseName = SCCBaseName
s.namespace = s.NSBaseName + UUIDgen.String()
s.mappedNS = s.namespace + "-mapped"
s.TestMsg = &TestMSG{
Desc: "Changing PV/PVC Storage Classes",
FailedMSG: "Failed to changing PV/PVC Storage Classes",
Text: "Change the storage class of persistent volumes and persistent" +
" volume claims during restores",
}
s.BackupName = "backup-sc-" + UUIDgen.String()
s.RestoreName = "restore-" + UUIDgen.String()
s.srcStorageClass = "default"
s.desStorageClass = "e2e-storage-class"
s.desStorageClass = StorageClassName
s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
"velero.io/plugin-config": ""}
s.data = map[string]string{s.srcStorageClass: s.desStorageClass}
s.configmaptName = "change-storage-class-config"
s.volume = "volume-1"
s.pvcName = fmt.Sprintf("pvc-%s", s.volume)
s.podName = "pod-1"
return nil
}
func (s *StorageClasssChanging) StartRun() error {
s.BackupName = s.BackupName + "backup-" + UUIDgen.String()
s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String()
s.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", s.BackupName,
"--include-namespaces", s.namespace,
@@ -74,9 +75,12 @@ func (s *StorageClasssChanging) StartRun() error {
return nil
}
func (s *StorageClasssChanging) CreateResources() error {
s.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
label := map[string]string{
"app": "test",
}
s.Ctx, _ = context.WithTimeout(context.Background(), 10*time.Minute)
By(fmt.Sprintf("Create a storage class %s", s.desStorageClass), func() {
Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml",
Expect(InstallStorageClass(s.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml",
s.VeleroCfg.CloudProvider))).To(Succeed())
})
By(fmt.Sprintf("Create namespace %s", s.namespace), func() {
@@ -84,10 +88,20 @@ func (s *StorageClasssChanging) CreateResources() error {
fmt.Sprintf("Failed to create namespace %s", s.namespace))
})
By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() {
_, err := CreatePod(s.Client, s.namespace, s.podName, s.srcStorageClass, "", []string{s.volume}, nil, nil)
By(fmt.Sprintf("Create a deployment in namespace %s", s.VeleroCfg.VeleroNamespace), func() {
pvc, err := CreatePVC(s.Client, s.namespace, s.pvcName, s.srcStorageClass, nil)
Expect(err).To(Succeed())
vols := CreateVolumes(pvc.Name, []string{s.volume})
deployment := NewDeployment(s.CaseBaseName, s.namespace, 1, label, nil).WithVolume(vols).Result()
deployment, err = CreateDeployment(s.Client.ClientGo, s.namespace, deployment)
Expect(err).To(Succeed())
s.deploymentName = deployment.Name
err = WaitForReadyDeployment(s.Client.ClientGo, s.namespace, s.deploymentName)
Expect(err).To(Succeed())
})
By(fmt.Sprintf("Create ConfigMap %s in namespace %s", s.configmaptName, s.VeleroCfg.VeleroNamespace), func() {
_, err := CreateConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName, s.labels, s.data)
Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", s.VeleroCfg.VeleroNamespace))
@@ -97,19 +111,14 @@ func (s *StorageClasssChanging) CreateResources() error {
func (s *StorageClasssChanging) Destroy() error {
By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.srcStorageClass), func() {
pvName, err := GetPVByPodName(s.Client, s.namespace, s.volume)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName))
pvName, err := GetPVByPVCName(s.Client, s.namespace, s.pvcName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by PVC name %s", s.pvcName))
pv, err := GetPersistentVolume(s.Ctx, s.Client, s.namespace, pvName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName))
fmt.Println(pv)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by name %s", pvName))
Expect(pv.Spec.StorageClassName).To(Equal(s.srcStorageClass),
fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.srcStorageClass))
})
By(fmt.Sprintf("Start to destroy namespace %s......", s.NSBaseName), func() {
Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.NSBaseName)).To(Succeed(),
fmt.Sprintf("Failed to delete namespace %s", s.NSBaseName))
})
return nil
}
@@ -129,14 +138,26 @@ func (s *StorageClasssChanging) Restore() error {
}
func (s *StorageClasssChanging) Verify() error {
By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.desStorageClass), func() {
time.Sleep(1 * time.Minute)
pvName, err := GetPVByPodName(s.Client, s.mappedNS, s.volume)
Expect(WaitForReadyDeployment(s.Client.ClientGo, s.mappedNS, s.deploymentName)).To(Succeed())
pvName, err := GetPVByPVCName(s.Client, s.mappedNS, s.pvcName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName))
pv, err := GetPersistentVolume(s.Ctx, s.Client, s.mappedNS, pvName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName))
fmt.Println(pv)
Expect(pv.Spec.StorageClassName).To(Equal(s.desStorageClass),
fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.desStorageClass))
})
return nil
}
func (s *StorageClasssChanging) Clean() error {
if !s.VeleroCfg.Debug {
By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() {
Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(),
fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName))
})
DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName)
DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass)
s.TestCase.Clean()
}
return nil
}

4
test/e2e/bsl-mgmt/deletion.go
View File

@@ -169,12 +169,12 @@ func BslDeletionTest(useVolumeSnapshots bool) {
})
By("Get all 2 PVCs of Kibishii and label them seprately ", func() {
pvc, err := GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_1)
pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1)
Expect(err).To(Succeed())
fmt.Println(pvc)
Expect(len(pvc)).To(Equal(1))
pvc1 := pvc[0]
pvc, err = GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_2)
pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2)
Expect(err).To(Succeed())
fmt.Println(pvc)
Expect(len(pvc)).To(Equal(1))

6
test/e2e/pv-backup/pv-backup-filter.go
View File

@@ -124,7 +124,7 @@ func (p *PVBackupFiltering) CreateResources() error {
WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])
for i, pod := range p.podsList[index] {
for j := range p.volumesList[i] {
Expect(CreateFileToPod(p.Ctx, ns, pod, p.volumesList[i][j],
Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j],
FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed())
}
}
@@ -182,7 +182,7 @@ func fileContent(namespace, podName, volume string) string {
}
func fileExist(ctx context.Context, namespace, podName, volume string) error {
c, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
FILE_NAME, volume, podName, namespace))
@@ -197,7 +197,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
}
}
func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
_, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
if err != nil {
return nil
} else {

2
test/e2e/resource-filtering/base.go
View File

@@ -84,7 +84,7 @@ func (f *FilteringCase) CreateResources() error {
}
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil)
deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result()
deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

2
test/e2e/resource-filtering/exclude_label.go
View File

@@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error {
}
//Create deployment: to be included
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil)
deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result()
deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

2
test/e2e/resource-filtering/label_selector.go
View File

@@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error {
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil)
deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result()
deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

2
test/e2e/schedule/ordered_resources.go
View File

@@ -159,7 +159,7 @@ func (o *OrderedResources) CreateResources() error {
//Create deployment
deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil)
deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result()
deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))

4
test/e2e/test/test.go
View File

@@ -34,6 +34,8 @@ import (
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
const StorageClassName = "e2e-storage-class"
/*
The VeleroBackupRestoreTest interface is just could be suit for the cases that follow the test flow of
create resources, backup, delete test resource, restore and verify.
@@ -83,7 +85,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
flag.Parse()
veleroCfg := test.GetTestCase().VeleroCfg
// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") {
if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") {
Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
}
if veleroCfg.InstallVelero {

1
test/e2e/upgrade/upgrade.go
View File

@@ -120,6 +120,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.RestoreHelperImage = ""
tmpCfgForOldVeleroInstall.Plugins = ""
tmpCfgForOldVeleroInstall.UploaderType = ""
tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots
if supportUploaderType {
tmpCfgForOldVeleroInstall.UseRestic = false
tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots

58
test/e2e/util/csi/common.go
View File

@@ -93,7 +93,7 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
return snapshotHandleList, nil
}
func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName)
if err != nil {
return "", err
}
@@ -129,13 +129,61 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
}
func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) {
var err error
var snapshotContentNameList []string
if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil ||
len(snapshotContentNameList) != expectedCount {
return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
if apiVersion == "v1beta1" {
if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
}
} else if apiVersion == "v1" {
if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
}
} else {
return nil, errors.New("API version is invalid")
}
if len(snapshotContentNameList) != expectedCount {
return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))
}
fmt.Println(snapshotContentNameList)
return snapshotContentNameList, nil
}
func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) {
_, snapshotClient, err := GetClients()
if err != nil {
return nil, err
}
vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
if err1 != nil {
return nil, err
}
var snapshotHandleList []string
for _, i := range vscList.Items {
if i.Status == nil {
fmt.Println("SnapshotHandle Status s nil")
continue
}
if i.Status.SnapshotHandle == nil {
fmt.Println("SnapshotHandle is nil")
continue
}
if i.Labels == nil {
fmt.Println("VolumeSnapshotContents label is nil")
continue
}
if i.Labels["velero.io/backup-name"] == backupName {
tmp := strings.Split(*i.Status.SnapshotHandle, "/")
snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
}
}
if len(snapshotHandleList) == 0 {
fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
}
return snapshotHandleList, nil
}

92
test/e2e/util/k8s/common.go
View File

@@ -33,7 +33,7 @@ import (
"github.com/vmware-tanzu/velero/test/e2e/util/common"
)
// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
// ensureClusterExists returns whether or not a Kubernetes cluster exists for tests to be run on.
func EnsureClusterExists(ctx context.Context) error {
return exec.CommandContext(ctx, "kubectl", "cluster-info").Run()
}
@@ -81,7 +81,7 @@ func WaitForPods(ctx context.Context, client TestClient, namespace string, pods
return nil
}
func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, error) {
func GetPvcByPVCName(ctx context.Context, namespace, pvcName string) ([]string, error) {
// Example:
// NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
// kibishii-data-kibishii-deployment-0 Bound pvc-94b9fdf2-c30f-4a7b-87bf-06eadca0d5b6 1Gi RWO kibishii-storage-class 115s
@@ -94,7 +94,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string,
cmd = &common.OsCommandLine{
Cmd: "grep",
Args: []string{podName},
Args: []string{pvcName},
}
cmds = append(cmds, cmd)
@@ -200,6 +200,7 @@ func AddLabelToCRD(ctx context.Context, crd, label string) error {
func KubectlApplyByFile(ctx context.Context, file string) error {
args := []string{"apply", "-f", file, "--force=true"}
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}
@@ -213,39 +214,20 @@ func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error {
return err
}
func GetAPIVersions(client *TestClient, name string) ([]string, error) {
var version []string
APIGroup, err := client.ClientGo.Discovery().ServerGroups()
if err != nil {
return nil, errors.Wrap(err, "Fail to get server API groups")
}
for _, group := range APIGroup.Groups {
fmt.Println(group.Name)
if group.Name == name {
for _, v := range group.Versions {
fmt.Println(v.Version)
version = append(version, v.Version)
}
return version, nil
}
}
return nil, errors.New("Server API groups is empty")
}
func GetPVByPodName(client TestClient, namespace, podName string) (string, error) {
pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
func GetPVByPVCName(client TestClient, namespace, pvcName string) (string, error) {
pvcList, err := GetPvcByPVCName(context.Background(), namespace, pvcName)
if err != nil {
return "", err
}
if len(pvcList) != 1 {
return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace))
return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s but got %v", pvcName, namespace, pvcList))
}
pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
if err != nil {
return "", err
}
if len(pvList) != 1 {
return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace))
return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace))
}
pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0])
fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"])
@@ -255,15 +237,30 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error
return pv_value.Name, nil
}
func CreateFileToPod(ctx context.Context, namespace, podName, volume, filename, content string) error {
arg := []string{"exec", "-n", namespace, "-c", podName, podName,
func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) {
for i, volume := range volumeNameList {
vols = append(vols, &corev1.Volume{
Name: volume,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: fmt.Sprintf("pvc-%d", i),
ReadOnly: false,
},
},
})
}
return
}
func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error {
arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
"--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)}
cmd := exec.CommandContext(ctx, "kubectl", arg...)
fmt.Printf("Kubectl exec cmd =%v\n", cmd)
return cmd.Run()
}
func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, filename string) (string, error) {
arg := []string{"exec", "-n", namespace, "-c", podName, podName,
func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) {
arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
cmd := exec.CommandContext(ctx, "kubectl", arg...)
fmt.Printf("Kubectl exec cmd =%v\n", cmd)
@@ -331,3 +328,38 @@ func GetAllService(ctx context.Context) (string, error) {
}
return stdout, nil
}
func CreateVolumes(pvcName string, volumeNameList []string) (vols []*corev1.Volume) {
vols = []*corev1.Volume{}
for _, volume := range volumeNameList {
vols = append(vols, &corev1.Volume{
Name: volume,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
ReadOnly: false,
},
},
})
}
return
}
func GetAPIVersions(client *TestClient, name string) ([]string, error) {
var version []string
APIGroup, err := client.ClientGo.Discovery().ServerGroups()
if err != nil {
return nil, errors.Wrap(err, "Fail to get server API groups")
}
for _, group := range APIGroup.Groups {
fmt.Println(group.Name)
if group.Name == name {
for _, v := range group.Versions {
fmt.Println(v.Version)
version = append(version, v.Version)
}
return version, nil
}
}
return nil, errors.New("Server API groups is empty")
}

24
test/e2e/util/k8s/configmap.go
View File

@@ -42,6 +42,20 @@ func CreateConfigMap(c clientset.Interface, ns, name string, labels, data map[st
return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{})
}
func CreateConfigMapFromYAMLData(c clientset.Interface, yamlData, cmName, namespace string) error {
cmData := make(map[string]string)
cmData[cmName] = yamlData
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
Namespace: namespace,
},
Data: cmData,
}
_, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), cm, metav1.CreateOptions{})
return err
}
// WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns.
func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error {
return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
@@ -57,10 +71,18 @@ func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap,
return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
}
func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
func DeleteConfigmap(c clientset.Interface, ns, name string) error {
if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns))
}
return nil
}
func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
if err := DeleteConfigmap(c, ns, name); err != nil {
return err
}
return waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {

92
test/e2e/util/k8s/deployment.go
View File

@@ -26,6 +26,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
)
const (
@@ -35,46 +37,90 @@ const (
PollTimeout = 15 * time.Minute
)
// DeploymentBuilder builds Deployment objects.
type DeploymentBuilder struct {
*apps.Deployment
}
func (d *DeploymentBuilder) Result() *apps.Deployment {
return d.Deployment
}
// newDeployment returns a RollingUpdate Deployment with a fake container image
func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment {
func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *DeploymentBuilder {
if containers == nil {
containers = []v1.Container{
{
Name: fmt.Sprintf("container-%s", "busybox"),
Name: "container-busybox",
Image: "gcr.io/velero-gcp/busybox:latest",
Command: []string{"sleep", "1000000"},
// Make pod obeys the restricted pod security standards.
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: boolptr.False(),
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
RunAsNonRoot: boolptr.True(),
RunAsUser: func(i int64) *int64 { return &i }(65534),
RunAsGroup: func(i int64) *int64 { return &i }(65534),
SeccompProfile: &v1.SeccompProfile{
Type: v1.SeccompProfileTypeRuntimeDefault,
},
},
},
}
}
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: name,
Labels: labels,
},
Spec: apps.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{MatchLabels: labels},
Strategy: apps.DeploymentStrategy{
Type: apps.RollingUpdateDeploymentStrategyType,
RollingUpdate: new(apps.RollingUpdateDeployment),
return &DeploymentBuilder{
&apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: name,
Labels: labels,
},
Spec: apps.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{MatchLabels: labels},
Strategy: apps.DeploymentStrategy{
Type: apps.RollingUpdateDeploymentStrategyType,
RollingUpdate: new(apps.RollingUpdateDeployment),
},
Spec: v1.PodSpec{
Containers: containers,
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
FSGroup: func(i int64) *int64 { return &i }(65534),
FSGroupChangePolicy: func(policy v1.PodFSGroupChangePolicy) *v1.PodFSGroupChangePolicy { return &policy }(v1.FSGroupChangeAlways),
},
Containers: containers,
},
},
},
},
}
}
func (d *DeploymentBuilder) WithVolume(volumes []*v1.Volume) *DeploymentBuilder {
vmList := []v1.VolumeMount{}
for _, v := range volumes {
vmList = append(vmList, v1.VolumeMount{
Name: v.Name,
MountPath: "/" + v.Name,
})
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, *v)
}
// NOTE here just mount volumes to the first container
d.Spec.Template.Spec.Containers[0].VolumeMounts = vmList
return d
}
func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error {
_, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
return err

10
test/e2e/util/k8s/sc.go
View File

@@ -3,6 +3,9 @@ package k8s
import (
"context"
"fmt"
"github.com/pkg/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func InstallStorageClass(ctx context.Context, yaml string) error {
@@ -10,3 +13,10 @@ func InstallStorageClass(ctx context.Context, yaml string) error {
err := KubectlApplyByFile(ctx, yaml)
return err
}
func DeleteStorageClass(ctx context.Context, client TestClient, name string) error {
if err := client.ClientGo.StorageV1().StorageClasses().Delete(ctx, name, v1.DeleteOptions{}); err != nil {
return errors.Wrapf(err, "Could not retrieve storage classes %s", name)
}
return nil
}

36
test/e2e/util/velero/velero_utils.go
View File

@@ -1056,15 +1056,25 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
return common.GetListByCmdPipes(ctx, cmds)
}
func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
var snapshotCheckPoint SnapshotCheckPoint
var err error
snapshotCheckPoint.ExpectCount = expectCount
snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
snapshotCheckPoint.PodName = kibishiiPodNameList
snapshotCheckPoint.PodName = KibishiiPVCNameList
if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
snapshotCheckPoint.EnableCSI = true
if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
resourceName := "snapshot.storage.k8s.io"
srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
if err != nil {
return snapshotCheckPoint, err
}
if len(srcVersions) == 0 {
return snapshotCheckPoint, errors.New("Fail to get APIVersion")
}
if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil {
return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
}
}
@@ -1183,12 +1193,9 @@ func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]stri
Args: []string{"get", "deploy", "-n", veleroCfg.VeleroNamespace, "-ojson"},
}
cmds = append(cmds, cmd)
var args string
if veleroCfg.CloudProvider == "vsphere" {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
} else {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
}
args := fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
cmd = &common.OsCommandLine{
Cmd: "sed",
Args: []string{args},
@@ -1236,12 +1243,9 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string)
Args: []string{dsjson},
}
cmds = append(cmds, cmd)
var args string
if veleroCfg.CloudProvider == "vsphere" {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
} else {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
}
args := fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
cmd = &common.OsCommandLine{
Cmd: "sed",
Args: []string{args},