Support using image registry proxy in more cases.

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>
This commit is contained in:
Xun Jiang
2025-05-09 19:27:03 +08:00
parent d5a2e7e6b9
commit 14e1055a9a
25 changed files with 387 additions and 107 deletions

View File

@@ -115,8 +115,17 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
}()
}
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, ns,
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
if err := KibishiiPrepareBeforeBackup(
oneHourTimeout,
client,
providerName,
ns,
registryCredentialFile,
veleroFeatures,
kibishiiDirectory,
DefaultKibishiiData,
veleroCfg.ImageRegistryProxy,
); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns)
}
err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)

View File

@@ -100,9 +100,17 @@ func TTLTest() {
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(ctx, client, veleroCfg.CloudProvider,
test.testNS, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(
ctx,
client,
veleroCfg.CloudProvider,
test.testNS,
veleroCfg.RegistryCredentialFile,
veleroCfg.Features,
veleroCfg.KibishiiDirectory,
DefaultKibishiiData,
veleroCfg.ImageRegistryProxy,
)).To(Succeed())
})
var BackupCfg BackupConfig

View File

@@ -121,7 +121,7 @@ func (v *BackupVolumeInfo) CreateResources() error {
volumeName := fmt.Sprintf("volume-info-pv-%d", i)
vols = append(vols, CreateVolumes(pvc.Name, []string{volumeName})...)
}
deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, nil).WithVolume(vols).Result()
deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, v.VeleroCfg.ImageRegistryProxy).WithVolume(vols).Result()
deployment, err := CreateDeployment(v.Client.ClientGo, createNSName, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", createNSName))

View File

@@ -91,9 +91,17 @@ func (n *NamespaceMapping) CreateResources() error {
Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, n.VeleroCfg.CloudProvider,
ns, n.VeleroCfg.RegistryCredentialFile, n.VeleroCfg.Features,
n.VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(
n.Ctx,
n.Client,
n.VeleroCfg.CloudProvider,
ns,
n.VeleroCfg.RegistryCredentialFile,
n.VeleroCfg.Features,
n.VeleroCfg.KibishiiDirectory,
n.kibishiiData,
n.VeleroCfg.ImageRegistryProxy,
)).To(Succeed())
})
}
return nil

View File

@@ -75,7 +75,8 @@ func (p *PVCSelectedNodeChanging) CreateResources() error {
p.oldNodeName = nodeName
fmt.Printf("Create PVC on node %s\n", p.oldNodeName)
pvcAnn := map[string]string{p.ann: nodeName}
_, err := CreatePod(p.Client, p.namespace, p.podName, StorageClassName, p.pvcName, []string{p.volume}, pvcAnn, nil)
_, err := CreatePod(p.Client, p.namespace, p.podName, StorageClassName, p.pvcName, []string{p.volume},
pvcAnn, nil, p.VeleroCfg.ImageRegistryProxy)
Expect(err).To(Succeed())
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName})
Expect(err).To(Succeed())

View File

@@ -82,7 +82,7 @@ func (s *StorageClasssChanging) CreateResources() error {
Expect(err).To(Succeed())
vols := CreateVolumes(pvc.Name, []string{s.volume})
deployment := NewDeployment(s.CaseBaseName, s.namespace, 1, label, nil).WithVolume(vols).Result()
deployment := NewDeployment(s.CaseBaseName, s.namespace, 1, label, s.VeleroCfg.ImageRegistryProxy).WithVolume(vols).Result()
deployment, err = CreateDeployment(s.Client.ClientGo, s.namespace, deployment)
Expect(err).To(Succeed())
s.deploymentName = deployment.Name

View File

@@ -152,9 +152,17 @@ func BslDeletionTest(useVolumeSnapshots bool) {
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, veleroCfg.CloudProvider,
bslDeletionTestNs, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(
oneHourTimeout,
*veleroCfg.ClientToInstallVelero,
veleroCfg.CloudProvider,
bslDeletionTestNs,
veleroCfg.RegistryCredentialFile,
veleroCfg.Features,
veleroCfg.KibishiiDirectory,
DefaultKibishiiData,
veleroCfg.ImageRegistryProxy,
)).To(Succeed())
})
// Restic can not backup PV only, so pod need to be labeled also

View File

@@ -55,6 +55,7 @@ import (
func init() {
test.VeleroCfg.Options = install.Options{}
test.VeleroCfg.BackupRepoConfigMap = test.BackupRepositoryConfigName // Set to the default value
flag.StringVar(
&test.VeleroCfg.CloudProvider,
"cloud-provider",
@@ -699,6 +700,8 @@ func TestE2e(t *testing.T) {
t.FailNow()
}
veleroutil.UpdateImagesMatrixByProxy(test.VeleroCfg.ImageRegistryProxy)
RegisterFailHandler(Fail)
testSuitePassed = RunSpecs(t, "E2e Suite")
}

View File

@@ -195,8 +195,8 @@ func (m *migrationE2E) Backup() error {
OriginVeleroCfg.RegistryCredentialFile,
OriginVeleroCfg.Features,
OriginVeleroCfg.KibishiiDirectory,
OriginVeleroCfg.UseVolumeSnapshots,
&m.kibishiiData,
OriginVeleroCfg.ImageRegistryProxy,
)).To(Succeed())
})

View File

@@ -95,7 +95,8 @@ func (p *ParallelFilesDownload) CreateResources() error {
})
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume},
nil, nil, p.VeleroCfg.ImageRegistryProxy)
Expect(err).To(Succeed())
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
Expect(err).To(Succeed())

View File

@@ -86,7 +86,8 @@ func (p *ParallelFilesUpload) CreateResources() error {
})
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc,
[]string{p.volume}, nil, nil, p.VeleroCfg.ImageRegistryProxy)
Expect(err).To(Succeed())
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
Expect(err).To(Succeed())

View File

@@ -87,7 +87,8 @@ func (p *PVBackupFiltering) CreateResources() error {
podName := fmt.Sprintf("pod-%d", i)
pods = append(pods, podName)
By(fmt.Sprintf("Create pod %s in namespace %s", podName, ns), func() {
pod, err := CreatePod(p.Client, ns, podName, StorageClassName, "", volumes, nil, nil)
pod, err := CreatePod(p.Client, ns, podName, StorageClassName, "",
volumes, nil, nil, p.VeleroCfg.ImageRegistryProxy)
Expect(err).To(Succeed())
ann := map[string]string{
p.annotation: volumesToAnnotation,

View File

@@ -68,7 +68,7 @@ func (f *FilteringCase) CreateResources() error {
}
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(f.CaseBaseName, namespace, f.replica, f.labels, nil).Result()
deployment := NewDeployment(f.CaseBaseName, namespace, f.replica, f.labels, f.VeleroCfg.ImageRegistryProxy).Result()
deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

View File

@@ -88,7 +88,7 @@ func (e *ExcludeFromBackup) CreateResources() error {
}
//Create deployment: to be included
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(e.CaseBaseName, namespace, e.replica, label2, nil).Result()
deployment := NewDeployment(e.CaseBaseName, namespace, e.replica, label2, e.VeleroCfg.ImageRegistryProxy).Result()
deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

View File

@@ -88,7 +88,7 @@ func (l *LabelSelector) CreateResources() error {
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(l.CaseBaseName, namespace, l.replica, labels, nil).Result()
deployment := NewDeployment(l.CaseBaseName, namespace, l.replica, labels, l.VeleroCfg.ImageRegistryProxy).Result()
deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

View File

@@ -145,7 +145,7 @@ func (r *ResourceModifiersCase) Clean() error {
}
func (r *ResourceModifiersCase) createDeployment(namespace string) error {
deployment := NewDeployment(r.CaseBaseName, namespace, 1, map[string]string{"app": "test"}, nil).Result()
deployment := NewDeployment(r.CaseBaseName, namespace, 1, map[string]string{"app": "test"}, r.VeleroCfg.ImageRegistryProxy).Result()
deployment, err := CreateDeployment(r.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to create deloyment %s the namespace %q", deployment.Name, namespace))

View File

@@ -209,7 +209,7 @@ func (r *ResourcePoliciesCase) createPVC(index int, namespace string, volList []
}
func (r *ResourcePoliciesCase) createDeploymentWithVolume(namespace string, volList []*corev1api.Volume) error {
deployment := NewDeployment(r.CaseBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, nil).WithVolume(volList).Result()
deployment := NewDeployment(r.CaseBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, r.VeleroCfg.ImageRegistryProxy).WithVolume(volList).Result()
deployment, err := CreateDeployment(r.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to create deloyment %s the namespace %q", deployment.Name, namespace))

View File

@@ -84,6 +84,7 @@ func (s *InProgressCase) CreateResources() error {
[]string{s.volume},
nil,
s.podAnn,
s.VeleroCfg.ImageRegistryProxy,
)
Expect(err).To(Succeed())

View File

@@ -99,7 +99,7 @@ func (o *OrderedResources) CreateResources() error {
//Create deployment
deploymentName := fmt.Sprintf("deploy-%s", o.CaseBaseName)
fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
deployment := k8sutil.NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result()
deployment := k8sutil.NewDeployment(deploymentName, o.Namespace, 1, label, o.VeleroCfg.ImageRegistryProxy).Result()
_, err := k8sutil.CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))

View File

@@ -126,6 +126,15 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion
tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI
// CLI under version v1.14.x
if veleroCLI2Version.VeleroVersion < "v1.15" {
tmpCfgForOldVeleroInstall.BackupRepoConfigMap = ""
fmt.Printf(
"CLI version %s is lower than v1.15. Set BackupRepoConfigMap to empty, because it's not supported",
veleroCLI2Version.VeleroVersion,
)
}
tmpCfgForOldVeleroInstall, err = SetImagesToDefaultValues(
tmpCfgForOldVeleroInstall,
veleroCLI2Version.VeleroVersion,
@@ -157,9 +166,17 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, tmpCfg.CloudProvider,
upgradeNamespace, tmpCfg.RegistryCredentialFile, tmpCfg.Features,
tmpCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(
oneHourTimeout,
*veleroCfg.ClientToInstallVelero,
tmpCfg.CloudProvider,
upgradeNamespace,
tmpCfg.RegistryCredentialFile,
tmpCfg.Features,
tmpCfg.KibishiiDirectory,
DefaultKibishiiData,
tmpCfg.ImageRegistryProxy,
)).To(Succeed())
})
By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() {
@@ -239,6 +256,9 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
}
})
// Wait for 70s to make sure the backups are synced after Velero reinstall
time.Sleep(70 * time.Second)
By(fmt.Sprintf("Restore %s", upgradeNamespace), func() {
Expect(VeleroRestore(oneHourTimeout, tmpCfg.VeleroCLI,
tmpCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {

View File

@@ -18,6 +18,7 @@ package k8s
import (
"fmt"
"path"
"time"
"golang.org/x/net/context"
@@ -36,6 +37,7 @@ const (
PollInterval = 2 * time.Second
PollTimeout = 15 * time.Minute
DefaultContainerName = "container-busybox"
TestImage = "busybox:1.37.0"
)
// DeploymentBuilder builds Deployment objects.
@@ -48,29 +50,33 @@ func (d *DeploymentBuilder) Result() *appsv1api.Deployment {
}
// newDeployment returns a RollingUpdate Deployment with a fake container image
func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []corev1api.Container) *DeploymentBuilder {
if containers == nil {
containers = []corev1api.Container{
{
Name: DefaultContainerName,
Image: "busybox:1.37.0",
Command: []string{"sleep", "1000000"},
// Make pod obeys the restricted pod security standards.
SecurityContext: &corev1api.SecurityContext{
AllowPrivilegeEscalation: boolptr.False(),
Capabilities: &corev1api.Capabilities{
Drop: []corev1api.Capability{"ALL"},
},
RunAsNonRoot: boolptr.True(),
RunAsUser: func(i int64) *int64 { return &i }(65534),
RunAsGroup: func(i int64) *int64 { return &i }(65534),
SeccompProfile: &corev1api.SeccompProfile{
Type: corev1api.SeccompProfileTypeRuntimeDefault,
},
func NewDeployment(name, ns string, replicas int32, labels map[string]string, imageRegistryProxy string) *DeploymentBuilder {
imageAddress := TestImage
if imageRegistryProxy != "" {
imageAddress = path.Join(imageRegistryProxy, TestImage)
}
containers := []corev1api.Container{
{
Name: DefaultContainerName,
Image: imageAddress,
Command: []string{"sleep", "1000000"},
// Make pod obeys the restricted pod security standards.
SecurityContext: &corev1api.SecurityContext{
AllowPrivilegeEscalation: boolptr.False(),
Capabilities: &corev1api.Capabilities{
Drop: []corev1api.Capability{"ALL"},
},
RunAsNonRoot: boolptr.True(),
RunAsUser: func(i int64) *int64 { return &i }(65534),
RunAsGroup: func(i int64) *int64 { return &i }(65534),
SeccompProfile: &corev1api.SeccompProfile{
Type: corev1api.SeccompProfileTypeRuntimeDefault,
},
},
}
},
}
return &DeploymentBuilder{
&appsv1api.Deployment{
TypeMeta: metav1.TypeMeta{

View File

@@ -19,6 +19,7 @@ package k8s
import (
"context"
"fmt"
"path"
"github.com/pkg/errors"
corev1api "k8s.io/api/core/v1"
@@ -27,10 +28,22 @@ import (
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
)
func CreatePod(client TestClient, ns, name, sc, pvcName string, volumeNameList []string, pvcAnn, ann map[string]string) (*corev1api.Pod, error) {
func CreatePod(
client TestClient,
ns, name, sc, pvcName string,
volumeNameList []string,
pvcAnn, ann map[string]string,
imageRegistryProxy string,
) (*corev1api.Pod, error) {
if pvcName != "" && len(volumeNameList) != 1 {
return nil, errors.New("Volume name list should contain only 1 since PVC name is not empty")
}
imageAddress := TestImage
if imageRegistryProxy != "" {
imageAddress = path.Join(imageRegistryProxy, TestImage)
}
volumes := []corev1api.Volume{}
for _, volume := range volumeNameList {
var _pvcName string
@@ -76,7 +89,7 @@ func CreatePod(client TestClient, ns, name, sc, pvcName string, volumeNameList [
Containers: []corev1api.Container{
{
Name: name,
Image: "busybox:1.37.0",
Image: imageAddress,
Command: []string{"sleep", "3600"},
VolumeMounts: vmList,
// Make pod obeys the restricted pod security standards.

View File

@@ -18,6 +18,8 @@ package kibishii
import (
"fmt"
"html/template"
"os"
"os/exec"
"path"
"strconv"
@@ -27,7 +29,10 @@ import (
. "github.com/onsi/ginkgo/v2"
"github.com/pkg/errors"
"golang.org/x/net/context"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/yaml"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test"
@@ -102,9 +107,17 @@ func RunKibishiiTests(
}
}()
fmt.Printf("KibishiiPrepareBeforeBackup %s\n", time.Now().Format("2006-01-02 15:04:05"))
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName,
kibishiiNamespace, registryCredentialFile, veleroFeatures,
kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
if err := KibishiiPrepareBeforeBackup(
oneHourTimeout,
client,
providerName,
kibishiiNamespace,
registryCredentialFile,
veleroFeatures,
kibishiiDirectory,
DefaultKibishiiData,
veleroCfg.ImageRegistryProxy,
); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
}
fmt.Printf("KibishiiPrepareBeforeBackup done %s\n", time.Now().Format("2006-01-02 15:04:05"))
@@ -264,8 +277,15 @@ func RunKibishiiTests(
return nil
}
func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures,
kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error {
func installKibishii(
ctx context.Context,
namespace string,
cloudPlatform,
veleroFeatures,
kibishiiDirectory string,
workerReplicas int,
imageRegistryProxy string,
) error {
if strings.EqualFold(cloudPlatform, Azure) &&
strings.EqualFold(veleroFeatures, FeatureCSI) {
cloudPlatform = AzureCSI
@@ -274,6 +294,29 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
strings.EqualFold(veleroFeatures, FeatureCSI) {
cloudPlatform = AwsCSI
}
if strings.EqualFold(cloudPlatform, Vsphere) {
if strings.HasPrefix(kibishiiDirectory, "https://") {
return errors.New("vSphere needs to download the Kibishii repository first because it needs to inject some image patch file to work.")
}
kibishiiImage := readBaseKibishiiImage(path.Join(kibishiiDirectory, "base", "kibishii.yaml"))
if err := generateKibishiiImagePatch(
path.Join(imageRegistryProxy, kibishiiImage),
path.Join(kibishiiDirectory, cloudPlatform, "worker-image-patch.yaml"),
); err != nil {
return nil
}
jumpPadImage := readBaseJumpPadImage(path.Join(kibishiiDirectory, "base", "jump-pad.yaml"))
if err := generateJumpPadPatch(
path.Join(imageRegistryProxy, jumpPadImage),
path.Join(kibishiiDirectory, cloudPlatform, "jump-pad-image-patch.yaml"),
); err != nil {
return nil
}
}
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
path.Join(kibishiiDirectory, cloudPlatform), "--timeout=90s")
@@ -313,16 +356,134 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
return err
}
func readBaseKibishiiImage(kibishiiFilePath string) string {
bytes, err := os.ReadFile(kibishiiFilePath)
if err != nil {
return ""
}
sts := &appsv1api.StatefulSet{}
if err := yaml.UnmarshalStrict(bytes, sts); err != nil {
return ""
}
kibishiiImage := ""
if len(sts.Spec.Template.Spec.Containers) > 0 {
kibishiiImage = sts.Spec.Template.Spec.Containers[0].Image
}
return kibishiiImage
}
func readBaseJumpPadImage(jumpPadFilePath string) string {
bytes, err := os.ReadFile(jumpPadFilePath)
if err != nil {
return ""
}
pod := &corev1api.Pod{}
if err := yaml.UnmarshalStrict(bytes, pod); err != nil {
return ""
}
jumpPadImage := ""
if len(pod.Spec.Containers) > 0 {
jumpPadImage = pod.Spec.Containers[0].Image
}
return jumpPadImage
}
type patchImageData struct {
Image string
}
func generateKibishiiImagePatch(kibishiiImage string, patchDirectory string) error {
patchString := `
apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2
kind: StatefulSet
metadata:
name: kibishii-deployment
spec:
template:
spec:
containers:
- name: kibishii
image: {{.Image}}
`
file, err := os.OpenFile(patchDirectory, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
defer file.Close()
if err != nil {
return err
}
patchTemplate, err := template.New("imagePatch").Parse(patchString)
if err != nil {
return err
}
if err := patchTemplate.Execute(file, patchImageData{Image: kibishiiImage}); err != nil {
return err
}
return nil
}
func generateJumpPadPatch(jumpPadImage string, patchDirectory string) error {
patchString := `
apiVersion: v1
kind: Pod
metadata:
name: jump-pad
spec:
containers:
- name: jump-pad
image: {{.Image}}
`
file, err := os.OpenFile(patchDirectory, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
defer file.Close()
if err != nil {
return err
}
patchTemplate, err := template.New("imagePatch").Parse(patchString)
if err != nil {
return err
}
if err := patchTemplate.Execute(file, patchImageData{Image: jumpPadImage}); err != nil {
return err
}
return nil
}
func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
timeout := 30 * time.Minute
interval := 1 * time.Second
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
timeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*20)
defer ctxCancel()
kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
"/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),
strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes))
kibishiiGenerateCmd := exec.CommandContext(
timeout,
"kubectl",
"exec",
"-n",
namespace,
"jump-pad",
"--",
"/usr/local/bin/generate.sh",
strconv.Itoa(kibishiiData.Levels),
strconv.Itoa(kibishiiData.DirsPerLevel),
strconv.Itoa(kibishiiData.FilesPerLevel),
strconv.Itoa(kibishiiData.FileLength),
strconv.Itoa(kibishiiData.BlockSize),
strconv.Itoa(kibishiiData.PassNum),
strconv.Itoa(kibishiiData.ExpectedNodes),
)
fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd)
stdout, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd)
@@ -342,26 +503,44 @@ func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiD
func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
timeout := 10 * time.Minute
interval := 5 * time.Second
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
timeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*20)
defer ctxCancel()
kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
"/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),
strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum),
strconv.Itoa(kibishiiData.ExpectedNodes))
fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
err := wait.PollUntilContextTimeout(
ctx,
interval,
timeout,
true,
func(ctx context.Context) (bool, error) {
timeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*20)
defer ctxCancel()
kibishiiVerifyCmd := exec.CommandContext(
timeout,
"kubectl",
"exec",
"-n",
namespace,
"jump-pad",
"--",
"/usr/local/bin/verify.sh",
strconv.Itoa(kibishiiData.Levels),
strconv.Itoa(kibishiiData.DirsPerLevel),
strconv.Itoa(kibishiiData.FilesPerLevel),
strconv.Itoa(kibishiiData.FileLength),
strconv.Itoa(kibishiiData.BlockSize),
strconv.Itoa(kibishiiData.PassNum),
strconv.Itoa(kibishiiData.ExpectedNodes),
)
fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
if strings.Contains(stderr, "Timeout occurred") {
return false, nil
}
if err != nil {
fmt.Printf("Kibishi verify stdout Timeout occurred: %s stderr: %s err: %s\n", stdout, stderr, err)
return false, nil
}
return true, nil
})
stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
if strings.Contains(stderr, "Timeout occurred") {
return false, nil
}
if err != nil {
fmt.Printf("Kibishi verify stdout Timeout occurred: %s stderr: %s err: %s\n", stdout, stderr, err)
return false, nil
}
return true, nil
},
)
if err != nil {
return errors.Wrapf(err, "Failed to verify kibishii data in namespace %s\n", namespace)
@@ -371,7 +550,12 @@ func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiDat
}
func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespace string) error {
return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
return WaitForPods(
ctx,
client,
kibishiiNamespace,
[]string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"},
)
}
func KibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace string, kibishiiData *KibishiiData) error {
@@ -383,9 +567,17 @@ func KibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace stri
return nil
}
func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient,
providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures,
kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error {
func KibishiiPrepareBeforeBackup(
oneHourTimeout context.Context,
client TestClient,
providerName,
kibishiiNamespace,
registryCredentialFile,
veleroFeatures,
kibishiiDirectory string,
kibishiiData *KibishiiData,
imageRegistryProxy string,
) error {
fmt.Printf("installKibishii %s\n", time.Now().Format("2006-01-02 15:04:05"))
serviceAccountName := "default"
@@ -399,8 +591,15 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
}
if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures,
kibishiiDirectory, useVolumeSnapshots, kibishiiData.ExpectedNodes); err != nil {
if err := installKibishii(
oneHourTimeout,
kibishiiNamespace,
providerName,
veleroFeatures,
kibishiiDirectory,
kibishiiData.ExpectedNodes,
imageRegistryProxy,
); err != nil {
return errors.Wrap(err, "Failed to install Kibishii workload")
}
// wait for kibishii pod startup

View File

@@ -390,7 +390,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
if options.ItemBlockWorkerCount > 1 {
args = append(args, fmt.Sprintf("--item-block-worker-count=%d", options.ItemBlockWorkerCount))
}
args = append(args, fmt.Sprintf("--backup-repository-configmap=%s", test.BackupRepositoryConfigName))
if options.BackupRepoConfigMap != "" {
args = append(args, fmt.Sprintf("--backup-repository-configmap=%s", options.BackupRepoConfigMap))
}
if err := createVeleroResources(ctx, cli, namespace, args, options); err != nil {
return err

View File

@@ -61,7 +61,7 @@ var ImagesMatrix = map[string]map[string][]string{
"v1.13": {
"aws": {"velero/velero-plugin-for-aws:v1.9.2"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.2"},
"vsphere": {"velero/velero-plugin-for-vsphere:v1.5.2"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.9.2"},
"csi": {"velero/velero-plugin-for-csi:v0.7.1"},
"datamover": {"velero/velero-plugin-for-aws:v1.9.2"},
@@ -71,7 +71,7 @@ var ImagesMatrix = map[string]map[string][]string{
"v1.14": {
"aws": {"velero/velero-plugin-for-aws:v1.10.1"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.10.1"},
"vsphere": {"velero/velero-plugin-for-vsphere:v1.5.2"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.10.1"},
"datamover": {"velero/velero-plugin-for-aws:v1.10.1"},
"velero": {"velero/velero:v1.14.1"},
@@ -80,7 +80,7 @@ var ImagesMatrix = map[string]map[string][]string{
"v1.15": {
"aws": {"velero/velero-plugin-for-aws:v1.11.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.11.0"},
"vsphere": {"velero/velero-plugin-for-vsphere:v1.5.2"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.11.0"},
"datamover": {"velero/velero-plugin-for-aws:v1.11.0"},
"velero": {"velero/velero:v1.15.2"},
@@ -89,7 +89,7 @@ var ImagesMatrix = map[string]map[string][]string{
"v1.16": {
"aws": {"velero/velero-plugin-for-aws:v1.12.0"},
"azure": {"velero/velero-plugin-for-microsoft-azure:v1.12.0"},
"vsphere": {"velero/velero-plugin-for-vsphere:v1.5.2"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
"gcp": {"velero/velero-plugin-for-gcp:v1.12.0"},
"datamover": {"velero/velero-plugin-for-aws:v1.12.0"},
"velero": {"velero/velero:v1.15.0"},
@@ -98,7 +98,7 @@ var ImagesMatrix = map[string]map[string][]string{
"main": {
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"velero/velero-plugin-for-vsphere:v1.5.2"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
"datamover": {"velero/velero-plugin-for-aws:main"},
"velero": {"velero/velero:main"},
@@ -106,6 +106,17 @@ var ImagesMatrix = map[string]map[string][]string{
},
}
// UpdateImagesMatrixByProxy is used to append the proxy to the image lists.
func UpdateImagesMatrixByProxy(imageRegistryProxy string) {
if imageRegistryProxy != "" {
for i := range ImagesMatrix {
for j := range ImagesMatrix[i] {
ImagesMatrix[i][j][0] = path.Join(imageRegistryProxy, ImagesMatrix[i][j][0])
}
}
}
}
func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig, error) {
fmt.Printf("Get the images for version %s\n", version)
@@ -121,12 +132,6 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig
versionWithoutPatch)
}
if config.ImageRegistryProxy != "" {
for index := range images {
images[index][0] = path.Join(config.ImageRegistryProxy, images[index][0])
}
}
ret.VeleroImage = images[Velero][0]
ret.RestoreHelperImage = images[VeleroRestoreHelper][0]
@@ -157,7 +162,7 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig
return ret, nil
}
func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool, imageRegistryProxy string) ([]string, error) {
func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) {
var cloudMap map[string][]string
arr := strings.Split(version, ".")
if len(arr) >= 3 {
@@ -172,12 +177,6 @@ func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlug
var plugins []string
var ok bool
if imageRegistryProxy != "" {
for index := range cloudMap {
cloudMap[index][0] = path.Join(imageRegistryProxy, cloudMap[index][0])
}
}
if slices.Contains(LocalCloudProviders, cloudProvider) {
plugins, ok = cloudMap[AWS]
if !ok {
@@ -667,7 +666,6 @@ func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([
cloudProvider := veleroCfg.CloudProvider
objectStoreProvider := veleroCfg.ObjectStoreProvider
providerPlugins := veleroCfg.Plugins
imageRegistryProxy := veleroCfg.ImageRegistryProxy
needDataMoverPlugin := false
var plugins []string
@@ -686,9 +684,9 @@ func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([
return []string{}, errors.New("AdditionalBSLProvider should be provided.")
}
plugins, err = getPluginsByVersion(version, cloudProvider, false, imageRegistryProxy)
plugins, err = getPluginsByVersion(version, veleroCfg.AdditionalBSLProvider, false)
if err != nil {
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version)
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", veleroCfg.AdditionalBSLProvider, version)
}
} else {
plugins = append(plugins, veleroCfg.AddBSLPlugins)
@@ -716,7 +714,7 @@ func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([
needDataMoverPlugin = true
}
plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin, imageRegistryProxy)
plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin)
if err != nil {
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
}