Change the CreateFileToPod function's OS parameter as the E2E pass-in value.

Fix GetResourceWithLabel's bug: labels were not applied.
Add workOS for deployment and pod creationg.
Add OS label for select node.
Enlarge the context timeout to 10 minutes. 5 min is not enough for Windows.
Enlarge the Kibishii test context to 15 minutes for Windows.

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>

test/util/common/common.go

@@ -72,10 +72,13 @@ func GetListByCmdPipes(ctx context.Context, cmdLines []*OsCommandLine) ([]string
 
 func GetResourceWithLabel(ctx context.Context, namespace, resourceName string, labels map[string]string) ([]string, error) {
 	labelStr := ""
+	parts := make([]string, 0, len(labels))
 
 	for key, value := range labels {
 		strings.Join([]string{labelStr, key + "=" + value}, ",")
+		parts = append(parts, key+"="+value)
 	}
+	labelStr = strings.Join(parts, ",")
 
 	cmds := []*OsCommandLine{}
 	cmd := &OsCommandLine{

test/util/k8s/common.go

@@ -323,7 +323,6 @@ func WriteRandomDataToFileInPod(ctx context.Context, namespace, podName, contain
 }
 
 func CreateFileToPod(
-	ctx context.Context,
 	namespace string,
 	podName string,
 	containerName string,
@@ -345,8 +344,9 @@ func CreateFileToPod(
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", shell, shellParameter, fmt.Sprintf("echo ns-%s pod-%s volume-%s  > %s", namespace, podName, volume, filePath)}
 
-	cmd := exec.CommandContext(ctx, "kubectl", arg...)
+	cmd := exec.CommandContext(context.Background(), "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+
 	return cmd.Run()
 }
 
@@ -359,9 +359,9 @@ func FileExistInPV(
 	filename string,
 	workerOS string,
 ) (bool, error) {
-	stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename, workerOS)
+	stdout, stderr, err := ReadFileFromPodVolume(namespace, podName, containerName, volume, filename, workerOS)
 
-	output := fmt.Sprintf("%s:%s", stdout, stderr)
+	output := fmt.Sprintf("%s:%s:%s", stdout, stderr, err)
 
 	if workerOS == common.WorkerOSWindows {
 		if strings.Contains(output, "The system cannot find the file specified") {
@@ -380,8 +380,8 @@ func FileExistInPV(
 			filename, volume, podName, namespace))
 	}
 }
+
 func ReadFileFromPodVolume(
-	ctx context.Context,
 	namespace string,
 	podName string,
 	containerName string,
@@ -391,16 +391,20 @@ func ReadFileFromPodVolume(
 ) (string, string, error) {
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
+
 	if workerOS == common.WorkerOSWindows {
 		arg = []string{"exec", "-n", namespace, "-c", containerName, podName,
-			"--", "cmd", "/c", fmt.Sprintf("type C:\\%s\\%s", volume, filename)}
+			"--", "cmd", "/c", "type", fmt.Sprintf("C:\\%s\\%s", volume, filename)}
 	}
 
-	cmd := exec.CommandContext(ctx, "kubectl", arg...)
-	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	cmd := exec.CommandContext(context.Background(), "kubectl", arg...)
+	fmt.Printf("kubectl exec cmd =%v\n", cmd)
+
 	stdout, stderr, err := veleroexec.RunCommand(cmd)
 	fmt.Printf("stdout: %s\n", stdout)
 	fmt.Printf("stderr: %s\n", stderr)
+	fmt.Printf("err: %v\n", err)
+
 	return stdout, stderr, err
 }
 

test/util/k8s/deployment.go

@@ -29,6 +29,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+	common "github.com/vmware-tanzu/velero/test/util/common"
 )
 
 const (
@@ -37,7 +38,8 @@ const (
 	PollInterval         = 2 * time.Second
 	PollTimeout          = 15 * time.Minute
 	DefaultContainerName = "container-busybox"
-	TestImage            = "busybox:1.37.0"
+	LinuxTestImage       = "busybox:1.37.0"
+	WindowTestImage      = "mcr.microsoft.com/windows/nanoserver:ltsc2022"
 )
 
 // DeploymentBuilder builds Deployment objects.
@@ -50,30 +52,100 @@ func (d *DeploymentBuilder) Result() *appsv1api.Deployment {
 }
 
 // newDeployment returns a RollingUpdate Deployment with a fake container image
-func NewDeployment(name, ns string, replicas int32, labels map[string]string, imageRegistryProxy string) *DeploymentBuilder {
-	imageAddress := TestImage
-	if imageRegistryProxy != "" {
-		imageAddress = path.Join(imageRegistryProxy, TestImage)
+func NewDeployment(
+	name, ns string,
+	replicas int32,
+	labels map[string]string,
+	imageRegistryProxy string,
+	workerOS string,
+) *DeploymentBuilder {
+	// Default to Linux environment
+	imageAddress := LinuxTestImage
+	command := []string{"sleep", "infinity"}
+	args := make([]string, 0)
+	var affinity corev1api.Affinity
+	var tolerations []corev1api.Toleration
+
+	if workerOS == common.WorkerOSLinux && imageRegistryProxy != "" {
+		imageAddress = path.Join(imageRegistryProxy, LinuxTestImage)
+	}
+
+	containerSecurityContext := &corev1api.SecurityContext{
+		AllowPrivilegeEscalation: boolptr.False(),
+		Capabilities: &corev1api.Capabilities{
+			Drop: []corev1api.Capability{"ALL"},
+		},
+		RunAsNonRoot: boolptr.True(),
+		RunAsUser:    func(i int64) *int64 { return &i }(65534),
+		RunAsGroup:   func(i int64) *int64 { return &i }(65534),
+		SeccompProfile: &corev1api.SeccompProfile{
+			Type: corev1api.SeccompProfileTypeRuntimeDefault,
+		},
+	}
+
+	podSecurityContext := &corev1api.PodSecurityContext{
+		FSGroup:             func(i int64) *int64 { return &i }(65534),
+		FSGroupChangePolicy: func(policy corev1api.PodFSGroupChangePolicy) *corev1api.PodFSGroupChangePolicy { return &policy }(corev1api.FSGroupChangeAlways),
+	}
+
+	// Settings for Windows
+	if workerOS == common.WorkerOSWindows {
+		imageAddress = WindowTestImage
+		command = []string{"cmd"}
+		args = []string{"/c", "ping -t localhost > NUL"}
+
+		affinity = corev1api.Affinity{
+			NodeAffinity: &corev1api.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{
+					NodeSelectorTerms: []corev1api.NodeSelectorTerm{
+						{
+							MatchExpressions: []corev1api.NodeSelectorRequirement{
+								{
+									Key:      "kubernetes.io/os",
+									Values:   []string{common.WorkerOSWindows},
+									Operator: corev1api.NodeSelectorOpIn,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		tolerations = []corev1api.Toleration{
+			{
+				Effect: corev1api.TaintEffectNoSchedule,
+				Key:    "os",
+				Value:  common.WorkerOSWindows,
+			},
+			{
+				Effect: corev1api.TaintEffectNoExecute,
+				Key:    "os",
+				Value:  common.WorkerOSWindows,
+			},
+		}
+
+		whetherToRunAsRoot := false
+		containerSecurityContext = &corev1api.SecurityContext{
+			RunAsNonRoot: &whetherToRunAsRoot,
+		}
+
+		containerUserName := "ContainerAdministrator"
+		podSecurityContext = &corev1api.PodSecurityContext{
+			WindowsOptions: &corev1api.WindowsSecurityContextOptions{
+				RunAsUserName: &containerUserName,
+			},
+		}
 	}
 
 	containers := []corev1api.Container{
 		{
 			Name:    DefaultContainerName,
 			Image:   imageAddress,
-			Command: []string{"sleep", "1000000"},
+			Command: command,
+			Args:    args,
 			// Make pod obeys the restricted pod security standards.
-			SecurityContext: &corev1api.SecurityContext{
-				AllowPrivilegeEscalation: boolptr.False(),
-				Capabilities: &corev1api.Capabilities{
-					Drop: []corev1api.Capability{"ALL"},
-				},
-				RunAsNonRoot: boolptr.True(),
-				RunAsUser:    func(i int64) *int64 { return &i }(65534),
-				RunAsGroup:   func(i int64) *int64 { return &i }(65534),
-				SeccompProfile: &corev1api.SeccompProfile{
-					Type: corev1api.SeccompProfileTypeRuntimeDefault,
-				},
-			},
+			SecurityContext: containerSecurityContext,
 		},
 	}
 
@@ -100,11 +172,10 @@ func NewDeployment(name, ns string, replicas int32, labels map[string]string, im
 						Labels: labels,
 					},
 					Spec: corev1api.PodSpec{
-						SecurityContext: &corev1api.PodSecurityContext{
-							FSGroup:             func(i int64) *int64 { return &i }(65534),
-							FSGroupChangePolicy: func(policy corev1api.PodFSGroupChangePolicy) *corev1api.PodFSGroupChangePolicy { return &policy }(corev1api.FSGroupChangeAlways),
-						},
-						Containers: containers,
+						SecurityContext: podSecurityContext,
+						Containers:      containers,
+						Affinity:        &affinity,
+						Tolerations:     tolerations,
 					},
 				},
 			},
@@ -127,10 +198,6 @@ func (d *DeploymentBuilder) WithVolume(volumes []*corev1api.Volume) *DeploymentB
 	return d
 }
 
-func CreateDeploy(c clientset.Interface, ns string, deployment *appsv1api.Deployment) error {
-	_, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
-	return err
-}
 func CreateDeployment(c clientset.Interface, ns string, deployment *appsv1api.Deployment) (*appsv1api.Deployment, error) {
 	return c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
 }

test/util/k8s/namespace.go

@@ -53,8 +53,12 @@ func CreateNamespaceWithLabel(ctx context.Context, client TestClient, namespace
 	ns := builder.ForNamespace(namespace).Result()
 	ns.Labels = label
 	// Add label to avoid PSA check.
-	ns.Labels["pod-security.kubernetes.io/enforce"] = "baseline"
-	ns.Labels["pod-security.kubernetes.io/enforce-version"] = "latest"
+	if _, ok := ns.Labels["pod-security.kubernetes.io/enforce"]; !ok {
+		ns.Labels["pod-security.kubernetes.io/enforce"] = "baseline"
+	}
+	if _, ok := ns.Labels["pod-security.kubernetes.io/enforce-version"]; !ok {
+		ns.Labels["pod-security.kubernetes.io/enforce-version"] = "latest"
+	}
 	_, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
 	if apierrors.IsAlreadyExists(err) {
 		return nil

test/util/k8s/node.go

@@ -11,8 +11,13 @@ import (
 	common "github.com/vmware-tanzu/velero/test/util/common"
 )
 
-func GetWorkerNodes(ctx context.Context) ([]string, error) {
-	getCMD := exec.CommandContext(ctx, "kubectl", "get", "node", "-o", "json")
+func GetWorkerNodes(ctx context.Context, workerOS string) ([]string, error) {
+	getCMD := exec.CommandContext(
+		ctx,
+		"kubectl", "get", "node", "-l",
+		fmt.Sprintf("kubernetes.io/os=%s", workerOS),
+		"-o", "json",
+	)
 
 	fmt.Printf("kubectl get node cmd =%v\n", getCMD)
 	jsonBuf, err := common.CMDExecWithOutput(getCMD)

test/util/k8s/pod.go

@@ -18,14 +18,17 @@ package k8s
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"path"
 
 	"github.com/pkg/errors"
 	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+	common "github.com/vmware-tanzu/velero/test/util/common"
 )
 
 func CreatePod(
@@ -34,14 +37,88 @@ func CreatePod(
 	volumeNameList []string,
 	pvcAnn, ann map[string]string,
 	imageRegistryProxy string,
+	workerOS string,
 ) (*corev1api.Pod, error) {
 	if pvcName != "" && len(volumeNameList) != 1 {
 		return nil, errors.New("Volume name list should contain only 1 since PVC name is not empty")
 	}
 
-	imageAddress := TestImage
-	if imageRegistryProxy != "" {
-		imageAddress = path.Join(imageRegistryProxy, TestImage)
+	// Default to Linux environment
+	imageAddress := LinuxTestImage
+	command := []string{"sleep", "infinity"}
+	args := make([]string, 0)
+	var affinity corev1api.Affinity
+	var tolerations []corev1api.Toleration
+
+	if workerOS == common.WorkerOSLinux && imageRegistryProxy != "" {
+		imageAddress = path.Join(imageRegistryProxy, LinuxTestImage)
+	}
+
+	containerSecurityContext := &corev1api.SecurityContext{
+		AllowPrivilegeEscalation: boolptr.False(),
+		Capabilities: &corev1api.Capabilities{
+			Drop: []corev1api.Capability{"ALL"},
+		},
+		RunAsNonRoot: boolptr.True(),
+		RunAsUser:    func(i int64) *int64 { return &i }(65534),
+		RunAsGroup:   func(i int64) *int64 { return &i }(65534),
+		SeccompProfile: &corev1api.SeccompProfile{
+			Type: corev1api.SeccompProfileTypeRuntimeDefault,
+		},
+	}
+
+	podSecurityContext := &corev1api.PodSecurityContext{
+		FSGroup:             func(i int64) *int64 { return &i }(65534),
+		FSGroupChangePolicy: func(policy corev1api.PodFSGroupChangePolicy) *corev1api.PodFSGroupChangePolicy { return &policy }(corev1api.FSGroupChangeAlways),
+	}
+
+	// Settings for Windows
+	if workerOS == common.WorkerOSWindows {
+		imageAddress = WindowTestImage
+		command = []string{"cmd"}
+		args = []string{"/c", "ping -t localhost > NUL"}
+		affinity = corev1api.Affinity{
+			NodeAffinity: &corev1api.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{
+					NodeSelectorTerms: []corev1api.NodeSelectorTerm{
+						{
+							MatchExpressions: []corev1api.NodeSelectorRequirement{
+								{
+									Key:      "kubernetes.io/os",
+									Values:   []string{common.WorkerOSWindows},
+									Operator: corev1api.NodeSelectorOpIn,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		tolerations = []corev1api.Toleration{
+			{
+				Effect: corev1api.TaintEffectNoSchedule,
+				Key:    "os",
+				Value:  common.WorkerOSWindows,
+			},
+			{
+				Effect: corev1api.TaintEffectNoExecute,
+				Key:    "os",
+				Value:  common.WorkerOSWindows,
+			},
+		}
+
+		whetherToRunAsRoot := false
+		containerSecurityContext = &corev1api.SecurityContext{
+			RunAsNonRoot: &whetherToRunAsRoot,
+		}
+
+		containerUserName := "ContainerAdministrator"
+		podSecurityContext = &corev1api.PodSecurityContext{
+			WindowsOptions: &corev1api.WindowsSecurityContextOptions{
+				RunAsUserName: &containerUserName,
+			},
+		}
 	}
 
 	volumes := []corev1api.Volume{}
@@ -82,32 +159,20 @@ func CreatePod(
 			Annotations: ann,
 		},
 		Spec: corev1api.PodSpec{
-			SecurityContext: &corev1api.PodSecurityContext{
-				FSGroup:             func(i int64) *int64 { return &i }(65534),
-				FSGroupChangePolicy: func(policy corev1api.PodFSGroupChangePolicy) *corev1api.PodFSGroupChangePolicy { return &policy }(corev1api.FSGroupChangeAlways),
-			},
+			SecurityContext: podSecurityContext,
 			Containers: []corev1api.Container{
 				{
-					Name:         name,
-					Image:        imageAddress,
-					Command:      []string{"sleep", "3600"},
-					VolumeMounts: vmList,
-					// Make pod obeys the restricted pod security standards.
-					SecurityContext: &corev1api.SecurityContext{
-						AllowPrivilegeEscalation: boolptr.False(),
-						Capabilities: &corev1api.Capabilities{
-							Drop: []corev1api.Capability{"ALL"},
-						},
-						RunAsNonRoot: boolptr.True(),
-						RunAsUser:    func(i int64) *int64 { return &i }(65534),
-						RunAsGroup:   func(i int64) *int64 { return &i }(65534),
-						SeccompProfile: &corev1api.SeccompProfile{
-							Type: corev1api.SeccompProfileTypeRuntimeDefault,
-						},
-					},
+					Name:            name,
+					Image:           imageAddress,
+					Command:         command,
+					Args:            args,
+					VolumeMounts:    vmList,
+					SecurityContext: containerSecurityContext,
 				},
 			},
-			Volumes: volumes,
+			Volumes:     volumes,
+			Affinity:    &affinity,
+			Tolerations: tolerations,
 		},
 	}
 
@@ -134,7 +199,25 @@ func AddAnnotationToPod(ctx context.Context, client TestClient, namespace, podNa
 	newPod.Annotations = newAnn
 	fmt.Println(newPod.Annotations)
 
-	return client.ClientGo.CoreV1().Pods(namespace).Update(ctx, newPod, metav1.UpdateOptions{})
+	// Strategic merge patch to add/update label
+	patch := map[string]any{
+		"metadata": map[string]any{
+			"annotations": newAnn,
+		},
+	}
+	patchBytes, err := json.Marshal(patch)
+	if err != nil {
+		fmt.Println("fail to marshal patch for pod: ", err.Error())
+		return nil, err
+	}
+
+	return client.ClientGo.CoreV1().Pods(namespace).Patch(
+		ctx,
+		newPod.Name,
+		types.StrategicMergePatchType,
+		patchBytes,
+		metav1.PatchOptions{},
+	)
 }
 
 func ListPods(ctx context.Context, client TestClient, namespace string) (*corev1api.PodList, error) {

test/util/kibishii/kibishii_utils.go

@@ -83,7 +83,7 @@ func RunKibishiiTests(
 ) error {
 	pvCount := len(KibishiiPVCNameList)
 	client := *veleroCfg.ClientToInstallVelero
-	timeOutContext, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+	timeOutContext, ctxCancel := context.WithTimeout(context.Background(), time.Minute*15)
 	defer ctxCancel()
 	veleroCLI := veleroCfg.VeleroCLI
 	providerName := veleroCfg.CloudProvider
@@ -208,11 +208,10 @@ func RunKibishiiTests(
 	fmt.Printf("Re-populate volume  %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	for _, pod := range KibishiiPodNameList {
 		// To ensure Kibishii verification result is accurate
-		ClearKibishiiData(timeOutContext, kibishiiNamespace, pod, "kibishii", "data")
+		ClearKibishiiData(kibishiiNamespace, pod, "kibishii", "data", veleroCfg.WorkerOS)
 
 		CreateFileContent := fileBaseContent + pod
 		err := CreateFileToPod(
-			timeOutContext,
 			kibishiiNamespace,
 			pod,
 			"kibishii",
@@ -789,6 +788,13 @@ func KibishiiVerifyAfterRestore(
 	if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
 	}
+
+	// TODO - check that namespace exists
+	fmt.Printf("running kibishii verify\n")
+	if err := verifyData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
+		return errors.Wrap(err, "Failed to verify data generated by kibishii")
+	}
+
 	if incrementalFileName != "" {
 		for _, pod := range KibishiiPodNameList {
 			exist, err := FileExistInPV(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", incrementalFileName, workerOS)
@@ -801,19 +807,18 @@ func KibishiiVerifyAfterRestore(
 			}
 		}
 	}
-
-	// TODO - check that namespace exists
-	fmt.Printf("running kibishii verify\n")
-	if err := verifyData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
-		return errors.Wrap(err, "Failed to verify data generated by kibishii")
-	}
 	return nil
 }
 
-func ClearKibishiiData(ctx context.Context, namespace, podName, containerName, dir string) error {
+func ClearKibishiiData(namespace, podName, containerName, dir, workerOS string) error {
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "/bin/sh", "-c", "rm -rf /" + dir + "/*"}
-	cmd := exec.CommandContext(ctx, "kubectl", arg...)
+
+	if workerOS == common.WorkerOSWindows {
+		arg = []string{"exec", "-n", namespace, "-c", containerName, podName,
+			"--", "cmd", "/c", fmt.Sprintf("del /Q C:\\%s\\*", dir)}
+	}
+	cmd := exec.CommandContext(context.Background(), "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	return cmd.Run()
 }