Merge pull request #7852 from reasonerjt/fix-7849-1.14

Use PVC to track the CSI snapshot in restore

.github/workflows/crds-verify-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.22.2'
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI

.github/workflows/e2e-test-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.22.2'
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
@@ -82,7 +82,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.22.2'
         id: go
       - name: Check out the code
         uses: actions/checkout@v4

.github/workflows/pr-ci-check.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.22.2'
         id: go
       - name: Check out the code
         uses: actions/checkout@v4

.github/workflows/pr-codespell.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere.
         skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE
-        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast
+        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme
         check_filenames: true
         check_hidden: true
 

.github/workflows/push.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.22'
+        go-version: '1.22.2'
       id: go
 
     - uses: actions/checkout@v4

CODE_OF_CONDUCT.md

@@ -5,7 +5,7 @@
 We as members, contributors, and leaders pledge to make participation in the Velero project and our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
+identity and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity
 and orientation.
 

Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Velero binary build section
-FROM --platform=$BUILDPLATFORM golang:1.22-bookworm as velero-builder
+FROM --platform=$BUILDPLATFORM golang:1.22.2-bookworm as velero-builder
 
 ARG GOPROXY
 ARG BIN
@@ -47,7 +47,7 @@ RUN mkdir -p /output/usr/bin && \
     go clean -modcache -cache
 
 # Restic binary build section
-FROM --platform=$BUILDPLATFORM golang:1.22-bookworm as restic-builder
+FROM --platform=$BUILDPLATFORM golang:1.22.2-bookworm as restic-builder
 
 ARG BIN
 ARG TARGETOS
@@ -70,7 +70,7 @@ RUN mkdir -p /output/usr/bin && \
     go clean -modcache -cache
 
 # Velero image packing section
-FROM paketobuildpacks/run-jammy-tiny:latest
+FROM paketobuildpacks/run-jammy-tiny:0.2.38
 
 LABEL maintainer="Xun Jiang <jxun@vmware.com>"
 

Tiltfile

@@ -52,7 +52,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip(
 
 tilt_helper_dockerfile_header = """
 # Tilt image
-FROM golang:1.22 as tilt-helper
+FROM golang:1.22.2 as tilt-helper
 
 # Support live reloading with Tilt
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \

design/Implemented/move-gh-org.md

@@ -65,7 +65,7 @@ This page contains a pre-migration checklist for ensuring a repo migration goes
 
 #### Updating Netlify
 
-The settings for Netflify should remain the same, except that it now needs to be installed in the new repo. The instructions on how to install Netlify on the new repo are here: https://www.netlify.com/docs/github-permissions/.
+The settings for Netlify should remain the same, except that it now needs to be installed in the new repo. The instructions on how to install Netlify on the new repo are here: https://www.netlify.com/docs/github-permissions/.
 
 #### Communication strategy
 

design/volume-snapshot-data-movement/volume-snapshot-data-movement.md

@@ -27,7 +27,7 @@ Moreover, we would like to create a general workflow to variations during the da
 - Support different data accesses, i.e., file system level and block level
 - Support different snapshot types, i.e., CSI snapshot, volume snapshot API from storage vendors
 - Support different snapshot accesses, i.e., through PV generated from snapshots, and through direct access API from storage vendors
-- Reuse the existing Velero generic data path as creatd in [Unified Repository design][1]
+- Reuse the existing Velero generic data path as created in [Unified Repository design][1]
 
 ## Non-Goals
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/vmware-tanzu/velero
 
-go 1.22.0
+go 1.22.2
 
 require (
 	cloud.google.com/go/storage v1.40.0

hack/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$TARGETPLATFORM golang:1.22-bookworm
+FROM --platform=$TARGETPLATFORM golang:1.22.2-bookworm
 
 ARG GOPROXY
 
@@ -99,3 +99,6 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl
 RUN chmod +x ./kubectl
 RUN mv ./kubectl /usr/local/bin
+
+# Fix the "dubious ownership" issue from git when running goreleaser.sh
+RUN echo "[safe] \n\t directory = *" > /.gitconfig

internal/resourcepolicies/resource_policies_test.go

@@ -182,7 +182,7 @@ func TestGetResourceMatchedAction(t *testing.T) {
 			expectedAction: &Action{Type: "snapshot"},
 		},
 		{
-			name: "dismatch all policies",
+			name: "mismatch all policies",
 			volume: &structuredVolume{
 				capacity:     *resource.NewQuantity(50<<30, resource.BinarySI),
 				storageClass: "ebs-sc",
@@ -394,7 +394,7 @@ volumePolicies:
 			skip: true,
 		},
 		{
-			name: "dismatch volume by types",
+			name: "mismatch volume by types",
 			yamlData: `version: v1
 volumePolicies:
 - conditions:

internal/resourcepolicies/volume_resources_test.go

@@ -165,7 +165,7 @@ func TestNFSConditionMatch(t *testing.T) {
 			expectedMatch: true,
 		},
 		{
-			name:          "server dismatch",
+			name:          "server mismatch",
 			condition:     &nfsCondition{&nFSVolumeSource{Server: "192.168.10.20", Path: ""}},
 			volume:        setStructuredVolume(*resource.NewQuantity(0, resource.BinarySI), "", &nFSVolumeSource{Server: ""}, nil),
 			expectedMatch: false,

internal/volume/volumes_information.go

@@ -662,17 +662,17 @@ type RestoreVolumeInfoTracker struct {
 
 	// map of PV name to the NativeSnapshotInfo from which the PV is restored
 	pvNativeSnapshotMap map[string]*NativeSnapshotInfo
-	// map of PV name to the CSISnapshot object from which the PV is restored
-	pvCSISnapshotMap map[string]snapshotv1api.VolumeSnapshot
-	datadownloadList *velerov2alpha1.DataDownloadList
-	pvrs             []*velerov1api.PodVolumeRestore
+	// map of PVC object to the CSISnapshot object from which the PV is restored
+	// the key is in the form of $pvc-ns/$pvc-name
+	pvcCSISnapshotMap map[string]snapshotv1api.VolumeSnapshot
+	datadownloadList  *velerov2alpha1.DataDownloadList
+	pvrs              []*velerov1api.PodVolumeRestore
 }
 
 // Populate data objects in the tracker, which will be used to generate the RestoreVolumeInfo array in Result()
 // The input param resourceList should be the final result of the restore.
 func (t *RestoreVolumeInfoTracker) Populate(ctx context.Context, restoredResourceList map[string][]string) {
 	pvcs := RestoredPVCFromRestoredResourceList(restoredResourceList)
-
 	t.Lock()
 	defer t.Unlock()
 	for item := range pvcs {
@@ -684,25 +684,26 @@ func (t *RestoreVolumeInfoTracker) Populate(ctx context.Context, restoredResourc
 			log.WithError(err).Error("Failed to get PVC")
 			continue
 		}
-		if pvc.Status.Phase != corev1api.ClaimBound || pvc.Spec.VolumeName == "" {
-			log.Info("PVC is not bound or has no volume name")
-			continue
-		}
-		pv := &corev1api.PersistentVolume{}
-		if err := t.client.Get(ctx, kbclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv); err != nil {
-			log.WithError(err).Error("Failed to get PV")
-		} else {
-			t.pvPvc.insert(*pv, pvcName, pvcNS)
-		}
 		// Collect the CSI VolumeSnapshot objects referenced by the restored PVCs,
 		if pvc.Spec.DataSource != nil && pvc.Spec.DataSource.Kind == "VolumeSnapshot" {
 			vs := &snapshotv1api.VolumeSnapshot{}
 			if err := t.client.Get(ctx, kbclient.ObjectKey{Namespace: pvcNS, Name: pvc.Spec.DataSource.Name}, vs); err != nil {
 				log.WithError(err).Error("Failed to get VolumeSnapshot")
 			} else {
-				t.pvCSISnapshotMap[pv.Name] = *vs
+				t.pvcCSISnapshotMap[pvc.Namespace+"/"+pvcName] = *vs
 			}
 		}
+		if pvc.Status.Phase == corev1api.ClaimBound && pvc.Spec.VolumeName != "" {
+			pv := &corev1api.PersistentVolume{}
+			if err := t.client.Get(ctx, kbclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv); err != nil {
+				log.WithError(err).Error("Failed to get PV")
+			} else {
+				t.pvPvc.insert(*pv, pvcName, pvcNS)
+			}
+		} else {
+			log.Warn("PVC is not bound or has no volume name")
+			continue
+		}
 	}
 	if err := t.client.List(ctx, t.datadownloadList, &kbclient.ListOptions{
 		Namespace:     t.restore.Namespace,
@@ -761,9 +762,16 @@ func (t *RestoreVolumeInfoTracker) Result() []*RestoreVolumeInfo {
 	}
 
 	// Generate RestoreVolumeInfo for PVs restored from CSISnapshots
-	for pvName, csiSnapshot := range t.pvCSISnapshotMap {
+	for pvc, csiSnapshot := range t.pvcCSISnapshotMap {
+		n := strings.Split(pvc, "/")
+		if len(n) != 2 {
+			t.log.Warnf("Invalid PVC key '%s' in the pvc-CSISnapshot map, skip populating it to volume info", pvc)
+			continue
+		}
+		pvcNS, pvcName := n[0], n[1]
 		volumeInfo := &RestoreVolumeInfo{
-			PVName:            pvName,
+			PVCNamespace:      pvcNS,
+			PVCName:           pvcName,
 			SnapshotDataMoved: false,
 			RestoreMethod:     CSISnapshot,
 			CSISnapshotInfo: &CSISnapshotInfo{
@@ -773,9 +781,8 @@ func (t *RestoreVolumeInfoTracker) Result() []*RestoreVolumeInfo {
 				VSCName:        *csiSnapshot.Spec.Source.VolumeSnapshotContentName,
 			},
 		}
-		if pvcPVInfo := t.pvPvc.retrieve(pvName, "", ""); pvcPVInfo != nil {
-			volumeInfo.PVCName = pvcPVInfo.PVCName
-			volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace
+		if pvcPVInfo := t.pvPvc.retrieve("", pvcName, pvcNS); pvcPVInfo != nil {
+			volumeInfo.PVName = pvcPVInfo.PV.Name
 		}
 		volumeInfos = append(volumeInfos, volumeInfo)
 	}
@@ -829,7 +836,7 @@ func NewRestoreVolInfoTracker(restore *velerov1api.Restore, logger logrus.FieldL
 			data: make(map[string]pvcPvInfo),
 		},
 		pvNativeSnapshotMap: make(map[string]*NativeSnapshotInfo),
-		pvCSISnapshotMap:    make(map[string]snapshotv1api.VolumeSnapshot),
+		pvcCSISnapshotMap:   make(map[string]snapshotv1api.VolumeSnapshot),
 		datadownloadList:    &velerov2alpha1.DataDownloadList{},
 	}
 }

internal/volume/volumes_information_test.go

@@ -933,7 +933,7 @@ func TestRestoreVolumeInfoResult(t *testing.T) {
 					data: make(map[string]pvcPvInfo),
 				},
 				pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{},
-				pvCSISnapshotMap:    map[string]snapshotv1api.VolumeSnapshot{},
+				pvcCSISnapshotMap:   map[string]snapshotv1api.VolumeSnapshot{},
 				datadownloadList:    &velerov2alpha1.DataDownloadList{},
 				pvrs:                []*velerov1api.PodVolumeRestore{},
 			},
@@ -968,8 +968,8 @@ func TestRestoreVolumeInfoResult(t *testing.T) {
 						IOPS:           "10000",
 					},
 				},
-				pvCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{},
-				datadownloadList: &velerov2alpha1.DataDownloadList{},
+				pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{},
+				datadownloadList:  &velerov2alpha1.DataDownloadList{},
 				pvrs: []*velerov1api.PodVolumeRestore{
 					builder.ForPodVolumeRestore("velero", "testRestore-1234").
 						PodNamespace("testNS").
@@ -1031,8 +1031,8 @@ func TestRestoreVolumeInfoResult(t *testing.T) {
 					},
 				},
 				pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{},
-				pvCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{
-					"testPV": *builder.ForVolumeSnapshot("sourceNS", "testCSISnapshot").
+				pvcCSISnapshotMap: map[string]snapshotv1api.VolumeSnapshot{
+					"testNS/testPVC": *builder.ForVolumeSnapshot("sourceNS", "testCSISnapshot").
 						ObjectMeta(
 							builder.WithAnnotations(VolumeSnapshotHandleAnnotation, "csi-snap-001",
 								CSIDriverNameAnnotation, "test-csi-driver"),
@@ -1101,7 +1101,7 @@ func TestRestoreVolumeInfoResult(t *testing.T) {
 					},
 				},
 				pvNativeSnapshotMap: map[string]*NativeSnapshotInfo{},
-				pvCSISnapshotMap:    map[string]snapshotv1api.VolumeSnapshot{},
+				pvcCSISnapshotMap:   map[string]snapshotv1api.VolumeSnapshot{},
 				datadownloadList: &velerov2alpha1.DataDownloadList{
 					Items: []velerov2alpha1.DataDownload{
 						*builder.ForDataDownload("velero", "testDataDownload-1").

pkg/controller/data_upload_controller_test.go

@@ -427,7 +427,7 @@ func TestReconcile(t *testing.T) {
 			notCreateFSBR:     true,
 		},
 		{
-			name: "Dataupload should not be cancel with dismatch node",
+			name: "Dataupload should not be cancel with mismatch node",
 			pod:  builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(),
 			du: func() *velerov2alpha1api.DataUpload {
 				du := dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).SnapshotType(fakeSnapshotType).Cancel(true).Result()

pkg/repository/manager.go

@@ -182,10 +182,6 @@ func (m *manager) PruneRepo(repo *velerov1api.BackupRepository) error {
 	m.repoLocker.LockExclusive(repo.Name)
 	defer m.repoLocker.UnlockExclusive(repo.Name)
 
-	prd, err := m.getRepositoryProvider(repo)
-	if err != nil {
-		return errors.WithStack(err)
-	}
 	param, err := m.assembleRepoParam(repo)
 	if err != nil {
 		return errors.WithStack(err)
@@ -208,11 +204,7 @@ func (m *manager) PruneRepo(repo *velerov1api.BackupRepository) error {
 		return nil
 	}
 
-	if err := prd.BoostRepoConnect(context.Background(), param); err != nil {
-		return errors.WithStack(err)
-	}
-
-	log.Info("Start to maintence repo")
+	log.Info("Start to maintenance repo")
 
 	maintenanceJob, err := buildMaintenanceJob(m.maintenanceCfg, param, m.client, m.namespace)
 	if err != nil {

pkg/util/podvolume/pod_volume_test.go

@@ -581,7 +581,7 @@ func TestGetPodVolumeNameForPVC(t *testing.T) {
 		expectedVolumeName string
 	}{
 		{
-			name: "should get volume name for pod with multuple PVCs",
+			name: "should get volume name for pod with multiple PVCs",
 			pod: v1.Pod{
 				Spec: v1.PodSpec{
 					Volumes: []v1.Volume{