Merge pull request #4721 from ywk253100/220307_golang_1.8

Bump up golang to 1.17.8

.github/workflows/push.yml

@@ -2,7 +2,9 @@ name: Main CI
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - 'main'
+      - 'release-**'
     tags:
       - '*'
 

Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.17.6 as builder-env
+FROM --platform=$BUILDPLATFORM golang:1.17.8 as builder-env
 
 ARG GOPROXY
 ARG PKG

changelogs/CHANGELOG-1.8.md

@@ -1,3 +1,23 @@
+## v1.8.1
+### 2022-03-04
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.8.1
+
+### Container Image
+`velero/velero:v1.8.1`
+
+### Documentation
+https://velero.io/docs/v1.8
+
+### Upgrading
+https://velero.io/docs/v1.8/upgrade-to-1.8/
+
+### All changes
+* Bypass the remap CRD version plugin when v1beta1 CRD is not supported (#4706, @reasonerjt)
+* Support regional pv for GKE (#4691, @jxun)
+* Bump up golang to 1.17.8 (#4721, @ywk253100)
+
 ## v1.8.0
 ### 2022-01-14
 

hack/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.17.6
+FROM golang:1.17.8
 
 ARG GOPROXY
 

hack/docker-push.sh

@@ -56,26 +56,24 @@ elif [[ "$triggeredBy" == "tags" ]]; then
     TAG=$(echo $GITHUB_REF | cut -d / -f 3)
 fi
 
-if [[ "$BRANCH" == "main" ]]; then
-    VERSION="$BRANCH"
-elif [[ ! -z "$TAG" ]]; then
+TAG_LATEST=false
+if [[ ! -z "$TAG" ]]; then
+    echo "We're building tag $TAG"
+    VERSION="$TAG"
     # Explicitly checkout tags when building from a git tag.
     # This is not needed when building from main
     git fetch --tags
     # Calculate the latest release if there's a tag.
     highest_release
-    VERSION="$TAG"
+    if [[ "$TAG" == "$HIGHEST" ]]; then
+      TAG_LATEST=true
+    fi
 else
-    echo "We're not on main and we're not building a tag, exit early."
-    exit 0
-fi
-
-# Assume we're not tagging `latest` by default, and never on main.
-TAG_LATEST=false
-if [[ "$BRANCH" == "main" ]]; then
-    echo "Building main, not tagging latest."
-elif [[ "$TAG" == "$HIGHEST" ]]; then
-    TAG_LATEST=true
+    echo "We're on branch $BRANCH"
+    VERSION="$BRANCH"
+    if [[ "$VERSION" == release-* ]]; then
+      VERSION=${VERSION}-dev
+    fi
 fi
 
 if [[ -z "$BUILDX_PLATFORMS" ]]; then
@@ -87,6 +85,7 @@ echo "Highest tag found: $HIGHEST"
 echo "BRANCH: $BRANCH"
 echo "TAG: $TAG"
 echo "TAG_LATEST: $TAG_LATEST"
+echo "VERSION: $VERSION"
 echo "BUILDX_PLATFORMS: $BUILDX_PLATFORMS"
 
 echo "Building and pushing container images."

pkg/backup/backup.go

@@ -47,6 +47,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/podexec"
 	"github.com/vmware-tanzu/velero/pkg/restic"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 )
 
@@ -200,7 +201,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 	backupRequest.ResourceIncludesExcludes = collections.GetResourceIncludesExcludes(kb.discoveryHelper, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources)
 	log.Infof("Including resources: %s", backupRequest.ResourceIncludesExcludes.IncludesString())
 	log.Infof("Excluding resources: %s", backupRequest.ResourceIncludesExcludes.ExcludesString())
-	log.Infof("Backing up all pod volumes using restic: %t", *backupRequest.Backup.Spec.DefaultVolumesToRestic)
+	log.Infof("Backing up all pod volumes using restic: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToRestic))
 
 	var err error
 	backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper)

pkg/backup/item_backupper.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -384,6 +385,7 @@ const (
 	awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone"
 	azureCsiZoneKey  = "topology.disk.csi.azure.com/zone"
 	gkeCsiZoneKey    = "topology.gke.io/zone"
+	gkeZoneSeparator = "__"
 )
 
 // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided
@@ -539,15 +541,27 @@ func zoneFromPVNodeAffinity(res *corev1api.PersistentVolume, topologyKeys ...str
 		return "", ""
 	}
 	keySet := sets.NewString(topologyKeys...)
+	providerGke := false
+	zones := make([]string, 0)
 	for _, term := range nodeAffinity.Required.NodeSelectorTerms {
 		if term.MatchExpressions == nil {
 			continue
 		}
 		for _, exp := range term.MatchExpressions {
 			if keySet.Has(exp.Key) && exp.Operator == "In" && len(exp.Values) > 0 {
-				return exp.Key, exp.Values[0]
+				if exp.Key == gkeCsiZoneKey {
+					providerGke = true
+					zones = append(zones, exp.Values[0])
+				} else {
+					return exp.Key, exp.Values[0]
+				}
 			}
 		}
 	}
+
+	if providerGke {
+		return gkeCsiZoneKey, strings.Join(zones, gkeZoneSeparator)
+	}
+
 	return "", ""
 }

pkg/backup/item_backupper_test.go

@@ -131,6 +131,36 @@ func Test_zoneFromPVNodeAffinity(t *testing.T) {
 			wantKey:   "topology.disk.csi.azure.com/zone",
 			wantValue: "us-central",
 		},
+		{
+			/* an valid example of node affinity in a GKE's regional PV
+			nodeAffinity:
+			  required:
+			    nodeSelectorTerms:
+			    - matchExpressions:
+			      - key: topology.gke.io/zone
+			        operator: In
+			        values:
+			        - us-central1-a
+			    - matchExpressions:
+			      - key: topology.gke.io/zone
+			        operator: In
+			        values:
+			        - us-central1-c
+			*/
+			name: "Volume with multiple valid keys, and provider is gke, returns all valid entries's first zone value",
+			pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired(
+				builder.ForNodeSelector(
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "us-central1-c").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "us-east-2c", "us-east-2b").Result(),
+					*builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone",
+						"In", "europe-north1-a").Result(),
+				).Result(),
+			).Result(),
+			wantKey:   "topology.gke.io/zone",
+			wantValue: "us-central1-c__us-east-2c__europe-north1-a",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/backup/remap_crd_version_action.go

@@ -30,6 +30,8 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
+
 	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 )
@@ -37,13 +39,14 @@ import (
 // RemapCRDVersionAction inspects CustomResourceDefinition and decides if it is a v1
 // CRD that needs to be backed up as v1beta1.
 type RemapCRDVersionAction struct {
-	logger        logrus.FieldLogger
-	betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface
+	logger          logrus.FieldLogger
+	betaCRDClient   apiextv1beta1client.CustomResourceDefinitionInterface
+	discoveryHelper velerodiscovery.Helper
 }
 
 // NewRemapCRDVersionAction instantiates a new RemapCRDVersionAction plugin.
-func NewRemapCRDVersionAction(logger logrus.FieldLogger, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface) *RemapCRDVersionAction {
-	return &RemapCRDVersionAction{logger: logger, betaCRDClient: betaCRDClient}
+func NewRemapCRDVersionAction(logger logrus.FieldLogger, betaCRDClient apiextv1beta1client.CustomResourceDefinitionInterface, discoveryHelper velerodiscovery.Helper) *RemapCRDVersionAction {
+	return &RemapCRDVersionAction{logger: logger, betaCRDClient: betaCRDClient, discoveryHelper: discoveryHelper}
 }
 
 // AppliesTo selects the resources the plugin should run against. In this case, CustomResourceDefinitions.
@@ -68,7 +71,26 @@ func (a *RemapCRDVersionAction) Execute(item runtime.Unstructured, backup *v1.Ba
 		return item, nil, nil
 	}
 
-	// We've got a v1 CRD, so proceed.
+	// This plugin will exit if the CRD was installed via v1beta1 but the cluster does not support v1beta1 CRD
+	supportv1b1 := false
+CheckVersion:
+	for _, g := range a.discoveryHelper.APIGroups() {
+		if g.Name == apiextv1.GroupName {
+			for _, v := range g.Versions {
+				if v.Version == apiextv1beta1.SchemeGroupVersion.Version {
+					supportv1b1 = true
+					break CheckVersion
+				}
+			}
+
+		}
+	}
+	if !supportv1b1 {
+		a.logger.Info("Exiting RemapCRDVersionAction, the cluster does not support v1beta1 CRD")
+		return item, nil, nil
+	}
+
+	// We've got a v1 CRD and the cluster supports v1beta1 CRD, so proceed.
 	var crd apiextv1.CustomResourceDefinition
 
 	// Do not use runtime.DefaultUnstructuredConverter.FromUnstructured here because it has a bug when converting integers/whole

pkg/backup/remap_crd_version_action_test.go

@@ -32,6 +32,8 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
+
 	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
@@ -48,8 +50,7 @@ func TestRemapCRDVersionAction(t *testing.T) {
 	c := b.Result()
 	_, err := betaClient.Create(context.TODO(), c, metav1.CreateOptions{})
 	require.NoError(t, err)
-
-	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient)
+	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper())
 
 	t.Run("Test a v1 CRD without any Schema information", func(t *testing.T) {
 		b := builder.ForV1CustomResourceDefinition("test.velero.io")
@@ -109,6 +110,33 @@ func TestRemapCRDVersionAction(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"])
 	})
+
+	t.Run("When the cluster only supports v1 CRD, v1 CRD will be returned even the input has Spec.PreserveUnknownFields set to true (issue 4080)", func(t *testing.T) {
+		a.discoveryHelper = &velerotest.FakeDiscoveryHelper{
+			APIGroupsList: []metav1.APIGroup{
+				{
+					Name: apiextv1.GroupName,
+					Versions: []metav1.GroupVersionForDiscovery{
+						{
+							Version: apiextv1.SchemeGroupVersion.Version,
+						},
+					},
+				},
+			},
+		}
+		b := builder.ForV1CustomResourceDefinition("test.velero.io")
+		b.PreserveUnknownFields(true)
+		c := b.Result()
+		obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&c)
+		require.NoError(t, err)
+
+		item, _, err := a.Execute(&unstructured.Unstructured{Object: obj}, backup)
+		require.NoError(t, err)
+		assert.Equal(t, "apiextensions.k8s.io/v1", item.UnstructuredContent()["apiVersion"])
+		// set it back to the default one
+		a.discoveryHelper = fakeDiscoveryHelper()
+	})
+
 }
 
 // TestRemapCRDVersionActionData tests the RemapCRDVersionAction plugin against actual CRD to confirm that the v1beta1 version is returned when the v1 version is passed in to the plugin.
@@ -116,8 +144,7 @@ func TestRemapCRDVersionActionData(t *testing.T) {
 	backup := &v1.Backup{}
 	clientset := apiextfakes.NewSimpleClientset()
 	betaClient := clientset.ApiextensionsV1beta1().CustomResourceDefinitions()
-
-	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient)
+	a := NewRemapCRDVersionAction(velerotest.NewLogger(), betaClient, fakeDiscoveryHelper())
 
 	tests := []struct {
 		crd                     string
@@ -192,3 +219,21 @@ func TestRemapCRDVersionActionData(t *testing.T) {
 	}
 
 }
+
+func fakeDiscoveryHelper() velerodiscovery.Helper {
+	return &velerotest.FakeDiscoveryHelper{
+		APIGroupsList: []metav1.APIGroup{
+			{
+				Name: apiextv1.GroupName,
+				Versions: []metav1.GroupVersionForDiscovery{
+					{
+						Version: apiextv1beta1.SchemeGroupVersion.Version,
+					},
+					{
+						Version: apiextv1.SchemeGroupVersion.Version,
+					},
+				},
+			},
+		},
+	}
+}

pkg/cmd/server/plugin/plugin.go

@@ -110,7 +110,16 @@ func newRemapCRDVersionAction(f client.Factory) veleroplugin.HandlerInitializer
 			return nil, err
 		}
 
-		return backup.NewRemapCRDVersionAction(logger, client.ApiextensionsV1beta1().CustomResourceDefinitions()), nil
+		clientset, err := f.KubeClient()
+		if err != nil {
+			return nil, err
+		}
+		discoveryHelper, err := velerodiscovery.NewHelper(clientset.Discovery(), logger)
+		if err != nil {
+			return nil, err
+		}
+
+		return backup.NewRemapCRDVersionAction(logger, client.ApiextensionsV1beta1().CustomResourceDefinitions(), discoveryHelper), nil
 	}
 }