Wait for CRDs to be ready before restoring CRs (#1937)

* Wait for CRDs to be available and ready When restoring CRDs, we should wait for the definition to be ready and available before moving on to restoring specific CRs. While the CRDs are often ready by the time we get to restoring a CR, there is a race condition where the CRD isn't ready. This change waits on each CRD at restore time. Signed-off-by: Nolan Brubaker <brubakern@vmware.com>
2026-09-20 23:14:17 +00:00 · 2020-01-30 09:19:13 -08:00
parent 710beb96c2
commit 6745979a7b
8 changed files with 392 additions and 30 deletions
@@ -1,5 +1,5 @@
 /*
-Copyright 2017 the Velero contributors.
+Copyright 2017, 2019 the Velero contributors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -22,8 +22,10 @@ import (

 	"github.com/pkg/errors"
 	corev1api "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
@@ -135,3 +137,73 @@ func GetVolumeDirectory(pod *corev1api.Pod, volumeName string, pvcLister corev1l

 	return pvc.Spec.VolumeName, nil
 }
+
+// IsCRDReady checks a CRD to see if it's ready, with both the Established and NamesAccepted conditions.
+func IsCRDReady(crd *apiextv1beta1.CustomResourceDefinition) bool {
+	var isEstablished, namesAccepted bool
+	for _, cond := range crd.Status.Conditions {
+		if cond.Type == apiextv1beta1.Established && cond.Status == apiextv1beta1.ConditionTrue {
+			isEstablished = true
+		}
+		if cond.Type == apiextv1beta1.NamesAccepted && cond.Status == apiextv1beta1.ConditionTrue {
+			namesAccepted = true
+		}
+	}
+
+	return (isEstablished && namesAccepted)
+}
+
+// IsUnstructuredCRDReady checks an unstructured CRD to see if it's ready, with both the Established and NamesAccepted conditions.
+// TODO: Delete this function and use IsCRDReady when the upstream runtime.FromUnstructured function properly handles int64 field conversions.
+// Duplicated function because the velero install package uses IsCRDReady with the beta types.
+// See https://github.com/kubernetes/kubernetes/issues/87675
+func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) {
+	var isEstablished, namesAccepted bool
+
+	conditions, ok, err := unstructured.NestedSlice(crd.UnstructuredContent(), "status", "conditions")
+	if !ok {
+		return false, nil
+	}
+	if err != nil {
+		return false, errors.Wrap(err, "unable to access CRD's conditions")
+	}
+
+	for _, c := range conditions {
+		// Unlike the typed version of this function, we need to cast the Condition since it's an interface{} here,
+		// then we fetch the type and status of the Condition before inspecting them for relevant values
+		cond, ok := c.(map[string]interface{})
+		if !ok {
+			return false, errors.New("unable to convert condition to map[string]interface{}")
+		}
+		conditionType, ok, err := unstructured.NestedString(cond, "type")
+		if !ok {
+			// This should never happen unless someone manually edits the serialized data.
+			return false, errors.New("condition missing a type")
+		}
+
+		if err != nil {
+			return false, errors.Wrap(err, "unable to access condition's type")
+		}
+
+		status, ok, err := unstructured.NestedString(cond, "status")
+		if !ok {
+			// This should never happen unless someone manually edits the serialized data.
+			return false, errors.New("condition missing a status")
+		}
+
+		if err != nil {
+			return false, errors.Wrap(err, "unable to access condition's status")
+		}
+
+		// Here is the actual logic of the function
+		// Cast the API's types into strings since we're pulling strings out of the unstructured data.
+		if conditionType == string(apiextv1beta1.Established) && status == string(apiextv1beta1.ConditionTrue) {
+			isEstablished = true
+		}
+		if conditionType == string(apiextv1beta1.NamesAccepted) && status == string(apiextv1beta1.ConditionTrue) {
+			namesAccepted = true
+		}
+	}
+
+	return (isEstablished && namesAccepted), nil
+}
@@ -17,6 +17,7 @@ limitations under the License.
 package kube

 import (
+	"encoding/json"
 	"testing"
 	"time"

@@ -24,8 +25,11 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	kubeinformers "k8s.io/client-go/informers"

@@ -197,3 +201,137 @@ func TestGetVolumeDirectorySuccess(t *testing.T) {
 		assert.Equal(t, tc.want, dir)
 	}
 }
+
+func TestIsCRDReady(t *testing.T) {
+	tests := []struct {
+		name string
+		crd  *apiextv1beta1.CustomResourceDefinition
+		want bool
+	}{
+		{
+			name: "CRD is not established & not accepting names - not ready",
+			crd:  builder.ForCustomResourceDefinition("MyCRD").Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & not accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is not established & accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & accepting names - ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).
+				Result(),
+			want: true,
+		},
+	}
+
+	for _, tc := range tests {
+		result := IsCRDReady(tc.crd)
+		assert.Equal(t, tc.want, result)
+	}
+}
+
+func TestIsUnstructuredCRDReady(t *testing.T) {
+	tests := []struct {
+		name string
+		crd  *apiextv1beta1.CustomResourceDefinition
+		want bool
+	}{
+		{
+			name: "CRD is not established & not accepting names - not ready",
+			crd:  builder.ForCustomResourceDefinition("MyCRD").Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & not accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is not established & accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & accepting names - ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).
+				Result(),
+			want: true,
+		},
+	}
+
+	for _, tc := range tests {
+		m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.crd)
+		require.NoError(t, err)
+		result, err := IsUnstructuredCRDReady(&unstructured.Unstructured{Object: m})
+		require.NoError(t, err)
+		assert.Equal(t, tc.want, result)
+	}
+}
+
+// TestFromUnstructuredIntToFloatBug tests for a bug where runtime.DefaultUnstructuredConverter.FromUnstructured can't take a whole number into a float.
+// This test should fail when https://github.com/kubernetes/kubernetes/issues/87675 is fixed upstream, letting us know we can remove the IsUnstructuredCRDReady function.
+func TestFromUnstructuredIntToFloatBug(t *testing.T) {
+	b := []byte(`
+{
+	"apiVersion": "apiextensions.k8s.io/v1beta1",
+	"kind": "CustomResourceDefinition",
+	"metadata": {
+	  "name": "foos.example.foo.com"
+	},
+	"spec": {
+	  "group": "example.foo.com",
+	  "version": "v1alpha1",
+	  "scope": "Namespaced",
+	  "names": {
+		"plural": "foos",
+		"singular": "foo",
+		"kind": "Foo"
+	  },
+	  "validation": {
+		"openAPIV3Schema": {
+		  "required": [
+			"spec"
+		  ],
+		  "properties": {
+			"spec": {
+			  "required": [
+				"bar"
+			  ],
+			  "properties": {
+				"bar": {
+				  "type": "integer",
+				  "minimum": 1
+				}
+			  }
+			}
+		  }
+		}
+	  }
+	}
+  }
+`)
+
+	var obj unstructured.Unstructured
+	err := json.Unmarshal(b, &obj)
+	require.NoError(t, err)
+
+	var newCRD apiextv1beta1.CustomResourceDefinition
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &newCRD)
+	// If there's no error, then the upstream issue is fixed, and we need to remove our workarounds.
+	require.Error(t, err)
+}