Merge pull request #405 from nrb/ignore-duplicates

Compare for duplicates before logging object exists errors

Gopkg.lock

@@ -583,6 +583,7 @@
 [[projects]]
   name = "k8s.io/apimachinery"
   packages = [
+    "pkg/api/equality",
     "pkg/api/errors",
     "pkg/api/meta",
     "pkg/api/resource",
@@ -721,6 +722,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "4b39ff1fb04dedf398ef1e5695bb6b34491b312c55a2626f153f25a6ce3ffc95"
+  inputs-digest = "1ff4ae8599f679fbc275103407cfda3293b431888877531339c789749659b943"
   solver-name = "gps-cdcl"
   solver-version = 1

pkg/restore/restore.go

@@ -31,6 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -634,20 +635,37 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
 		}
 
 		// necessary because we may have remapped the namespace
-		obj.SetNamespace(namespace)
+		// if the namespace is blank, don't create the key
+		if namespace != "" {
+			obj.SetNamespace(namespace)
+		}
 
 		// add an ark-restore label to each resource for easy ID
 		addLabel(obj, api.RestoreLabelKey, ctx.restore.Name)
 
 		ctx.infof("Restoring %s: %v", obj.GroupVersionKind().Kind, obj.GetName())
-		_, err = resourceClient.Create(obj)
-		if apierrors.IsAlreadyExists(err) {
-			addToResult(&warnings, namespace, err)
+		_, restoreErr := resourceClient.Create(obj)
+		if apierrors.IsAlreadyExists(restoreErr) {
+			equal := false
+			if fromCluster, err := resourceClient.Get(obj.GetName(), metav1.GetOptions{}); err == nil {
+				equal, err = objectsAreEqual(fromCluster, obj)
+				// Log any errors trying to check equality
+				if err != nil {
+					ctx.infof("error checking %s against cluster: %v", obj.GetName(), err)
+				}
+			} else {
+				ctx.infof("Error retrieving cluster version of %s: %v", obj.GetName(), err)
+			}
+			if !equal {
+				e := errors.Errorf("not restored: %s and is different from backed up version.", restoreErr)
+				addToResult(&warnings, namespace, e)
+			}
 			continue
 		}
-		if err != nil {
+		// Error was something other than an AlreadyExists
+		if restoreErr != nil {
 			ctx.infof("error restoring %s: %v", obj.GetName(), err)
-			addToResult(&errs, namespace, fmt.Errorf("error restoring %s: %v", fullPath, err))
+			addToResult(&errs, namespace, fmt.Errorf("error restoring %s: %v", fullPath, restoreErr))
 			continue
 		}
 
@@ -722,6 +740,25 @@ func (ctx *context) executePVAction(obj *unstructured.Unstructured) (*unstructur
 	return updated2, nil
 }
 
+// objectsAreEqual takes two unstructured objects and checks for equality.
+// The fromCluster object is mutated to remove any insubstantial runtime
+// information that won't match
+func objectsAreEqual(fromCluster, fromBackup *unstructured.Unstructured) (bool, error) {
+	// Remove insubstantial metadata
+	fromCluster, err := resetMetadataAndStatus(fromCluster)
+	if err != nil {
+		return false, err
+	}
+
+	// We know the cluster won't have the restore name label, so
+	// copy it over from the backup
+	restoreName := fromBackup.GetLabels()[api.RestoreLabelKey]
+	addLabel(fromCluster, api.RestoreLabelKey, restoreName)
+
+	// If there are no specific actions needed based on the type, simply check for equality.
+	return equality.Semantic.DeepEqual(fromBackup, fromCluster), nil
+}
+
 func isPVReady(obj runtime.Unstructured) bool {
 	phase, err := collections.GetString(obj.UnstructuredContent(), "status.phase")
 	if err != nil {

pkg/restore/restore_test.go

@@ -676,6 +676,61 @@ func TestResetMetadataAndStatus(t *testing.T) {
 	}
 }
 
+func TestObjectsAreEqual(t *testing.T) {
+	tests := []struct {
+		name        string
+		backupObj   *unstructured.Unstructured
+		clusterObj  *unstructured.Unstructured
+		expectedErr bool
+		expectedRes bool
+	}{
+		{
+			name:        "objects are already equal",
+			backupObj:   NewTestUnstructured().WithName("obj").WithArkLabel("test").Unstructured,
+			clusterObj:  NewTestUnstructured().WithName("obj").Unstructured,
+			expectedErr: false,
+			expectedRes: true,
+		},
+		{
+			name:        "objects reset correctly",
+			backupObj:   NewTestUnstructured().WithName("obj").WithArkLabel("test").Unstructured,
+			clusterObj:  NewTestUnstructured().WithMetadata("blah", "foo").WithName("obj").Unstructured,
+			expectedErr: false,
+			expectedRes: true,
+		},
+		{
+			name:        "cluster object has no metadata to reset",
+			backupObj:   NewTestUnstructured().WithName("obj").WithArkLabel("test").Unstructured,
+			clusterObj:  NewTestUnstructured().Unstructured,
+			expectedErr: true,
+			expectedRes: false,
+		},
+		{
+			name:        "Test JSON objects",
+			backupObj:   unstructuredOrDie(`{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"default","namespace":"nginx-example", "labels": {"ark-restore": "test"}},"secrets":[{"name":"default-token-xhjjc"}]}`),
+			clusterObj:  unstructuredOrDie(`{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"creationTimestamp":"2018-04-05T20:12:21Z","name":"default","namespace":"nginx-example","resourceVersion":"650","selfLink":"/api/v1/namespaces/nginx-example/serviceaccounts/default","uid":"a5a3d2a2-390d-11e8-9644-42010a960002"},"secrets":[{"name":"default-token-xhjjc"}]}`),
+			expectedErr: false,
+			expectedRes: true,
+		},
+		{
+			name:        "Test ServiceAccount secrets mismatch",
+			backupObj:   unstructuredOrDie(`{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"default","namespace":"nginx-example", "labels": {"ark-restore": "test"}},"secrets":[{"name":"default-token-abcde"}]}`),
+			clusterObj:  unstructuredOrDie(`{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"creationTimestamp":"2018-04-05T20:12:21Z","name":"default","namespace":"nginx-example","resourceVersion":"650","selfLink":"/api/v1/namespaces/nginx-example/serviceaccounts/default","uid":"a5a3d2a2-390d-11e8-9644-42010a960002"},"secrets":[{"name":"default-token-xhjjc"}]}`),
+			expectedErr: false,
+			expectedRes: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			res, err := objectsAreEqual(test.clusterObj, test.backupObj)
+			if assert.Equal(t, test.expectedErr, err != nil) {
+				assert.Equal(t, test.expectedRes, res)
+			}
+		})
+	}
+}
+
 func TestExecutePVAction(t *testing.T) {
 	iops := int64(1000)
 
@@ -844,6 +899,16 @@ func TestIsPVReady(t *testing.T) {
 	}
 }
 
+// Copied from backup/backup_test.go for JSON testing.
+// TODO: move this into util/test for re-use.
+func unstructuredOrDie(data string) *unstructured.Unstructured {
+	o, _, err := unstructured.UnstructuredJSONScheme.Decode([]byte(data), nil, nil)
+	if err != nil {
+		panic(err)
+	}
+	return o.(*unstructured.Unstructured)
+}
+
 type testUnstructured struct {
 	*unstructured.Unstructured
 }
@@ -897,6 +962,17 @@ func (obj *testUnstructured) WithName(name string) *testUnstructured {
 	return obj.WithMetadataField("name", name)
 }
 
+func (obj *testUnstructured) WithArkLabel(restoreName string) *testUnstructured {
+	ls := obj.GetLabels()
+	if ls == nil {
+		ls = make(map[string]string)
+	}
+	ls[api.RestoreLabelKey] = restoreName
+	obj.SetLabels(ls)
+
+	return obj
+}
+
 func (obj *testUnstructured) withMap(name string, fields ...string) *testUnstructured {
 	m := make(map[string]interface{})
 	obj.Object[name] = m
@@ -942,10 +1018,6 @@ func toUnstructured(objs ...runtime.Object) []unstructured.Unstructured {
 
 		delete(metadata, "creationTimestamp")
 
-		if _, exists := metadata["namespace"]; !exists {
-			metadata["namespace"] = ""
-		}
-
 		delete(unstructuredObj.Object, "status")
 
 		res = append(res, unstructuredObj)

vendor/k8s.io/apimachinery/pkg/api/equality/semantic.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package equality
+
+import (
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+// Semantic can do semantic deep equality checks for api objects.
+// Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true
+var Semantic = conversion.EqualitiesOrDie(
+	func(a, b resource.Quantity) bool {
+		// Ignore formatting, only care that numeric value stayed the same.
+		// TODO: if we decide it's important, it should be safe to start comparing the format.
+		//
+		// Uninitialized quantities are equivalent to 0 quantities.
+		return a.Cmp(b) == 0
+	},
+	func(a, b metav1.MicroTime) bool {
+		return a.UTC() == b.UTC()
+	},
+	func(a, b metav1.Time) bool {
+		return a.UTC() == b.UTC()
+	},
+	func(a, b labels.Selector) bool {
+		return a.String() == b.String()
+	},
+	func(a, b fields.Selector) bool {
+		return a.String() == b.String()
+	},
+)