data mover restore expose

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
Lyndon-Li
2023-06-06 16:14:58 +08:00
parent 8427a9fdb3
commit 2ea6ffb63f
8 changed files with 1210 additions and 0 deletions
+29
View File
@@ -17,12 +17,14 @@ package kube
import (
"context"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
)
@@ -81,3 +83,30 @@ func DeletePodIfAny(ctx context.Context, podGetter corev1client.CoreV1Interface,
}
}
}
// EnsureDeletePod asserts the existence of a pod by name, deletes it and waits for its disappearance and returns errors on any failure
func EnsureDeletePod(ctx context.Context, podGetter corev1client.CoreV1Interface, pod string, namespace string, timeout time.Duration) error {
err := podGetter.Pods(namespace).Delete(ctx, pod, metav1.DeleteOptions{})
if err != nil {
return errors.Wrapf(err, "error to delete pod %s", pod)
}
err = wait.PollImmediate(waitInternal, timeout, func() (bool, error) {
_, err := podGetter.Pods(namespace).Get(ctx, pod, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, errors.Wrapf(err, "error to get pod %s", pod)
}
return false, nil
})
if err != nil {
return errors.Wrapf(err, "error to assure pod is deleted, %s", pod)
}
return nil
}
+93
View File
@@ -0,0 +1,93 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"context"
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
clientTesting "k8s.io/client-go/testing"
)
func TestEnsureDeletePod(t *testing.T) {
podObject := &corev1api.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-ns",
Name: "fake-pod",
},
}
tests := []struct {
name string
clientObj []runtime.Object
podName string
namespace string
reactors []reactor
err string
}{
{
name: "delete fail",
podName: "fake-pod",
namespace: "fake-ns",
err: "error to delete pod fake-pod: pods \"fake-pod\" not found",
},
{
name: "wait fail",
podName: "fake-pod",
namespace: "fake-ns",
clientObj: []runtime.Object{podObject},
reactors: []reactor{
{
verb: "get",
resource: "pods",
reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
return true, nil, errors.New("fake-get-error")
},
},
},
err: "error to assure pod is deleted, fake-pod: error to get pod fake-pod: fake-get-error",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.clientObj...)
for _, reactor := range test.reactors {
fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
}
var kubeClient kubernetes.Interface = fakeKubeClient
err := EnsureDeletePod(context.Background(), kubeClient.CoreV1(), test.podName, test.namespace, time.Millisecond)
if err != nil {
assert.EqualError(t, err, test.err)
} else {
assert.NoError(t, err)
}
})
}
}
+225
View File
@@ -18,17 +18,23 @@ package kube
import (
"context"
"encoding/json"
"fmt"
"time"
jsonpatch "github.com/evanphx/json-patch"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
storagev1api "k8s.io/api/storage/v1"
storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1"
)
const (
@@ -77,3 +83,222 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
return pv, err
}
// DeletePVIfAny deletes a PV by name if it exists, and log an error when the deletion fails
func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, log logrus.FieldLogger) {
err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
log.WithError(err).Debugf("Abort deleting PV, it doesn't exist, %s", pvName)
} else {
log.WithError(err).Errorf("Failed to delete PV %s", pvName)
}
}
}
// EnsureDeletePVC asserts the existence of a PVC by name, deletes it and waits for its disappearance and returns errors on any failure
func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvc string, namespace string, timeout time.Duration) error {
err := pvcGetter.PersistentVolumeClaims(namespace).Delete(ctx, pvc, metav1.DeleteOptions{})
if err != nil {
return errors.Wrapf(err, "error to delete pvc %s", pvc)
}
err = wait.PollImmediate(waitInternal, timeout, func() (bool, error) {
_, err := pvcGetter.PersistentVolumeClaims(namespace).Get(ctx, pvc, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, errors.Wrapf(err, "error to get pvc %s", pvc)
}
return false, nil
})
if err != nil {
return errors.Wrapf(err, "error to retrieve pvc info for %s", pvc)
}
return nil
}
// RebindPVC rebinds a PVC by modifying its VolumeName to the specific PV
func RebindPVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface,
pvc *corev1api.PersistentVolumeClaim, pv string) (*corev1api.PersistentVolumeClaim, error) {
origBytes, err := json.Marshal(pvc)
if err != nil {
return nil, errors.Wrap(err, "error marshaling original PVC")
}
updated := pvc.DeepCopy()
updated.Spec.VolumeName = pv
delete(updated.Annotations, KubeAnnBindCompleted)
delete(updated.Annotations, KubeAnnBoundByController)
updatedBytes, err := json.Marshal(updated)
if err != nil {
return nil, errors.Wrap(err, "error marshaling updated PV")
}
patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes)
if err != nil {
return nil, errors.Wrap(err, "error creating json merge patch for PV")
}
updated, err = pvcGetter.PersistentVolumeClaims(pvc.Namespace).Patch(ctx, pvc.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return nil, errors.Wrap(err, "error patching PVC")
}
return updated, nil
}
// ResetPVBinding resets the binding info of a PV and adds the required labels so as to make it ready for binding
func ResetPVBinding(ctx context.Context, pvGetter corev1client.CoreV1Interface, pv *corev1api.PersistentVolume, labels map[string]string) (*corev1api.PersistentVolume, error) {
origBytes, err := json.Marshal(pv)
if err != nil {
return nil, errors.Wrap(err, "error marshaling original PV")
}
updated := pv.DeepCopy()
updated.Spec.ClaimRef = nil
delete(updated.Annotations, KubeAnnBoundByController)
if labels != nil {
if updated.Labels == nil {
updated.Labels = make(map[string]string)
}
for k, v := range labels {
if _, ok := updated.Labels[k]; !ok {
updated.Labels[k] = v
}
}
}
updatedBytes, err := json.Marshal(updated)
if err != nil {
return nil, errors.Wrap(err, "error marshaling updated PV")
}
patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes)
if err != nil {
return nil, errors.Wrap(err, "error creating json merge patch for PV")
}
updated, err = pvGetter.PersistentVolumes().Patch(ctx, pv.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return nil, errors.Wrap(err, "error patching PV")
}
return updated, nil
}
// SetPVReclaimPolicy sets the specified reclaim policy to a PV
func SetPVReclaimPolicy(ctx context.Context, pvGetter corev1client.CoreV1Interface, pv *corev1api.PersistentVolume,
policy corev1api.PersistentVolumeReclaimPolicy) (*corev1api.PersistentVolume, error) {
if pv.Spec.PersistentVolumeReclaimPolicy == policy {
return nil, nil
}
origBytes, err := json.Marshal(pv)
if err != nil {
return nil, errors.Wrap(err, "error marshaling original PV")
}
updated := pv.DeepCopy()
updated.Spec.PersistentVolumeReclaimPolicy = policy
updatedBytes, err := json.Marshal(updated)
if err != nil {
return nil, errors.Wrap(err, "error marshaling updated PV")
}
patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes)
if err != nil {
return nil, errors.Wrap(err, "error creating json merge patch for PV")
}
updated, err = pvGetter.PersistentVolumes().Patch(ctx, pv.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return nil, errors.Wrap(err, "error patching PV")
}
return updated, nil
}
// WaitPVCConsumed waits for a PVC to be consumed by a pod so that the selected node is set by the pod scheduling; or does
// nothing if the consuming doesn't affect the PV provision.
// The latest PVC and the selected node will be returned.
func WaitPVCConsumed(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvc string, namespace string,
storageClient storagev1.StorageV1Interface, timeout time.Duration) (string, *corev1api.PersistentVolumeClaim, error) {
selectedNode := ""
var updated *corev1api.PersistentVolumeClaim
var storageClass *storagev1api.StorageClass
err := wait.PollImmediate(waitInternal, timeout, func() (bool, error) {
tmpPVC, err := pvcGetter.PersistentVolumeClaims(namespace).Get(ctx, pvc, metav1.GetOptions{})
if err != nil {
return false, errors.Wrapf(err, "error to get pvc %s/%s", namespace, pvc)
}
if tmpPVC.Spec.StorageClassName != nil && storageClass == nil {
storageClass, err = storageClient.StorageClasses().Get(ctx, *tmpPVC.Spec.StorageClassName, metav1.GetOptions{})
if err != nil {
return false, errors.Wrapf(err, "error to get storage class %s", *tmpPVC.Spec.StorageClassName)
}
}
if storageClass != nil {
if storageClass.VolumeBindingMode != nil && *storageClass.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer {
selectedNode = tmpPVC.Annotations[KubeAnnSelectedNode]
if selectedNode == "" {
return false, nil
}
}
}
updated = tmpPVC
return true, nil
})
if err != nil {
return "", nil, errors.Wrap(err, "error to wait for PVC")
}
return selectedNode, updated, err
}
// WaitPVBound wait for binding of a PV specified by name and returns the bound PV object
func WaitPVBound(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, pvcName string, pvcNamespace string, timeout time.Duration) (*corev1api.PersistentVolume, error) {
var updated *corev1api.PersistentVolume
err := wait.PollImmediate(waitInternal, timeout, func() (bool, error) {
tmpPV, err := pvGetter.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
if err != nil {
return false, errors.Wrapf(err, fmt.Sprintf("failed to get pv %s", pvName))
}
if tmpPV.Spec.ClaimRef == nil {
return false, nil
}
if tmpPV.Spec.ClaimRef.Name != pvcName {
return false, nil
}
if tmpPV.Spec.ClaimRef.Namespace != pvcNamespace {
return false, nil
}
updated = tmpPV
return true, nil
})
if err != nil {
return nil, errors.Wrap(err, "error to wait for bound of PV")
} else {
return updated, nil
}
}
+155
View File
@@ -28,6 +28,7 @@ import (
"k8s.io/client-go/kubernetes/fake"
corev1api "k8s.io/api/core/v1"
storagev1api "k8s.io/api/storage/v1"
clientTesting "k8s.io/client-go/testing"
)
@@ -129,3 +130,157 @@ func TestWaitPVCBound(t *testing.T) {
})
}
}
func TestWaitPVCConsumed(t *testing.T) {
storageClass := "fake-storage-class"
bindModeImmediate := storagev1api.VolumeBindingImmediate
bindModeWait := storagev1api.VolumeBindingWaitForFirstConsumer
pvcObject := &corev1api.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-namespace",
Name: "fake-pvc-1",
},
}
pvcObjectWithSC := &corev1api.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-namespace",
Name: "fake-pvc-2",
},
Spec: corev1api.PersistentVolumeClaimSpec{
StorageClassName: &storageClass,
},
}
scObjWithoutBindMode := &storagev1api.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "fake-storage-class",
},
}
scObjWaitBind := &storagev1api.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "fake-storage-class",
},
VolumeBindingMode: &bindModeWait,
}
scObjWithImmidateBinding := &storagev1api.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "fake-storage-class",
},
VolumeBindingMode: &bindModeImmediate,
}
pvcObjectWithSCAndAnno := &corev1api.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Namespace: "fake-namespace",
Name: "fake-pvc-3",
Annotations: map[string]string{"volume.kubernetes.io/selected-node": "fake-node-1"},
},
Spec: corev1api.PersistentVolumeClaimSpec{
StorageClassName: &storageClass,
},
}
tests := []struct {
name string
pvcName string
pvcNamespace string
kubeClientObj []runtime.Object
kubeReactors []reactor
expectedPVC *corev1api.PersistentVolumeClaim
selectedNode string
err string
}{
{
name: "get pvc error",
pvcName: "fake-pvc",
pvcNamespace: "fake-namespace",
err: "error to wait for PVC: error to get pvc fake-namespace/fake-pvc: persistentvolumeclaims \"fake-pvc\" not found",
},
{
name: "success when no sc",
pvcName: "fake-pvc-1",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObject,
},
expectedPVC: pvcObject,
},
{
name: "get sc fail",
pvcName: "fake-pvc-2",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObjectWithSC,
},
err: "error to wait for PVC: error to get storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found",
},
{
name: "success on sc without binding mode",
pvcName: "fake-pvc-2",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObjectWithSC,
scObjWithoutBindMode,
},
expectedPVC: pvcObjectWithSC,
},
{
name: "success on sc without immediate binding mode",
pvcName: "fake-pvc-2",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObjectWithSC,
scObjWithImmidateBinding,
},
expectedPVC: pvcObjectWithSC,
},
{
name: "pvc annotation miss",
pvcName: "fake-pvc-2",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObjectWithSC,
scObjWaitBind,
},
err: "error to wait for PVC: timed out waiting for the condition",
},
{
name: "success on sc without wait binding mode",
pvcName: "fake-pvc-3",
pvcNamespace: "fake-namespace",
kubeClientObj: []runtime.Object{
pvcObjectWithSCAndAnno,
scObjWaitBind,
},
expectedPVC: pvcObjectWithSCAndAnno,
selectedNode: "fake-node-1",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
for _, reactor := range test.kubeReactors {
fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
}
var kubeClient kubernetes.Interface = fakeKubeClient
selectedNode, pvc, err := WaitPVCConsumed(context.Background(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, kubeClient.StorageV1(), time.Millisecond)
if err != nil {
assert.EqualError(t, err, test.err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, test.expectedPVC, pvc)
assert.Equal(t, test.selectedNode, selectedNode)
})
}
}
+1
View File
@@ -47,6 +47,7 @@ const (
KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller"
KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
KubeAnnMigratedTo = "pv.kubernetes.io/migrated-to"
KubeAnnSelectedNode = "volume.kubernetes.io/selected-node"
)
// NamespaceAndName returns a string in the format <namespace>/<name>