Merge branch 'main' into issue-fix-5123

Dockerfile

@@ -68,7 +68,7 @@ RUN mkdir -p /output/usr/bin && \
 # Velero image packing section
 FROM gcr.io/distroless/base-nossl-debian11:nonroot
 
-LABEL maintainer="Nolan Brubaker <brubakern@vmware.com>"
+LABEL maintainer="Xun Jiang <jxun@vmware.com>"
 
 COPY --from=velero-builder /output /
 

Tiltfile

@@ -60,7 +60,7 @@ RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com
 
 additional_docker_helper_commands = """
 # Install delve to allow debugging
-RUN go get github.com/go-delve/delve/cmd/dlv
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
 RUN wget -qO- https://dl.k8s.io/v1.25.2/kubernetes-client-linux-amd64.tar.gz | tar xvz
 RUN wget -qO- https://get.docker.com | sh

changelogs/unreleased/6232-kaovilai

@@ -0,0 +1 @@
+log volumes to backup to help debug why `IsPodRunning` is called.

hack/update-1fmt.sh

@@ -35,13 +35,11 @@ fi
 
 files="$(find . -type f -name '*.go' -not -path './.go/*' -not -path './vendor/*' -not -path './site/*' -not -path '*/generated/*' -not -name 'zz_generated*' -not -path '*/mocks/*')"
 echo "${ACTION} gofmt"
-for file in ${files}; do
-  output=$(gofmt "${MODE}" -s "${file}")
-  if [[ -n "${output}" ]]; then
-    VERIFY_FMT_FAILED=1
-    echo "${output}"
-  fi
-done
+output=$(printf '%s\n' "${files}" | xargs gofmt "${MODE}" -s)
+if [[ -n "${output}" ]]; then
+  VERIFY_FMT_FAILED=1
+  echo "${output}"
+fi
 if [[ -n "${VERIFY_FMT_FAILED:-}" ]]; then
   echo "${ACTION} gofmt - failed! Please run 'make update'."
 else
@@ -49,13 +47,11 @@ else
 fi
 
 echo "${ACTION} goimports"
-for file in ${files}; do
-  output=$(goimports "${MODE}" -local github.com/vmware-tanzu/velero "${file}")
-  if [[ -n "${output}" ]]; then
-    VERIFY_IMPORTS_FAILED=1
-    echo "${output}"
-  fi
-done
+output=$(printf '%s\n' "${files}" | xargs goimports "${MODE}" -local github.com/vmware-tanzu/velero)
+if [[ -n "${output}" ]]; then
+  VERIFY_IMPORTS_FAILED=1
+  echo "${output}"
+fi
 if [[ -n "${VERIFY_IMPORTS_FAILED:-}" ]]; then
   echo "${ACTION} goimports - failed! Please run 'make update'."
 else

pkg/podvolume/backupper.go

@@ -131,7 +131,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 	if len(volumesToBackup) == 0 {
 		return nil, nil
 	}
-
+	log.Infof("pod %s/%s has volumes to backup: %v", pod.Namespace, pod.Name, volumesToBackup)
 	err := kube.IsPodRunning(pod)
 	if err != nil {
 		for _, volumeName := range volumesToBackup {

pkg/podvolume/backupper_test.go

@@ -17,13 +17,19 @@ limitations under the License.
 package podvolume
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 )
 
 func TestIsHostPathVolume(t *testing.T) {
@@ -139,3 +145,62 @@ func (g *fakePVGetter) Get(ctx context.Context, name string, opts metav1.GetOpti
 
 	return nil, errors.New("item not found")
 }
+
+func Test_backupper_BackupPodVolumes_log_test(t *testing.T) {
+	type args struct {
+		backup          *velerov1api.Backup
+		pod             *corev1api.Pod
+		volumesToBackup []string
+		resPolicies     *resourcepolicies.Policies
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantLog string
+	}{
+		{
+			name: "backup pod volumes should log volume names",
+			args: args{
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "backup-1",
+						Namespace: "ns-1",
+					},
+				},
+				pod: &corev1api.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "pod-1",
+						Namespace: "ns-1",
+					},
+					Spec: corev1api.PodSpec{
+						Volumes: []corev1api.Volume{
+							{
+								Name: "vol-1",
+							},
+							{
+								Name: "vol-2",
+							},
+						},
+					},
+				},
+				volumesToBackup: []string{"vol-1", "vol-2"},
+				resPolicies:     nil,
+			},
+			wantLog: "pod ns-1/pod-1 has volumes to backup: [vol-1 vol-2]",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &backupper{
+				ctx: context.Background(),
+			}
+			logOutput := bytes.Buffer{}
+			var log = logrus.New()
+			log.SetOutput(&logOutput)
+			b.BackupPodVolumes(tt.args.backup, tt.args.pod, tt.args.volumesToBackup, tt.args.resPolicies, log)
+			fmt.Println(logOutput.String())
+			assert.Contains(t, logOutput.String(), tt.wantLog)
+
+		})
+	}
+}

pkg/uploader/kopia/shim.go

@@ -174,6 +174,7 @@ func (sr *shimRepository) NewObjectWriter(ctx context.Context, option object.Wri
 	opt.Prefix = udmrepo.ID(option.Prefix)
 	opt.FullPath = ""
 	opt.AccessMode = udmrepo.ObjectDataAccessModeFile
+	opt.AsyncWrites = option.AsyncWrites
 
 	if strings.HasPrefix(option.Description, "DIR:") {
 		opt.DataType = udmrepo.ObjectDataTypeMetadata

site/content/posts/2023-04-26-Velero-1.11.md

@@ -0,0 +1,109 @@
+---
+title: "Velero 1.11: New Actions, New Horizons"
+excerpt: In this release, we've grown the team and continue to welcome new members to our community.We're thrilled to have such significant contributions from the community and we're proud to deliver Velero 1.11.
+author_name: Orlin Vasilev
+slug: Velero-1.11
+categories: ['velero','release']
+image: /img/posts/post-1.11.jpg
+# Tag should match author to drive author pages
+tags: ['Velero Team', 'Orlin Vasilev', 'Velero Release']
+---
+
+We haven't posted for while, but this deserves your attention!
+Last week during KubeCon Europe in Amsterdam we released [Velero v1.11](https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0), which brings significant improvements in its functionality, flexibility, and performance. In this blog post, we will discuss the new features and changes that come with Velero v1.11 and how they can benefit users.
+
+The theme of the v1.11 release is most definitely more flexibility, not only with the features added, but also in terms of the Velero contribution, development, and quality assurance processes.
+
+In case you missed it we have new Product Manager - [Pradeep Kumar Chaturvedi](https://github.com/pradeepkchaturvedi) and few new contributors from companies like DELL and Microsoft.
+
+### Full list of changes can be found [here](https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0)
+
+## Release Highlights
+
+### BackupItemAction v2
+This feature implements the BackupItemAction v2. BIA v2 has two new methods: Progress() and Cancel() and modifies the Execute() return value.
+
+The API change is needed to facilitate long-running BackupItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running BackupItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item.
+[https://github.com/vmware-tanzu/velero/pull/5442](https://github.com/vmware-tanzu/velero/pull/5442)
+
+### RestoreItemAction v2
+This feature implemented the RestoreItemAction v2. RIA v2 has three new methods: Progress(), Cancel(), and AreAdditionalItemsReady(), and it modifies RestoreItemActionExecuteOutput() structure in the RIA return value.
+
+The Progress() and Cancel() methods are needed to facilitate long-running RestoreItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running RestoreItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. The AreAdditionalItemsReady() method is needed to allow plugins to tell Velero to wait until the returned additional items have been restored and are ready for use in the cluster before restoring the current item.
+
+[https://github.com/vmware-tanzu/velero/pull/5569](https://github.com/vmware-tanzu/velero/pull/5569)
+
+### Plugin Progress Monitoring
+This is intended as a replacement for the previously-approved Upload Progress Monitoring design ([Upload Progress Monitoring](https://github.com/vmware-tanzu/velero/blob/main/design/upload-progress.md)) to expand the supported use cases beyond snapshot upload to include what was previously called Async Backup/Restore Item Actions.
+
+### Flexible resource policy that can filter volumes to skip in the backup
+This feature provides a flexible policy to filter volumes in the backup without requiring patching any labels or annotations to the pods or volumes. This policy is configured as k8s ConfigMap and maintained by the users themselves, and it can be extended to more scenarios in the future. By now, the policy rules out volumes from backup depending on the CSI driver, NFS setting, volume size, and StorageClass setting. Please refer to [Resource policies rules](https://velero.io/docs/v1.11/resource-filtering/#resource-policies) for the policy's ConifgMap format. It is not guaranteed to work on unofficial third-party plugins as it may not follow the existing backup workflow code logic of Velero.
+
+### Resource Filters that can distinguish cluster scope and namespace scope resources
+This feature adds four new resource filters for backup. The new filters are separated into cluster scope and namespace scope. Before this feature, Velero could not filter cluster scope resources precisely. This feature provides the ability and refactors existing resource filter parameters.
+
+### New parameter in installation to customize the ServiceAccount name
+The `velero install` sub-command now includes a new parameter,`--service-account-name`, which allows users to specify the ServiceAccountName for the Velero and node-agent pods. This feature may be particularly useful for users who utilize IRSA (IAM Roles for Service Accounts) in Amazon EKS (Elastic Kubernetes Service)."
+
+### Add a parameter for setting the Velero server connection with the k8s API server's timeout
+In Velero, some code pieces need to communicate with the k8s API server. Before v1.11, these code pieces used hard-code timeout settings. This feature adds a resource-timeout parameter in the velero server binary to make it configurable.
+
+### Add resource list in the output of the restore describe command
+Before this feature, Velero restore didn't have a restored resources list as the Velero backup. It's not convenient for users to learn what is restored. This feature adds the resources list and the handling result of the resources (including created, updated, failed, and skipped).
+
+### Support JSON format output of backup describe command
+Before the Velero v1.11 release, users could not choose Velero's backup describe command's output format. The command output format is friendly for human reading, but it's not a structured output, and it's not easy for other programs to get information from it. Velero v1.11 adds a JSON format output for the backup describe command.
+
+### Refactor controllers with controller-runtime
+In v1.11, Backup Controller and Restore controller are refactored with controller-runtime. Till v1.11, all Velero controllers use the controller-runtime framework.
+
+### Runtime and dependencies
+To fix CVEs and keep pace with Golang, Velero made changes as follows:
+* Bump Golang runtime to v1.19.8.
+* Bump several dependent libraries to new versions.
+* Compile Restic (v0.15.0) with Golang v1.19.8 instead of packaging the official binary.
+
+
+## Breaking changes
+* The Velero CSI plugin now determines whether to restore Volume's data from snapshots on the restore's restorePVs setting. Before v1.11, the CSI plugin doesn't check the restorePVs parameter setting. 
+
+
+## Limitations/Known issues
+* The Flexible resource policy that can filter volumes to skip in the backup is not guaranteed to work on unofficial third-party plugins because the plugins may not follow the existing backup workflow code logic of Velero. The ConfigMap used as the policy is supposed to be maintained by users.
+
+
+### Improving Developer Experience
+
+As we continue to grow our community of contributors, we want to lower the barrier to entry for making contributions to the Velero project.
+We’ve made huge improvements to the developer experience during this release cycle by introducing Tilt to the developer workflow.
+Using Tilt enables developers to make changes to Velero and its plugins, and have those changes automatically built and deployed to your cluster.
+This removes the need for any manual building or pushing of images, and provides a faster and much simpler workflow.
+Our Tilt configuration also enables contributors to more easily debug the Velero process using Delve, which has integrations with many editors and IDEs.
+If you would like to try it out, please see our [documentation](https://velero.io/docs/v1.11/tilt/).
+
+### Looking Forward
+
+We have more exciting additions and improvements to Velero earmarked for future releases.
+For v1.12, we would like to have your input again [here](https://github.com/vmware-tanzu/velero/discussions/6217)
+See our [1.12 RoadMap](https://github.com/vmware-tanzu/velero/wiki/1.12-Roadmap) for the complete list.
+
+
+### Join the Community and Make Velero Better
+
+Velero is better because of our contributors and maintainers.
+It is because of you that we can bring great software to the community.
+Please join us during our online [community meetings every Tuesday](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA?view) and catch up with past meetings on YouTube on the [Velero Community Meetings playlist](https://www.youtube.com/watch?v=nc48ocI-6go&list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM).
+
+You can always find the latest project information at [velero.io](https://velero.io).
+Look for issues on GitHub marked ["Good first issue"](https://github.com/vmware-tanzu/velero/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+first+issue%22) or ["Help wanted"](https://github.com/vmware-tanzu/velero/issues?q=is%3Aopen+is%3Aissue+label%3A%22Help+wanted%22+) if you want to roll up your sleeves and write some code with us.
+
+For opportunities to help and be helped, visit our [Community Support Q&A on GitHub](https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a).
+
+You can chat with us on [Kubernetes Slack in the #velero channel](https://kubernetes.slack.com/messages/C6VCGP4MT) and follow us on Twitter at [@projectvelero](https://twitter.com/projectvelero).
+
+Orlin Vasilev
+Velero Community Lead
+
+
+Photo by [Markus Spiske on Unsplash](https://unsplash.com/@markusspiske?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText)
+