From 0a7e5d600bc70eed1de4fa025cd106158b223019 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 28 May 2026 14:38:26 +0800 Subject: [PATCH] remove restic command package Signed-off-by: Lyndon-Li --- changelogs/unreleased/9861-Lyndon-Li | 1 + pkg/restic/command.go | 104 ---------- pkg/restic/command_factory.go | 125 ------------ pkg/restic/command_factory_test.go | 131 ------------ pkg/restic/command_test.go | 106 ---------- pkg/restic/common.go | 159 --------------- pkg/restic/common_test.go | 141 ------------- pkg/restic/exec_commands.go | 292 --------------------------- pkg/restic/exec_commands_test.go | 111 ---------- 9 files changed, 1 insertion(+), 1169 deletions(-) create mode 100644 changelogs/unreleased/9861-Lyndon-Li delete mode 100644 pkg/restic/command.go delete mode 100644 pkg/restic/command_factory.go delete mode 100644 pkg/restic/command_factory_test.go delete mode 100644 pkg/restic/command_test.go delete mode 100644 pkg/restic/common.go delete mode 100644 pkg/restic/common_test.go delete mode 100644 pkg/restic/exec_commands.go delete mode 100644 pkg/restic/exec_commands_test.go diff --git a/changelogs/unreleased/9861-Lyndon-Li b/changelogs/unreleased/9861-Lyndon-Li new file mode 100644 index 000000000..452765c61 --- /dev/null +++ b/changelogs/unreleased/9861-Lyndon-Li @@ -0,0 +1 @@ +Remove restic command package \ No newline at end of file diff --git a/pkg/restic/command.go b/pkg/restic/command.go deleted file mode 100644 index feb41642d..000000000 --- a/pkg/restic/command.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2020 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" -) - -// Command represents a restic command. -type Command struct { - Command string - RepoIdentifier string - PasswordFile string - CACertFile string - Dir string - Args []string - ExtraFlags []string - Env []string -} - -func (c *Command) RepoName() string { - if c.RepoIdentifier == "" { - return "" - } - - return c.RepoIdentifier[strings.LastIndex(c.RepoIdentifier, "/")+1:] -} - -// StringSlice returns the command as a slice of strings. -func (c *Command) StringSlice() []string { - res := []string{"restic"} - - res = append(res, c.Command, repoFlag(c.RepoIdentifier)) - if c.PasswordFile != "" { - res = append(res, passwordFlag(c.PasswordFile)) - } - if c.CACertFile != "" { - res = append(res, cacertFlag(c.CACertFile)) - } - - // If VELERO_SCRATCH_DIR is defined, put the restic cache within it. If not, - // allow restic to choose the location. This makes running either in-cluster - // or local (dev) work properly. - if scratch := os.Getenv("VELERO_SCRATCH_DIR"); scratch != "" { - res = append(res, cacheDirFlag(filepath.Join(scratch, ".cache", "restic"))) - } - - res = append(res, c.Args...) - res = append(res, c.ExtraFlags...) - - return res -} - -// String returns the command as a string. -func (c *Command) String() string { - return strings.Join(c.StringSlice(), " ") -} - -// Cmd returns an exec.Cmd for the command. -func (c *Command) Cmd() *exec.Cmd { - parts := c.StringSlice() - cmd := exec.Command(parts[0], parts[1:]...) //nolint:gosec,noctx // Internal call. No need to check the parameter. No to add context for deprecated Restic. - cmd.Dir = c.Dir - - if len(c.Env) > 0 { - cmd.Env = c.Env - } - - return cmd -} - -func repoFlag(repoIdentifier string) string { - return fmt.Sprintf("--repo=%s", repoIdentifier) -} - -func passwordFlag(file string) string { - return fmt.Sprintf("--password-file=%s", file) -} - -func cacheDirFlag(dir string) string { - return fmt.Sprintf("--cache-dir=%s", dir) -} - -func cacertFlag(path string) string { - return fmt.Sprintf("--cacert=%s", path) -} diff --git a/pkg/restic/command_factory.go b/pkg/restic/command_factory.go deleted file mode 100644 index 98bb6ffef..000000000 --- a/pkg/restic/command_factory.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2018, 2019 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "fmt" - "strings" -) - -// BackupCommand returns a Command for running a restic backup. -func BackupCommand(repoIdentifier, passwordFile, path string, tags map[string]string) *Command { - // --host flag is provided with a generic value because restic uses the host - // to find a parent snapshot, and by default it will be the name of the daemonset pod - // where the `restic backup` command is run. If this pod is recreated, we want to continue - // taking incremental backups rather than triggering a full one due to a new pod name. - - return &Command{ - Command: "backup", - RepoIdentifier: repoIdentifier, - PasswordFile: passwordFile, - Dir: path, - Args: []string{"."}, - ExtraFlags: append(backupTagFlags(tags), "--host=velero", "--json"), - } -} - -func backupTagFlags(tags map[string]string) []string { - var flags []string - for k, v := range tags { - flags = append(flags, fmt.Sprintf("--tag=%s=%s", k, v)) - } - return flags -} - -// RestoreCommand returns a Command for running a restic restore. -func RestoreCommand(repoIdentifier, passwordFile, snapshotID, target string) *Command { - return &Command{ - Command: "restore", - RepoIdentifier: repoIdentifier, - PasswordFile: passwordFile, - Dir: target, - Args: []string{snapshotID}, - ExtraFlags: []string{"--target=."}, - } -} - -// GetSnapshotCommand returns a Command for running a restic (get) snapshots. -func GetSnapshotCommand(repoIdentifier, passwordFile string, tags map[string]string) *Command { - return &Command{ - Command: "snapshots", - RepoIdentifier: repoIdentifier, - PasswordFile: passwordFile, - // "--last" is replaced by "--latest=1" in restic v0.12.1 - ExtraFlags: []string{"--json", "--latest=1", getSnapshotTagFlag(tags)}, - } -} - -func getSnapshotTagFlag(tags map[string]string) string { - var tagFilters []string - for k, v := range tags { - tagFilters = append(tagFilters, fmt.Sprintf("%s=%s", k, v)) - } - - return fmt.Sprintf("--tag=%s", strings.Join(tagFilters, ",")) -} - -func InitCommand(repoIdentifier string) *Command { - return &Command{ - Command: "init", - RepoIdentifier: repoIdentifier, - } -} - -func SnapshotsCommand(repoIdentifier string) *Command { - return &Command{ - Command: "snapshots", - RepoIdentifier: repoIdentifier, - } -} - -func PruneCommand(repoIdentifier string) *Command { - return &Command{ - Command: "prune", - RepoIdentifier: repoIdentifier, - } -} - -func ForgetCommand(repoIdentifier, snapshotID string) *Command { - return &Command{ - Command: "forget", - RepoIdentifier: repoIdentifier, - Args: []string{snapshotID}, - } -} - -func UnlockCommand(repoIdentifier string) *Command { - return &Command{ - Command: "unlock", - RepoIdentifier: repoIdentifier, - } -} - -func StatsCommand(repoIdentifier, passwordFile, snapshotID string) *Command { - return &Command{ - Command: "stats", - RepoIdentifier: repoIdentifier, - PasswordFile: passwordFile, - Args: []string{snapshotID}, - ExtraFlags: []string{"--json"}, - } -} diff --git a/pkg/restic/command_factory_test.go b/pkg/restic/command_factory_test.go deleted file mode 100644 index 75f20d47a..000000000 --- a/pkg/restic/command_factory_test.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2018 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package restic - -import ( - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestBackupCommand(t *testing.T) { - c := BackupCommand("repo-id", "password-file", "path", map[string]string{"foo": "bar", "c": "d"}) - - assert.Equal(t, "backup", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) - assert.Equal(t, "password-file", c.PasswordFile) - assert.Equal(t, "path", c.Dir) - assert.Equal(t, []string{"."}, c.Args) - - expected := []string{"--tag=foo=bar", "--tag=c=d", "--host=velero", "--json"} - sort.Strings(expected) - sort.Strings(c.ExtraFlags) - assert.Equal(t, expected, c.ExtraFlags) -} - -func TestRestoreCommand(t *testing.T) { - c := RestoreCommand("repo-id", "password-file", "snapshot-id", "target") - - assert.Equal(t, "restore", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) - assert.Equal(t, "password-file", c.PasswordFile) - assert.Equal(t, "target", c.Dir) - assert.Equal(t, []string{"snapshot-id"}, c.Args) - assert.Equal(t, []string{"--target=."}, c.ExtraFlags) -} - -func TestGetSnapshotCommand(t *testing.T) { - expectedTags := map[string]string{"foo": "bar", "c": "d"} - c := GetSnapshotCommand("repo-id", "password-file", expectedTags) - - assert.Equal(t, "snapshots", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) - assert.Equal(t, "password-file", c.PasswordFile) - - // set up expected flag names - expectedFlags := []string{"--json", "--latest=1", "--tag"} - // for tracking actual flag names - actualFlags := []string{} - // for tracking actual --tag values as a map - actualTags := make(map[string]string) - - // loop through actual flags - for _, flag := range c.ExtraFlags { - // split into 2 parts from the first = sign (if any) - parts := strings.SplitN(flag, "=", 2) - - // convert --tag data to a map - if parts[0] == "--tag" { - actualFlags = append(actualFlags, parts[0]) - - // split based on , - tags := strings.Split(parts[1], ",") - // loop through each key-value tag pair - for _, tag := range tags { - // split the pair on = - kvs := strings.Split(tag, "=") - // record actual key & value - actualTags[kvs[0]] = kvs[1] - } - } else { - actualFlags = append(actualFlags, flag) - } - } - - assert.Equal(t, expectedFlags, actualFlags) - assert.Equal(t, expectedTags, actualTags) -} - -func TestInitCommand(t *testing.T) { - c := InitCommand("repo-id") - - assert.Equal(t, "init", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) -} - -func TestSnapshotsCommand(t *testing.T) { - c := SnapshotsCommand("repo-id") - - assert.Equal(t, "snapshots", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) -} - -func TestPruneCommand(t *testing.T) { - c := PruneCommand("repo-id") - - assert.Equal(t, "prune", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) -} - -func TestForgetCommand(t *testing.T) { - c := ForgetCommand("repo-id", "snapshot-id") - - assert.Equal(t, "forget", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) - assert.Equal(t, []string{"snapshot-id"}, c.Args) -} - -func TestStatsCommand(t *testing.T) { - c := StatsCommand("repo-id", "password-file", "snapshot-id") - - assert.Equal(t, "stats", c.Command) - assert.Equal(t, "repo-id", c.RepoIdentifier) - assert.Equal(t, "password-file", c.PasswordFile) - assert.Equal(t, []string{"snapshot-id"}, c.Args) - assert.Equal(t, []string{"--json"}, c.ExtraFlags) -} diff --git a/pkg/restic/command_test.go b/pkg/restic/command_test.go deleted file mode 100644 index 6048478d7..000000000 --- a/pkg/restic/command_test.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2020 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRepoName(t *testing.T) { - c := &Command{RepoIdentifier: ""} - assert.Empty(t, c.RepoName()) - - c.RepoIdentifier = "s3:s3.amazonaws.com/bucket/prefix/repo" - assert.Equal(t, "repo", c.RepoName()) - - c.RepoIdentifier = "azure:bucket:/repo" - assert.Equal(t, "repo", c.RepoName()) - - c.RepoIdentifier = "gs:bucket:/prefix/repo" - assert.Equal(t, "repo", c.RepoName()) -} - -func TestStringSlice(t *testing.T) { - c := &Command{ - Command: "cmd", - RepoIdentifier: "repo-id", - PasswordFile: "/path/to/password-file", - Dir: "/some/pwd", - Args: []string{"arg-1", "arg-2"}, - ExtraFlags: []string{"--foo=bar"}, - } - - require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) - assert.Equal(t, []string{ - "restic", - "cmd", - "--repo=repo-id", - "--password-file=/path/to/password-file", - "arg-1", - "arg-2", - "--foo=bar", - }, c.StringSlice()) - - os.Setenv("VELERO_SCRATCH_DIR", "/foo") - assert.Equal(t, []string{ - "restic", - "cmd", - "--repo=repo-id", - "--password-file=/path/to/password-file", - "--cache-dir=/foo/.cache/restic", - "arg-1", - "arg-2", - "--foo=bar", - }, c.StringSlice()) - - require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) -} - -func TestString(t *testing.T) { - c := &Command{ - Command: "cmd", - RepoIdentifier: "repo-id", - PasswordFile: "/path/to/password-file", - Dir: "/some/pwd", - Args: []string{"arg-1", "arg-2"}, - ExtraFlags: []string{"--foo=bar"}, - } - - require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) - assert.Equal(t, "restic cmd --repo=repo-id --password-file=/path/to/password-file arg-1 arg-2 --foo=bar", c.String()) -} - -func TestCmd(t *testing.T) { - c := &Command{ - Command: "cmd", - RepoIdentifier: "repo-id", - PasswordFile: "/path/to/password-file", - Dir: "/some/pwd", - Args: []string{"arg-1", "arg-2"}, - ExtraFlags: []string{"--foo=bar"}, - } - - require.NoError(t, os.Unsetenv("VELERO_SCRATCH_DIR")) - execCmd := c.Cmd() - - assert.Equal(t, c.StringSlice(), execCmd.Args) - assert.Equal(t, c.Dir, execCmd.Dir) -} diff --git a/pkg/restic/common.go b/pkg/restic/common.go deleted file mode 100644 index a5bf05c44..000000000 --- a/pkg/restic/common.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "fmt" - "os" - "strconv" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -const ( - - // DefaultMaintenanceFrequency is the default time interval - // at which restic prune is run. - DefaultMaintenanceFrequency = 7 * 24 * time.Hour - - // insecureSkipTLSVerifyKey is the flag in BackupStorageLocation's config - // to indicate whether to skip TLS verify to setup insecure HTTPS connection. - insecureSkipTLSVerifyKey = "insecureSkipTLSVerify" - - // resticInsecureTLSFlag is the flag for Restic command line to indicate - // skip TLS verify on https connection. - resticInsecureTLSFlag = "--insecure-tls" -) - -// TempCACertFile creates a temp file containing a CA bundle -// and returns its path. The caller should generally call os.Remove() -// to remove the file when done with it. -func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - file, err := fs.TempFile("", fmt.Sprintf("cacert-%s", bsl)) - if err != nil { - return "", errors.WithStack(err) - } - - if _, err := file.Write(caCert); err != nil { - // nothing we can do about an error closing the file here, and we're - // already returning an error about the write failing. - file.Close() - return "", errors.WithStack(err) - } - - name := file.Name() - - if err := file.Close(); err != nil { - return "", errors.WithStack(err) - } - - return name, nil -} - -// environ is a slice of strings representing the environment, in the form "key=value". -type environ []string - -// Unset a single environment variable. -func (e *environ) Unset(key string) { - for i := range *e { - if strings.HasPrefix((*e)[i], key+"=") { - (*e)[i] = (*e)[len(*e)-1] - *e = (*e)[:len(*e)-1] - break - } - } -} - -// CmdEnv returns a list of environment variables (in the format var=val) that -// should be used when running a restic command for a particular backend provider. -// This list is the current environment, plus any provider-specific variables restic needs. -func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - var env environ - env = os.Environ() - customEnv := map[string]string{} - var err error - - config := backupLocation.Spec.Config - if config == nil { - config = map[string]string{} - } - - if backupLocation.Spec.Credential != nil { - credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential) - if err != nil { - return []string{}, errors.WithStack(err) - } - config[repoconfig.CredentialsFileKey] = credsFile - } - - backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config) - - switch backendType { - case repoconfig.AWSBackend: - customEnv, err = repoconfig.GetS3ResticEnvVars(config) - if err != nil { - return []string{}, err - } - case repoconfig.AzureBackend: - customEnv, err = repoconfig.GetAzureResticEnvVars(config) - if err != nil { - return []string{}, err - } - case repoconfig.GCPBackend: - customEnv, err = repoconfig.GetGCPResticEnvVars(config) - if err != nil { - return []string{}, err - } - } - - for k, v := range customEnv { - env.Unset(k) - if v == "" { - continue - } - env = append(env, fmt.Sprintf("%s=%s", k, v)) - } - - return env, nil -} - -// GetInsecureSkipTLSVerifyFromBSL get insecureSkipTLSVerify flag from BSL configuration, -// Then return --insecure-tls flag with boolean value as result. -func GetInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string { - result := "" - - if backupLocation == nil { - logger.Info("bsl is nil. return empty.") - return result - } - - if insecure, _ := strconv.ParseBool(backupLocation.Spec.Config[insecureSkipTLSVerifyKey]); insecure { - logger.Debugf("set --insecure-tls=true for Restic command according to BSL %s config", backupLocation.Name) - result = resticInsecureTLSFlag + "=true" - return result - } - - return result -} diff --git a/pkg/restic/common_test.go b/pkg/restic/common_test.go deleted file mode 100644 index 97363340c..000000000 --- a/pkg/restic/common_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "os" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - velerotest "github.com/vmware-tanzu/velero/pkg/test" -) - -func TestTempCACertFile(t *testing.T) { - var ( - fs = velerotest.NewFakeFileSystem() - caCertData = []byte("cacert") - ) - - fileName, err := TempCACertFile(caCertData, "default", fs) - require.NoError(t, err) - - contents, err := fs.ReadFile(fileName) - require.NoError(t, err) - - assert.Equal(t, string(caCertData), string(contents)) - - os.Remove(fileName) -} - -func TestGetInsecureSkipTLSVerifyFromBSL(t *testing.T) { - log := logrus.StandardLogger() - tests := []struct { - name string - backupLocation *velerov1api.BackupStorageLocation - logger logrus.FieldLogger - expected string - }{ - { - "Test with nil BSL. Should return empty string.", - nil, - log, - "", - }, - { - "Test BSL with no configuration. Should return empty string.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "azure", - }, - }, - log, - "", - }, - { - "Test with AWS BSL's insecureSkipTLSVerify set to false.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - Config: map[string]string{ - "insecureSkipTLSVerify": "false", - }, - }, - }, - log, - "", - }, - { - "Test with AWS BSL's insecureSkipTLSVerify set to true.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - Config: map[string]string{ - "insecureSkipTLSVerify": "true", - }, - }, - }, - log, - "--insecure-tls=true", - }, - { - "Test with Azure BSL's insecureSkipTLSVerify set to invalid.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "azure", - Config: map[string]string{ - "insecureSkipTLSVerify": "invalid", - }, - }, - }, - log, - "", - }, - { - "Test with GCP without insecureSkipTLSVerify.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "gcp", - Config: map[string]string{}, - }, - }, - log, - "", - }, - { - "Test with AWS without config.", - &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - }, - }, - log, - "", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res := GetInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger) - - assert.Equal(t, test.expected, res) - }) - } -} diff --git a/pkg/restic/exec_commands.go b/pkg/restic/exec_commands.go deleted file mode 100644 index 94c17c04a..000000000 --- a/pkg/restic/exec_commands.go +++ /dev/null @@ -1,292 +0,0 @@ -/* -Copyright The Velero Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/pkg/uploader" - "github.com/vmware-tanzu/velero/pkg/util/exec" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -const restoreProgressCheckInterval = 10 * time.Second -const backupProgressCheckInterval = 10 * time.Second - -var fileSystem = filesystem.NewFileSystem() - -type backupStatusLine struct { - MessageType string `json:"message_type"` - // seen in status lines - TotalBytes int64 `json:"total_bytes"` - BytesDone int64 `json:"bytes_done"` - // seen in summary line at the end - TotalBytesProcessed int64 `json:"total_bytes_processed"` -} - -// GetSnapshotID runs provided 'restic snapshots' command to get the ID of a snapshot -// and an error if a unique snapshot cannot be identified. -func GetSnapshotID(snapshotIDCmd *Command) (string, error) { - stdout, stderr, err := exec.RunCommand(snapshotIDCmd.Cmd()) - if err != nil { - return "", errors.Wrapf(err, "error running command, stderr=%s", stderr) - } - - type snapshotID struct { - ShortID string `json:"short_id"` - } - - var snapshots []snapshotID - if err := json.Unmarshal([]byte(stdout), &snapshots); err != nil { - return "", errors.Wrap(err, "error unmarshaling restic snapshots result") - } - - if len(snapshots) != 1 { - return "", errors.Errorf("expected one matching snapshot by command: %s, got %d", snapshotIDCmd.String(), len(snapshots)) - } - - return snapshots[0].ShortID, nil -} - -// RunBackup runs a `restic backup` command and watches the output to provide -// progress updates to the caller. -func RunBackup(backupCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) { - // buffers for copying command stdout/err output into - stdoutBuf := new(bytes.Buffer) - stderrBuf := new(bytes.Buffer) - - // create a channel to signal when to end the goroutine scanning for progress - // updates - quit := make(chan struct{}) - - cmd := backupCmd.Cmd() - cmd.Stdout = stdoutBuf - cmd.Stderr = stderrBuf - - err := cmd.Start() - if err != nil { - exec.LogErrorAsExitCode(err, log) - return stdoutBuf.String(), stderrBuf.String(), err - } - - go func() { - ticker := time.NewTicker(backupProgressCheckInterval) - for { - select { - case <-ticker.C: - lastLine := getLastLine(stdoutBuf.Bytes()) - if len(lastLine) > 0 { - stat, err := decodeBackupStatusLine(lastLine) - if err != nil { - log.WithError(err).Errorf("error getting restic backup progress") - } - - // if the line contains a non-empty bytes_done field, we can update the - // caller with the progress - if stat.BytesDone != 0 { - updater.UpdateProgress(&uploader.Progress{ - TotalBytes: stat.TotalBytes, - BytesDone: stat.BytesDone, - }) - } - } - case <-quit: - ticker.Stop() - return - } - } - }() - - err = cmd.Wait() - if err != nil { - exec.LogErrorAsExitCode(err, log) - return stdoutBuf.String(), stderrBuf.String(), err - } - quit <- struct{}{} - - summary, err := getSummaryLine(stdoutBuf.Bytes()) - if err != nil { - return stdoutBuf.String(), stderrBuf.String(), err - } - stat, err := decodeBackupStatusLine(summary) - if err != nil { - return stdoutBuf.String(), stderrBuf.String(), err - } - if stat.MessageType != "summary" { - return stdoutBuf.String(), stderrBuf.String(), errors.WithStack(fmt.Errorf("error getting restic backup summary: %s", string(summary))) - } - - // update progress to 100% - updater.UpdateProgress(&uploader.Progress{ - TotalBytes: stat.TotalBytesProcessed, - BytesDone: stat.TotalBytesProcessed, - }) - - return string(summary), stderrBuf.String(), nil -} - -func decodeBackupStatusLine(lastLine []byte) (backupStatusLine, error) { - var stat backupStatusLine - if err := json.Unmarshal(lastLine, &stat); err != nil { - return stat, errors.Wrapf(err, "unable to decode backup JSON line: %s", string(lastLine)) - } - return stat, nil -} - -// getLastLine returns the last line of a byte array. The string is assumed to -// have a newline at the end of it, so this returns the substring between the -// last two newlines. -func getLastLine(b []byte) []byte { - if len(b) == 0 { - return []byte("") - } - // subslice the byte array to ignore the newline at the end of the string - lastNewLineIdx := bytes.LastIndex(b[:len(b)-1], []byte("\n")) - return b[lastNewLineIdx+1 : len(b)-1] -} - -// getSummaryLine looks for the summary JSON line -// (`{"message_type:"summary",...`) in the restic backup command output. Due to -// an issue in Restic, this might not always be the last line -// (https://github.com/restic/restic/issues/2389). It returns an error if it -// can't be found. -func getSummaryLine(b []byte) ([]byte, error) { - summaryLineIdx := bytes.LastIndex(b, []byte(`{"message_type":"summary"`)) - if summaryLineIdx < 0 { - return nil, errors.New("unable to find summary in restic backup command output") - } - // find the end of the summary line - newLineIdx := bytes.Index(b[summaryLineIdx:], []byte("\n")) - if newLineIdx < 0 { - return nil, errors.New("unable to get summary line from restic backup command output") - } - return b[summaryLineIdx : summaryLineIdx+newLineIdx], nil -} - -// RunRestore runs a `restic restore` command and monitors the volume size to -// provide progress updates to the caller. -func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updater uploader.ProgressUpdater) (string, string, error) { - insecureTLSFlag := "" - - for _, extraFlag := range restoreCmd.ExtraFlags { - if strings.Contains(extraFlag, resticInsecureTLSFlag) { - insecureTLSFlag = extraFlag - } - } - - snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env, insecureTLSFlag) - if err != nil { - return "", "", errors.Wrap(err, "error getting snapshot size") - } - - updater.UpdateProgress(&uploader.Progress{ - TotalBytes: snapshotSize, - }) - - // create a channel to signal when to end the goroutine scanning for progress - // updates - quit := make(chan struct{}) - - go func() { - ticker := time.NewTicker(restoreProgressCheckInterval) - for { - select { - case <-ticker.C: - volumeSize, err := getVolumeSize(restoreCmd.Dir) - if err != nil { - log.WithError(err).Errorf("error getting restic restore progress") - } - - if volumeSize != 0 { - updater.UpdateProgress(&uploader.Progress{ - TotalBytes: snapshotSize, - BytesDone: volumeSize, - }) - } - case <-quit: - ticker.Stop() - return - } - } - }() - - stdout, stderr, err := exec.RunCommandWithLog(restoreCmd.Cmd(), log) - quit <- struct{}{} - - // update progress to 100% - updater.UpdateProgress(&uploader.Progress{ - TotalBytes: snapshotSize, - BytesDone: snapshotSize, - }) - - return stdout, stderr, err -} - -func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string, insecureTLS string) (int64, error) { - cmd := StatsCommand(repoIdentifier, passwordFile, snapshotID) - cmd.Env = env - cmd.CACertFile = caCertFile - - if len(insecureTLS) > 0 { - cmd.ExtraFlags = append(cmd.ExtraFlags, insecureTLS) - } - - stdout, stderr, err := exec.RunCommand(cmd.Cmd()) - if err != nil { - return 0, errors.Wrapf(err, "error running command, stderr=%s", stderr) - } - - var snapshotStats struct { - TotalSize int64 `json:"total_size"` - } - - if err := json.Unmarshal([]byte(stdout), &snapshotStats); err != nil { - return 0, errors.Wrapf(err, "error unmarshaling restic stats result, stdout=%s", stdout) - } - - return snapshotStats.TotalSize, nil -} - -func getVolumeSize(path string) (int64, error) { - var size int64 - - files, err := fileSystem.ReadDir(path) - if err != nil { - return 0, errors.Wrapf(err, "error reading directory %s", path) - } - - for _, file := range files { - if file.IsDir() { - s, err := getVolumeSize(fmt.Sprintf("%s/%s", path, file.Name())) - if err != nil { - return 0, err - } - size += s - } else { - size += file.Size() - } - } - - return size, nil -} diff --git a/pkg/restic/exec_commands_test.go b/pkg/restic/exec_commands_test.go deleted file mode 100644 index 370f6653c..000000000 --- a/pkg/restic/exec_commands_test.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2019 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/vmware-tanzu/velero/pkg/test" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -func Test_getSummaryLine(t *testing.T) { - summaryLine := `{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"}` - tests := []struct { - name string - output string - wantErr bool - }{ - {"no summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} -{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} -`, true}, - {"no newline after summary", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} -{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} -{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0`, true}, - {"summary at end", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} -{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} -{"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000} -{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"} -`, false}, - {"summary before status", `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":10485760000} -{"message_type":"status","percent_done":0,"total_files":3,"files_done":1,"total_bytes":13238272000} -{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":3,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":3,"total_bytes_processed":13238272000,"total_duration":0.319265105,"snapshot_id":"38515bb5"} -{"message_type":"status","percent_done":1,"total_files":3,"files_done":3,"total_bytes":13238272000,"bytes_done":13238272000} -`, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - summary, err := getSummaryLine([]byte(tt.output)) - if tt.wantErr { - assert.Error(t, err) - } else { - require.NoError(t, err) - assert.Equal(t, summaryLine, string(summary)) - } - }) - } -} - -func Test_getLastLine(t *testing.T) { - tests := []struct { - output []byte - want string - }{ - {[]byte(`last line -`), "last line"}, - {[]byte(`first line -second line -third line -`), "third line"}, - {[]byte(""), ""}, - {nil, ""}, - } - for _, tt := range tests { - t.Run(tt.want, func(t *testing.T) { - assert.Equal(t, []byte(tt.want), getLastLine(tt.output)) - }) - } -} - -func Test_getVolumeSize(t *testing.T) { - files := map[string][]byte{ - "/file1.txt": []byte("file1"), - "/file2.txt": []byte("file2"), - "/file3.txt": []byte("file3"), - "/files/file4.txt": []byte("file4"), - "/files/nested/file5.txt": []byte("file5"), - } - fakefs := test.NewFakeFileSystem() - - var expectedSize int64 - for path, content := range files { - fakefs.WithFile(path, content) - expectedSize += int64(len(content)) - } - - fileSystem = fakefs - defer func() { fileSystem = filesystem.NewFileSystem() }() - - actualSize, err := getVolumeSize("/") - - require.NoError(t, err) - assert.Equal(t, expectedSize, actualSize) -}