diff --git a/changelogs/unreleased/10260-Lyndon-Li b/changelogs/unreleased/10260-Lyndon-Li new file mode 100644 index 000000000..418371f47 --- /dev/null +++ b/changelogs/unreleased/10260-Lyndon-Li @@ -0,0 +1 @@ +Add cap for backup data extraction \ No newline at end of file diff --git a/pkg/archive/extractor.go b/pkg/archive/extractor.go index aae9a7a85..15cacde22 100644 --- a/pkg/archive/extractor.go +++ b/pkg/archive/extractor.go @@ -32,14 +32,27 @@ import ( // Extractor unzips/extracts a backup tarball to a local // temp directory. type Extractor struct { - log logrus.FieldLogger - fs filesystem.Interface + log logrus.FieldLogger + fs filesystem.Interface + maxExtractionSize int64 + totalExtractedSize int64 +} + +var maxExtractionSize = int64(16) << 30 + +// SetMaxExtractionSize sets the maximum extraction size. It is normally called at server startup. +func SetMaxExtractionSize(size int64) { + if size > 0 { + maxExtractionSize = size + } } func NewExtractor(log logrus.FieldLogger, fs filesystem.Interface) *Extractor { return &Extractor{ - log: log, - fs: fs, + log: log, + fs: fs, + maxExtractionSize: maxExtractionSize, + totalExtractedSize: 0, } } @@ -96,6 +109,15 @@ func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { return "", err } + // Enforce maximum extraction size to prevent memory/storage exhaustion and zip bombs. + maxSize := e.maxExtractionSize + e.totalExtractedSize += header.Size + if e.totalExtractedSize > maxSize { + err := fmt.Errorf("decompressed backup exceeds maximum allowed size of %d bytes", maxSize) + e.log.Infof("error checking extracted size: %v", err) + return "", err + } + target, err := sanitizeArchivePath(dir, header.Name) if err != nil { e.log.Infof("error sanitizing archive path: %s", err.Error()) diff --git a/pkg/archive/extractor_test.go b/pkg/archive/extractor_test.go index a4daf02ca..d87f787c3 100644 --- a/pkg/archive/extractor_test.go +++ b/pkg/archive/extractor_test.go @@ -20,6 +20,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "fmt" "io" "os" "testing" @@ -113,6 +114,66 @@ func TestUnzipAndExtractBackupRejectsPathTraversal(t *testing.T) { require.Contains(t, err.Error(), "invalid archive path") } +func TestUnzipAndExtractBackupRejectsLargeFile(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + data := make([]byte, 2048) // 2KB data + err := tw.WriteHeader(&tar.Header{ + Name: "large.txt", + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err = ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + +func TestUnzipAndExtractBackupRejectsManySmallFiles(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + // Create 100 files of 20 bytes each (total 2000 bytes, exceeding the 1024 byte limit) + for i := 0; i < 100; i++ { + data := make([]byte, 20) + err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("small_%d.txt", i), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + } + + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err := ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + func createArchive(files []string, fs filesystem.Interface) (string, error) { outName := "output.tar.gz" out, err := fs.Create(outName) diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index 08b58a1bd..2cc7bac4e 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -183,6 +183,7 @@ type Config struct { ConcurrentBackups int GlobalBackupVolumePoliciesConfigMap string DefaultResourceModifierConfigMap string + MaxBackupExtractionSize int } func GetDefaultConfig() *Config { @@ -289,4 +290,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.DefaultResourceModifierConfigMap, "The name of a ConfigMap in the Velero namespace containing default resource modifier rules applied to all restores. Ignored when a per-restore resource modifier is specified.", ) + flags.IntVar( + &c.MaxBackupExtractionSize, + "max-backup-extraction-size", + c.MaxBackupExtractionSize, + "Maximum size of a backup extraction in megabytes. If not set, default value (16GB) will be used.", + ) } diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 7aff5e946..665577672 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -61,6 +61,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" @@ -935,6 +936,11 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + if s.config.MaxBackupExtractionSize > 0 { + s.logger.Infof("Setting backup data extraction cap as %v MB", s.config.MaxBackupExtractionSize) + archive.SetMaxExtractionSize(int64(s.config.MaxBackupExtractionSize) * 1024 * 1024) + } + s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index 194d947eb..4b4a11f30 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -348,6 +348,21 @@ By default, only one backup is processed in the `InProgress` phase at a time. Th Enabling parallel backups can provide a significant performance benefit for backups which contain a large number of Kubernetes resources or ones which contain a large number of smaller volumes. Backups dominated by large volumes will not see as much benefit, since the majority of time for those backups is spent waiting for the async phase to complete. A larger `concurrent-backups` configuration may require additional memory and CPU resources for the velero container. +## Limiting Resource Backup Data Cache Size +For Kubernetes resource data (non volume data), for some operations like Restores or Backup Deletions, etc., Velero uses local cache (in the root file system of the cluster node) to download and extract the data from the backup storage location, Velero sets a limit for the cache size. If the cache size exceeds the limit, the specific operation would fail. +By default Velero sets the limit as 16GB, if your backup data is large, you can change the Velero server parameter `max-backup-extraction-size`. Here is an example to set the limit to 32GB: + +```yaml +containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --max-backup-extraction-size=32768 +``` + ## Additional options Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options.