diff --git a/cmd/stbak/cmd/archive.go b/cmd/stbak/cmd/archive.go index e6578f9..b93ee68 100644 --- a/cmd/stbak/cmd/archive.go +++ b/cmd/stbak/cmd/archive.go @@ -11,6 +11,7 @@ import ( "strconv" "github.com/klauspost/pgzip" + "github.com/pierrec/lz4/v4" "github.com/pojntfx/stfs/pkg/adapters" "github.com/pojntfx/stfs/pkg/controllers" "github.com/pojntfx/stfs/pkg/counters" @@ -218,6 +219,40 @@ func archive( hdr.Size = int64(fileSizeCounter.BytesRead) hdr.Name += compressionFormatGZipSuffix + case compressionFormatLZ4Key: + // Get the compressed size for the header + file, err := os.Open(path) + if err != nil { + return err + } + + fileSizeCounter := counters.CounterWriter{ + Writer: io.Discard, + } + + lz := lz4.NewWriter(&fileSizeCounter) + if err := lz.Apply(lz4.ConcurrencyOption(-1)); err != nil { + return err + } + + if _, err := io.Copy(lz, file); err != nil { + return err + } + + if err := lz.Close(); err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } + + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) + hdr.Size = int64(fileSizeCounter.BytesRead) + + hdr.Name += compressionFormatLZ4Suffix case compressionFormatNoneKey: default: return errUnsupportedCompressionFormat @@ -284,6 +319,39 @@ func archive( if err := file.Close(); err != nil { return err } + case compressionFormatLZ4Key: + // Compress and write the file + file, err := os.Open(path) + if err != nil { + return err + } + + lz := lz4.NewWriter(tw) + if err := lz.Apply(lz4.ConcurrencyOption(-1)); err != nil { + return err + } + + if _, err := io.Copy(lz, file); err != nil { + return err + } + + if isRegular { + if _, err := io.Copy(lz, file); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(lz, file, buf); err != nil { + return err + } + } + + if err := lz.Close(); err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } case compressionFormatNoneKey: // Write the file file, err := os.Open(path) diff --git a/cmd/stbak/cmd/recovery_fetch.go b/cmd/stbak/cmd/recovery_fetch.go index 40de66b..1976b0d 100644 --- a/cmd/stbak/cmd/recovery_fetch.go +++ b/cmd/stbak/cmd/recovery_fetch.go @@ -9,6 +9,7 @@ import ( "path/filepath" "github.com/klauspost/pgzip" + "github.com/pierrec/lz4/v4" "github.com/pojntfx/stfs/pkg/controllers" "github.com/pojntfx/stfs/pkg/formatting" "github.com/spf13/cobra" @@ -150,6 +151,15 @@ func restoreFromRecordAndBlock( if _, err := io.Copy(dstFile, gz); err != nil { return err } + case compressionFormatLZ4Key: + lz := lz4.NewReader(tr) + if err := lz.Apply(lz4.ConcurrencyOption(-1)); err != nil { + return err + } + + if _, err := io.Copy(dstFile, lz); err != nil { + return err + } case compressionFormatNoneKey: if _, err := io.Copy(dstFile, tr); err != nil { return err diff --git a/cmd/stbak/cmd/recovery_index.go b/cmd/stbak/cmd/recovery_index.go index bb7c970..030ffca 100644 --- a/cmd/stbak/cmd/recovery_index.go +++ b/cmd/stbak/cmd/recovery_index.go @@ -274,6 +274,10 @@ func indexHeader( if hdr.FileInfo().Mode().IsRegular() { hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatGZipSuffix) } + case compressionFormatLZ4Key: + if hdr.FileInfo().Mode().IsRegular() { + hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatLZ4Suffix) + } case compressionFormatNoneKey: default: return errUnsupportedCompressionFormat diff --git a/cmd/stbak/cmd/root.go b/cmd/stbak/cmd/root.go index 98a30a6..8fd7d78 100644 --- a/cmd/stbak/cmd/root.go +++ b/cmd/stbak/cmd/root.go @@ -23,10 +23,13 @@ const ( compressionFormatGZipSuffix = ".gz" compressionFormatParallelGZipKey = "parallelgzip" + + compressionFormatLZ4Key = "lz4" + compressionFormatLZ4Suffix = ".lz4" ) var ( - knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey} + knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key} errUnknownCompressionFormat = errors.New("unknown compression format") errUnsupportedCompressionFormat = errors.New("unsupported compression format") diff --git a/go.mod b/go.mod index b6f7231..d80eabe 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pierrec/lz4/v4 v4.1.11 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/go.sum b/go.sum index 2504814..d8faa69 100644 --- a/go.sum +++ b/go.sum @@ -327,6 +327,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pierrec/lz4/v4 v4.1.11 h1:LVs17FAZJFOjgmJXl9Tf13WfLUvZq7/RjfEJrnwZ9OE= +github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=