feat: Add brotli compression support
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/klauspost/pgzip"
|
||||
"github.com/pierrec/lz4/v4"
|
||||
@@ -291,6 +292,40 @@ func archive(
|
||||
hdr.Size = int64(fileSizeCounter.BytesRead)
|
||||
|
||||
hdr.Name += compressionFormatZStandardSuffix
|
||||
case compressionFormatBrotliKey:
|
||||
// Get the compressed size for the header
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileSizeCounter := counters.CounterWriter{
|
||||
Writer: io.Discard,
|
||||
}
|
||||
|
||||
br := brotli.NewWriter(&fileSizeCounter)
|
||||
|
||||
if _, err := io.Copy(br, file); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := br.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := br.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if hdr.PAXRecords == nil {
|
||||
hdr.PAXRecords = map[string]string{}
|
||||
}
|
||||
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
|
||||
hdr.Size = int64(fileSizeCounter.BytesRead)
|
||||
|
||||
hdr.Name += compressionFormatBrotliSuffix
|
||||
case compressionFormatNoneKey:
|
||||
default:
|
||||
return errUnsupportedCompressionFormat
|
||||
@@ -426,6 +461,39 @@ func archive(
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
case compressionFormatBrotliKey:
|
||||
// Compress and write the file
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
br := brotli.NewWriter(tw)
|
||||
|
||||
if _, err := io.Copy(br, file); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isRegular {
|
||||
if _, err := io.Copy(br, file); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
buf := make([]byte, controllers.BlockSize*recordSize)
|
||||
if _, err := io.CopyBuffer(br, file, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := br.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := br.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
case compressionFormatNoneKey:
|
||||
// Write the file
|
||||
file, err := os.Open(path)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/klauspost/pgzip"
|
||||
"github.com/pierrec/lz4/v4"
|
||||
@@ -170,6 +171,12 @@ func restoreFromRecordAndBlock(
|
||||
if _, err := io.Copy(dstFile, zz); err != nil {
|
||||
return err
|
||||
}
|
||||
case compressionFormatBrotliKey:
|
||||
br := brotli.NewReader(tr)
|
||||
|
||||
if _, err := io.Copy(dstFile, br); err != nil {
|
||||
return err
|
||||
}
|
||||
case compressionFormatNoneKey:
|
||||
if _, err := io.Copy(dstFile, tr); err != nil {
|
||||
return err
|
||||
|
||||
@@ -277,6 +277,8 @@ func indexHeader(
|
||||
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatLZ4Suffix)
|
||||
case compressionFormatZStandardKey:
|
||||
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatZStandardSuffix)
|
||||
case compressionFormatBrotliKey:
|
||||
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatBrotliSuffix)
|
||||
case compressionFormatNoneKey:
|
||||
default:
|
||||
return errUnsupportedCompressionFormat
|
||||
|
||||
@@ -29,10 +29,13 @@ const (
|
||||
|
||||
compressionFormatZStandardKey = "zstandard"
|
||||
compressionFormatZStandardSuffix = ".zst"
|
||||
|
||||
compressionFormatBrotliKey = "brotli"
|
||||
compressionFormatBrotliSuffix = ".br"
|
||||
)
|
||||
|
||||
var (
|
||||
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey}
|
||||
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey, compressionFormatBrotliKey}
|
||||
|
||||
errUnknownCompressionFormat = errors.New("unknown compression format")
|
||||
errUnsupportedCompressionFormat = errors.New("unsupported compression format")
|
||||
|
||||
Reference in New Issue
Block a user