feat: Add brotli compression support

This commit is contained in:
Felicitas Pojtinger
2021-11-30 19:09:48 +01:00
parent b3a45ae9b2
commit 5bb1b341ea
6 changed files with 84 additions and 1 deletions

View File

@@ -10,6 +10,7 @@ import (
"path/filepath"
"strconv"
"github.com/andybalholm/brotli"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
@@ -291,6 +292,40 @@ func archive(
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name += compressionFormatZStandardSuffix
case compressionFormatBrotliKey:
// Get the compressed size for the header
file, err := os.Open(path)
if err != nil {
return err
}
fileSizeCounter := counters.CounterWriter{
Writer: io.Discard,
}
br := brotli.NewWriter(&fileSizeCounter)
if _, err := io.Copy(br, file); err != nil {
return err
}
if err := br.Flush(); err != nil {
return err
}
if err := br.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name += compressionFormatBrotliSuffix
case compressionFormatNoneKey:
default:
return errUnsupportedCompressionFormat
@@ -426,6 +461,39 @@ func archive(
if err := file.Close(); err != nil {
return err
}
case compressionFormatBrotliKey:
// Compress and write the file
file, err := os.Open(path)
if err != nil {
return err
}
br := brotli.NewWriter(tw)
if _, err := io.Copy(br, file); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(br, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(br, file, buf); err != nil {
return err
}
}
if err := br.Flush(); err != nil {
return err
}
if err := br.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
case compressionFormatNoneKey:
// Write the file
file, err := os.Open(path)

View File

@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"github.com/andybalholm/brotli"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
@@ -170,6 +171,12 @@ func restoreFromRecordAndBlock(
if _, err := io.Copy(dstFile, zz); err != nil {
return err
}
case compressionFormatBrotliKey:
br := brotli.NewReader(tr)
if _, err := io.Copy(dstFile, br); err != nil {
return err
}
case compressionFormatNoneKey:
if _, err := io.Copy(dstFile, tr); err != nil {
return err

View File

@@ -277,6 +277,8 @@ func indexHeader(
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatLZ4Suffix)
case compressionFormatZStandardKey:
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatZStandardSuffix)
case compressionFormatBrotliKey:
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatBrotliSuffix)
case compressionFormatNoneKey:
default:
return errUnsupportedCompressionFormat

View File

@@ -29,10 +29,13 @@ const (
compressionFormatZStandardKey = "zstandard"
compressionFormatZStandardSuffix = ".zst"
compressionFormatBrotliKey = "brotli"
compressionFormatBrotliSuffix = ".br"
)
var (
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey}
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey, compressionFormatBrotliKey}
errUnknownCompressionFormat = errors.New("unknown compression format")
errUnsupportedCompressionFormat = errors.New("unsupported compression format")