feat: Add bzip2 compression support

This commit is contained in:
Felicitas Pojtinger
2021-11-30 19:24:39 +01:00
parent 5bb1b341ea
commit 6b26046bad
6 changed files with 91 additions and 1 deletions
+68
View File
@@ -11,6 +11,7 @@ import (
"strconv"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
@@ -326,6 +327,40 @@ func archive(
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name += compressionFormatBrotliSuffix
case compressionFormatBzip2Key:
// Get the compressed size for the header
file, err := os.Open(path)
if err != nil {
return err
}
fileSizeCounter := counters.CounterWriter{
Writer: io.Discard,
}
bz, err := bzip2.NewWriter(&fileSizeCounter, nil)
if err != nil {
return err
}
if _, err := io.Copy(bz, file); err != nil {
return err
}
if err := bz.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name += compressionFormatBzip2Suffix
case compressionFormatNoneKey:
default:
return errUnsupportedCompressionFormat
@@ -494,6 +529,39 @@ func archive(
if err := file.Close(); err != nil {
return err
}
case compressionFormatBzip2Key:
// Compress and write the file
file, err := os.Open(path)
if err != nil {
return err
}
lz, err := bzip2.NewWriter(tw, nil)
if err != nil {
return err
}
if _, err := io.Copy(lz, file); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(lz, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(lz, file, buf); err != nil {
return err
}
}
if err := lz.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
case compressionFormatNoneKey:
// Write the file
file, err := os.Open(path)
+10
View File
@@ -9,6 +9,7 @@ import (
"path/filepath"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
@@ -177,6 +178,15 @@ func restoreFromRecordAndBlock(
if _, err := io.Copy(dstFile, br); err != nil {
return err
}
case compressionFormatBzip2Key:
bz, err := bzip2.NewReader(tr, nil)
if err != nil {
return err
}
if _, err := io.Copy(dstFile, bz); err != nil {
return err
}
case compressionFormatNoneKey:
if _, err := io.Copy(dstFile, tr); err != nil {
return err
+2
View File
@@ -279,6 +279,8 @@ func indexHeader(
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatZStandardSuffix)
case compressionFormatBrotliKey:
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatBrotliSuffix)
case compressionFormatBzip2Key:
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatBzip2Suffix)
case compressionFormatNoneKey:
default:
return errUnsupportedCompressionFormat
+4 -1
View File
@@ -32,10 +32,13 @@ const (
compressionFormatBrotliKey = "brotli"
compressionFormatBrotliSuffix = ".br"
compressionFormatBzip2Key = "bzip2"
compressionFormatBzip2Suffix = ".bz2"
)
var (
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey, compressionFormatBrotliKey}
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey, compressionFormatLZ4Key, compressionFormatZStandardKey, compressionFormatBrotliKey, compressionFormatBzip2Key}
errUnknownCompressionFormat = errors.New("unknown compression format")
errUnsupportedCompressionFormat = errors.New("unsupported compression format")