mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add parallel GZip compression and decompression support

Browse Source
This commit is contained in:
Felicitas Pojtinger
2021-11-30 18:16:36 +01:00
parent 82621af612
commit 1c544c3400
6 changed files with 55 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/stbak/cmd/archive.go
View File

@@ -10,6 +10,7 @@ import (
"path/filepath"
"strconv"
"github.com/klauspost/pgzip"
"github.com/pojntfx/stfs/pkg/adapters"
"github.com/pojntfx/stfs/pkg/controllers"
"github.com/pojntfx/stfs/pkg/counters"
@@ -27,6 +28,12 @@ const (
overwriteFlag = "overwrite"
)
type flusher interface {
io.WriteCloser
Flush() error
}
var archiveCmd = &cobra.Command{
Use: "archive",
Aliases: []string{"arc", "a", "c"},
@@ -172,6 +179,8 @@ func archive(
if info.Mode().IsRegular() {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
// Get the compressed size for the header
file, err := os.Open(path)
if err != nil {
@@ -182,7 +191,12 @@ func archive(
Writer: io.Discard,
}
gz := gzip.NewWriter(&fileSizeCounter)
var gz flusher
if compressionFormat == compressionFormatGZipKey {
gz = gzip.NewWriter(&fileSizeCounter)
} else {
gz = pgzip.NewWriter(&fileSizeCounter)
}
if _, err := io.Copy(gz, file); err != nil {
return err
}
@@ -232,13 +246,23 @@ func archive(
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
// Compress and write the file
file, err := os.Open(path)
if err != nil {
return err
}
gz := gzip.NewWriter(tw)
var gz flusher
if compressionFormat == compressionFormatGZipKey {
gz = gzip.NewWriter(tw)
} else {
gz = pgzip.NewWriter(tw)
}
if _, err := io.Copy(gz, file); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(gz, file); err != nil {

17
cmd/stbak/cmd/recovery_fetch.go
View File

@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"github.com/klauspost/pgzip"
"github.com/pojntfx/stfs/pkg/controllers"
"github.com/pojntfx/stfs/pkg/formatting"
"github.com/spf13/cobra"
@@ -130,9 +131,19 @@ func restoreFromRecordAndBlock(
switch compressionFormat {
case compressionFormatGZipKey:
gz, err := gzip.NewReader(tr)
if err != nil {
return err
fallthrough
case compressionFormatParallelGZipKey:
var gz io.ReadCloser
if compressionFormat == compressionFormatGZipKey {
gz, err = gzip.NewReader(tr)
if err != nil {
return err
}
} else {
gz, err = pgzip.NewReader(tr)
if err != nil {
return err
}
}
defer gz.Close()

2
cmd/stbak/cmd/recovery_index.go
View File

@@ -269,6 +269,8 @@ func indexHeader(
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
if hdr.FileInfo().Mode().IsRegular() {
hdr.Name = strings.TrimSuffix(hdr.Name, compressionFormatGZipSuffix)
}

10
cmd/stbak/cmd/root.go
View File

@@ -2,6 +2,7 @@ package cmd
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -16,13 +17,16 @@ const (
verboseFlag = "verbose"
compressionFlag = "compression"
compressionFormatNoneKey = ""
compressionFormatNoneKey = "none"
compressionFormatGZipKey = "gzip"
compressionFormatGZipSuffix = ".gz"
compressionFormatParallelGZipKey = "parallelgzip"
)
var (
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey}
knownCompressionFormats = []string{compressionFormatNoneKey, compressionFormatGZipKey, compressionFormatParallelGZipKey}
errUnknownCompressionFormat = errors.New("unknown compression format")
errUnsupportedCompressionFormat = errors.New("unsupported compression format")
@@ -67,7 +71,7 @@ func Execute() {
rootCmd.PersistentFlags().StringP(tapeFlag, "t", "/dev/nst0", "Tape or tar file to use")
rootCmd.PersistentFlags().StringP(metadataFlag, "m", metadataPath, "Metadata database to use")
rootCmd.PersistentFlags().BoolP(verboseFlag, "v", false, "Enable verbose logging")
rootCmd.PersistentFlags().StringP(compressionFlag, "c", "", "Compression format to use (default none, available are none, gzip)")
rootCmd.PersistentFlags().StringP(compressionFlag, "c", compressionFormatNoneKey, fmt.Sprintf("Compression format to use (default none, available are %v)", knownCompressionFormats))
if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
panic(err)

2
go.mod
View File

@@ -4,6 +4,7 @@ go 1.17
require (
github.com/friendsofgo/errors v0.9.2
github.com/klauspost/pgzip v1.2.5
github.com/mattn/go-sqlite3 v1.14.9
github.com/pkg/errors v0.9.1
github.com/rubenv/sql-migrate v0.0.0-20211023115951-9f02b1e13857
@@ -21,6 +22,7 @@ require (
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mitchellh/mapstructure v1.4.2 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect

4
go.sum
View File

@@ -255,6 +255,10 @@ github.com/kat-co/vala v0.0.0-20170210184112-42e1d8b61f12/go.mod h1:u9MdXq/QageO
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=