feat: Add trannsparent decryption support based on age and enable streaming encryption & compression

This commit is contained in:
Felicitas Pojtinger
2021-12-02 00:38:11 +01:00
parent 9f61c37c9d
commit f6e27af47f
5 changed files with 359 additions and 267 deletions

View File

@@ -67,6 +67,16 @@ type nopCloser struct {
func (nopCloser) Close() error { return nil }
func nopFlusherWriter(w io.WriteCloser) nopFlusher {
return nopFlusher{w}
}
type nopFlusher struct {
io.WriteCloser
}
func (nopFlusher) Flush() error { return nil }
var archiveCmd = &cobra.Command{
Use: "archive",
Aliases: []string{"arc", "a", "c"},
@@ -141,6 +151,7 @@ var archiveCmd = &cobra.Command{
int(lastIndexedBlock),
viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(encryptionFlag),
)
},
}
@@ -240,11 +251,6 @@ func archive(
if info.Mode().IsRegular() {
// Get the compressed size for the header
file, err := os.Open(path)
if err != nil {
return err
}
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
@@ -254,14 +260,42 @@ func archive(
return err
}
if err := compress(
file,
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
); err != nil {
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
@@ -269,10 +303,6 @@ func archive(
return err
}
if err := file.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
@@ -298,6 +328,14 @@ func archive(
default:
return errUnsupportedCompressionFormat
}
switch encryptionFormat {
case encryptionFormatAgeKey:
hdr.Name += encryptionFormatAgeSuffix
case compressionFormatNoneKey:
default:
return errUnsupportedEncryptionFormat
}
}
if first {
@@ -321,24 +359,47 @@ func archive(
}
// Compress and write the file
file, err := os.Open(path)
if err != nil {
return err
}
encryptor, err := encrypt(tw, encryptionFormat, pubkey)
if err != nil {
return err
}
if err := compress(
file,
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
); err != nil {
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
@@ -346,10 +407,6 @@ func archive(
return err
}
if err := file.Close(); err != nil {
return err
}
dirty = true
return nil
@@ -393,18 +450,16 @@ func encrypt(
}
func compress(
src io.Reader,
dst io.Writer,
compressionFormat string,
compressionLevel string,
isRegular bool,
recordSize int,
) error {
) (flusher, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
var gz flusher
if compressionFormat == compressionFormatGZipKey {
l := gzip.DefaultCompression
switch compressionLevel {
@@ -415,55 +470,25 @@ func compress(
case compressionLevelSmallest:
l = gzip.BestCompression
default:
return errUnsupportedCompressionLevel
return nil, errUnsupportedCompressionLevel
}
g, err := gzip.NewWriterLevel(dst, l)
if err != nil {
return err
}
gz = g
} else {
l := pgzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = pgzip.BestSpeed
case compressionLevelBalanced:
l = pgzip.DefaultCompression
case compressionLevelSmallest:
l = pgzip.BestCompression
default:
return errUnsupportedCompressionLevel
}
g, err := pgzip.NewWriterLevel(dst, l)
if err != nil {
return err
}
gz = g
return gzip.NewWriterLevel(dst, l)
}
if _, err := io.Copy(gz, src); err != nil {
return err
l := pgzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = pgzip.BestSpeed
case compressionLevelBalanced:
l = pgzip.DefaultCompression
case compressionLevelSmallest:
l = pgzip.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
if isRegular {
if _, err := io.Copy(gz, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(gz, src, buf); err != nil {
return err
}
}
if err := gz.Flush(); err != nil {
return err
}
if err := gz.Close(); err != nil {
return err
}
return pgzip.NewWriterLevel(dst, l)
case compressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
@@ -474,32 +499,15 @@ func compress(
case compressionLevelSmallest:
l = lz4.Level9
default:
return errUnsupportedCompressionLevel
return nil, errUnsupportedCompressionLevel
}
lz := lz4.NewWriter(dst)
if err := lz.Apply(lz4.ConcurrencyOption(-1), lz4.CompressionLevelOption(l)); err != nil {
return err
return nil, err
}
if _, err := io.Copy(lz, src); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(lz, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(lz, src, buf); err != nil {
return err
}
}
if err := lz.Close(); err != nil {
return err
}
return nopFlusherWriter(lz), nil
case compressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
@@ -510,35 +518,15 @@ func compress(
case compressionLevelSmallest:
l = zstd.SpeedBestCompression
default:
return errUnsupportedCompressionLevel
return nil, errUnsupportedCompressionLevel
}
zz, err := zstd.NewWriter(dst, zstd.WithEncoderLevel(l))
if err != nil {
return err
return nil, err
}
if _, err := io.Copy(zz, src); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(zz, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(zz, src, buf); err != nil {
return err
}
}
if err := zz.Flush(); err != nil {
return err
}
if err := zz.Close(); err != nil {
return err
}
return zz, nil
case compressionFormatBrotliKey:
l := brotli.DefaultCompression
switch compressionLevel {
@@ -549,32 +537,12 @@ func compress(
case compressionLevelSmallest:
l = brotli.BestCompression
default:
return errUnsupportedCompressionLevel
return nil, errUnsupportedCompressionLevel
}
br := brotli.NewWriterLevel(dst, l)
if _, err := io.Copy(br, src); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(br, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(br, src, buf); err != nil {
return err
}
}
if err := br.Flush(); err != nil {
return err
}
if err := br.Close(); err != nil {
return err
}
return br, nil
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
@@ -587,50 +555,22 @@ func compress(
case compressionLevelSmallest:
l = bzip2.BestCompression
default:
return errUnsupportedCompressionLevel
return nil, errUnsupportedCompressionLevel
}
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
Level: l,
})
if err != nil {
return err
return nil, err
}
if _, err := io.Copy(bz, src); err != nil {
return err
}
if isRegular {
if _, err := io.Copy(bz, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(bz, src, buf); err != nil {
return err
}
}
if err := bz.Close(); err != nil {
return err
}
return nopFlusherWriter(bz), nil
case compressionFormatNoneKey:
if isRegular {
if _, err := io.Copy(dst, src); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(dst, src, buf); err != nil {
return err
}
}
return nopFlusherWriter(nopCloserWriter(dst)), nil
default:
return errUnsupportedCompressionFormat
return nil, errUnsupportedCompressionFormat
}
return nil
}
func init() {

View File

@@ -6,9 +6,11 @@ import (
"compress/gzip"
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"filippo.io/age"
"github.com/andybalholm/brotli"
"github.com/cosnicolaou/pbzip2"
"github.com/dsnet/compress/bzip2"
@@ -32,6 +34,19 @@ const (
var recoveryFetchCmd = &cobra.Command{
Use: "fetch",
Short: "Fetch a file or directory from tape or tar file by record and block",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil {
return err
}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
if _, err := os.Stat(viper.GetString(keyFlag)); err != nil {
return errKeyNotAccessible
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil {
return err
@@ -41,6 +56,16 @@ var recoveryFetchCmd = &cobra.Command{
boil.DebugMode = true
}
privkey := []byte{}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
p, err := ioutil.ReadFile(viper.GetString(keyFlag))
if err != nil {
return err
}
privkey = p
}
return restoreFromRecordAndBlock(
viper.GetString(tapeFlag),
viper.GetInt(recordSizeFlag),
@@ -50,6 +75,8 @@ var recoveryFetchCmd = &cobra.Command{
viper.GetBool(previewFlag),
true,
viper.GetString(compressionFlag),
viper.GetString(encryptionFlag),
privkey,
)
},
}
@@ -63,6 +90,8 @@ func restoreFromRecordAndBlock(
preview bool,
showHeader bool,
compressionFormat string,
encryptionFormat string,
privkey []byte,
) error {
f, isRegular, err := openTapeReadOnly(tape)
if err != nil {
@@ -135,11 +164,31 @@ func restoreFromRecordAndBlock(
return nil
}
return decompress(
tr,
dstFile,
compressionFormat,
)
decryptor, err := decrypt(tr, encryptionFormat, privkey)
if err != nil {
return err
}
decompressor, err := decompress(decryptor, compressionFormat)
if err != nil {
return err
}
if _, err := io.Copy(dstFile, decompressor); err != nil {
return err
}
if err := decryptor.Close(); err != nil {
return err
}
if err := decompressor.Close(); err != nil {
return err
}
if err := dstFile.Close(); err != nil {
return err
}
}
return nil
@@ -147,108 +196,71 @@ func restoreFromRecordAndBlock(
func decompress(
src io.Reader,
dst io.WriteCloser,
compressionFormat string,
) error {
) (io.ReadCloser, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
var gz io.ReadCloser
if compressionFormat == compressionFormatGZipKey {
g, err := gzip.NewReader(src)
if err != nil {
return err
}
gz = g
} else {
g, err := pgzip.NewReader(src)
if err != nil {
return err
}
gz = g
}
defer gz.Close()
if _, err := io.Copy(dst, gz); err != nil {
return err
return gzip.NewReader(src)
}
if err := dst.Close(); err != nil {
return err
}
return pgzip.NewReader(src)
case compressionFormatLZ4Key:
lz := lz4.NewReader(src)
if err := lz.Apply(lz4.ConcurrencyOption(-1)); err != nil {
return err
return nil, err
}
if _, err := io.Copy(dst, lz); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return io.NopCloser(lz), nil
case compressionFormatZStandardKey:
zz, err := zstd.NewReader(src)
if err != nil {
return err
return nil, err
}
if _, err := io.Copy(dst, zz); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return io.NopCloser(zz), nil
case compressionFormatBrotliKey:
br := brotli.NewReader(src)
if _, err := io.Copy(dst, br); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return io.NopCloser(br), nil
case compressionFormatBzip2Key:
bz, err := bzip2.NewReader(src, nil)
if err != nil {
return err
}
if _, err := io.Copy(dst, bz); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return bzip2.NewReader(src, nil)
case compressionFormatBzip2ParallelKey:
bz := pbzip2.NewReader(context.Background(), src)
if _, err := io.Copy(dst, bz); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return io.NopCloser(bz), nil
case compressionFormatNoneKey:
if _, err := io.Copy(dst, src); err != nil {
return err
}
if err := dst.Close(); err != nil {
return err
}
return io.NopCloser(src), nil
default:
return errUnsupportedCompressionFormat
return nil, errUnsupportedCompressionFormat
}
}
return nil
func decrypt(
src io.Reader,
encryptionFormat string,
privkey []byte,
) (io.ReadCloser, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
identity, err := age.ParseX25519Identity(string(privkey))
if err != nil {
return nil, err
}
r, err := age.Decrypt(src, identity)
if err != nil {
return nil, err
}
return io.NopCloser(r), nil
case encryptionFormatNoneKey:
return io.NopCloser(src), nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func init() {
@@ -257,6 +269,7 @@ func init() {
recoveryFetchCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too")
recoveryFetchCmd.PersistentFlags().StringP(dstFlag, "d", "", "File to restore to (archived name by default)")
recoveryFetchCmd.PersistentFlags().BoolP(previewFlag, "p", false, "Only read the header")
recoveryFetchCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to private key of recipient that has been encrypted for")
viper.AutomaticEnv()

View File

@@ -43,6 +43,7 @@ var recoveryIndexCmd = &cobra.Command{
viper.GetInt(blockFlag),
viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(encryptionFlag),
)
},
}
@@ -55,6 +56,7 @@ func index(
block int,
overwrite bool,
compressionFormat string,
encryptionFormat string,
) error {
if overwrite {
f, err := os.OpenFile(metadata, os.O_WRONLY|os.O_CREATE, 0600)
@@ -142,7 +144,7 @@ func index(
break
}
if err := indexHeader(record, block, hdr, metadataPersister, compressionFormat); err != nil {
if err := indexHeader(record, block, hdr, metadataPersister, compressionFormat, encryptionFormat); err != nil {
return nil
}
@@ -215,7 +217,7 @@ func index(
}
}
if err := indexHeader(record, block, hdr, metadataPersister, compressionFormat); err != nil {
if err := indexHeader(record, block, hdr, metadataPersister, compressionFormat, encryptionFormat); err != nil {
return nil
}
@@ -257,6 +259,7 @@ func indexHeader(
hdr *tar.Header,
metadataPersister *persisters.MetadataPersister,
compressionFormat string,
encryptionFormat string,
) error {
if record == 0 && block == 0 {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
@@ -275,6 +278,14 @@ func indexHeader(
}
if hdr.FileInfo().Mode().IsRegular() {
switch encryptionFormat {
case encryptionFormatAgeKey:
hdr.Name = strings.TrimSuffix(hdr.Name, encryptionFormatAgeSuffix)
case encryptionFormatNoneKey:
default:
return errUnsupportedEncryptionFormat
}
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough

View File

@@ -4,6 +4,8 @@ import (
"archive/tar"
"context"
"database/sql"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
@@ -25,6 +27,19 @@ var restoreCmd = &cobra.Command{
Use: "restore",
Aliases: []string{"res", "r", "x"},
Short: "Restore a file or directory from tape or tar file",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil {
return err
}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
if _, err := os.Stat(viper.GetString(keyFlag)); err != nil {
return errKeyNotAccessible
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil {
return err
@@ -39,6 +54,16 @@ var restoreCmd = &cobra.Command{
return err
}
privkey := []byte{}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
p, err := ioutil.ReadFile(viper.GetString(keyFlag))
if err != nil {
return err
}
privkey = p
}
headersToRestore := []*models.Header{}
src := strings.TrimSuffix(viper.GetString(srcFlag), "/")
dbhdr, err := metadataPersister.GetHeader(context.Background(), src)
@@ -104,6 +129,8 @@ var restoreCmd = &cobra.Command{
false,
false,
viper.GetString(compressionFlag),
viper.GetString(encryptionFlag),
privkey,
); err != nil {
return err
}
@@ -118,6 +145,7 @@ func init() {
restoreCmd.PersistentFlags().StringP(srcFlag, "s", "", "File or directory to restore")
restoreCmd.PersistentFlags().StringP(dstFlag, "d", "", "File or directory restore to (archived name by default)")
restoreCmd.PersistentFlags().BoolP(flattenFlag, "f", false, "Ignore the folder hierarchy on the tape or tar file")
restoreCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to private key of recipient that has been encrypted for")
viper.AutomaticEnv()

View File

@@ -6,11 +6,13 @@ import (
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"github.com/pojntfx/stfs/pkg/adapters"
"github.com/pojntfx/stfs/pkg/controllers"
"github.com/pojntfx/stfs/pkg/counters"
"github.com/pojntfx/stfs/pkg/formatting"
"github.com/pojntfx/stfs/pkg/pax"
@@ -29,7 +31,17 @@ var updateCmd = &cobra.Command{
return err
}
return checkCompressionLevel(viper.GetString(compressionLevelFlag))
if err := checkCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil {
return err
}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
if _, err := os.Stat(viper.GetString(keyFlag)); err != nil {
return errKeyNotAccessible
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil {
@@ -50,6 +62,16 @@ var updateCmd = &cobra.Command{
return err
}
pubkey := []byte{}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
p, err := ioutil.ReadFile(viper.GetString(keyFlag))
if err != nil {
return err
}
pubkey = p
}
if err := update(
viper.GetString(tapeFlag),
viper.GetInt(recordSizeFlag),
@@ -57,6 +79,8 @@ var updateCmd = &cobra.Command{
viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(compressionLevelFlag),
viper.GetString(encryptionFlag),
pubkey,
); err != nil {
return err
}
@@ -69,6 +93,7 @@ var updateCmd = &cobra.Command{
int(lastIndexedBlock),
false,
viper.GetString(compressionFlag),
viper.GetString(encryptionFlag),
)
},
}
@@ -80,6 +105,8 @@ func update(
replacesContent bool,
compressionFormat string,
compressionLevel string,
encryptionFormat string,
pubkey []byte,
) error {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape)
@@ -120,23 +147,55 @@ func update(
if info.Mode().IsRegular() && replacesContent {
// Get the compressed size for the header
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, pubkey)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
fileSizeCounter := counters.CounterWriter{
Writer: io.Discard,
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := compress(
file,
&fileSizeCounter,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
); err != nil {
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
@@ -165,6 +224,14 @@ func update(
default:
return errUnsupportedCompressionFormat
}
switch encryptionFormat {
case encryptionFormatAgeKey:
hdr.Name += encryptionFormatAgeSuffix
case compressionFormatNoneKey:
default:
return errUnsupportedEncryptionFormat
}
}
if first {
@@ -191,19 +258,51 @@ func update(
}
// Compress and write the file
encryptor, err := encrypt(tw, encryptionFormat, pubkey)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if err := compress(
file,
tw,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
); err != nil {
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
} else {
@@ -229,6 +328,7 @@ func init() {
updateCmd.PersistentFlags().StringP(srcFlag, "s", "", "Path of the file or directory to update")
updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
updateCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to public key of recipient to encrypt for")
viper.AutomaticEnv()