From effd4df3e7e00b741657eb62fd330b679cda1488 Mon Sep 17 00:00:00 2001 From: Felix Pojtinger Date: Wed, 1 Dec 2021 23:27:43 +0100 Subject: [PATCH] feat: Start implementation of transparent encryption based on `age` --- cmd/stbak/cmd/archive.go | 117 +++++++++++++++++++++++++------- cmd/stbak/cmd/delete.go | 2 +- cmd/stbak/cmd/find.go | 2 +- cmd/stbak/cmd/move.go | 2 +- cmd/stbak/cmd/recovery_fetch.go | 2 +- cmd/stbak/cmd/recovery_index.go | 2 +- cmd/stbak/cmd/recovery_query.go | 2 +- cmd/stbak/cmd/restore.go | 2 +- cmd/stbak/cmd/root.go | 26 +++++++ cmd/stbak/cmd/update.go | 2 +- go.mod | 2 + go.sum | 7 +- 12 files changed, 134 insertions(+), 34 deletions(-) diff --git a/cmd/stbak/cmd/archive.go b/cmd/stbak/cmd/archive.go index 4b6afa1..6ec43d4 100644 --- a/cmd/stbak/cmd/archive.go +++ b/cmd/stbak/cmd/archive.go @@ -8,10 +8,12 @@ import ( "fmt" "io" "io/fs" + "io/ioutil" "os" "path/filepath" "strconv" + "filippo.io/age" "github.com/andybalholm/brotli" "github.com/dsnet/compress/bzip2" "github.com/klauspost/compress/zstd" @@ -33,6 +35,7 @@ const ( srcFlag = "src" overwriteFlag = "overwrite" compressionLevelFlag = "compression-level" + keyFlag = "key" compressionLevelFastest = "fastest" compressionLevelBalanced = "balanced" @@ -44,6 +47,8 @@ var ( errUnknownCompressionLevel = errors.New("unknown compression level") errUnsupportedCompressionLevel = errors.New("unsupported compression level") + + errKeyNotAccessible = errors.New("key not found or accessible") ) type flusher interface { @@ -52,6 +57,16 @@ type flusher interface { Flush() error } +func nopCloserWriter(w io.Writer) nopCloser { + return nopCloser{w} +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + var archiveCmd = &cobra.Command{ Use: "archive", Aliases: []string{"arc", "a", "c"}, @@ -61,7 +76,17 @@ var archiveCmd = &cobra.Command{ return err } - return checkCompressionLevel(viper.GetString(compressionLevelFlag)) + if err := checkCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil { + return err + } + + if viper.GetString(encryptionFlag) != encryptionFormatNoneKey { + if _, err := os.Stat(viper.GetString(keyFlag)); err != nil { + return errKeyNotAccessible + } + } + + return nil }, RunE: func(cmd *cobra.Command, args []string) error { if viper.GetBool(verboseFlag) { @@ -85,6 +110,16 @@ var archiveCmd = &cobra.Command{ lastIndexedBlock = b } + pubkey := []byte{} + if viper.GetString(encryptionFlag) != encryptionFormatNoneKey { + p, err := ioutil.ReadFile(viper.GetString(keyFlag)) + if err != nil { + return err + } + + pubkey = p + } + if err := archive( viper.GetString(tapeFlag), viper.GetInt(recordSizeFlag), @@ -92,6 +127,8 @@ var archiveCmd = &cobra.Command{ viper.GetBool(overwriteFlag), viper.GetString(compressionFlag), viper.GetString(compressionLevelFlag), + viper.GetString(encryptionFlag), + pubkey, ); err != nil { return err } @@ -115,6 +152,8 @@ func archive( overwrite bool, compressionFormat string, compressionLevel string, + encryptionFormat string, + pubkey []byte, ) error { dirty := false tw, isRegular, cleanup, err := openTapeWriter(tape) @@ -206,13 +245,18 @@ func archive( return err } - fileSizeCounter := counters.CounterWriter{ + fileSizeCounter := &counters.CounterWriter{ Writer: io.Discard, } + encryptor, err := encrypt(fileSizeCounter, encryptionFormat, pubkey) + if err != nil { + return err + } + if err := compress( file, - &fileSizeCounter, + encryptor, compressionFormat, compressionLevel, isRegular, @@ -221,6 +265,14 @@ func archive( return err } + if err := encryptor.Close(); err != nil { + return err + } + + if err := file.Close(); err != nil { + return err + } + if hdr.PAXRecords == nil { hdr.PAXRecords = map[string]string{} } @@ -274,9 +326,14 @@ func archive( return err } + encryptor, err := encrypt(tw, encryptionFormat, pubkey) + if err != nil { + return err + } + if err := compress( file, - tw, + encryptor, compressionFormat, compressionLevel, isRegular, @@ -285,6 +342,14 @@ func archive( return err } + if err := encryptor.Close(); err != nil { + return err + } + + if err := file.Close(); err != nil { + return err + } + dirty = true return nil @@ -307,8 +372,28 @@ func checkCompressionLevel(compressionLevel string) error { return nil } +func encrypt( + dst io.Writer, + encryptionFormat string, + pubkey []byte, +) (io.WriteCloser, error) { + switch encryptionFormat { + case encryptionFormatAgeKey: + recipient, err := age.ParseX25519Recipient(string(pubkey)) + if err != nil { + return nil, err + } + + return age.Encrypt(dst, recipient) + case encryptionFormatNoneKey: + return nopCloserWriter(dst), nil + default: + return nil, errUnsupportedEncryptionFormat + } +} + func compress( - src io.ReadCloser, + src io.Reader, dst io.Writer, compressionFormat string, compressionLevel string, @@ -379,9 +464,6 @@ func compress( if err := gz.Close(); err != nil { return err } - if err := src.Close(); err != nil { - return err - } case compressionFormatLZ4Key: l := lz4.Level5 switch compressionLevel { @@ -418,9 +500,6 @@ func compress( if err := lz.Close(); err != nil { return err } - if err := src.Close(); err != nil { - return err - } case compressionFormatZStandardKey: l := zstd.SpeedDefault switch compressionLevel { @@ -460,9 +539,6 @@ func compress( if err := zz.Close(); err != nil { return err } - if err := src.Close(); err != nil { - return err - } case compressionFormatBrotliKey: l := brotli.DefaultCompression switch compressionLevel { @@ -499,9 +575,6 @@ func compress( if err := br.Close(); err != nil { return err } - if err := src.Close(); err != nil { - return err - } case compressionFormatBzip2Key: fallthrough case compressionFormatBzip2ParallelKey: @@ -542,9 +615,6 @@ func compress( if err := bz.Close(); err != nil { return err } - if err := src.Close(); err != nil { - return err - } case compressionFormatNoneKey: if isRegular { if _, err := io.Copy(dst, src); err != nil { @@ -556,10 +626,6 @@ func compress( return err } } - - if err := src.Close(); err != nil { - return err - } default: return errUnsupportedCompressionFormat } @@ -568,10 +634,11 @@ func compress( } func init() { - archiveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") archiveCmd.PersistentFlags().StringP(srcFlag, "s", ".", "File or directory to archive") archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file") archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) + archiveCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to public key of recipient to encrypt for") viper.AutomaticEnv() diff --git a/cmd/stbak/cmd/delete.go b/cmd/stbak/cmd/delete.go index d8e22a1..110ee64 100644 --- a/cmd/stbak/cmd/delete.go +++ b/cmd/stbak/cmd/delete.go @@ -167,7 +167,7 @@ func openTapeWriter(tape string) (tw *tar.Writer, isRegular bool, cleanup func(d } func init() { - deleteCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove") viper.AutomaticEnv() diff --git a/cmd/stbak/cmd/find.go b/cmd/stbak/cmd/find.go index e889a1b..0a3b6c9 100644 --- a/cmd/stbak/cmd/find.go +++ b/cmd/stbak/cmd/find.go @@ -66,7 +66,7 @@ var findCmd = &cobra.Command{ } func init() { - findCmd.PersistentFlags().StringP(expressionFlag, "e", "", "Regex to match the file/directory name against") + findCmd.PersistentFlags().StringP(expressionFlag, "x", "", "Regex to match the file/directory name against") viper.AutomaticEnv() diff --git a/cmd/stbak/cmd/move.go b/cmd/stbak/cmd/move.go index 4468dbf..c1aa016 100644 --- a/cmd/stbak/cmd/move.go +++ b/cmd/stbak/cmd/move.go @@ -95,7 +95,7 @@ var moveCmd = &cobra.Command{ } func init() { - moveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + moveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") moveCmd.PersistentFlags().StringP(srcFlag, "s", "", "Current path of the file or directory to move") moveCmd.PersistentFlags().StringP(dstFlag, "d", "", "Path to move the file or directory to") diff --git a/cmd/stbak/cmd/recovery_fetch.go b/cmd/stbak/cmd/recovery_fetch.go index 0ed1352..a988aaf 100644 --- a/cmd/stbak/cmd/recovery_fetch.go +++ b/cmd/stbak/cmd/recovery_fetch.go @@ -252,7 +252,7 @@ func decompress( } func init() { - recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") recoveryFetchCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too") recoveryFetchCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too") recoveryFetchCmd.PersistentFlags().StringP(dstFlag, "d", "", "File to restore to (archived name by default)") diff --git a/cmd/stbak/cmd/recovery_index.go b/cmd/stbak/cmd/recovery_index.go index f320e88..b5575bc 100644 --- a/cmd/stbak/cmd/recovery_index.go +++ b/cmd/stbak/cmd/recovery_index.go @@ -242,7 +242,7 @@ func index( } func init() { - recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") recoveryIndexCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting") recoveryIndexCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting") recoveryIndexCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Remove the old index before starting to index") diff --git a/cmd/stbak/cmd/recovery_query.go b/cmd/stbak/cmd/recovery_query.go index bdb15fd..e6b9e86 100644 --- a/cmd/stbak/cmd/recovery_query.go +++ b/cmd/stbak/cmd/recovery_query.go @@ -204,7 +204,7 @@ var recoveryQueryCmd = &cobra.Command{ } func init() { - recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") recoveryQueryCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting") recoveryQueryCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting") diff --git a/cmd/stbak/cmd/restore.go b/cmd/stbak/cmd/restore.go index fde1d53..2e6653d 100644 --- a/cmd/stbak/cmd/restore.go +++ b/cmd/stbak/cmd/restore.go @@ -114,7 +114,7 @@ var restoreCmd = &cobra.Command{ } func init() { - restoreCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + restoreCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") restoreCmd.PersistentFlags().StringP(srcFlag, "s", "", "File or directory to restore") restoreCmd.PersistentFlags().StringP(dstFlag, "d", "", "File or directory restore to (archived name by default)") restoreCmd.PersistentFlags().BoolP(flattenFlag, "f", false, "Ignore the folder hierarchy on the tape or tar file") diff --git a/cmd/stbak/cmd/root.go b/cmd/stbak/cmd/root.go index 2774737..80a6fad 100644 --- a/cmd/stbak/cmd/root.go +++ b/cmd/stbak/cmd/root.go @@ -37,6 +37,13 @@ const ( compressionFormatBzip2Suffix = ".bz2" compressionFormatBzip2ParallelKey = "parallelbzip2" + + encryptionFlag = "encryption" + + encryptionFormatNoneKey = "none" + + encryptionFormatAgeKey = "age" + encryptionFormatAgeSuffix = ".age" ) var ( @@ -44,6 +51,11 @@ var ( errUnknownCompressionFormat = errors.New("unknown compression format") errUnsupportedCompressionFormat = errors.New("unsupported compression format") + + knownEncryptionFormats = []string{encryptionFormatNoneKey, encryptionFormatAgeKey} + + errUnknownEncryptionFormat = errors.New("unknown encryption format") + errUnsupportedEncryptionFormat = errors.New("unsupported encryption format") ) var rootCmd = &cobra.Command{ @@ -70,6 +82,19 @@ https://github.com/pojntfx/stfs`, return errUnknownCompressionFormat } + encryptionFormatIsKnown := false + encryptionFormat := viper.GetString(encryptionFlag) + + for _, candidate := range knownEncryptionFormats { + if encryptionFormat == candidate { + encryptionFormatIsKnown = true + } + } + + if !encryptionFormatIsKnown { + return errUnknownEncryptionFormat + } + return nil }, } @@ -86,6 +111,7 @@ func Execute() { rootCmd.PersistentFlags().StringP(metadataFlag, "m", metadataPath, "Metadata database to use") rootCmd.PersistentFlags().BoolP(verboseFlag, "v", false, "Enable verbose logging") rootCmd.PersistentFlags().StringP(compressionFlag, "c", compressionFormatNoneKey, fmt.Sprintf("Compression format to use (default %v, available are %v)", compressionFormatNoneKey, knownCompressionFormats)) + rootCmd.PersistentFlags().StringP(encryptionFlag, "e", encryptionFormatNoneKey, fmt.Sprintf("Encryption format to use (default %v, available are %v)", encryptionFormatNoneKey, knownEncryptionFormats)) if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil { panic(err) diff --git a/cmd/stbak/cmd/update.go b/cmd/stbak/cmd/update.go index 9cc1695..947ee90 100644 --- a/cmd/stbak/cmd/update.go +++ b/cmd/stbak/cmd/update.go @@ -225,7 +225,7 @@ func update( } func init() { - updateCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") updateCmd.PersistentFlags().StringP(srcFlag, "s", "", "Path of the file or directory to update") updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file") updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) diff --git a/go.mod b/go.mod index 54289f9..3e713cc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/pojntfx/stfs go 1.17 require ( + filippo.io/age v1.0.0 github.com/andybalholm/brotli v1.0.4 github.com/cosnicolaou/pbzip2 v1.0.1 github.com/dsnet/compress v0.0.1 @@ -37,6 +38,7 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect github.com/volatiletech/inflect v0.0.1 // indirect github.com/ziutek/mymysql v1.5.4 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect diff --git a/go.sum b/go.sum index 312ba44..f145ba1 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/age v1.0.0 h1:V6q14n0mqYU3qKFkZ6oOaF9oXneOviS3ubXsSVBRSzc= +filippo.io/age v1.0.0/go.mod h1:PaX+Si/Sd5G8LgfCwldsSba3H1DDQZhIhFGkhbHaBq8= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -624,10 +627,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=