feat: Start implementation of transparent encryption based on age

cmd/stbak/cmd/archive.go

@@ -8,10 +8,12 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
 
+	"filippo.io/age"
 	"github.com/andybalholm/brotli"
 	"github.com/dsnet/compress/bzip2"
 	"github.com/klauspost/compress/zstd"
@@ -33,6 +35,7 @@ const (
 	srcFlag              = "src"
 	overwriteFlag        = "overwrite"
 	compressionLevelFlag = "compression-level"
+	keyFlag              = "key"
 
 	compressionLevelFastest  = "fastest"
 	compressionLevelBalanced = "balanced"
@@ -44,6 +47,8 @@ var (
 
 	errUnknownCompressionLevel     = errors.New("unknown compression level")
 	errUnsupportedCompressionLevel = errors.New("unsupported compression level")
+
+	errKeyNotAccessible = errors.New("key not found or accessible")
 )
 
 type flusher interface {
@@ -52,6 +57,16 @@ type flusher interface {
 	Flush() error
 }
 
+func nopCloserWriter(w io.Writer) nopCloser {
+	return nopCloser{w}
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (nopCloser) Close() error { return nil }
+
 var archiveCmd = &cobra.Command{
 	Use:     "archive",
 	Aliases: []string{"arc", "a", "c"},
@@ -61,7 +76,17 @@ var archiveCmd = &cobra.Command{
 			return err
 		}
 
-		return checkCompressionLevel(viper.GetString(compressionLevelFlag))
+		if err := checkCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil {
+			return err
+		}
+
+		if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
+			if _, err := os.Stat(viper.GetString(keyFlag)); err != nil {
+				return errKeyNotAccessible
+			}
+		}
+
+		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if viper.GetBool(verboseFlag) {
@@ -85,6 +110,16 @@ var archiveCmd = &cobra.Command{
 			lastIndexedBlock = b
 		}
 
+		pubkey := []byte{}
+		if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
+			p, err := ioutil.ReadFile(viper.GetString(keyFlag))
+			if err != nil {
+				return err
+			}
+
+			pubkey = p
+		}
+
 		if err := archive(
 			viper.GetString(tapeFlag),
 			viper.GetInt(recordSizeFlag),
@@ -92,6 +127,8 @@ var archiveCmd = &cobra.Command{
 			viper.GetBool(overwriteFlag),
 			viper.GetString(compressionFlag),
 			viper.GetString(compressionLevelFlag),
+			viper.GetString(encryptionFlag),
+			pubkey,
 		); err != nil {
 			return err
 		}
@@ -115,6 +152,8 @@ func archive(
 	overwrite bool,
 	compressionFormat string,
 	compressionLevel string,
+	encryptionFormat string,
+	pubkey []byte,
 ) error {
 	dirty := false
 	tw, isRegular, cleanup, err := openTapeWriter(tape)
@@ -206,13 +245,18 @@ func archive(
 				return err
 			}
 
-			fileSizeCounter := counters.CounterWriter{
+			fileSizeCounter := &counters.CounterWriter{
 				Writer: io.Discard,
 			}
 
+			encryptor, err := encrypt(fileSizeCounter, encryptionFormat, pubkey)
+			if err != nil {
+				return err
+			}
+
 			if err := compress(
 				file,
-				&fileSizeCounter,
+				encryptor,
 				compressionFormat,
 				compressionLevel,
 				isRegular,
@@ -221,6 +265,14 @@ func archive(
 				return err
 			}
 
+			if err := encryptor.Close(); err != nil {
+				return err
+			}
+
+			if err := file.Close(); err != nil {
+				return err
+			}
+
 			if hdr.PAXRecords == nil {
 				hdr.PAXRecords = map[string]string{}
 			}
@@ -274,9 +326,14 @@ func archive(
 			return err
 		}
 
+		encryptor, err := encrypt(tw, encryptionFormat, pubkey)
+		if err != nil {
+			return err
+		}
+
 		if err := compress(
 			file,
-			tw,
+			encryptor,
 			compressionFormat,
 			compressionLevel,
 			isRegular,
@@ -285,6 +342,14 @@ func archive(
 			return err
 		}
 
+		if err := encryptor.Close(); err != nil {
+			return err
+		}
+
+		if err := file.Close(); err != nil {
+			return err
+		}
+
 		dirty = true
 
 		return nil
@@ -307,8 +372,28 @@ func checkCompressionLevel(compressionLevel string) error {
 	return nil
 }
 
+func encrypt(
+	dst io.Writer,
+	encryptionFormat string,
+	pubkey []byte,
+) (io.WriteCloser, error) {
+	switch encryptionFormat {
+	case encryptionFormatAgeKey:
+		recipient, err := age.ParseX25519Recipient(string(pubkey))
+		if err != nil {
+			return nil, err
+		}
+
+		return age.Encrypt(dst, recipient)
+	case encryptionFormatNoneKey:
+		return nopCloserWriter(dst), nil
+	default:
+		return nil, errUnsupportedEncryptionFormat
+	}
+}
+
 func compress(
-	src io.ReadCloser,
+	src io.Reader,
 	dst io.Writer,
 	compressionFormat string,
 	compressionLevel string,
@@ -379,9 +464,6 @@ func compress(
 		if err := gz.Close(); err != nil {
 			return err
 		}
-		if err := src.Close(); err != nil {
-			return err
-		}
 	case compressionFormatLZ4Key:
 		l := lz4.Level5
 		switch compressionLevel {
@@ -418,9 +500,6 @@ func compress(
 		if err := lz.Close(); err != nil {
 			return err
 		}
-		if err := src.Close(); err != nil {
-			return err
-		}
 	case compressionFormatZStandardKey:
 		l := zstd.SpeedDefault
 		switch compressionLevel {
@@ -460,9 +539,6 @@ func compress(
 		if err := zz.Close(); err != nil {
 			return err
 		}
-		if err := src.Close(); err != nil {
-			return err
-		}
 	case compressionFormatBrotliKey:
 		l := brotli.DefaultCompression
 		switch compressionLevel {
@@ -499,9 +575,6 @@ func compress(
 		if err := br.Close(); err != nil {
 			return err
 		}
-		if err := src.Close(); err != nil {
-			return err
-		}
 	case compressionFormatBzip2Key:
 		fallthrough
 	case compressionFormatBzip2ParallelKey:
@@ -542,9 +615,6 @@ func compress(
 		if err := bz.Close(); err != nil {
 			return err
 		}
-		if err := src.Close(); err != nil {
-			return err
-		}
 	case compressionFormatNoneKey:
 		if isRegular {
 			if _, err := io.Copy(dst, src); err != nil {
@@ -556,10 +626,6 @@ func compress(
 				return err
 			}
 		}
-
-		if err := src.Close(); err != nil {
-			return err
-		}
 	default:
 		return errUnsupportedCompressionFormat
 	}
@@ -568,10 +634,11 @@ func compress(
 }
 
 func init() {
-	archiveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	archiveCmd.PersistentFlags().StringP(srcFlag, "s", ".", "File or directory to archive")
 	archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file")
 	archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
+	archiveCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to public key of recipient to encrypt for")
 
 	viper.AutomaticEnv()
 

cmd/stbak/cmd/delete.go

@@ -167,7 +167,7 @@ func openTapeWriter(tape string) (tw *tar.Writer, isRegular bool, cleanup func(d
 }
 
 func init() {
-	deleteCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove")
 
 	viper.AutomaticEnv()

cmd/stbak/cmd/find.go

@@ -66,7 +66,7 @@ var findCmd = &cobra.Command{
 }
 
 func init() {
-	findCmd.PersistentFlags().StringP(expressionFlag, "e", "", "Regex to match the file/directory name against")
+	findCmd.PersistentFlags().StringP(expressionFlag, "x", "", "Regex to match the file/directory name against")
 
 	viper.AutomaticEnv()
 

cmd/stbak/cmd/move.go

@@ -95,7 +95,7 @@ var moveCmd = &cobra.Command{
 }
 
 func init() {
-	moveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	moveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	moveCmd.PersistentFlags().StringP(srcFlag, "s", "", "Current path of the file or directory to move")
 	moveCmd.PersistentFlags().StringP(dstFlag, "d", "", "Path to move the file or directory to")
 

cmd/stbak/cmd/recovery_fetch.go

@@ -252,7 +252,7 @@ func decompress(
 }
 
 func init() {
-	recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	recoveryFetchCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too")
 	recoveryFetchCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too")
 	recoveryFetchCmd.PersistentFlags().StringP(dstFlag, "d", "", "File to restore to (archived name by default)")

cmd/stbak/cmd/recovery_index.go

@@ -242,7 +242,7 @@ func index(
 }
 
 func init() {
-	recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	recoveryIndexCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting")
 	recoveryIndexCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting")
 	recoveryIndexCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Remove the old index before starting to index")

cmd/stbak/cmd/recovery_query.go

@@ -204,7 +204,7 @@ var recoveryQueryCmd = &cobra.Command{
 }
 
 func init() {
-	recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	recoveryQueryCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting")
 	recoveryQueryCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting")
 

cmd/stbak/cmd/restore.go

@@ -114,7 +114,7 @@ var restoreCmd = &cobra.Command{
 }
 
 func init() {
-	restoreCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	restoreCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	restoreCmd.PersistentFlags().StringP(srcFlag, "s", "", "File or directory to restore")
 	restoreCmd.PersistentFlags().StringP(dstFlag, "d", "", "File or directory restore to (archived name by default)")
 	restoreCmd.PersistentFlags().BoolP(flattenFlag, "f", false, "Ignore the folder hierarchy on the tape or tar file")

cmd/stbak/cmd/root.go

@@ -37,6 +37,13 @@ const (
 	compressionFormatBzip2Suffix = ".bz2"
 
 	compressionFormatBzip2ParallelKey = "parallelbzip2"
+
+	encryptionFlag = "encryption"
+
+	encryptionFormatNoneKey = "none"
+
+	encryptionFormatAgeKey    = "age"
+	encryptionFormatAgeSuffix = ".age"
 )
 
 var (
@@ -44,6 +51,11 @@ var (
 
 	errUnknownCompressionFormat     = errors.New("unknown compression format")
 	errUnsupportedCompressionFormat = errors.New("unsupported compression format")
+
+	knownEncryptionFormats = []string{encryptionFormatNoneKey, encryptionFormatAgeKey}
+
+	errUnknownEncryptionFormat     = errors.New("unknown encryption format")
+	errUnsupportedEncryptionFormat = errors.New("unsupported encryption format")
 )
 
 var rootCmd = &cobra.Command{
@@ -70,6 +82,19 @@ https://github.com/pojntfx/stfs`,
 			return errUnknownCompressionFormat
 		}
 
+		encryptionFormatIsKnown := false
+		encryptionFormat := viper.GetString(encryptionFlag)
+
+		for _, candidate := range knownEncryptionFormats {
+			if encryptionFormat == candidate {
+				encryptionFormatIsKnown = true
+			}
+		}
+
+		if !encryptionFormatIsKnown {
+			return errUnknownEncryptionFormat
+		}
+
 		return nil
 	},
 }
@@ -86,6 +111,7 @@ func Execute() {
 	rootCmd.PersistentFlags().StringP(metadataFlag, "m", metadataPath, "Metadata database to use")
 	rootCmd.PersistentFlags().BoolP(verboseFlag, "v", false, "Enable verbose logging")
 	rootCmd.PersistentFlags().StringP(compressionFlag, "c", compressionFormatNoneKey, fmt.Sprintf("Compression format to use (default %v, available are %v)", compressionFormatNoneKey, knownCompressionFormats))
+	rootCmd.PersistentFlags().StringP(encryptionFlag, "e", encryptionFormatNoneKey, fmt.Sprintf("Encryption format to use (default %v, available are %v)", encryptionFormatNoneKey, knownEncryptionFormats))
 
 	if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
 		panic(err)

cmd/stbak/cmd/update.go

@@ -225,7 +225,7 @@ func update(
 }
 
 func init() {
-	updateCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
+	updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
 	updateCmd.PersistentFlags().StringP(srcFlag, "s", "", "Path of the file or directory to update")
 	updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
 	updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))

go.mod

@@ -3,6 +3,7 @@ module github.com/pojntfx/stfs
 go 1.17
 
 require (
+	filippo.io/age v1.0.0
 	github.com/andybalholm/brotli v1.0.4
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dsnet/compress v0.0.1
@@ -37,6 +38,7 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/volatiletech/inflect v0.0.1 // indirect
 	github.com/ziutek/mymysql v1.5.4 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect

go.sum

@@ -43,6 +43,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+filippo.io/age v1.0.0 h1:V6q14n0mqYU3qKFkZ6oOaF9oXneOviS3ubXsSVBRSzc=
+filippo.io/age v1.0.0/go.mod h1:PaX+Si/Sd5G8LgfCwldsSba3H1DDQZhIhFGkhbHaBq8=
+filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM=
@@ -624,10 +627,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI=
 golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=