mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Start implementation of transparent encryption based on age

Browse Source
This commit is contained in:
Felix Pojtinger
2021-12-01 23:27:43 +01:00
parent db336ba9a5
commit effd4df3e7
12 changed files with 134 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/stbak/cmd/archive.go
+92 -25
View File
@@ -8,10 +8,12 @@ import (
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"filippo.io/age"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
@@ -33,6 +35,7 @@ const (
srcFlag = "src"
overwriteFlag = "overwrite"
compressionLevelFlag = "compression-level"
keyFlag = "key"
compressionLevelFastest = "fastest"
compressionLevelBalanced = "balanced"
@@ -44,6 +47,8 @@ var (
errUnknownCompressionLevel = errors.New("unknown compression level")
errUnsupportedCompressionLevel = errors.New("unsupported compression level")
errKeyNotAccessible = errors.New("key not found or accessible")
)
type flusher interface {
@@ -52,6 +57,16 @@ type flusher interface {
Flush() error
}
func nopCloserWriter(w io.Writer) nopCloser {
return nopCloser{w}
}
type nopCloser struct {
io.Writer
}
func (nopCloser) Close() error { return nil }
var archiveCmd = &cobra.Command{
Use: "archive",
Aliases: []string{"arc", "a", "c"},
@@ -61,7 +76,17 @@ var archiveCmd = &cobra.Command{
return err
}
return checkCompressionLevel(viper.GetString(compressionLevelFlag))
if err := checkCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil {
return err
}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
if _, err := os.Stat(viper.GetString(keyFlag)); err != nil {
return errKeyNotAccessible
}
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if viper.GetBool(verboseFlag) {
@@ -85,6 +110,16 @@ var archiveCmd = &cobra.Command{
lastIndexedBlock = b
}
pubkey := []byte{}
if viper.GetString(encryptionFlag) != encryptionFormatNoneKey {
p, err := ioutil.ReadFile(viper.GetString(keyFlag))
if err != nil {
return err
}
pubkey = p
}
if err := archive(
viper.GetString(tapeFlag),
viper.GetInt(recordSizeFlag),
@@ -92,6 +127,8 @@ var archiveCmd = &cobra.Command{
viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(compressionLevelFlag),
viper.GetString(encryptionFlag),
pubkey,
); err != nil {
return err
}
@@ -115,6 +152,8 @@ func archive(
overwrite bool,
compressionFormat string,
compressionLevel string,
encryptionFormat string,
pubkey []byte,
) error {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape)
@@ -206,13 +245,18 @@ func archive(
return err
}
fileSizeCounter := counters.CounterWriter{
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, pubkey)
if err != nil {
return err
}
if err := compress(
file,
&fileSizeCounter,
encryptor,
compressionFormat,
compressionLevel,
isRegular,
@@ -221,6 +265,14 @@ func archive(
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
@@ -274,9 +326,14 @@ func archive(
return err
}
encryptor, err := encrypt(tw, encryptionFormat, pubkey)
if err != nil {
return err
}
if err := compress(
file,
tw,
encryptor,
compressionFormat,
compressionLevel,
isRegular,
@@ -285,6 +342,14 @@ func archive(
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
dirty = true
return nil
@@ -307,8 +372,28 @@ func checkCompressionLevel(compressionLevel string) error {
return nil
}
func encrypt(
dst io.Writer,
encryptionFormat string,
pubkey []byte,
) (io.WriteCloser, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
recipient, err := age.ParseX25519Recipient(string(pubkey))
if err != nil {
return nil, err
}
return age.Encrypt(dst, recipient)
case encryptionFormatNoneKey:
return nopCloserWriter(dst), nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func compress(
src io.ReadCloser,
src io.Reader,
dst io.Writer,
compressionFormat string,
compressionLevel string,
@@ -379,9 +464,6 @@ func compress(
if err := gz.Close(); err != nil {
return err
}
if err := src.Close(); err != nil {
return err
}
case compressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
@@ -418,9 +500,6 @@ func compress(
if err := lz.Close(); err != nil {
return err
}
if err := src.Close(); err != nil {
return err
}
case compressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
@@ -460,9 +539,6 @@ func compress(
if err := zz.Close(); err != nil {
return err
}
if err := src.Close(); err != nil {
return err
}
case compressionFormatBrotliKey:
l := brotli.DefaultCompression
switch compressionLevel {
@@ -499,9 +575,6 @@ func compress(
if err := br.Close(); err != nil {
return err
}
if err := src.Close(); err != nil {
return err
}
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
@@ -542,9 +615,6 @@ func compress(
if err := bz.Close(); err != nil {
return err
}
if err := src.Close(); err != nil {
return err
}
case compressionFormatNoneKey:
if isRegular {
if _, err := io.Copy(dst, src); err != nil {
@@ -556,10 +626,6 @@ func compress(
return err
}
}
if err := src.Close(); err != nil {
return err
}
default:
return errUnsupportedCompressionFormat
}
@@ -568,10 +634,11 @@ func compress(
}
func init() {
archiveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
archiveCmd.PersistentFlags().StringP(srcFlag, "s", ".", "File or directory to archive")
archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file")
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
archiveCmd.PersistentFlags().StringP(keyFlag, "k", "", "Path to public key of recipient to encrypt for")
viper.AutomaticEnv()
cmd/stbak/cmd/delete.go
+1 -1
View File
@@ -167,7 +167,7 @@ func openTapeWriter(tape string) (tw *tar.Writer, isRegular bool, cleanup func(d
}
func init() {
deleteCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove")
viper.AutomaticEnv()
cmd/stbak/cmd/find.go
+1 -1
View File
@@ -66,7 +66,7 @@ var findCmd = &cobra.Command{
}
func init() {
findCmd.PersistentFlags().StringP(expressionFlag, "e", "", "Regex to match the file/directory name against")
findCmd.PersistentFlags().StringP(expressionFlag, "x", "", "Regex to match the file/directory name against")
viper.AutomaticEnv()
cmd/stbak/cmd/move.go
+1 -1
View File
@@ -95,7 +95,7 @@ var moveCmd = &cobra.Command{
}
func init() {
moveCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
moveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
moveCmd.PersistentFlags().StringP(srcFlag, "s", "", "Current path of the file or directory to move")
moveCmd.PersistentFlags().StringP(dstFlag, "d", "", "Path to move the file or directory to")
cmd/stbak/cmd/recovery_fetch.go
+1 -1
View File
@@ -252,7 +252,7 @@ func decompress(
}
func init() {
recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
recoveryFetchCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
recoveryFetchCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too")
recoveryFetchCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too")
recoveryFetchCmd.PersistentFlags().StringP(dstFlag, "d", "", "File to restore to (archived name by default)")
cmd/stbak/cmd/recovery_index.go
+1 -1
View File
@@ -242,7 +242,7 @@ func index(
}
func init() {
recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
recoveryIndexCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
recoveryIndexCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting")
recoveryIndexCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting")
recoveryIndexCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Remove the old index before starting to index")
cmd/stbak/cmd/recovery_query.go
+1 -1
View File
@@ -204,7 +204,7 @@ var recoveryQueryCmd = &cobra.Command{
}
func init() {
recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
recoveryQueryCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
recoveryQueryCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting")
recoveryQueryCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting")
cmd/stbak/cmd/restore.go
+1 -1
View File
@@ -114,7 +114,7 @@ var restoreCmd = &cobra.Command{
}
func init() {
restoreCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
restoreCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
restoreCmd.PersistentFlags().StringP(srcFlag, "s", "", "File or directory to restore")
restoreCmd.PersistentFlags().StringP(dstFlag, "d", "", "File or directory restore to (archived name by default)")
restoreCmd.PersistentFlags().BoolP(flattenFlag, "f", false, "Ignore the folder hierarchy on the tape or tar file")
cmd/stbak/cmd/root.go
+26
View File
@@ -37,6 +37,13 @@ const (
compressionFormatBzip2Suffix = ".bz2"
compressionFormatBzip2ParallelKey = "parallelbzip2"
encryptionFlag = "encryption"
encryptionFormatNoneKey = "none"
encryptionFormatAgeKey = "age"
encryptionFormatAgeSuffix = ".age"
)
var (
@@ -44,6 +51,11 @@ var (
errUnknownCompressionFormat = errors.New("unknown compression format")
errUnsupportedCompressionFormat = errors.New("unsupported compression format")
knownEncryptionFormats = []string{encryptionFormatNoneKey, encryptionFormatAgeKey}
errUnknownEncryptionFormat = errors.New("unknown encryption format")
errUnsupportedEncryptionFormat = errors.New("unsupported encryption format")
)
var rootCmd = &cobra.Command{
@@ -70,6 +82,19 @@ https://github.com/pojntfx/stfs`,
return errUnknownCompressionFormat
}
encryptionFormatIsKnown := false
encryptionFormat := viper.GetString(encryptionFlag)
for _, candidate := range knownEncryptionFormats {
if encryptionFormat == candidate {
encryptionFormatIsKnown = true
}
}
if !encryptionFormatIsKnown {
return errUnknownEncryptionFormat
}
return nil
},
}
@@ -86,6 +111,7 @@ func Execute() {
rootCmd.PersistentFlags().StringP(metadataFlag, "m", metadataPath, "Metadata database to use")
rootCmd.PersistentFlags().BoolP(verboseFlag, "v", false, "Enable verbose logging")
rootCmd.PersistentFlags().StringP(compressionFlag, "c", compressionFormatNoneKey, fmt.Sprintf("Compression format to use (default %v, available are %v)", compressionFormatNoneKey, knownCompressionFormats))
rootCmd.PersistentFlags().StringP(encryptionFlag, "e", encryptionFormatNoneKey, fmt.Sprintf("Encryption format to use (default %v, available are %v)", encryptionFormatNoneKey, knownEncryptionFormats))
if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
panic(err)
cmd/stbak/cmd/update.go
+1 -1
View File
@@ -225,7 +225,7 @@ func update(
}
func init() {
updateCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record")
updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
updateCmd.PersistentFlags().StringP(srcFlag, "s", "", "Path of the file or directory to update")
updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))