diff --git a/cmd/stbak/cmd/archive.go b/cmd/stbak/cmd/archive.go index c8cf60d..481c9b8 100644 --- a/cmd/stbak/cmd/archive.go +++ b/cmd/stbak/cmd/archive.go @@ -2,39 +2,16 @@ package cmd import ( "archive/tar" - "bytes" - "compress/gzip" "context" - "encoding/base64" - "encoding/json" "errors" "fmt" - "io" - "io/fs" "io/ioutil" - "math" "os" - "path/filepath" - "strconv" - "aead.dev/minisign" - "filippo.io/age" - "github.com/ProtonMail/go-crypto/openpgp" - "github.com/ProtonMail/go-crypto/openpgp/packet" - "github.com/andybalholm/brotli" - "github.com/dsnet/compress/bzip2" - "github.com/klauspost/compress/zstd" - "github.com/klauspost/pgzip" - "github.com/pierrec/lz4/v4" - "github.com/pojntfx/stfs/internal/adapters" - "github.com/pojntfx/stfs/internal/controllers" - "github.com/pojntfx/stfs/internal/counters" - "github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/keys" - "github.com/pojntfx/stfs/internal/noop" - "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/pkg/config" + "github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -50,14 +27,10 @@ const ( recipientFlag = "recipient" identityFlag = "identity" passwordFlag = "password" - - compressionLevelFastest = "fastest" - compressionLevelBalanced = "balanced" - compressionLevelSmallest = "smallest" ) var ( - knownCompressionLevels = []string{compressionLevelFastest, compressionLevelBalanced, compressionLevelSmallest} + knownCompressionLevels = []string{config.CompressionLevelFastest, config.CompressionLevelBalanced, config.CompressionLevelSmallest} errUnknownCompressionLevel = errors.New("unknown compression level") errUnsupportedCompressionLevel = errors.New("unsupported compression level") @@ -121,7 +94,7 @@ var archiveCmd = &cobra.Command{ return err } - recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) + recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey) if err != nil { return err } @@ -136,21 +109,27 @@ var archiveCmd = &cobra.Command{ return err } - hdrs, err := archive( - viper.GetString(driveFlag), + hdrs, err := operations.Archive( + config.StateConfig{ + Drive: viper.GetString(driveFlag), + Metadata: viper.GetString(metadataFlag), + }, + config.PipeConfig{ + Compression: viper.GetString(compressionFlag), + Encryption: viper.GetString(encryptionFlag), + Signature: viper.GetString(signatureFlag), + }, + config.CryptoConfig{ + Recipient: recipient, + Identity: identity, + Password: viper.GetString(passwordFlag), + }, + viper.GetInt(recordSizeFlag), viper.GetString(fromFlag), viper.GetBool(overwriteFlag), - viper.GetString(compressionFlag), viper.GetString(compressionLevelFlag), - viper.GetString(encryptionFlag), - recipient, - viper.GetString(signatureFlag), - identity, ) - if err != nil { - return err - } return recovery.Index( config.StateConfig{ @@ -194,267 +173,6 @@ var archiveCmd = &cobra.Command{ }, } -func archive( - tape string, - recordSize int, - src string, - overwrite bool, - compressionFormat string, - compressionLevel string, - encryptionFormat string, - recipient interface{}, - signatureFormat string, - identity interface{}, -) ([]*tar.Header, error) { - dirty := false - tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, overwrite) - if err != nil { - return []*tar.Header{}, err - } - - if overwrite { - if isRegular { - if err := cleanup(&dirty); err != nil { // dirty will always be false here - return []*tar.Header{}, err - } - - f, err := os.OpenFile(tape, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return []*tar.Header{}, err - } - - // Clear the file's content - if err := f.Truncate(0); err != nil { - return []*tar.Header{}, err - } - - if err := f.Close(); err != nil { - return []*tar.Header{}, err - } - - tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite) - if err != nil { - return []*tar.Header{}, err - } - } else { - if err := cleanup(&dirty); err != nil { // dirty will always be false here - return []*tar.Header{}, err - } - - f, err := os.OpenFile(tape, os.O_WRONLY, os.ModeCharDevice) - if err != nil { - return []*tar.Header{}, err - } - - // Seek to the start of the tape - if err := controllers.SeekToRecordOnTape(f, 0); err != nil { - return []*tar.Header{}, err - } - - if err := f.Close(); err != nil { - return []*tar.Header{}, err - } - - tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite) - if err != nil { - return []*tar.Header{}, err - } - } - } - - defer cleanup(&dirty) - - headers := []*tar.Header{} - first := true - return headers, filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - link := "" - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - if link, err = os.Readlink(path); err != nil { - return err - } - } - - hdr, err := tar.FileInfoHeader(info, link) - if err != nil { - return err - } - - if err := adapters.EnhanceHeader(path, hdr); err != nil { - return err - } - - hdr.Name = path - hdr.Format = tar.FormatPAX - - if info.Mode().IsRegular() { - // Get the compressed size for the header - fileSizeCounter := &counters.CounterWriter{ - Writer: io.Discard, - } - - encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient) - if err != nil { - return err - } - - compressor, err := compress( - encryptor, - compressionFormat, - compressionLevel, - isRegular, - recordSize, - ) - if err != nil { - return err - } - - file, err := os.Open(path) - if err != nil { - return err - } - - signer, sign, err := sign(file, isRegular, signatureFormat, identity) - if err != nil { - return err - } - - if isRegular { - if _, err := io.Copy(compressor, signer); err != nil { - return err - } - } else { - buf := make([]byte, controllers.BlockSize*recordSize) - if _, err := io.CopyBuffer(compressor, signer, buf); err != nil { - return err - } - } - - if err := file.Close(); err != nil { - return err - } - - if err := compressor.Flush(); err != nil { - return err - } - - if err := compressor.Close(); err != nil { - return err - } - - if err := encryptor.Close(); err != nil { - return err - } - - if hdr.PAXRecords == nil { - hdr.PAXRecords = map[string]string{} - } - hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) - signature, err := sign() - if err != nil { - return err - } - - if signature != "" { - hdr.PAXRecords[pax.STFSRecordSignature] = signature - } - hdr.Size = int64(fileSizeCounter.BytesRead) - - hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat) - if err != nil { - return err - } - } - - if first { - if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { - return err - } - - first = false - } - - if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { - return err - } - - hdrToAppend := *hdr - headers = append(headers, &hdrToAppend) - - if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { - return err - } - - if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { - return err - } - - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - if !info.Mode().IsRegular() { - return nil - } - - // Compress and write the file - encryptor, err := encrypt(tw, encryptionFormat, recipient) - if err != nil { - return err - } - - compressor, err := compress( - encryptor, - compressionFormat, - compressionLevel, - isRegular, - recordSize, - ) - if err != nil { - return err - } - - file, err := os.Open(path) - if err != nil { - return err - } - - if isRegular { - if _, err := io.Copy(compressor, file); err != nil { - return err - } - } else { - buf := make([]byte, controllers.BlockSize*recordSize) - if _, err := io.CopyBuffer(compressor, file, buf); err != nil { - return err - } - } - - if err := file.Close(); err != nil { - return err - } - - if err := compressor.Flush(); err != nil { - return err - } - - if err := compressor.Close(); err != nil { - return err - } - - if err := encryptor.Close(); err != nil { - return err - } - - dirty = true - - return nil - }) -} - func checkKeyAccessible(encryptionFormat string, pathToKey string) error { if encryptionFormat == noneKey { return nil @@ -491,489 +209,11 @@ func checkCompressionLevel(compressionLevel string) error { return nil } -func encryptHeader( - hdr *tar.Header, - encryptionFormat string, - recipient interface{}, -) error { - if encryptionFormat == noneKey { - return nil - } - - newHdr := &tar.Header{ - Format: tar.FormatPAX, - Size: hdr.Size, - PAXRecords: map[string]string{}, - } - - wrappedHeader, err := json.Marshal(hdr) - if err != nil { - return err - } - - newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = encryptString(string(wrappedHeader), encryptionFormat, recipient) - if err != nil { - return err - } - - *hdr = *newHdr - - return nil -} - -func signHeader( - hdr *tar.Header, - isRegular bool, - signatureFormat string, - identity interface{}, -) error { - if signatureFormat == noneKey { - return nil - } - - newHdr := &tar.Header{ - Format: tar.FormatPAX, - Size: hdr.Size, - PAXRecords: map[string]string{}, - } - - wrappedHeader, err := json.Marshal(hdr) - if err != nil { - return err - } - - newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader) - newHdr.PAXRecords[pax.STFSRecordSignature], err = signString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity) - if err != nil { - return err - } - - *hdr = *newHdr - - return nil -} - -func addSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) { - switch compressionFormat { - case compressionFormatGZipKey: - fallthrough - case compressionFormatParallelGZipKey: - name += compressionFormatGZipSuffix - case compressionFormatLZ4Key: - name += compressionFormatLZ4Suffix - case compressionFormatZStandardKey: - name += compressionFormatZStandardSuffix - case compressionFormatBrotliKey: - name += compressionFormatBrotliSuffix - case compressionFormatBzip2Key: - fallthrough - case compressionFormatBzip2ParallelKey: - name += compressionFormatBzip2Suffix - case noneKey: - default: - return "", errUnsupportedCompressionFormat - } - - switch encryptionFormat { - case encryptionFormatAgeKey: - name += encryptionFormatAgeSuffix - case encryptionFormatPGPKey: - name += encryptionFormatPGPSuffix - case noneKey: - default: - return "", errUnsupportedEncryptionFormat - } - - return name, nil -} - -func parseRecipient( - encryptionFormat string, - pubkey []byte, -) (interface{}, error) { - switch encryptionFormat { - case encryptionFormatAgeKey: - return age.ParseX25519Recipient(string(pubkey)) - case encryptionFormatPGPKey: - return openpgp.ReadKeyRing(bytes.NewBuffer(pubkey)) - case noneKey: - return pubkey, nil - default: - return nil, errUnsupportedEncryptionFormat - } -} - -func encrypt( - dst io.Writer, - encryptionFormat string, - recipient interface{}, -) (io.WriteCloser, error) { - switch encryptionFormat { - case encryptionFormatAgeKey: - recipient, ok := recipient.(*age.X25519Recipient) - if !ok { - return nil, errRecipientUnparsable - } - - return age.Encrypt(dst, recipient) - case encryptionFormatPGPKey: - recipient, ok := recipient.(openpgp.EntityList) - if !ok { - return nil, errRecipientUnparsable - } - - return openpgp.Encrypt(dst, recipient, nil, nil, nil) - case noneKey: - return noop.AddClose(dst), nil - default: - return nil, errUnsupportedEncryptionFormat - } -} - -func sign( - src io.Reader, - isRegular bool, - signatureFormat string, - identity interface{}, -) (io.Reader, func() (string, error), error) { - switch signatureFormat { - case signatureFormatMinisignKey: - if !isRegular { - return nil, nil, errSignatureFormatOnlyRegularSupport - } - - identity, ok := identity.(minisign.PrivateKey) - if !ok { - return nil, nil, errIdentityUnparsable - } - - signer := minisign.NewReader(src) - - return signer, func() (string, error) { - return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil - }, nil - case signatureFormatPGPKey: - identities, ok := identity.(openpgp.EntityList) - if !ok { - return nil, nil, errIdentityUnparsable - } - - if len(identities) < 1 { - return nil, nil, errIdentityUnparsable - } - - // See openpgp.DetachSign - var config *packet.Config - signingKey, ok := identities[0].SigningKeyById(config.Now(), config.SigningKey()) - if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil { - return nil, nil, errIdentityUnparsable - } - - sig := new(packet.Signature) - sig.SigType = packet.SigTypeBinary - sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo - sig.Hash = config.Hash() - sig.CreationTime = config.Now() - sigLifetimeSecs := config.SigLifetime() - sig.SigLifetimeSecs = &sigLifetimeSecs - sig.IssuerKeyId = &signingKey.PrivateKey.KeyId - - hash := sig.Hash.New() - - return io.TeeReader(src, hash), func() (string, error) { - if err := sig.Sign(hash, signingKey.PrivateKey, config); err != nil { - return "", err - } - - out := &bytes.Buffer{} - if err := sig.Serialize(out); err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(out.Bytes()), nil - }, nil - case noneKey: - return src, func() (string, error) { - return "", nil - }, nil - default: - return nil, nil, errUnsupportedSignatureFormat - } -} - -func encryptString( - src string, - encryptionFormat string, - recipient interface{}, -) (string, error) { - switch encryptionFormat { - case encryptionFormatAgeKey: - recipient, ok := recipient.(*age.X25519Recipient) - if !ok { - return "", errRecipientUnparsable - } - - out := &bytes.Buffer{} - w, err := age.Encrypt(out, recipient) - if err != nil { - return "", err - } - - if _, err := io.WriteString(w, src); err != nil { - return "", err - } - - if err := w.Close(); err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(out.Bytes()), nil - case encryptionFormatPGPKey: - recipient, ok := recipient.(openpgp.EntityList) - if !ok { - return "", errRecipientUnparsable - } - - out := &bytes.Buffer{} - w, err := openpgp.Encrypt(out, recipient, nil, nil, nil) - if err != nil { - return "", err - } - - if _, err := io.WriteString(w, src); err != nil { - return "", err - } - - if err := w.Close(); err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(out.Bytes()), nil - case noneKey: - return src, nil - default: - return "", errUnsupportedEncryptionFormat - } -} - -func signString( - src string, - isRegular bool, - signatureFormat string, - identity interface{}, -) (string, error) { - switch signatureFormat { - case signatureFormatMinisignKey: - if !isRegular { - return "", errSignatureFormatOnlyRegularSupport - } - - identity, ok := identity.(minisign.PrivateKey) - if !ok { - return "", errIdentityUnparsable - } - - return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil - case signatureFormatPGPKey: - identities, ok := identity.(openpgp.EntityList) - if !ok { - return "", errIdentityUnparsable - } - - if len(identities) < 1 { - return "", errIdentityUnparsable - } - - out := &bytes.Buffer{} - if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(out.Bytes()), nil - case noneKey: - return src, nil - default: - return "", errUnsupportedSignatureFormat - } -} - -func compress( - dst io.Writer, - compressionFormat string, - compressionLevel string, - isRegular bool, - recordSize int, -) (noop.Flusher, error) { - switch compressionFormat { - case compressionFormatGZipKey: - fallthrough - case compressionFormatParallelGZipKey: - if compressionFormat == compressionFormatGZipKey { - if !isRegular { - maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize) - - if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header. - return nil, errCompressionFormatRequiresLargerRecordSize - } - } - - l := gzip.DefaultCompression - switch compressionLevel { - case compressionLevelFastest: - l = gzip.BestSpeed - case compressionLevelBalanced: - l = gzip.DefaultCompression - case compressionLevelSmallest: - l = gzip.BestCompression - default: - return nil, errUnsupportedCompressionLevel - } - - return gzip.NewWriterLevel(dst, l) - } - - if !isRegular { - return nil, errCompressionFormatOnlyRegularSupport // "device or resource busy" - } - - l := pgzip.DefaultCompression - switch compressionLevel { - case compressionLevelFastest: - l = pgzip.BestSpeed - case compressionLevelBalanced: - l = pgzip.DefaultCompression - case compressionLevelSmallest: - l = pgzip.BestCompression - default: - return nil, errUnsupportedCompressionLevel - } - - return pgzip.NewWriterLevel(dst, l) - case compressionFormatLZ4Key: - l := lz4.Level5 - switch compressionLevel { - case compressionLevelFastest: - l = lz4.Level1 - case compressionLevelBalanced: - l = lz4.Level5 - case compressionLevelSmallest: - l = lz4.Level9 - default: - return nil, errUnsupportedCompressionLevel - } - - opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)} - if !isRegular { - maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize) - - if uint32(maxSize) < uint32(lz4.Block64Kb) { - return nil, errCompressionFormatRequiresLargerRecordSize - } - - if uint32(maxSize) < uint32(lz4.Block256Kb) { - opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb)) - } else if uint32(maxSize) < uint32(lz4.Block1Mb) { - opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb)) - } else if uint32(maxSize) < uint32(lz4.Block4Mb) { - opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb)) - } else { - opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb)) - } - } - - lz := lz4.NewWriter(dst) - if err := lz.Apply(opts...); err != nil { - return nil, err - } - - return noop.AddFlush(lz), nil - case compressionFormatZStandardKey: - l := zstd.SpeedDefault - switch compressionLevel { - case compressionLevelFastest: - l = zstd.SpeedFastest - case compressionLevelBalanced: - l = zstd.SpeedDefault - case compressionLevelSmallest: - l = zstd.SpeedBestCompression - default: - return nil, errUnsupportedCompressionLevel - } - - opts := []zstd.EOption{zstd.WithEncoderLevel(l)} - if !isRegular { - opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize))) - } - - zz, err := zstd.NewWriter(dst, opts...) - if err != nil { - return nil, err - } - - return zz, nil - case compressionFormatBrotliKey: - if !isRegular { - return nil, errCompressionFormatOnlyRegularSupport // "cannot allocate memory" - } - - l := brotli.DefaultCompression - switch compressionLevel { - case compressionLevelFastest: - l = brotli.BestSpeed - case compressionLevelBalanced: - l = brotli.DefaultCompression - case compressionLevelSmallest: - l = brotli.BestCompression - default: - return nil, errUnsupportedCompressionLevel - } - - br := brotli.NewWriterLevel(dst, l) - - return br, nil - case compressionFormatBzip2Key: - fallthrough - case compressionFormatBzip2ParallelKey: - l := bzip2.DefaultCompression - switch compressionLevel { - case compressionLevelFastest: - l = bzip2.BestSpeed - case compressionLevelBalanced: - l = bzip2.DefaultCompression - case compressionLevelSmallest: - l = bzip2.BestCompression - default: - return nil, errUnsupportedCompressionLevel - } - - bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{ - Level: l, - }) - if err != nil { - return nil, err - } - - return noop.AddFlush(bz), nil - case noneKey: - return noop.AddFlush(noop.AddClose(dst)), nil - default: - return nil, errUnsupportedCompressionFormat - } -} - -func getNearestPowerOf2Lower(n int) int { - return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/ -} - -func getNearestLogOf2Lower(n int) int { - return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/ -} - func init() { archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") archiveCmd.PersistentFlags().StringP(fromFlag, "f", ".", "File or directory to archive") archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file") - archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) + archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels)) archiveCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for") archiveCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with") archiveCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key") diff --git a/cmd/stbak/cmd/delete.go b/cmd/stbak/cmd/delete.go index b5dcdfa..bb71ac3 100644 --- a/cmd/stbak/cmd/delete.go +++ b/cmd/stbak/cmd/delete.go @@ -2,18 +2,17 @@ package cmd import ( "archive/tar" - "bufio" "context" - "os" - "github.com/pojntfx/stfs/internal/controllers" "github.com/pojntfx/stfs/internal/converters" - "github.com/pojntfx/stfs/internal/counters" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" + "github.com/pojntfx/stfs/internal/encryption" "github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" + "github.com/pojntfx/stfs/internal/signature" + "github.com/pojntfx/stfs/internal/tape" "github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" @@ -54,7 +53,7 @@ var deleteCmd = &cobra.Command{ return err } - recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) + recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey) if err != nil { return err } @@ -83,7 +82,7 @@ var deleteCmd = &cobra.Command{ } func delete( - tape string, + drive string, recordSize int, metadata string, name string, @@ -93,7 +92,7 @@ func delete( identity interface{}, ) error { dirty := false - tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) + tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false) if err != nil { return err } @@ -142,11 +141,11 @@ func delete( hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1 hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete - if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { + if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { return err } - if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { + if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { return err } @@ -204,75 +203,6 @@ func delete( ) } -func openTapeWriter(tape string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) { - stat, err := os.Stat(tape) - if err == nil { - isRegular = stat.Mode().IsRegular() - } else { - if os.IsNotExist(err) { - isRegular = true - } else { - return nil, false, nil, err - } - } - - var f *os.File - if isRegular { - f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return nil, false, nil, err - } - - // No need to go to end manually due to `os.O_APPEND` - } else { - f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) - if err != nil { - return nil, false, nil, err - } - - if !overwrite { - // Go to end of tape - if err := controllers.GoToEndOfTape(f); err != nil { - return nil, false, nil, err - } - } - } - - var bw *bufio.Writer - var counter *counters.CounterWriter - if isRegular { - tw = tar.NewWriter(f) - } else { - bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize) - counter = &counters.CounterWriter{Writer: bw, BytesRead: 0} - tw = tar.NewWriter(counter) - } - - return tw, isRegular, func(dirty *bool) error { - // Only write the trailer if we wrote to the archive - if *dirty { - if err := tw.Close(); err != nil { - return err - } - - if !isRegular { - if controllers.BlockSize*recordSize-counter.BytesRead > 0 { - // Fill the rest of the record with zeros - if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil { - return err - } - } - - if err := bw.Flush(); err != nil { - return err - } - } - } - - return f.Close() - }, nil -} - func init() { deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove") diff --git a/cmd/stbak/cmd/move.go b/cmd/stbak/cmd/move.go index 4030d08..7e01bf9 100644 --- a/cmd/stbak/cmd/move.go +++ b/cmd/stbak/cmd/move.go @@ -7,10 +7,13 @@ import ( "github.com/pojntfx/stfs/internal/converters" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" + "github.com/pojntfx/stfs/internal/encryption" "github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" + "github.com/pojntfx/stfs/internal/signature" + "github.com/pojntfx/stfs/internal/tape" "github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" @@ -47,7 +50,7 @@ var moveCmd = &cobra.Command{ return err } - recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) + recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey) if err != nil { return err } @@ -77,7 +80,7 @@ var moveCmd = &cobra.Command{ } func move( - tape string, + drive string, recordSize int, metadata string, src string, @@ -88,7 +91,7 @@ func move( identity interface{}, ) error { dirty := false - tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) + tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false) if err != nil { return err } @@ -139,11 +142,11 @@ func move( hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate hdr.PAXRecords[pax.STFSRecordReplacesName] = dbhdr.Name - if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { + if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { return err } - if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { + if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { return err } diff --git a/cmd/stbak/cmd/update.go b/cmd/stbak/cmd/update.go index fab7c3e..9a1e6bd 100644 --- a/cmd/stbak/cmd/update.go +++ b/cmd/stbak/cmd/update.go @@ -11,12 +11,17 @@ import ( "strconv" "github.com/pojntfx/stfs/internal/adapters" + "github.com/pojntfx/stfs/internal/compression" "github.com/pojntfx/stfs/internal/controllers" "github.com/pojntfx/stfs/internal/counters" + "github.com/pojntfx/stfs/internal/encryption" "github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" + "github.com/pojntfx/stfs/internal/signature" + "github.com/pojntfx/stfs/internal/suffix" + "github.com/pojntfx/stfs/internal/tape" "github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" @@ -67,7 +72,7 @@ var updateCmd = &cobra.Command{ return err } - recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) + recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey) if err != nil { return err } @@ -141,7 +146,7 @@ var updateCmd = &cobra.Command{ } func update( - tape string, + drive string, recordSize int, src string, replacesContent bool, @@ -153,7 +158,7 @@ func update( identity interface{}, ) ([]*tar.Header, error) { dirty := false - tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) + tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false) if err != nil { return []*tar.Header{}, err } @@ -196,12 +201,12 @@ func update( Writer: io.Discard, } - encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient) + encryptor, err := encryption.Encrypt(fileSizeCounter, encryptionFormat, recipient) if err != nil { return err } - compressor, err := compress( + compressor, err := compression.Compress( encryptor, compressionFormat, compressionLevel, @@ -217,7 +222,7 @@ func update( return err } - signer, sign, err := sign(file, isRegular, signatureFormat, identity) + signer, sign, err := signature.Sign(file, isRegular, signatureFormat, identity) if err != nil { return err } @@ -263,7 +268,7 @@ func update( } hdr.Size = int64(fileSizeCounter.BytesRead) - hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat) + hdr.Name, err = suffix.AddSuffix(hdr.Name, compressionFormat, encryptionFormat) if err != nil { return err } @@ -287,11 +292,11 @@ func update( hdrToAppend := *hdr headers = append(headers, &hdrToAppend) - if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { + if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { return err } - if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { + if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { return err } @@ -304,12 +309,12 @@ func update( } // Compress and write the file - encryptor, err := encrypt(tw, encryptionFormat, recipient) + encryptor, err := encryption.Encrypt(tw, encryptionFormat, recipient) if err != nil { return err } - compressor, err := compress( + compressor, err := compression.Compress( encryptor, compressionFormat, compressionLevel, @@ -361,11 +366,11 @@ func update( hdrToAppend := *hdr headers = append(headers, &hdrToAppend) - if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { + if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { return err } - if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { + if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { return err } @@ -384,7 +389,7 @@ func init() { updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update") updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file") - updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) + updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels)) updateCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for") updateCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with") updateCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key") diff --git a/internal/compression/compress.go b/internal/compression/compress.go new file mode 100644 index 0000000..28ec3c6 --- /dev/null +++ b/internal/compression/compress.go @@ -0,0 +1,188 @@ +package compression + +import ( + "compress/gzip" + "io" + "math" + + "github.com/andybalholm/brotli" + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/klauspost/pgzip" + "github.com/pierrec/lz4/v4" + "github.com/pojntfx/stfs/internal/controllers" + "github.com/pojntfx/stfs/internal/noop" + "github.com/pojntfx/stfs/pkg/config" +) + +func Compress( + dst io.Writer, + compressionFormat string, + compressionLevel string, + isRegular bool, + recordSize int, +) (noop.Flusher, error) { + switch compressionFormat { + case config.CompressionFormatGZipKey: + fallthrough + case config.CompressionFormatParallelGZipKey: + if compressionFormat == config.CompressionFormatGZipKey { + if !isRegular { + maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize) + + if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header. + return nil, config.ErrCompressionFormatRequiresLargerRecordSize + } + } + + l := gzip.DefaultCompression + switch compressionLevel { + case config.CompressionLevelFastest: + l = gzip.BestSpeed + case config.CompressionLevelBalanced: + l = gzip.DefaultCompression + case config.CompressionLevelSmallest: + l = gzip.BestCompression + default: + return nil, config.ErrCompressionLevelUnsupported + } + + return gzip.NewWriterLevel(dst, l) + } + + if !isRegular { + return nil, config.ErrCompressionFormatOnlyRegularSupport // "device or resource busy" + } + + l := pgzip.DefaultCompression + switch compressionLevel { + case config.CompressionLevelFastest: + l = pgzip.BestSpeed + case config.CompressionLevelBalanced: + l = pgzip.DefaultCompression + case config.CompressionLevelSmallest: + l = pgzip.BestCompression + default: + return nil, config.ErrCompressionLevelUnsupported + } + + return pgzip.NewWriterLevel(dst, l) + case config.CompressionFormatLZ4Key: + l := lz4.Level5 + switch compressionLevel { + case config.CompressionLevelFastest: + l = lz4.Level1 + case config.CompressionLevelBalanced: + l = lz4.Level5 + case config.CompressionLevelSmallest: + l = lz4.Level9 + default: + return nil, config.ErrCompressionLevelUnsupported + } + + opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)} + if !isRegular { + maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize) + + if uint32(maxSize) < uint32(lz4.Block64Kb) { + return nil, config.ErrCompressionFormatRequiresLargerRecordSize + } + + if uint32(maxSize) < uint32(lz4.Block256Kb) { + opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb)) + } else if uint32(maxSize) < uint32(lz4.Block1Mb) { + opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb)) + } else if uint32(maxSize) < uint32(lz4.Block4Mb) { + opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb)) + } else { + opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb)) + } + } + + lz := lz4.NewWriter(dst) + if err := lz.Apply(opts...); err != nil { + return nil, err + } + + return noop.AddFlush(lz), nil + case config.CompressionFormatZStandardKey: + l := zstd.SpeedDefault + switch compressionLevel { + case config.CompressionLevelFastest: + l = zstd.SpeedFastest + case config.CompressionLevelBalanced: + l = zstd.SpeedDefault + case config.CompressionLevelSmallest: + l = zstd.SpeedBestCompression + default: + return nil, config.ErrCompressionLevelUnsupported + } + + opts := []zstd.EOption{zstd.WithEncoderLevel(l)} + if !isRegular { + opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize))) + } + + zz, err := zstd.NewWriter(dst, opts...) + if err != nil { + return nil, err + } + + return zz, nil + case config.CompressionFormatBrotliKey: + if !isRegular { + return nil, config.ErrCompressionFormatOnlyRegularSupport // "cannot allocate memory" + } + + l := brotli.DefaultCompression + switch compressionLevel { + case config.CompressionLevelFastest: + l = brotli.BestSpeed + case config.CompressionLevelBalanced: + l = brotli.DefaultCompression + case config.CompressionLevelSmallest: + l = brotli.BestCompression + default: + return nil, config.ErrCompressionLevelUnsupported + } + + br := brotli.NewWriterLevel(dst, l) + + return br, nil + case config.CompressionFormatBzip2Key: + fallthrough + case config.CompressionFormatBzip2ParallelKey: + l := bzip2.DefaultCompression + switch compressionLevel { + case config.CompressionLevelFastest: + l = bzip2.BestSpeed + case config.CompressionLevelBalanced: + l = bzip2.DefaultCompression + case config.CompressionLevelSmallest: + l = bzip2.BestCompression + default: + return nil, config.ErrCompressionLevelUnsupported + } + + bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{ + Level: l, + }) + if err != nil { + return nil, err + } + + return noop.AddFlush(bz), nil + case config.NoneKey: + return noop.AddFlush(noop.AddClose(dst)), nil + default: + return nil, config.ErrCompressionFormatUnsupported + } +} + +func getNearestPowerOf2Lower(n int) int { + return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/ +} + +func getNearestLogOf2Lower(n int) int { + return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/ +} diff --git a/internal/compression/decompress.go b/internal/compression/decompress.go index 7c1eb8d..d027378 100644 --- a/internal/compression/decompress.go +++ b/internal/compression/decompress.go @@ -54,6 +54,6 @@ func Decompress( case config.NoneKey: return io.NopCloser(src), nil default: - return nil, config.ErrUnsupportedCompressionFormat + return nil, config.ErrCompressionFormatUnsupported } } diff --git a/internal/encryption/decrypt.go b/internal/encryption/decrypt.go index 775b6dd..4b80246 100644 --- a/internal/encryption/decrypt.go +++ b/internal/encryption/decrypt.go @@ -46,7 +46,7 @@ func Decrypt( case config.NoneKey: return io.NopCloser(src), nil default: - return nil, config.ErrUnsupportedEncryptionFormat + return nil, config.ErrEncryptionFormatUnsupported } } @@ -136,6 +136,6 @@ func DecryptString( case config.NoneKey: return src, nil default: - return "", config.ErrUnsupportedEncryptionFormat + return "", config.ErrEncryptionFormatUnsupported } } diff --git a/internal/encryption/encrypt.go b/internal/encryption/encrypt.go new file mode 100644 index 0000000..437eeee --- /dev/null +++ b/internal/encryption/encrypt.go @@ -0,0 +1,127 @@ +package encryption + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "encoding/json" + "io" + + "filippo.io/age" + "github.com/pojntfx/stfs/internal/noop" + "github.com/pojntfx/stfs/internal/pax" + "github.com/pojntfx/stfs/pkg/config" + "golang.org/x/crypto/openpgp" +) + +func Encrypt( + dst io.Writer, + encryptionFormat string, + recipient interface{}, +) (io.WriteCloser, error) { + switch encryptionFormat { + case config.EncryptionFormatAgeKey: + recipient, ok := recipient.(*age.X25519Recipient) + if !ok { + return nil, config.ErrRecipientUnparsable + } + + return age.Encrypt(dst, recipient) + case config.EncryptionFormatPGPKey: + recipient, ok := recipient.(openpgp.EntityList) + if !ok { + return nil, config.ErrRecipientUnparsable + } + + return openpgp.Encrypt(dst, recipient, nil, nil, nil) + case config.NoneKey: + return noop.AddClose(dst), nil + default: + return nil, config.ErrEncryptionFormatUnsupported + } +} + +func EncryptHeader( + hdr *tar.Header, + encryptionFormat string, + recipient interface{}, +) error { + if encryptionFormat == config.NoneKey { + return nil + } + + newHdr := &tar.Header{ + Format: tar.FormatPAX, + Size: hdr.Size, + PAXRecords: map[string]string{}, + } + + wrappedHeader, err := json.Marshal(hdr) + if err != nil { + return err + } + + newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = EncryptString(string(wrappedHeader), encryptionFormat, recipient) + if err != nil { + return err + } + + *hdr = *newHdr + + return nil +} + +func EncryptString( + src string, + encryptionFormat string, + recipient interface{}, +) (string, error) { + switch encryptionFormat { + case config.EncryptionFormatAgeKey: + recipient, ok := recipient.(*age.X25519Recipient) + if !ok { + return "", config.ErrRecipientUnparsable + } + + out := &bytes.Buffer{} + w, err := age.Encrypt(out, recipient) + if err != nil { + return "", err + } + + if _, err := io.WriteString(w, src); err != nil { + return "", err + } + + if err := w.Close(); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(out.Bytes()), nil + case config.EncryptionFormatPGPKey: + recipient, ok := recipient.(openpgp.EntityList) + if !ok { + return "", config.ErrRecipientUnparsable + } + + out := &bytes.Buffer{} + w, err := openpgp.Encrypt(out, recipient, nil, nil, nil) + if err != nil { + return "", err + } + + if _, err := io.WriteString(w, src); err != nil { + return "", err + } + + if err := w.Close(); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(out.Bytes()), nil + case config.NoneKey: + return src, nil + default: + return "", config.ErrEncryptionFormatUnsupported + } +} diff --git a/internal/keys/identity.go b/internal/keys/identity.go index 83f9dcf..116a022 100644 --- a/internal/keys/identity.go +++ b/internal/keys/identity.go @@ -65,7 +65,7 @@ func ParseIdentity( case config.NoneKey: return privkey, nil default: - return nil, config.ErrUnsupportedEncryptionFormat + return nil, config.ErrEncryptionFormatUnsupported } } @@ -82,6 +82,6 @@ func ParseSignerIdentity( case config.NoneKey: return privkey, nil default: - return nil, config.ErrUnsupportedSignatureFormat + return nil, config.ErrSignatureFormatUnsupported } } diff --git a/internal/keys/recipient.go b/internal/keys/recipient.go index cec9000..04fa4c3 100644 --- a/internal/keys/recipient.go +++ b/internal/keys/recipient.go @@ -21,7 +21,7 @@ func ParseRecipient( case config.NoneKey: return pubkey, nil default: - return nil, config.ErrUnsupportedEncryptionFormat + return nil, config.ErrEncryptionFormatUnsupported } } @@ -42,6 +42,6 @@ func ParseSignerRecipient( case config.NoneKey: return pubkey, nil default: - return nil, config.ErrUnsupportedSignatureFormat + return nil, config.ErrSignatureFormatUnsupported } } diff --git a/internal/signature/sign.go b/internal/signature/sign.go new file mode 100644 index 0000000..0d5cb6b --- /dev/null +++ b/internal/signature/sign.go @@ -0,0 +1,159 @@ +package signature + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "encoding/json" + "io" + + "aead.dev/minisign" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/pojntfx/stfs/internal/pax" + "github.com/pojntfx/stfs/pkg/config" +) + +func Sign( + src io.Reader, + isRegular bool, + signatureFormat string, + identity interface{}, +) (io.Reader, func() (string, error), error) { + switch signatureFormat { + case config.SignatureFormatMinisignKey: + if !isRegular { + return nil, nil, config.ErrSignatureFormatOnlyRegularSupport + } + + identity, ok := identity.(minisign.PrivateKey) + if !ok { + return nil, nil, config.ErrIdentityUnparsable + } + + signer := minisign.NewReader(src) + + return signer, func() (string, error) { + return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil + }, nil + case config.SignatureFormatPGPKey: + identities, ok := identity.(openpgp.EntityList) + if !ok { + return nil, nil, config.ErrIdentityUnparsable + } + + if len(identities) < 1 { + return nil, nil, config.ErrIdentityUnparsable + } + + // See openpgp.DetachSign + var c *packet.Config + signingKey, ok := identities[0].SigningKeyById(c.Now(), c.SigningKey()) + if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil { + return nil, nil, config.ErrIdentityUnparsable + } + + sig := new(packet.Signature) + sig.SigType = packet.SigTypeBinary + sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo + sig.Hash = c.Hash() + sig.CreationTime = c.Now() + sigLifetimeSecs := c.SigLifetime() + sig.SigLifetimeSecs = &sigLifetimeSecs + sig.IssuerKeyId = &signingKey.PrivateKey.KeyId + + hash := sig.Hash.New() + + return io.TeeReader(src, hash), func() (string, error) { + if err := sig.Sign(hash, signingKey.PrivateKey, c); err != nil { + return "", err + } + + out := &bytes.Buffer{} + if err := sig.Serialize(out); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(out.Bytes()), nil + }, nil + case config.NoneKey: + return src, func() (string, error) { + return "", nil + }, nil + default: + return nil, nil, config.ErrSignatureFormatUnsupported + } +} + +func SignHeader( + hdr *tar.Header, + isRegular bool, + signatureFormat string, + identity interface{}, +) error { + if signatureFormat == config.NoneKey { + return nil + } + + newHdr := &tar.Header{ + Format: tar.FormatPAX, + Size: hdr.Size, + PAXRecords: map[string]string{}, + } + + wrappedHeader, err := json.Marshal(hdr) + if err != nil { + return err + } + + newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader) + newHdr.PAXRecords[pax.STFSRecordSignature], err = SignString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity) + if err != nil { + return err + } + + *hdr = *newHdr + + return nil +} + +func SignString( + src string, + isRegular bool, + signatureFormat string, + identity interface{}, +) (string, error) { + switch signatureFormat { + case config.SignatureFormatMinisignKey: + if !isRegular { + return "", config.ErrSignatureFormatOnlyRegularSupport + } + + identity, ok := identity.(minisign.PrivateKey) + if !ok { + return "", config.ErrIdentityUnparsable + } + + return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil + case config.SignatureFormatPGPKey: + identities, ok := identity.(openpgp.EntityList) + if !ok { + return "", config.ErrIdentityUnparsable + } + + if len(identities) < 1 { + return "", config.ErrIdentityUnparsable + } + + out := &bytes.Buffer{} + if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(out.Bytes()), nil + case config.NoneKey: + return src, nil + default: + return "", config.ErrSignatureFormatUnsupported + } +} diff --git a/internal/signature/verify.go b/internal/signature/verify.go index 286b4c6..31b4362 100644 --- a/internal/signature/verify.go +++ b/internal/signature/verify.go @@ -84,7 +84,7 @@ func Verify( return nil }, nil default: - return nil, nil, config.ErrUnsupportedSignatureFormat + return nil, nil, config.ErrSignatureFormatUnsupported } } @@ -190,6 +190,6 @@ func VerifyString( case config.NoneKey: return nil default: - return config.ErrUnsupportedSignatureFormat + return config.ErrSignatureFormatUnsupported } } diff --git a/internal/suffix/add.go b/internal/suffix/add.go new file mode 100644 index 0000000..0b60f80 --- /dev/null +++ b/internal/suffix/add.go @@ -0,0 +1,37 @@ +package suffix + +import "github.com/pojntfx/stfs/pkg/config" + +func AddSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) { + switch compressionFormat { + case config.CompressionFormatGZipKey: + fallthrough + case config.CompressionFormatParallelGZipKey: + name += CompressionFormatGZipSuffix + case config.CompressionFormatLZ4Key: + name += CompressionFormatLZ4Suffix + case config.CompressionFormatZStandardKey: + name += CompressionFormatZStandardSuffix + case config.CompressionFormatBrotliKey: + name += CompressionFormatBrotliSuffix + case config.CompressionFormatBzip2Key: + fallthrough + case config.CompressionFormatBzip2ParallelKey: + name += CompressionFormatBzip2Suffix + case config.NoneKey: + default: + return "", config.ErrCompressionFormatUnsupported + } + + switch encryptionFormat { + case config.EncryptionFormatAgeKey: + name += EncryptionFormatAgeSuffix + case config.EncryptionFormatPGPKey: + name += EncryptionFormatPGPSuffix + case config.NoneKey: + default: + return "", config.ErrEncryptionFormatUnsupported + } + + return name, nil +} diff --git a/internal/suffix/remove.go b/internal/suffix/remove.go index acc27dd..c46914f 100644 --- a/internal/suffix/remove.go +++ b/internal/suffix/remove.go @@ -14,7 +14,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string name = strings.TrimSuffix(name, EncryptionFormatPGPSuffix) case config.NoneKey: default: - return "", config.ErrUnsupportedEncryptionFormat + return "", config.ErrEncryptionFormatUnsupported } switch compressionFormat { @@ -34,7 +34,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string name = strings.TrimSuffix(name, CompressionFormatBzip2Suffix) case config.NoneKey: default: - return "", config.ErrUnsupportedCompressionFormat + return "", config.ErrCompressionFormatUnsupported } return name, nil diff --git a/internal/tape/read.go b/internal/tape/read.go index 324fe0e..319430b 100644 --- a/internal/tape/read.go +++ b/internal/tape/read.go @@ -2,15 +2,15 @@ package tape import "os" -func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) { - fileDescription, err := os.Stat(tape) +func OpenTapeReadOnly(drive string) (f *os.File, isRegular bool, err error) { + fileDescription, err := os.Stat(drive) if err != nil { return nil, false, err } isRegular = fileDescription.Mode().IsRegular() if isRegular { - f, err = os.Open(tape) + f, err = os.Open(drive) if err != nil { return f, isRegular, err } @@ -18,7 +18,7 @@ func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) { return f, isRegular, nil } - f, err = os.OpenFile(tape, os.O_RDONLY, os.ModeCharDevice) + f, err = os.OpenFile(drive, os.O_RDONLY, os.ModeCharDevice) if err != nil { return f, isRegular, err } diff --git a/internal/tape/write.go b/internal/tape/write.go new file mode 100644 index 0000000..b242fb0 --- /dev/null +++ b/internal/tape/write.go @@ -0,0 +1,79 @@ +package tape + +import ( + "archive/tar" + "bufio" + "os" + + "github.com/pojntfx/stfs/internal/controllers" + "github.com/pojntfx/stfs/internal/counters" +) + +func OpenTapeWriteOnly(drive string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) { + stat, err := os.Stat(drive) + if err == nil { + isRegular = stat.Mode().IsRegular() + } else { + if os.IsNotExist(err) { + isRegular = true + } else { + return nil, false, nil, err + } + } + + var f *os.File + if isRegular { + f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return nil, false, nil, err + } + + // No need to go to end manually due to `os.O_APPEND` + } else { + f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) + if err != nil { + return nil, false, nil, err + } + + if !overwrite { + // Go to end of tape + if err := controllers.GoToEndOfTape(f); err != nil { + return nil, false, nil, err + } + } + } + + var bw *bufio.Writer + var counter *counters.CounterWriter + if isRegular { + tw = tar.NewWriter(f) + } else { + bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize) + counter = &counters.CounterWriter{Writer: bw, BytesRead: 0} + tw = tar.NewWriter(counter) + } + + return tw, isRegular, func(dirty *bool) error { + // Only write the trailer if we wrote to the archive + if *dirty { + if err := tw.Close(); err != nil { + return err + } + + if !isRegular { + if controllers.BlockSize*recordSize-counter.BytesRead > 0 { + // Fill the rest of the record with zeros + if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil { + return err + } + } + + if err := bw.Flush(); err != nil { + return err + } + } + } + + return f.Close() + }, nil +} diff --git a/pkg/config/constants.go b/pkg/config/constants.go index ef6dc59..531203b 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -16,4 +16,8 @@ const ( SignatureFormatMinisignKey = "minisign" SignatureFormatPGPKey = "pgp" + + CompressionLevelFastest = "fastest" + CompressionLevelBalanced = "balanced" + CompressionLevelSmallest = "smallest" ) diff --git a/pkg/config/error.go b/pkg/config/error.go index 3e1f887..59ba2ef 100644 --- a/pkg/config/error.go +++ b/pkg/config/error.go @@ -3,18 +3,22 @@ package config import "errors" var ( - ErrUnsupportedCompressionFormat = errors.New("unsupported compression format") - ErrUnsupportedEncryptionFormat = errors.New("unsupported encryption format") - ErrUnsupportedSignatureFormat = errors.New("unsupported signature format") + ErrEncryptionFormatUnsupported = errors.New("unsupported encryption format") ErrIdentityUnparsable = errors.New("recipient could not be parsed") ErrRecipientUnparsable = errors.New("recipient could not be parsed") ErrEmbeddedHeaderMissing = errors.New("embedded header is missing") + ErrSignatureFormatUnsupported = errors.New("unsupported signature format") ErrSignatureFormatOnlyRegularSupport = errors.New("this signature format only supports regular files, not i.e. tape drives") - ErrSignatureInvalid = errors.New("signature invalid") - ErrSignatureMissing = errors.New("signature missing") + ErrSignatureInvalid = errors.New("signature is invalid") + ErrSignatureMissing = errors.New("signature is missing") ErrKeygenForFormatUnsupported = errors.New("can not generate keys for this format") + + ErrCompressionFormatUnsupported = errors.New("unsupported compression format") + ErrCompressionFormatOnlyRegularSupport = errors.New("this compression format only supports regular files, not i.e. tape drives") + ErrCompressionFormatRequiresLargerRecordSize = errors.New("this compression format requires a larger record size") + ErrCompressionLevelUnsupported = errors.New("compression level is unsupported") ) diff --git a/pkg/operations/archive.go b/pkg/operations/archive.go new file mode 100644 index 0000000..6dbf401 --- /dev/null +++ b/pkg/operations/archive.go @@ -0,0 +1,281 @@ +package operations + +import ( + "archive/tar" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + + "github.com/pojntfx/stfs/internal/adapters" + "github.com/pojntfx/stfs/internal/compression" + "github.com/pojntfx/stfs/internal/controllers" + "github.com/pojntfx/stfs/internal/counters" + "github.com/pojntfx/stfs/internal/encryption" + "github.com/pojntfx/stfs/internal/formatting" + "github.com/pojntfx/stfs/internal/pax" + "github.com/pojntfx/stfs/internal/signature" + "github.com/pojntfx/stfs/internal/suffix" + "github.com/pojntfx/stfs/internal/tape" + "github.com/pojntfx/stfs/pkg/config" +) + +func Archive( + state config.StateConfig, + pipes config.PipeConfig, + crypto config.CryptoConfig, + + recordSize int, + from string, + overwrite bool, + compressionLevel string, +) ([]*tar.Header, error) { + dirty := false + tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite) + if err != nil { + return []*tar.Header{}, err + } + + if overwrite { + if isRegular { + if err := cleanup(&dirty); err != nil { // dirty will always be false here + return []*tar.Header{}, err + } + + f, err := os.OpenFile(state.Drive, os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return []*tar.Header{}, err + } + + // Clear the file's content + if err := f.Truncate(0); err != nil { + return []*tar.Header{}, err + } + + if err := f.Close(); err != nil { + return []*tar.Header{}, err + } + + tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite) + if err != nil { + return []*tar.Header{}, err + } + } else { + if err := cleanup(&dirty); err != nil { // dirty will always be false here + return []*tar.Header{}, err + } + + f, err := os.OpenFile(state.Drive, os.O_WRONLY, os.ModeCharDevice) + if err != nil { + return []*tar.Header{}, err + } + + // Seek to the start of the tape + if err := controllers.SeekToRecordOnTape(f, 0); err != nil { + return []*tar.Header{}, err + } + + if err := f.Close(); err != nil { + return []*tar.Header{}, err + } + + tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite) + if err != nil { + return []*tar.Header{}, err + } + } + } + + defer cleanup(&dirty) + + headers := []*tar.Header{} + first := true + return headers, filepath.Walk(from, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + link := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + + if err := adapters.EnhanceHeader(path, hdr); err != nil { + return err + } + + hdr.Name = path + hdr.Format = tar.FormatPAX + + if info.Mode().IsRegular() { + // Get the compressed size for the header + fileSizeCounter := &counters.CounterWriter{ + Writer: io.Discard, + } + + encryptor, err := encryption.Encrypt(fileSizeCounter, pipes.Encryption, crypto.Recipient) + if err != nil { + return err + } + + compressor, err := compression.Compress( + encryptor, + pipes.Encryption, + compressionLevel, + isRegular, + recordSize, + ) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + + signer, sign, err := signature.Sign(file, isRegular, pipes.Signature, crypto.Identity) + if err != nil { + return err + } + + if isRegular { + if _, err := io.Copy(compressor, signer); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(compressor, signer, buf); err != nil { + return err + } + } + + if err := file.Close(); err != nil { + return err + } + + if err := compressor.Flush(); err != nil { + return err + } + + if err := compressor.Close(); err != nil { + return err + } + + if err := encryptor.Close(); err != nil { + return err + } + + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) + signature, err := sign() + if err != nil { + return err + } + + if signature != "" { + hdr.PAXRecords[pax.STFSRecordSignature] = signature + } + hdr.Size = int64(fileSizeCounter.BytesRead) + + hdr.Name, err = suffix.AddSuffix(hdr.Name, pipes.Compression, pipes.Encryption) + if err != nil { + return err + } + } + + if first { + if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { + return err + } + + first = false + } + + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { + return err + } + + hdrToAppend := *hdr + headers = append(headers, &hdrToAppend) + + if err := signature.SignHeader(hdr, isRegular, pipes.Signature, crypto.Identity); err != nil { + return err + } + + if err := encryption.EncryptHeader(hdr, pipes.Encryption, crypto.Recipient); err != nil { + return err + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + if !info.Mode().IsRegular() { + return nil + } + + // Compress and write the file + encryptor, err := encryption.Encrypt(tw, pipes.Encryption, crypto.Recipient) + if err != nil { + return err + } + + compressor, err := compression.Compress( + encryptor, + pipes.Compression, + compressionLevel, + isRegular, + recordSize, + ) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + + if isRegular { + if _, err := io.Copy(compressor, file); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(compressor, file, buf); err != nil { + return err + } + } + + if err := file.Close(); err != nil { + return err + } + + if err := compressor.Flush(); err != nil { + return err + } + + if err := compressor.Close(); err != nil { + return err + } + + if err := encryptor.Close(); err != nil { + return err + } + + dirty = true + + return nil + }) +} diff --git a/pkg/operations/operations.go b/pkg/operations/operations.go index 7449a60..f810bd5 100644 --- a/pkg/operations/operations.go +++ b/pkg/operations/operations.go @@ -1,22 +1,9 @@ package operations import ( - "archive/tar" - "github.com/pojntfx/stfs/pkg/config" ) -func Archive( - state config.StateConfig, - pipes config.PipeConfig, - crypto config.CryptoConfig, - - recordSize int, - from string, - overwrite bool, - compressionLevel string, -) ([]*tar.Header, error) - func Restore( state config.StateConfig, pipes config.PipeConfig, @@ -26,7 +13,9 @@ func Restore( from string, to string, flatten bool, -) error +) error { + return nil +} func Update( state config.StateConfig, @@ -37,7 +26,9 @@ func Update( from string, overwrite bool, compressionLevel string, -) error +) error { + return nil +} func Delete( state config.StateConfig, @@ -46,7 +37,9 @@ func Delete( recordSize int, name string, -) error +) error { + return nil +} func Move( state config.StateConfig, @@ -56,4 +49,6 @@ func Move( recordSize int, from string, to string, -) error +) error { + return nil +}