mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Decompose archive func

Browse Source
This commit is contained in:
Felicitas Pojtinger
2021-12-07 21:12:23 +01:00
parent b700a767cb
commit db69d5f68c
20 changed files with 966 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

800
cmd/stbak/cmd/archive.go
View File

@@ -2,39 +2,16 @@ package cmd
import ( import (
"archive/tar" "archive/tar"
"bytes"
"compress/gzip"
"context" "context"
"encoding/base64"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"io/fs"
"io/ioutil" "io/ioutil"
"math"
"os" "os"
"path/filepath"
"strconv"
"aead.dev/minisign"
"filippo.io/age"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
"github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/recovery" "github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -50,14 +27,10 @@ const (
recipientFlag = "recipient" recipientFlag = "recipient"
identityFlag = "identity" identityFlag = "identity"
passwordFlag = "password" passwordFlag = "password"
compressionLevelFastest = "fastest"
compressionLevelBalanced = "balanced"
compressionLevelSmallest = "smallest"
) )
var ( var (
knownCompressionLevels = []string{compressionLevelFastest, compressionLevelBalanced, compressionLevelSmallest} knownCompressionLevels = []string{config.CompressionLevelFastest, config.CompressionLevelBalanced, config.CompressionLevelSmallest}
errUnknownCompressionLevel = errors.New("unknown compression level") errUnknownCompressionLevel = errors.New("unknown compression level")
errUnsupportedCompressionLevel = errors.New("unsupported compression level") errUnsupportedCompressionLevel = errors.New("unsupported compression level")
@@ -121,7 +94,7 @@ var archiveCmd = &cobra.Command{
return err return err
} }
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil { if err != nil {
return err return err
} }
@@ -136,21 +109,27 @@ var archiveCmd = &cobra.Command{
return err return err
} }
hdrs, err := archive( hdrs, err := operations.Archive(
viper.GetString(driveFlag), config.StateConfig{
Drive: viper.GetString(driveFlag),
Metadata: viper.GetString(metadataFlag),
},
config.PipeConfig{
Compression: viper.GetString(compressionFlag),
Encryption: viper.GetString(encryptionFlag),
Signature: viper.GetString(signatureFlag),
},
config.CryptoConfig{
Recipient: recipient,
Identity: identity,
Password: viper.GetString(passwordFlag),
},
viper.GetInt(recordSizeFlag), viper.GetInt(recordSizeFlag),
viper.GetString(fromFlag), viper.GetString(fromFlag),
viper.GetBool(overwriteFlag), viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(compressionLevelFlag), viper.GetString(compressionLevelFlag),
viper.GetString(encryptionFlag),
recipient,
viper.GetString(signatureFlag),
identity,
) )
if err != nil {
return err
}
return recovery.Index( return recovery.Index(
config.StateConfig{ config.StateConfig{
@@ -194,267 +173,6 @@ var archiveCmd = &cobra.Command{
}, },
} }
func archive(
tape string,
recordSize int,
src string,
overwrite bool,
compressionFormat string,
compressionLevel string,
encryptionFormat string,
recipient interface{},
signatureFormat string,
identity interface{},
) ([]*tar.Header, error) {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
if overwrite {
if isRegular {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(tape, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return []*tar.Header{}, err
}
// Clear the file's content
if err := f.Truncate(0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
} else {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(tape, os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return []*tar.Header{}, err
}
// Seek to the start of the tape
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
}
}
defer cleanup(&dirty)
headers := []*tar.Header{}
first := true
return headers, filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
hdr, err := tar.FileInfoHeader(info, link)
if err != nil {
return err
}
if err := adapters.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Format = tar.FormatPAX
if info.Mode().IsRegular() {
// Get the compressed size for the header
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
signer, sign, err := sign(file, isRegular, signatureFormat, identity)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, signer); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
signature, err := sign()
if err != nil {
return err
}
if signature != "" {
hdr.PAXRecords[pax.STFSRecordSignature] = signature
}
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat)
if err != nil {
return err
}
}
if first {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
first = false
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
return err
}
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
// Compress and write the file
encryptor, err := encrypt(tw, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
dirty = true
return nil
})
}
func checkKeyAccessible(encryptionFormat string, pathToKey string) error { func checkKeyAccessible(encryptionFormat string, pathToKey string) error {
if encryptionFormat == noneKey { if encryptionFormat == noneKey {
return nil return nil
@@ -491,489 +209,11 @@ func checkCompressionLevel(compressionLevel string) error {
return nil return nil
} }
func encryptHeader(
hdr *tar.Header,
encryptionFormat string,
recipient interface{},
) error {
if encryptionFormat == noneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = encryptString(string(wrappedHeader), encryptionFormat, recipient)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func signHeader(
hdr *tar.Header,
isRegular bool,
signatureFormat string,
identity interface{},
) error {
if signatureFormat == noneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
newHdr.PAXRecords[pax.STFSRecordSignature], err = signString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func addSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
name += compressionFormatGZipSuffix
case compressionFormatLZ4Key:
name += compressionFormatLZ4Suffix
case compressionFormatZStandardKey:
name += compressionFormatZStandardSuffix
case compressionFormatBrotliKey:
name += compressionFormatBrotliSuffix
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
name += compressionFormatBzip2Suffix
case noneKey:
default:
return "", errUnsupportedCompressionFormat
}
switch encryptionFormat {
case encryptionFormatAgeKey:
name += encryptionFormatAgeSuffix
case encryptionFormatPGPKey:
name += encryptionFormatPGPSuffix
case noneKey:
default:
return "", errUnsupportedEncryptionFormat
}
return name, nil
}
func parseRecipient(
encryptionFormat string,
pubkey []byte,
) (interface{}, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
return age.ParseX25519Recipient(string(pubkey))
case encryptionFormatPGPKey:
return openpgp.ReadKeyRing(bytes.NewBuffer(pubkey))
case noneKey:
return pubkey, nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func encrypt(
dst io.Writer,
encryptionFormat string,
recipient interface{},
) (io.WriteCloser, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return nil, errRecipientUnparsable
}
return age.Encrypt(dst, recipient)
case encryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return nil, errRecipientUnparsable
}
return openpgp.Encrypt(dst, recipient, nil, nil, nil)
case noneKey:
return noop.AddClose(dst), nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func sign(
src io.Reader,
isRegular bool,
signatureFormat string,
identity interface{},
) (io.Reader, func() (string, error), error) {
switch signatureFormat {
case signatureFormatMinisignKey:
if !isRegular {
return nil, nil, errSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return nil, nil, errIdentityUnparsable
}
signer := minisign.NewReader(src)
return signer, func() (string, error) {
return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
}, nil
case signatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return nil, nil, errIdentityUnparsable
}
if len(identities) < 1 {
return nil, nil, errIdentityUnparsable
}
// See openpgp.DetachSign
var config *packet.Config
signingKey, ok := identities[0].SigningKeyById(config.Now(), config.SigningKey())
if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
return nil, nil, errIdentityUnparsable
}
sig := new(packet.Signature)
sig.SigType = packet.SigTypeBinary
sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
sig.Hash = config.Hash()
sig.CreationTime = config.Now()
sigLifetimeSecs := config.SigLifetime()
sig.SigLifetimeSecs = &sigLifetimeSecs
sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
hash := sig.Hash.New()
return io.TeeReader(src, hash), func() (string, error) {
if err := sig.Sign(hash, signingKey.PrivateKey, config); err != nil {
return "", err
}
out := &bytes.Buffer{}
if err := sig.Serialize(out); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
}, nil
case noneKey:
return src, func() (string, error) {
return "", nil
}, nil
default:
return nil, nil, errUnsupportedSignatureFormat
}
}
func encryptString(
src string,
encryptionFormat string,
recipient interface{},
) (string, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return "", errRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := age.Encrypt(out, recipient)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case encryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return "", errRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case noneKey:
return src, nil
default:
return "", errUnsupportedEncryptionFormat
}
}
func signString(
src string,
isRegular bool,
signatureFormat string,
identity interface{},
) (string, error) {
switch signatureFormat {
case signatureFormatMinisignKey:
if !isRegular {
return "", errSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return "", errIdentityUnparsable
}
return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
case signatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return "", errIdentityUnparsable
}
if len(identities) < 1 {
return "", errIdentityUnparsable
}
out := &bytes.Buffer{}
if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case noneKey:
return src, nil
default:
return "", errUnsupportedSignatureFormat
}
}
func compress(
dst io.Writer,
compressionFormat string,
compressionLevel string,
isRegular bool,
recordSize int,
) (noop.Flusher, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
if compressionFormat == compressionFormatGZipKey {
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
return nil, errCompressionFormatRequiresLargerRecordSize
}
}
l := gzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = gzip.BestSpeed
case compressionLevelBalanced:
l = gzip.DefaultCompression
case compressionLevelSmallest:
l = gzip.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
return gzip.NewWriterLevel(dst, l)
}
if !isRegular {
return nil, errCompressionFormatOnlyRegularSupport // "device or resource busy"
}
l := pgzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = pgzip.BestSpeed
case compressionLevelBalanced:
l = pgzip.DefaultCompression
case compressionLevelSmallest:
l = pgzip.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
return pgzip.NewWriterLevel(dst, l)
case compressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
case compressionLevelFastest:
l = lz4.Level1
case compressionLevelBalanced:
l = lz4.Level5
case compressionLevelSmallest:
l = lz4.Level9
default:
return nil, errUnsupportedCompressionLevel
}
opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if uint32(maxSize) < uint32(lz4.Block64Kb) {
return nil, errCompressionFormatRequiresLargerRecordSize
}
if uint32(maxSize) < uint32(lz4.Block256Kb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
} else {
opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
}
}
lz := lz4.NewWriter(dst)
if err := lz.Apply(opts...); err != nil {
return nil, err
}
return noop.AddFlush(lz), nil
case compressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
case compressionLevelFastest:
l = zstd.SpeedFastest
case compressionLevelBalanced:
l = zstd.SpeedDefault
case compressionLevelSmallest:
l = zstd.SpeedBestCompression
default:
return nil, errUnsupportedCompressionLevel
}
opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
if !isRegular {
opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
}
zz, err := zstd.NewWriter(dst, opts...)
if err != nil {
return nil, err
}
return zz, nil
case compressionFormatBrotliKey:
if !isRegular {
return nil, errCompressionFormatOnlyRegularSupport // "cannot allocate memory"
}
l := brotli.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = brotli.BestSpeed
case compressionLevelBalanced:
l = brotli.DefaultCompression
case compressionLevelSmallest:
l = brotli.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
br := brotli.NewWriterLevel(dst, l)
return br, nil
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
l := bzip2.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = bzip2.BestSpeed
case compressionLevelBalanced:
l = bzip2.DefaultCompression
case compressionLevelSmallest:
l = bzip2.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
Level: l,
})
if err != nil {
return nil, err
}
return noop.AddFlush(bz), nil
case noneKey:
return noop.AddFlush(noop.AddClose(dst)), nil
default:
return nil, errUnsupportedCompressionFormat
}
}
func getNearestPowerOf2Lower(n int) int {
return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func getNearestLogOf2Lower(n int) int {
return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func init() { func init() {
archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
archiveCmd.PersistentFlags().StringP(fromFlag, "f", ".", "File or directory to archive") archiveCmd.PersistentFlags().StringP(fromFlag, "f", ".", "File or directory to archive")
archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file") archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file")
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
archiveCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for") archiveCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
archiveCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with") archiveCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
archiveCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key") archiveCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")

86
cmd/stbak/cmd/delete.go
View File

@@ -2,18 +2,17 @@ package cmd
import ( import (
"archive/tar" "archive/tar"
"bufio"
"context" "context"
"os"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/converters" "github.com/pojntfx/stfs/internal/converters"
"github.com/pojntfx/stfs/internal/counters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery" "github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -54,7 +53,7 @@ var deleteCmd = &cobra.Command{
return err return err
} }
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil { if err != nil {
return err return err
} }
@@ -83,7 +82,7 @@ var deleteCmd = &cobra.Command{
} }
func delete( func delete(
tape string, drive string,
recordSize int, recordSize int,
metadata string, metadata string,
name string, name string,
@@ -93,7 +92,7 @@ func delete(
identity interface{}, identity interface{},
) error { ) error {
dirty := false dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil { if err != nil {
return err return err
} }
@@ -142,11 +141,11 @@ func delete(
hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1 hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err return err
} }
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err return err
} }
@@ -204,75 +203,6 @@ func delete(
) )
} }
func openTapeWriter(tape string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
stat, err := os.Stat(tape)
if err == nil {
isRegular = stat.Mode().IsRegular()
} else {
if os.IsNotExist(err) {
isRegular = true
} else {
return nil, false, nil, err
}
}
var f *os.File
if isRegular {
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return nil, false, nil, err
}
// No need to go to end manually due to `os.O_APPEND`
} else {
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return nil, false, nil, err
}
if !overwrite {
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return nil, false, nil, err
}
}
}
var bw *bufio.Writer
var counter *counters.CounterWriter
if isRegular {
tw = tar.NewWriter(f)
} else {
bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
tw = tar.NewWriter(counter)
}
return tw, isRegular, func(dirty *bool) error {
// Only write the trailer if we wrote to the archive
if *dirty {
if err := tw.Close(); err != nil {
return err
}
if !isRegular {
if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
// Fill the rest of the record with zeros
if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
return err
}
}
if err := bw.Flush(); err != nil {
return err
}
}
}
return f.Close()
}, nil
}
func init() { func init() {
deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove") deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove")

13
cmd/stbak/cmd/move.go
View File

@@ -7,10 +7,13 @@ import (
"github.com/pojntfx/stfs/internal/converters" "github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery" "github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -47,7 +50,7 @@ var moveCmd = &cobra.Command{
return err return err
} }
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil { if err != nil {
return err return err
} }
@@ -77,7 +80,7 @@ var moveCmd = &cobra.Command{
} }
func move( func move(
tape string, drive string,
recordSize int, recordSize int,
metadata string, metadata string,
src string, src string,
@@ -88,7 +91,7 @@ func move(
identity interface{}, identity interface{},
) error { ) error {
dirty := false dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil { if err != nil {
return err return err
} }
@@ -139,11 +142,11 @@ func move(
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate
hdr.PAXRecords[pax.STFSRecordReplacesName] = dbhdr.Name hdr.PAXRecords[pax.STFSRecordReplacesName] = dbhdr.Name
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err return err
} }
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err return err
} }

33
cmd/stbak/cmd/update.go
View File

@@ -11,12 +11,17 @@ import (
"strconv" "strconv"
"github.com/pojntfx/stfs/internal/adapters" "github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/controllers" "github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters" "github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery" "github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -67,7 +72,7 @@ var updateCmd = &cobra.Command{
return err return err
} }
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey) recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil { if err != nil {
return err return err
} }
@@ -141,7 +146,7 @@ var updateCmd = &cobra.Command{
} }
func update( func update(
tape string, drive string,
recordSize int, recordSize int,
src string, src string,
replacesContent bool, replacesContent bool,
@@ -153,7 +158,7 @@ func update(
identity interface{}, identity interface{},
) ([]*tar.Header, error) { ) ([]*tar.Header, error) {
dirty := false dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false) tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil { if err != nil {
return []*tar.Header{}, err return []*tar.Header{}, err
} }
@@ -196,12 +201,12 @@ func update(
Writer: io.Discard, Writer: io.Discard,
} }
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient) encryptor, err := encryption.Encrypt(fileSizeCounter, encryptionFormat, recipient)
if err != nil { if err != nil {
return err return err
} }
compressor, err := compress( compressor, err := compression.Compress(
encryptor, encryptor,
compressionFormat, compressionFormat,
compressionLevel, compressionLevel,
@@ -217,7 +222,7 @@ func update(
return err return err
} }
signer, sign, err := sign(file, isRegular, signatureFormat, identity) signer, sign, err := signature.Sign(file, isRegular, signatureFormat, identity)
if err != nil { if err != nil {
return err return err
} }
@@ -263,7 +268,7 @@ func update(
} }
hdr.Size = int64(fileSizeCounter.BytesRead) hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat) hdr.Name, err = suffix.AddSuffix(hdr.Name, compressionFormat, encryptionFormat)
if err != nil { if err != nil {
return err return err
} }
@@ -287,11 +292,11 @@ func update(
hdrToAppend := *hdr hdrToAppend := *hdr
headers = append(headers, &hdrToAppend) headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err return err
} }
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err return err
} }
@@ -304,12 +309,12 @@ func update(
} }
// Compress and write the file // Compress and write the file
encryptor, err := encrypt(tw, encryptionFormat, recipient) encryptor, err := encryption.Encrypt(tw, encryptionFormat, recipient)
if err != nil { if err != nil {
return err return err
} }
compressor, err := compress( compressor, err := compression.Compress(
encryptor, encryptor,
compressionFormat, compressionFormat,
compressionLevel, compressionLevel,
@@ -361,11 +366,11 @@ func update(
hdrToAppend := *hdr hdrToAppend := *hdr
headers = append(headers, &hdrToAppend) headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil { if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err return err
} }
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil { if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err return err
} }
@@ -384,7 +389,7 @@ func init() {
updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update") updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update")
updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file") updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels)) updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
updateCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for") updateCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
updateCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with") updateCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
updateCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key") updateCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")

188
internal/compression/compress.go Normal file
View File

@@ -0,0 +1,188 @@
package compression
import (
"compress/gzip"
"io"
"math"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/pkg/config"
)
func Compress(
dst io.Writer,
compressionFormat string,
compressionLevel string,
isRegular bool,
recordSize int,
) (noop.Flusher, error) {
switch compressionFormat {
case config.CompressionFormatGZipKey:
fallthrough
case config.CompressionFormatParallelGZipKey:
if compressionFormat == config.CompressionFormatGZipKey {
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
return nil, config.ErrCompressionFormatRequiresLargerRecordSize
}
}
l := gzip.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = gzip.BestSpeed
case config.CompressionLevelBalanced:
l = gzip.DefaultCompression
case config.CompressionLevelSmallest:
l = gzip.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
return gzip.NewWriterLevel(dst, l)
}
if !isRegular {
return nil, config.ErrCompressionFormatOnlyRegularSupport // "device or resource busy"
}
l := pgzip.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = pgzip.BestSpeed
case config.CompressionLevelBalanced:
l = pgzip.DefaultCompression
case config.CompressionLevelSmallest:
l = pgzip.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
return pgzip.NewWriterLevel(dst, l)
case config.CompressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
case config.CompressionLevelFastest:
l = lz4.Level1
case config.CompressionLevelBalanced:
l = lz4.Level5
case config.CompressionLevelSmallest:
l = lz4.Level9
default:
return nil, config.ErrCompressionLevelUnsupported
}
opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if uint32(maxSize) < uint32(lz4.Block64Kb) {
return nil, config.ErrCompressionFormatRequiresLargerRecordSize
}
if uint32(maxSize) < uint32(lz4.Block256Kb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
} else {
opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
}
}
lz := lz4.NewWriter(dst)
if err := lz.Apply(opts...); err != nil {
return nil, err
}
return noop.AddFlush(lz), nil
case config.CompressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
case config.CompressionLevelFastest:
l = zstd.SpeedFastest
case config.CompressionLevelBalanced:
l = zstd.SpeedDefault
case config.CompressionLevelSmallest:
l = zstd.SpeedBestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
if !isRegular {
opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
}
zz, err := zstd.NewWriter(dst, opts...)
if err != nil {
return nil, err
}
return zz, nil
case config.CompressionFormatBrotliKey:
if !isRegular {
return nil, config.ErrCompressionFormatOnlyRegularSupport // "cannot allocate memory"
}
l := brotli.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = brotli.BestSpeed
case config.CompressionLevelBalanced:
l = brotli.DefaultCompression
case config.CompressionLevelSmallest:
l = brotli.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
br := brotli.NewWriterLevel(dst, l)
return br, nil
case config.CompressionFormatBzip2Key:
fallthrough
case config.CompressionFormatBzip2ParallelKey:
l := bzip2.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = bzip2.BestSpeed
case config.CompressionLevelBalanced:
l = bzip2.DefaultCompression
case config.CompressionLevelSmallest:
l = bzip2.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
Level: l,
})
if err != nil {
return nil, err
}
return noop.AddFlush(bz), nil
case config.NoneKey:
return noop.AddFlush(noop.AddClose(dst)), nil
default:
return nil, config.ErrCompressionFormatUnsupported
}
}
func getNearestPowerOf2Lower(n int) int {
return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func getNearestLogOf2Lower(n int) int {
return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}

2
internal/compression/decompress.go
View File

@@ -54,6 +54,6 @@ func Decompress(
case config.NoneKey: case config.NoneKey:
return io.NopCloser(src), nil return io.NopCloser(src), nil
default: default:
return nil, config.ErrUnsupportedCompressionFormat return nil, config.ErrCompressionFormatUnsupported
} }
} }

4
internal/encryption/decrypt.go
View File

@@ -46,7 +46,7 @@ func Decrypt(
case config.NoneKey: case config.NoneKey:
return io.NopCloser(src), nil return io.NopCloser(src), nil
default: default:
return nil, config.ErrUnsupportedEncryptionFormat return nil, config.ErrEncryptionFormatUnsupported
} }
} }
@@ -136,6 +136,6 @@ func DecryptString(
case config.NoneKey: case config.NoneKey:
return src, nil return src, nil
default: default:
return "", config.ErrUnsupportedEncryptionFormat return "", config.ErrEncryptionFormatUnsupported
} }
} }

127
internal/encryption/encrypt.go Normal file
View File

@@ -0,0 +1,127 @@
package encryption
import (
"archive/tar"
"bytes"
"encoding/base64"
"encoding/json"
"io"
"filippo.io/age"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/pkg/config"
"golang.org/x/crypto/openpgp"
)
func Encrypt(
dst io.Writer,
encryptionFormat string,
recipient interface{},
) (io.WriteCloser, error) {
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return nil, config.ErrRecipientUnparsable
}
return age.Encrypt(dst, recipient)
case config.EncryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return nil, config.ErrRecipientUnparsable
}
return openpgp.Encrypt(dst, recipient, nil, nil, nil)
case config.NoneKey:
return noop.AddClose(dst), nil
default:
return nil, config.ErrEncryptionFormatUnsupported
}
}
func EncryptHeader(
hdr *tar.Header,
encryptionFormat string,
recipient interface{},
) error {
if encryptionFormat == config.NoneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = EncryptString(string(wrappedHeader), encryptionFormat, recipient)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func EncryptString(
src string,
encryptionFormat string,
recipient interface{},
) (string, error) {
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return "", config.ErrRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := age.Encrypt(out, recipient)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.EncryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return "", config.ErrRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.NoneKey:
return src, nil
default:
return "", config.ErrEncryptionFormatUnsupported
}
}

4
internal/keys/identity.go
View File

@@ -65,7 +65,7 @@ func ParseIdentity(
case config.NoneKey: case config.NoneKey:
return privkey, nil return privkey, nil
default: default:
return nil, config.ErrUnsupportedEncryptionFormat return nil, config.ErrEncryptionFormatUnsupported
} }
} }
@@ -82,6 +82,6 @@ func ParseSignerIdentity(
case config.NoneKey: case config.NoneKey:
return privkey, nil return privkey, nil
default: default:
return nil, config.ErrUnsupportedSignatureFormat return nil, config.ErrSignatureFormatUnsupported
} }
} }

4
internal/keys/recipient.go
View File

@@ -21,7 +21,7 @@ func ParseRecipient(
case config.NoneKey: case config.NoneKey:
return pubkey, nil return pubkey, nil
default: default:
return nil, config.ErrUnsupportedEncryptionFormat return nil, config.ErrEncryptionFormatUnsupported
} }
} }
@@ -42,6 +42,6 @@ func ParseSignerRecipient(
case config.NoneKey: case config.NoneKey:
return pubkey, nil return pubkey, nil
default: default:
return nil, config.ErrUnsupportedSignatureFormat return nil, config.ErrSignatureFormatUnsupported
} }
} }

159
internal/signature/sign.go Normal file
View File

@@ -0,0 +1,159 @@
package signature
import (
"archive/tar"
"bytes"
"encoding/base64"
"encoding/json"
"io"
"aead.dev/minisign"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/pkg/config"
)
func Sign(
src io.Reader,
isRegular bool,
signatureFormat string,
identity interface{},
) (io.Reader, func() (string, error), error) {
switch signatureFormat {
case config.SignatureFormatMinisignKey:
if !isRegular {
return nil, nil, config.ErrSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return nil, nil, config.ErrIdentityUnparsable
}
signer := minisign.NewReader(src)
return signer, func() (string, error) {
return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
}, nil
case config.SignatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return nil, nil, config.ErrIdentityUnparsable
}
if len(identities) < 1 {
return nil, nil, config.ErrIdentityUnparsable
}
// See openpgp.DetachSign
var c *packet.Config
signingKey, ok := identities[0].SigningKeyById(c.Now(), c.SigningKey())
if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
return nil, nil, config.ErrIdentityUnparsable
}
sig := new(packet.Signature)
sig.SigType = packet.SigTypeBinary
sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
sig.Hash = c.Hash()
sig.CreationTime = c.Now()
sigLifetimeSecs := c.SigLifetime()
sig.SigLifetimeSecs = &sigLifetimeSecs
sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
hash := sig.Hash.New()
return io.TeeReader(src, hash), func() (string, error) {
if err := sig.Sign(hash, signingKey.PrivateKey, c); err != nil {
return "", err
}
out := &bytes.Buffer{}
if err := sig.Serialize(out); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
}, nil
case config.NoneKey:
return src, func() (string, error) {
return "", nil
}, nil
default:
return nil, nil, config.ErrSignatureFormatUnsupported
}
}
func SignHeader(
hdr *tar.Header,
isRegular bool,
signatureFormat string,
identity interface{},
) error {
if signatureFormat == config.NoneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
newHdr.PAXRecords[pax.STFSRecordSignature], err = SignString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func SignString(
src string,
isRegular bool,
signatureFormat string,
identity interface{},
) (string, error) {
switch signatureFormat {
case config.SignatureFormatMinisignKey:
if !isRegular {
return "", config.ErrSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return "", config.ErrIdentityUnparsable
}
return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
case config.SignatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return "", config.ErrIdentityUnparsable
}
if len(identities) < 1 {
return "", config.ErrIdentityUnparsable
}
out := &bytes.Buffer{}
if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.NoneKey:
return src, nil
default:
return "", config.ErrSignatureFormatUnsupported
}
}

4
internal/signature/verify.go
View File

@@ -84,7 +84,7 @@ func Verify(
return nil return nil
}, nil }, nil
default: default:
return nil, nil, config.ErrUnsupportedSignatureFormat return nil, nil, config.ErrSignatureFormatUnsupported
} }
} }
@@ -190,6 +190,6 @@ func VerifyString(
case config.NoneKey: case config.NoneKey:
return nil return nil
default: default:
return config.ErrUnsupportedSignatureFormat return config.ErrSignatureFormatUnsupported
} }
} }

37
internal/suffix/add.go Normal file
View File

@@ -0,0 +1,37 @@
package suffix
import "github.com/pojntfx/stfs/pkg/config"
func AddSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
switch compressionFormat {
case config.CompressionFormatGZipKey:
fallthrough
case config.CompressionFormatParallelGZipKey:
name += CompressionFormatGZipSuffix
case config.CompressionFormatLZ4Key:
name += CompressionFormatLZ4Suffix
case config.CompressionFormatZStandardKey:
name += CompressionFormatZStandardSuffix
case config.CompressionFormatBrotliKey:
name += CompressionFormatBrotliSuffix
case config.CompressionFormatBzip2Key:
fallthrough
case config.CompressionFormatBzip2ParallelKey:
name += CompressionFormatBzip2Suffix
case config.NoneKey:
default:
return "", config.ErrCompressionFormatUnsupported
}
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
name += EncryptionFormatAgeSuffix
case config.EncryptionFormatPGPKey:
name += EncryptionFormatPGPSuffix
case config.NoneKey:
default:
return "", config.ErrEncryptionFormatUnsupported
}
return name, nil
}

4
internal/suffix/remove.go
View File

@@ -14,7 +14,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
name = strings.TrimSuffix(name, EncryptionFormatPGPSuffix) name = strings.TrimSuffix(name, EncryptionFormatPGPSuffix)
case config.NoneKey: case config.NoneKey:
default: default:
return "", config.ErrUnsupportedEncryptionFormat return "", config.ErrEncryptionFormatUnsupported
} }
switch compressionFormat { switch compressionFormat {
@@ -34,7 +34,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
name = strings.TrimSuffix(name, CompressionFormatBzip2Suffix) name = strings.TrimSuffix(name, CompressionFormatBzip2Suffix)
case config.NoneKey: case config.NoneKey:
default: default:
return "", config.ErrUnsupportedCompressionFormat return "", config.ErrCompressionFormatUnsupported
} }
return name, nil return name, nil

8
internal/tape/read.go
View File

@@ -2,15 +2,15 @@ package tape
import "os" import "os"
func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) { func OpenTapeReadOnly(drive string) (f *os.File, isRegular bool, err error) {
fileDescription, err := os.Stat(tape) fileDescription, err := os.Stat(drive)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
isRegular = fileDescription.Mode().IsRegular() isRegular = fileDescription.Mode().IsRegular()
if isRegular { if isRegular {
f, err = os.Open(tape) f, err = os.Open(drive)
if err != nil { if err != nil {
return f, isRegular, err return f, isRegular, err
} }
@@ -18,7 +18,7 @@ func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) {
return f, isRegular, nil return f, isRegular, nil
} }
f, err = os.OpenFile(tape, os.O_RDONLY, os.ModeCharDevice) f, err = os.OpenFile(drive, os.O_RDONLY, os.ModeCharDevice)
if err != nil { if err != nil {
return f, isRegular, err return f, isRegular, err
} }

79
internal/tape/write.go Normal file
View File

@@ -0,0 +1,79 @@
package tape
import (
"archive/tar"
"bufio"
"os"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
)
func OpenTapeWriteOnly(drive string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
stat, err := os.Stat(drive)
if err == nil {
isRegular = stat.Mode().IsRegular()
} else {
if os.IsNotExist(err) {
isRegular = true
} else {
return nil, false, nil, err
}
}
var f *os.File
if isRegular {
f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return nil, false, nil, err
}
// No need to go to end manually due to `os.O_APPEND`
} else {
f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return nil, false, nil, err
}
if !overwrite {
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return nil, false, nil, err
}
}
}
var bw *bufio.Writer
var counter *counters.CounterWriter
if isRegular {
tw = tar.NewWriter(f)
} else {
bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
tw = tar.NewWriter(counter)
}
return tw, isRegular, func(dirty *bool) error {
// Only write the trailer if we wrote to the archive
if *dirty {
if err := tw.Close(); err != nil {
return err
}
if !isRegular {
if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
// Fill the rest of the record with zeros
if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
return err
}
}
if err := bw.Flush(); err != nil {
return err
}
}
}
return f.Close()
}, nil
}

4
pkg/config/constants.go
View File

@@ -16,4 +16,8 @@ const (
SignatureFormatMinisignKey = "minisign" SignatureFormatMinisignKey = "minisign"
SignatureFormatPGPKey = "pgp" SignatureFormatPGPKey = "pgp"
CompressionLevelFastest = "fastest"
CompressionLevelBalanced = "balanced"
CompressionLevelSmallest = "smallest"
) )

14
pkg/config/error.go
View File

@@ -3,18 +3,22 @@ package config
import "errors" import "errors"
var ( var (
ErrUnsupportedCompressionFormat = errors.New("unsupported compression format") ErrEncryptionFormatUnsupported = errors.New("unsupported encryption format")
ErrUnsupportedEncryptionFormat = errors.New("unsupported encryption format")
ErrUnsupportedSignatureFormat = errors.New("unsupported signature format")
ErrIdentityUnparsable = errors.New("recipient could not be parsed") ErrIdentityUnparsable = errors.New("recipient could not be parsed")
ErrRecipientUnparsable = errors.New("recipient could not be parsed") ErrRecipientUnparsable = errors.New("recipient could not be parsed")
ErrEmbeddedHeaderMissing = errors.New("embedded header is missing") ErrEmbeddedHeaderMissing = errors.New("embedded header is missing")
ErrSignatureFormatUnsupported = errors.New("unsupported signature format")
ErrSignatureFormatOnlyRegularSupport = errors.New("this signature format only supports regular files, not i.e. tape drives") ErrSignatureFormatOnlyRegularSupport = errors.New("this signature format only supports regular files, not i.e. tape drives")
ErrSignatureInvalid = errors.New("signature invalid") ErrSignatureInvalid = errors.New("signature is invalid")
ErrSignatureMissing = errors.New("signature missing") ErrSignatureMissing = errors.New("signature is missing")
ErrKeygenForFormatUnsupported = errors.New("can not generate keys for this format") ErrKeygenForFormatUnsupported = errors.New("can not generate keys for this format")
ErrCompressionFormatUnsupported = errors.New("unsupported compression format")
ErrCompressionFormatOnlyRegularSupport = errors.New("this compression format only supports regular files, not i.e. tape drives")
ErrCompressionFormatRequiresLargerRecordSize = errors.New("this compression format requires a larger record size")
ErrCompressionLevelUnsupported = errors.New("compression level is unsupported")
) )

281
pkg/operations/archive.go Normal file
View File

@@ -0,0 +1,281 @@
package operations
import (
"archive/tar"
"io"
"io/fs"
"os"
"path/filepath"
"strconv"
"github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config"
)
func Archive(
state config.StateConfig,
pipes config.PipeConfig,
crypto config.CryptoConfig,
recordSize int,
from string,
overwrite bool,
compressionLevel string,
) ([]*tar.Header, error) {
dirty := false
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
if overwrite {
if isRegular {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(state.Drive, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return []*tar.Header{}, err
}
// Clear the file's content
if err := f.Truncate(0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
} else {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(state.Drive, os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return []*tar.Header{}, err
}
// Seek to the start of the tape
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
}
}
defer cleanup(&dirty)
headers := []*tar.Header{}
first := true
return headers, filepath.Walk(from, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
hdr, err := tar.FileInfoHeader(info, link)
if err != nil {
return err
}
if err := adapters.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Format = tar.FormatPAX
if info.Mode().IsRegular() {
// Get the compressed size for the header
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encryption.Encrypt(fileSizeCounter, pipes.Encryption, crypto.Recipient)
if err != nil {
return err
}
compressor, err := compression.Compress(
encryptor,
pipes.Encryption,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
signer, sign, err := signature.Sign(file, isRegular, pipes.Signature, crypto.Identity)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, signer); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
signature, err := sign()
if err != nil {
return err
}
if signature != "" {
hdr.PAXRecords[pax.STFSRecordSignature] = signature
}
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = suffix.AddSuffix(hdr.Name, pipes.Compression, pipes.Encryption)
if err != nil {
return err
}
}
if first {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
first = false
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
return err
}
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signature.SignHeader(hdr, isRegular, pipes.Signature, crypto.Identity); err != nil {
return err
}
if err := encryption.EncryptHeader(hdr, pipes.Encryption, crypto.Recipient); err != nil {
return err
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
// Compress and write the file
encryptor, err := encryption.Encrypt(tw, pipes.Encryption, crypto.Recipient)
if err != nil {
return err
}
compressor, err := compression.Compress(
encryptor,
pipes.Compression,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
dirty = true
return nil
})
}

29
pkg/operations/operations.go
View File

@@ -1,22 +1,9 @@
package operations package operations
import ( import (
"archive/tar"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
) )
func Archive(
state config.StateConfig,
pipes config.PipeConfig,
crypto config.CryptoConfig,
recordSize int,
from string,
overwrite bool,
compressionLevel string,
) ([]*tar.Header, error)
func Restore( func Restore(
state config.StateConfig, state config.StateConfig,
pipes config.PipeConfig, pipes config.PipeConfig,
@@ -26,7 +13,9 @@ func Restore(
from string, from string,
to string, to string,
flatten bool, flatten bool,
) error ) error {
return nil
}
func Update( func Update(
state config.StateConfig, state config.StateConfig,
@@ -37,7 +26,9 @@ func Update(
from string, from string,
overwrite bool, overwrite bool,
compressionLevel string, compressionLevel string,
) error ) error {
return nil
}
func Delete( func Delete(
state config.StateConfig, state config.StateConfig,
@@ -46,7 +37,9 @@ func Delete(
recordSize int, recordSize int,
name string, name string,
) error ) error {
return nil
}
func Move( func Move(
state config.StateConfig, state config.StateConfig,
@@ -56,4 +49,6 @@ func Move(
recordSize int, recordSize int,
from string, from string,
to string, to string,
) error ) error {
return nil
}