mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Decompose archive func

Browse Source
This commit is contained in:
Felicitas Pojtinger
2021-12-07 21:12:23 +01:00
parent b700a767cb
commit db69d5f68c
20 changed files with 966 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

800
cmd/stbak/cmd/archive.go
View File

@@ -2,39 +2,16 @@ package cmd
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"io/ioutil"
"math"
"os"
"path/filepath"
"strconv"
"aead.dev/minisign"
"filippo.io/age"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
"github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -50,14 +27,10 @@ const (
recipientFlag = "recipient"
identityFlag = "identity"
passwordFlag = "password"
compressionLevelFastest = "fastest"
compressionLevelBalanced = "balanced"
compressionLevelSmallest = "smallest"
)
var (
knownCompressionLevels = []string{compressionLevelFastest, compressionLevelBalanced, compressionLevelSmallest}
knownCompressionLevels = []string{config.CompressionLevelFastest, config.CompressionLevelBalanced, config.CompressionLevelSmallest}
errUnknownCompressionLevel = errors.New("unknown compression level")
errUnsupportedCompressionLevel = errors.New("unsupported compression level")
@@ -121,7 +94,7 @@ var archiveCmd = &cobra.Command{
return err
}
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil {
return err
}
@@ -136,21 +109,27 @@ var archiveCmd = &cobra.Command{
return err
}
hdrs, err := archive(
viper.GetString(driveFlag),
hdrs, err := operations.Archive(
config.StateConfig{
Drive: viper.GetString(driveFlag),
Metadata: viper.GetString(metadataFlag),
},
config.PipeConfig{
Compression: viper.GetString(compressionFlag),
Encryption: viper.GetString(encryptionFlag),
Signature: viper.GetString(signatureFlag),
},
config.CryptoConfig{
Recipient: recipient,
Identity: identity,
Password: viper.GetString(passwordFlag),
},
viper.GetInt(recordSizeFlag),
viper.GetString(fromFlag),
viper.GetBool(overwriteFlag),
viper.GetString(compressionFlag),
viper.GetString(compressionLevelFlag),
viper.GetString(encryptionFlag),
recipient,
viper.GetString(signatureFlag),
identity,
)
if err != nil {
return err
}
return recovery.Index(
config.StateConfig{
@@ -194,267 +173,6 @@ var archiveCmd = &cobra.Command{
},
}
func archive(
tape string,
recordSize int,
src string,
overwrite bool,
compressionFormat string,
compressionLevel string,
encryptionFormat string,
recipient interface{},
signatureFormat string,
identity interface{},
) ([]*tar.Header, error) {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
if overwrite {
if isRegular {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(tape, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return []*tar.Header{}, err
}
// Clear the file's content
if err := f.Truncate(0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
} else {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(tape, os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return []*tar.Header{}, err
}
// Seek to the start of the tape
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
}
}
defer cleanup(&dirty)
headers := []*tar.Header{}
first := true
return headers, filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
hdr, err := tar.FileInfoHeader(info, link)
if err != nil {
return err
}
if err := adapters.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Format = tar.FormatPAX
if info.Mode().IsRegular() {
// Get the compressed size for the header
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
signer, sign, err := sign(file, isRegular, signatureFormat, identity)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, signer); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
signature, err := sign()
if err != nil {
return err
}
if signature != "" {
hdr.PAXRecords[pax.STFSRecordSignature] = signature
}
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat)
if err != nil {
return err
}
}
if first {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
first = false
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
return err
}
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
// Compress and write the file
encryptor, err := encrypt(tw, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
encryptor,
compressionFormat,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
dirty = true
return nil
})
}
func checkKeyAccessible(encryptionFormat string, pathToKey string) error {
if encryptionFormat == noneKey {
return nil
@@ -491,489 +209,11 @@ func checkCompressionLevel(compressionLevel string) error {
return nil
}
func encryptHeader(
hdr *tar.Header,
encryptionFormat string,
recipient interface{},
) error {
if encryptionFormat == noneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = encryptString(string(wrappedHeader), encryptionFormat, recipient)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func signHeader(
hdr *tar.Header,
isRegular bool,
signatureFormat string,
identity interface{},
) error {
if signatureFormat == noneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
newHdr.PAXRecords[pax.STFSRecordSignature], err = signString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func addSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
name += compressionFormatGZipSuffix
case compressionFormatLZ4Key:
name += compressionFormatLZ4Suffix
case compressionFormatZStandardKey:
name += compressionFormatZStandardSuffix
case compressionFormatBrotliKey:
name += compressionFormatBrotliSuffix
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
name += compressionFormatBzip2Suffix
case noneKey:
default:
return "", errUnsupportedCompressionFormat
}
switch encryptionFormat {
case encryptionFormatAgeKey:
name += encryptionFormatAgeSuffix
case encryptionFormatPGPKey:
name += encryptionFormatPGPSuffix
case noneKey:
default:
return "", errUnsupportedEncryptionFormat
}
return name, nil
}
func parseRecipient(
encryptionFormat string,
pubkey []byte,
) (interface{}, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
return age.ParseX25519Recipient(string(pubkey))
case encryptionFormatPGPKey:
return openpgp.ReadKeyRing(bytes.NewBuffer(pubkey))
case noneKey:
return pubkey, nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func encrypt(
dst io.Writer,
encryptionFormat string,
recipient interface{},
) (io.WriteCloser, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return nil, errRecipientUnparsable
}
return age.Encrypt(dst, recipient)
case encryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return nil, errRecipientUnparsable
}
return openpgp.Encrypt(dst, recipient, nil, nil, nil)
case noneKey:
return noop.AddClose(dst), nil
default:
return nil, errUnsupportedEncryptionFormat
}
}
func sign(
src io.Reader,
isRegular bool,
signatureFormat string,
identity interface{},
) (io.Reader, func() (string, error), error) {
switch signatureFormat {
case signatureFormatMinisignKey:
if !isRegular {
return nil, nil, errSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return nil, nil, errIdentityUnparsable
}
signer := minisign.NewReader(src)
return signer, func() (string, error) {
return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
}, nil
case signatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return nil, nil, errIdentityUnparsable
}
if len(identities) < 1 {
return nil, nil, errIdentityUnparsable
}
// See openpgp.DetachSign
var config *packet.Config
signingKey, ok := identities[0].SigningKeyById(config.Now(), config.SigningKey())
if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
return nil, nil, errIdentityUnparsable
}
sig := new(packet.Signature)
sig.SigType = packet.SigTypeBinary
sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
sig.Hash = config.Hash()
sig.CreationTime = config.Now()
sigLifetimeSecs := config.SigLifetime()
sig.SigLifetimeSecs = &sigLifetimeSecs
sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
hash := sig.Hash.New()
return io.TeeReader(src, hash), func() (string, error) {
if err := sig.Sign(hash, signingKey.PrivateKey, config); err != nil {
return "", err
}
out := &bytes.Buffer{}
if err := sig.Serialize(out); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
}, nil
case noneKey:
return src, func() (string, error) {
return "", nil
}, nil
default:
return nil, nil, errUnsupportedSignatureFormat
}
}
func encryptString(
src string,
encryptionFormat string,
recipient interface{},
) (string, error) {
switch encryptionFormat {
case encryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return "", errRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := age.Encrypt(out, recipient)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case encryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return "", errRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case noneKey:
return src, nil
default:
return "", errUnsupportedEncryptionFormat
}
}
func signString(
src string,
isRegular bool,
signatureFormat string,
identity interface{},
) (string, error) {
switch signatureFormat {
case signatureFormatMinisignKey:
if !isRegular {
return "", errSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return "", errIdentityUnparsable
}
return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
case signatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return "", errIdentityUnparsable
}
if len(identities) < 1 {
return "", errIdentityUnparsable
}
out := &bytes.Buffer{}
if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case noneKey:
return src, nil
default:
return "", errUnsupportedSignatureFormat
}
}
func compress(
dst io.Writer,
compressionFormat string,
compressionLevel string,
isRegular bool,
recordSize int,
) (noop.Flusher, error) {
switch compressionFormat {
case compressionFormatGZipKey:
fallthrough
case compressionFormatParallelGZipKey:
if compressionFormat == compressionFormatGZipKey {
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
return nil, errCompressionFormatRequiresLargerRecordSize
}
}
l := gzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = gzip.BestSpeed
case compressionLevelBalanced:
l = gzip.DefaultCompression
case compressionLevelSmallest:
l = gzip.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
return gzip.NewWriterLevel(dst, l)
}
if !isRegular {
return nil, errCompressionFormatOnlyRegularSupport // "device or resource busy"
}
l := pgzip.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = pgzip.BestSpeed
case compressionLevelBalanced:
l = pgzip.DefaultCompression
case compressionLevelSmallest:
l = pgzip.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
return pgzip.NewWriterLevel(dst, l)
case compressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
case compressionLevelFastest:
l = lz4.Level1
case compressionLevelBalanced:
l = lz4.Level5
case compressionLevelSmallest:
l = lz4.Level9
default:
return nil, errUnsupportedCompressionLevel
}
opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if uint32(maxSize) < uint32(lz4.Block64Kb) {
return nil, errCompressionFormatRequiresLargerRecordSize
}
if uint32(maxSize) < uint32(lz4.Block256Kb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
} else {
opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
}
}
lz := lz4.NewWriter(dst)
if err := lz.Apply(opts...); err != nil {
return nil, err
}
return noop.AddFlush(lz), nil
case compressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
case compressionLevelFastest:
l = zstd.SpeedFastest
case compressionLevelBalanced:
l = zstd.SpeedDefault
case compressionLevelSmallest:
l = zstd.SpeedBestCompression
default:
return nil, errUnsupportedCompressionLevel
}
opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
if !isRegular {
opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
}
zz, err := zstd.NewWriter(dst, opts...)
if err != nil {
return nil, err
}
return zz, nil
case compressionFormatBrotliKey:
if !isRegular {
return nil, errCompressionFormatOnlyRegularSupport // "cannot allocate memory"
}
l := brotli.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = brotli.BestSpeed
case compressionLevelBalanced:
l = brotli.DefaultCompression
case compressionLevelSmallest:
l = brotli.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
br := brotli.NewWriterLevel(dst, l)
return br, nil
case compressionFormatBzip2Key:
fallthrough
case compressionFormatBzip2ParallelKey:
l := bzip2.DefaultCompression
switch compressionLevel {
case compressionLevelFastest:
l = bzip2.BestSpeed
case compressionLevelBalanced:
l = bzip2.DefaultCompression
case compressionLevelSmallest:
l = bzip2.BestCompression
default:
return nil, errUnsupportedCompressionLevel
}
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
Level: l,
})
if err != nil {
return nil, err
}
return noop.AddFlush(bz), nil
case noneKey:
return noop.AddFlush(noop.AddClose(dst)), nil
default:
return nil, errUnsupportedCompressionFormat
}
}
func getNearestPowerOf2Lower(n int) int {
return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func getNearestLogOf2Lower(n int) int {
return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func init() {
archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
archiveCmd.PersistentFlags().StringP(fromFlag, "f", ".", "File or directory to archive")
archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file")
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
archiveCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
archiveCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
archiveCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")

86
cmd/stbak/cmd/delete.go
View File

@@ -2,18 +2,17 @@ package cmd
import (
"archive/tar"
"bufio"
"context"
"os"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/converters"
"github.com/pojntfx/stfs/internal/counters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra"
@@ -54,7 +53,7 @@ var deleteCmd = &cobra.Command{
return err
}
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil {
return err
}
@@ -83,7 +82,7 @@ var deleteCmd = &cobra.Command{
}
func delete(
tape string,
drive string,
recordSize int,
metadata string,
name string,
@@ -93,7 +92,7 @@ func delete(
identity interface{},
) error {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil {
return err
}
@@ -142,11 +141,11 @@ func delete(
hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}
@@ -204,75 +203,6 @@ func delete(
)
}
func openTapeWriter(tape string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
stat, err := os.Stat(tape)
if err == nil {
isRegular = stat.Mode().IsRegular()
} else {
if os.IsNotExist(err) {
isRegular = true
} else {
return nil, false, nil, err
}
}
var f *os.File
if isRegular {
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return nil, false, nil, err
}
// No need to go to end manually due to `os.O_APPEND`
} else {
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return nil, false, nil, err
}
if !overwrite {
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return nil, false, nil, err
}
}
}
var bw *bufio.Writer
var counter *counters.CounterWriter
if isRegular {
tw = tar.NewWriter(f)
} else {
bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
tw = tar.NewWriter(counter)
}
return tw, isRegular, func(dirty *bool) error {
// Only write the trailer if we wrote to the archive
if *dirty {
if err := tw.Close(); err != nil {
return err
}
if !isRegular {
if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
// Fill the rest of the record with zeros
if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
return err
}
}
if err := bw.Flush(); err != nil {
return err
}
}
}
return f.Close()
}, nil
}
func init() {
deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove")

13
cmd/stbak/cmd/move.go
View File

@@ -7,10 +7,13 @@ import (
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra"
@@ -47,7 +50,7 @@ var moveCmd = &cobra.Command{
return err
}
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil {
return err
}
@@ -77,7 +80,7 @@ var moveCmd = &cobra.Command{
}
func move(
tape string,
drive string,
recordSize int,
metadata string,
src string,
@@ -88,7 +91,7 @@ func move(
identity interface{},
) error {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil {
return err
}
@@ -139,11 +142,11 @@ func move(
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate
hdr.PAXRecords[pax.STFSRecordReplacesName] = dbhdr.Name
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}

33
cmd/stbak/cmd/update.go
View File

@@ -11,12 +11,17 @@ import (
"strconv"
"github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery"
"github.com/spf13/cobra"
@@ -67,7 +72,7 @@ var updateCmd = &cobra.Command{
return err
}
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
if err != nil {
return err
}
@@ -141,7 +146,7 @@ var updateCmd = &cobra.Command{
}
func update(
tape string,
drive string,
recordSize int,
src string,
replacesContent bool,
@@ -153,7 +158,7 @@ func update(
identity interface{},
) ([]*tar.Header, error) {
dirty := false
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
if err != nil {
return []*tar.Header{}, err
}
@@ -196,12 +201,12 @@ func update(
Writer: io.Discard,
}
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient)
encryptor, err := encryption.Encrypt(fileSizeCounter, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
compressor, err := compression.Compress(
encryptor,
compressionFormat,
compressionLevel,
@@ -217,7 +222,7 @@ func update(
return err
}
signer, sign, err := sign(file, isRegular, signatureFormat, identity)
signer, sign, err := signature.Sign(file, isRegular, signatureFormat, identity)
if err != nil {
return err
}
@@ -263,7 +268,7 @@ func update(
}
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat)
hdr.Name, err = suffix.AddSuffix(hdr.Name, compressionFormat, encryptionFormat)
if err != nil {
return err
}
@@ -287,11 +292,11 @@ func update(
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}
@@ -304,12 +309,12 @@ func update(
}
// Compress and write the file
encryptor, err := encrypt(tw, encryptionFormat, recipient)
encryptor, err := encryption.Encrypt(tw, encryptionFormat, recipient)
if err != nil {
return err
}
compressor, err := compress(
compressor, err := compression.Compress(
encryptor,
compressionFormat,
compressionLevel,
@@ -361,11 +366,11 @@ func update(
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
return err
}
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
return err
}
@@ -384,7 +389,7 @@ func init() {
updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update")
updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
updateCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
updateCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
updateCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")

188
internal/compression/compress.go Normal file
View File

@@ -0,0 +1,188 @@
package compression
import (
"compress/gzip"
"io"
"math"
"github.com/andybalholm/brotli"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4/v4"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/pkg/config"
)
func Compress(
dst io.Writer,
compressionFormat string,
compressionLevel string,
isRegular bool,
recordSize int,
) (noop.Flusher, error) {
switch compressionFormat {
case config.CompressionFormatGZipKey:
fallthrough
case config.CompressionFormatParallelGZipKey:
if compressionFormat == config.CompressionFormatGZipKey {
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
return nil, config.ErrCompressionFormatRequiresLargerRecordSize
}
}
l := gzip.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = gzip.BestSpeed
case config.CompressionLevelBalanced:
l = gzip.DefaultCompression
case config.CompressionLevelSmallest:
l = gzip.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
return gzip.NewWriterLevel(dst, l)
}
if !isRegular {
return nil, config.ErrCompressionFormatOnlyRegularSupport // "device or resource busy"
}
l := pgzip.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = pgzip.BestSpeed
case config.CompressionLevelBalanced:
l = pgzip.DefaultCompression
case config.CompressionLevelSmallest:
l = pgzip.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
return pgzip.NewWriterLevel(dst, l)
case config.CompressionFormatLZ4Key:
l := lz4.Level5
switch compressionLevel {
case config.CompressionLevelFastest:
l = lz4.Level1
case config.CompressionLevelBalanced:
l = lz4.Level5
case config.CompressionLevelSmallest:
l = lz4.Level9
default:
return nil, config.ErrCompressionLevelUnsupported
}
opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
if !isRegular {
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
if uint32(maxSize) < uint32(lz4.Block64Kb) {
return nil, config.ErrCompressionFormatRequiresLargerRecordSize
}
if uint32(maxSize) < uint32(lz4.Block256Kb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
} else {
opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
}
}
lz := lz4.NewWriter(dst)
if err := lz.Apply(opts...); err != nil {
return nil, err
}
return noop.AddFlush(lz), nil
case config.CompressionFormatZStandardKey:
l := zstd.SpeedDefault
switch compressionLevel {
case config.CompressionLevelFastest:
l = zstd.SpeedFastest
case config.CompressionLevelBalanced:
l = zstd.SpeedDefault
case config.CompressionLevelSmallest:
l = zstd.SpeedBestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
if !isRegular {
opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
}
zz, err := zstd.NewWriter(dst, opts...)
if err != nil {
return nil, err
}
return zz, nil
case config.CompressionFormatBrotliKey:
if !isRegular {
return nil, config.ErrCompressionFormatOnlyRegularSupport // "cannot allocate memory"
}
l := brotli.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = brotli.BestSpeed
case config.CompressionLevelBalanced:
l = brotli.DefaultCompression
case config.CompressionLevelSmallest:
l = brotli.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
br := brotli.NewWriterLevel(dst, l)
return br, nil
case config.CompressionFormatBzip2Key:
fallthrough
case config.CompressionFormatBzip2ParallelKey:
l := bzip2.DefaultCompression
switch compressionLevel {
case config.CompressionLevelFastest:
l = bzip2.BestSpeed
case config.CompressionLevelBalanced:
l = bzip2.DefaultCompression
case config.CompressionLevelSmallest:
l = bzip2.BestCompression
default:
return nil, config.ErrCompressionLevelUnsupported
}
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
Level: l,
})
if err != nil {
return nil, err
}
return noop.AddFlush(bz), nil
case config.NoneKey:
return noop.AddFlush(noop.AddClose(dst)), nil
default:
return nil, config.ErrCompressionFormatUnsupported
}
}
func getNearestPowerOf2Lower(n int) int {
return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}
func getNearestLogOf2Lower(n int) int {
return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
}

2
internal/compression/decompress.go
View File

@@ -54,6 +54,6 @@ func Decompress(
case config.NoneKey:
return io.NopCloser(src), nil
default:
return nil, config.ErrUnsupportedCompressionFormat
return nil, config.ErrCompressionFormatUnsupported
}
}

4
internal/encryption/decrypt.go
View File

@@ -46,7 +46,7 @@ func Decrypt(
case config.NoneKey:
return io.NopCloser(src), nil
default:
return nil, config.ErrUnsupportedEncryptionFormat
return nil, config.ErrEncryptionFormatUnsupported
}
}
@@ -136,6 +136,6 @@ func DecryptString(
case config.NoneKey:
return src, nil
default:
return "", config.ErrUnsupportedEncryptionFormat
return "", config.ErrEncryptionFormatUnsupported
}
}

127
internal/encryption/encrypt.go Normal file
View File

@@ -0,0 +1,127 @@
package encryption
import (
"archive/tar"
"bytes"
"encoding/base64"
"encoding/json"
"io"
"filippo.io/age"
"github.com/pojntfx/stfs/internal/noop"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/pkg/config"
"golang.org/x/crypto/openpgp"
)
func Encrypt(
dst io.Writer,
encryptionFormat string,
recipient interface{},
) (io.WriteCloser, error) {
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return nil, config.ErrRecipientUnparsable
}
return age.Encrypt(dst, recipient)
case config.EncryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return nil, config.ErrRecipientUnparsable
}
return openpgp.Encrypt(dst, recipient, nil, nil, nil)
case config.NoneKey:
return noop.AddClose(dst), nil
default:
return nil, config.ErrEncryptionFormatUnsupported
}
}
func EncryptHeader(
hdr *tar.Header,
encryptionFormat string,
recipient interface{},
) error {
if encryptionFormat == config.NoneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = EncryptString(string(wrappedHeader), encryptionFormat, recipient)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func EncryptString(
src string,
encryptionFormat string,
recipient interface{},
) (string, error) {
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
recipient, ok := recipient.(*age.X25519Recipient)
if !ok {
return "", config.ErrRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := age.Encrypt(out, recipient)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.EncryptionFormatPGPKey:
recipient, ok := recipient.(openpgp.EntityList)
if !ok {
return "", config.ErrRecipientUnparsable
}
out := &bytes.Buffer{}
w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
if err != nil {
return "", err
}
if _, err := io.WriteString(w, src); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.NoneKey:
return src, nil
default:
return "", config.ErrEncryptionFormatUnsupported
}
}

4
internal/keys/identity.go
View File

@@ -65,7 +65,7 @@ func ParseIdentity(
case config.NoneKey:
return privkey, nil
default:
return nil, config.ErrUnsupportedEncryptionFormat
return nil, config.ErrEncryptionFormatUnsupported
}
}
@@ -82,6 +82,6 @@ func ParseSignerIdentity(
case config.NoneKey:
return privkey, nil
default:
return nil, config.ErrUnsupportedSignatureFormat
return nil, config.ErrSignatureFormatUnsupported
}
}

4
internal/keys/recipient.go
View File

@@ -21,7 +21,7 @@ func ParseRecipient(
case config.NoneKey:
return pubkey, nil
default:
return nil, config.ErrUnsupportedEncryptionFormat
return nil, config.ErrEncryptionFormatUnsupported
}
}
@@ -42,6 +42,6 @@ func ParseSignerRecipient(
case config.NoneKey:
return pubkey, nil
default:
return nil, config.ErrUnsupportedSignatureFormat
return nil, config.ErrSignatureFormatUnsupported
}
}

159
internal/signature/sign.go Normal file
View File

@@ -0,0 +1,159 @@
package signature
import (
"archive/tar"
"bytes"
"encoding/base64"
"encoding/json"
"io"
"aead.dev/minisign"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/pkg/config"
)
func Sign(
src io.Reader,
isRegular bool,
signatureFormat string,
identity interface{},
) (io.Reader, func() (string, error), error) {
switch signatureFormat {
case config.SignatureFormatMinisignKey:
if !isRegular {
return nil, nil, config.ErrSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return nil, nil, config.ErrIdentityUnparsable
}
signer := minisign.NewReader(src)
return signer, func() (string, error) {
return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
}, nil
case config.SignatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return nil, nil, config.ErrIdentityUnparsable
}
if len(identities) < 1 {
return nil, nil, config.ErrIdentityUnparsable
}
// See openpgp.DetachSign
var c *packet.Config
signingKey, ok := identities[0].SigningKeyById(c.Now(), c.SigningKey())
if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
return nil, nil, config.ErrIdentityUnparsable
}
sig := new(packet.Signature)
sig.SigType = packet.SigTypeBinary
sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
sig.Hash = c.Hash()
sig.CreationTime = c.Now()
sigLifetimeSecs := c.SigLifetime()
sig.SigLifetimeSecs = &sigLifetimeSecs
sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
hash := sig.Hash.New()
return io.TeeReader(src, hash), func() (string, error) {
if err := sig.Sign(hash, signingKey.PrivateKey, c); err != nil {
return "", err
}
out := &bytes.Buffer{}
if err := sig.Serialize(out); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
}, nil
case config.NoneKey:
return src, func() (string, error) {
return "", nil
}, nil
default:
return nil, nil, config.ErrSignatureFormatUnsupported
}
}
func SignHeader(
hdr *tar.Header,
isRegular bool,
signatureFormat string,
identity interface{},
) error {
if signatureFormat == config.NoneKey {
return nil
}
newHdr := &tar.Header{
Format: tar.FormatPAX,
Size: hdr.Size,
PAXRecords: map[string]string{},
}
wrappedHeader, err := json.Marshal(hdr)
if err != nil {
return err
}
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
newHdr.PAXRecords[pax.STFSRecordSignature], err = SignString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
if err != nil {
return err
}
*hdr = *newHdr
return nil
}
func SignString(
src string,
isRegular bool,
signatureFormat string,
identity interface{},
) (string, error) {
switch signatureFormat {
case config.SignatureFormatMinisignKey:
if !isRegular {
return "", config.ErrSignatureFormatOnlyRegularSupport
}
identity, ok := identity.(minisign.PrivateKey)
if !ok {
return "", config.ErrIdentityUnparsable
}
return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
case config.SignatureFormatPGPKey:
identities, ok := identity.(openpgp.EntityList)
if !ok {
return "", config.ErrIdentityUnparsable
}
if len(identities) < 1 {
return "", config.ErrIdentityUnparsable
}
out := &bytes.Buffer{}
if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
case config.NoneKey:
return src, nil
default:
return "", config.ErrSignatureFormatUnsupported
}
}

4
internal/signature/verify.go
View File

@@ -84,7 +84,7 @@ func Verify(
return nil
}, nil
default:
return nil, nil, config.ErrUnsupportedSignatureFormat
return nil, nil, config.ErrSignatureFormatUnsupported
}
}
@@ -190,6 +190,6 @@ func VerifyString(
case config.NoneKey:
return nil
default:
return config.ErrUnsupportedSignatureFormat
return config.ErrSignatureFormatUnsupported
}
}

37
internal/suffix/add.go Normal file
View File

@@ -0,0 +1,37 @@
package suffix
import "github.com/pojntfx/stfs/pkg/config"
func AddSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
switch compressionFormat {
case config.CompressionFormatGZipKey:
fallthrough
case config.CompressionFormatParallelGZipKey:
name += CompressionFormatGZipSuffix
case config.CompressionFormatLZ4Key:
name += CompressionFormatLZ4Suffix
case config.CompressionFormatZStandardKey:
name += CompressionFormatZStandardSuffix
case config.CompressionFormatBrotliKey:
name += CompressionFormatBrotliSuffix
case config.CompressionFormatBzip2Key:
fallthrough
case config.CompressionFormatBzip2ParallelKey:
name += CompressionFormatBzip2Suffix
case config.NoneKey:
default:
return "", config.ErrCompressionFormatUnsupported
}
switch encryptionFormat {
case config.EncryptionFormatAgeKey:
name += EncryptionFormatAgeSuffix
case config.EncryptionFormatPGPKey:
name += EncryptionFormatPGPSuffix
case config.NoneKey:
default:
return "", config.ErrEncryptionFormatUnsupported
}
return name, nil
}

4
internal/suffix/remove.go
View File

@@ -14,7 +14,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
name = strings.TrimSuffix(name, EncryptionFormatPGPSuffix)
case config.NoneKey:
default:
return "", config.ErrUnsupportedEncryptionFormat
return "", config.ErrEncryptionFormatUnsupported
}
switch compressionFormat {
@@ -34,7 +34,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
name = strings.TrimSuffix(name, CompressionFormatBzip2Suffix)
case config.NoneKey:
default:
return "", config.ErrUnsupportedCompressionFormat
return "", config.ErrCompressionFormatUnsupported
}
return name, nil

8
internal/tape/read.go
View File

@@ -2,15 +2,15 @@ package tape
import "os"
func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) {
fileDescription, err := os.Stat(tape)
func OpenTapeReadOnly(drive string) (f *os.File, isRegular bool, err error) {
fileDescription, err := os.Stat(drive)
if err != nil {
return nil, false, err
}
isRegular = fileDescription.Mode().IsRegular()
if isRegular {
f, err = os.Open(tape)
f, err = os.Open(drive)
if err != nil {
return f, isRegular, err
}
@@ -18,7 +18,7 @@ func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) {
return f, isRegular, nil
}
f, err = os.OpenFile(tape, os.O_RDONLY, os.ModeCharDevice)
f, err = os.OpenFile(drive, os.O_RDONLY, os.ModeCharDevice)
if err != nil {
return f, isRegular, err
}

79
internal/tape/write.go Normal file
View File

@@ -0,0 +1,79 @@
package tape
import (
"archive/tar"
"bufio"
"os"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
)
func OpenTapeWriteOnly(drive string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
stat, err := os.Stat(drive)
if err == nil {
isRegular = stat.Mode().IsRegular()
} else {
if os.IsNotExist(err) {
isRegular = true
} else {
return nil, false, nil, err
}
}
var f *os.File
if isRegular {
f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return nil, false, nil, err
}
// No need to go to end manually due to `os.O_APPEND`
} else {
f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return nil, false, nil, err
}
if !overwrite {
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return nil, false, nil, err
}
}
}
var bw *bufio.Writer
var counter *counters.CounterWriter
if isRegular {
tw = tar.NewWriter(f)
} else {
bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
tw = tar.NewWriter(counter)
}
return tw, isRegular, func(dirty *bool) error {
// Only write the trailer if we wrote to the archive
if *dirty {
if err := tw.Close(); err != nil {
return err
}
if !isRegular {
if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
// Fill the rest of the record with zeros
if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
return err
}
}
if err := bw.Flush(); err != nil {
return err
}
}
}
return f.Close()
}, nil
}

4
pkg/config/constants.go
View File

@@ -16,4 +16,8 @@ const (
SignatureFormatMinisignKey = "minisign"
SignatureFormatPGPKey = "pgp"
CompressionLevelFastest = "fastest"
CompressionLevelBalanced = "balanced"
CompressionLevelSmallest = "smallest"
)

14
pkg/config/error.go
View File

@@ -3,18 +3,22 @@ package config
import "errors"
var (
ErrUnsupportedCompressionFormat = errors.New("unsupported compression format")
ErrUnsupportedEncryptionFormat = errors.New("unsupported encryption format")
ErrUnsupportedSignatureFormat = errors.New("unsupported signature format")
ErrEncryptionFormatUnsupported = errors.New("unsupported encryption format")
ErrIdentityUnparsable = errors.New("recipient could not be parsed")
ErrRecipientUnparsable = errors.New("recipient could not be parsed")
ErrEmbeddedHeaderMissing = errors.New("embedded header is missing")
ErrSignatureFormatUnsupported = errors.New("unsupported signature format")
ErrSignatureFormatOnlyRegularSupport = errors.New("this signature format only supports regular files, not i.e. tape drives")
ErrSignatureInvalid = errors.New("signature invalid")
ErrSignatureMissing = errors.New("signature missing")
ErrSignatureInvalid = errors.New("signature is invalid")
ErrSignatureMissing = errors.New("signature is missing")
ErrKeygenForFormatUnsupported = errors.New("can not generate keys for this format")
ErrCompressionFormatUnsupported = errors.New("unsupported compression format")
ErrCompressionFormatOnlyRegularSupport = errors.New("this compression format only supports regular files, not i.e. tape drives")
ErrCompressionFormatRequiresLargerRecordSize = errors.New("this compression format requires a larger record size")
ErrCompressionLevelUnsupported = errors.New("compression level is unsupported")
)

281
pkg/operations/archive.go Normal file
View File

@@ -0,0 +1,281 @@
package operations
import (
"archive/tar"
"io"
"io/fs"
"os"
"path/filepath"
"strconv"
"github.com/pojntfx/stfs/internal/adapters"
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/controllers"
"github.com/pojntfx/stfs/internal/counters"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/formatting"
"github.com/pojntfx/stfs/internal/pax"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tape"
"github.com/pojntfx/stfs/pkg/config"
)
func Archive(
state config.StateConfig,
pipes config.PipeConfig,
crypto config.CryptoConfig,
recordSize int,
from string,
overwrite bool,
compressionLevel string,
) ([]*tar.Header, error) {
dirty := false
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
if overwrite {
if isRegular {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(state.Drive, os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return []*tar.Header{}, err
}
// Clear the file's content
if err := f.Truncate(0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
} else {
if err := cleanup(&dirty); err != nil { // dirty will always be false here
return []*tar.Header{}, err
}
f, err := os.OpenFile(state.Drive, os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return []*tar.Header{}, err
}
// Seek to the start of the tape
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
return []*tar.Header{}, err
}
if err := f.Close(); err != nil {
return []*tar.Header{}, err
}
tw, isRegular, cleanup, err = tape.OpenTapeWriteOnly(state.Drive, recordSize, overwrite)
if err != nil {
return []*tar.Header{}, err
}
}
}
defer cleanup(&dirty)
headers := []*tar.Header{}
first := true
return headers, filepath.Walk(from, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
hdr, err := tar.FileInfoHeader(info, link)
if err != nil {
return err
}
if err := adapters.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Format = tar.FormatPAX
if info.Mode().IsRegular() {
// Get the compressed size for the header
fileSizeCounter := &counters.CounterWriter{
Writer: io.Discard,
}
encryptor, err := encryption.Encrypt(fileSizeCounter, pipes.Encryption, crypto.Recipient)
if err != nil {
return err
}
compressor, err := compression.Compress(
encryptor,
pipes.Encryption,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
signer, sign, err := signature.Sign(file, isRegular, pipes.Signature, crypto.Identity)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, signer); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
if hdr.PAXRecords == nil {
hdr.PAXRecords = map[string]string{}
}
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
signature, err := sign()
if err != nil {
return err
}
if signature != "" {
hdr.PAXRecords[pax.STFSRecordSignature] = signature
}
hdr.Size = int64(fileSizeCounter.BytesRead)
hdr.Name, err = suffix.AddSuffix(hdr.Name, pipes.Compression, pipes.Encryption)
if err != nil {
return err
}
}
if first {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
first = false
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
return err
}
hdrToAppend := *hdr
headers = append(headers, &hdrToAppend)
if err := signature.SignHeader(hdr, isRegular, pipes.Signature, crypto.Identity); err != nil {
return err
}
if err := encryption.EncryptHeader(hdr, pipes.Encryption, crypto.Recipient); err != nil {
return err
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
// Compress and write the file
encryptor, err := encryption.Encrypt(tw, pipes.Encryption, crypto.Recipient)
if err != nil {
return err
}
compressor, err := compression.Compress(
encryptor,
pipes.Compression,
compressionLevel,
isRegular,
recordSize,
)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
if isRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
}
} else {
buf := make([]byte, controllers.BlockSize*recordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
}
if err := compressor.Close(); err != nil {
return err
}
if err := encryptor.Close(); err != nil {
return err
}
dirty = true
return nil
})
}

29
pkg/operations/operations.go
View File

@@ -1,22 +1,9 @@
package operations
import (
"archive/tar"
"github.com/pojntfx/stfs/pkg/config"
)
func Archive(
state config.StateConfig,
pipes config.PipeConfig,
crypto config.CryptoConfig,
recordSize int,
from string,
overwrite bool,
compressionLevel string,
) ([]*tar.Header, error)
func Restore(
state config.StateConfig,
pipes config.PipeConfig,
@@ -26,7 +13,9 @@ func Restore(
from string,
to string,
flatten bool,
) error
) error {
return nil
}
func Update(
state config.StateConfig,
@@ -37,7 +26,9 @@ func Update(
from string,
overwrite bool,
compressionLevel string,
) error
) error {
return nil
}
func Delete(
state config.StateConfig,
@@ -46,7 +37,9 @@ func Delete(
recordSize int,
name string,
) error
) error {
return nil
}
func Move(
state config.StateConfig,
@@ -56,4 +49,6 @@ func Move(
recordSize int,
from string,
to string,
) error
) error {
return nil
}