refactor: Decompose archive func
This commit is contained in:
Felix Pojtinger
@@ -2,39 +2,16 @@ package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"aead.dev/minisign"
|
||||
"filippo.io/age"
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/dsnet/compress/bzip2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/klauspost/pgzip"
|
||||
"github.com/pierrec/lz4/v4"
|
||||
"github.com/pojntfx/stfs/internal/adapters"
|
||||
"github.com/pojntfx/stfs/internal/controllers"
|
||||
"github.com/pojntfx/stfs/internal/counters"
|
||||
"github.com/pojntfx/stfs/internal/formatting"
|
||||
"github.com/pojntfx/stfs/internal/keys"
|
||||
"github.com/pojntfx/stfs/internal/noop"
|
||||
"github.com/pojntfx/stfs/internal/pax"
|
||||
"github.com/pojntfx/stfs/internal/persisters"
|
||||
"github.com/pojntfx/stfs/pkg/config"
|
||||
"github.com/pojntfx/stfs/pkg/operations"
|
||||
"github.com/pojntfx/stfs/pkg/recovery"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
@@ -50,14 +27,10 @@ const (
|
||||
recipientFlag = "recipient"
|
||||
identityFlag = "identity"
|
||||
passwordFlag = "password"
|
||||
|
||||
compressionLevelFastest = "fastest"
|
||||
compressionLevelBalanced = "balanced"
|
||||
compressionLevelSmallest = "smallest"
|
||||
)
|
||||
|
||||
var (
|
||||
knownCompressionLevels = []string{compressionLevelFastest, compressionLevelBalanced, compressionLevelSmallest}
|
||||
knownCompressionLevels = []string{config.CompressionLevelFastest, config.CompressionLevelBalanced, config.CompressionLevelSmallest}
|
||||
|
||||
errUnknownCompressionLevel = errors.New("unknown compression level")
|
||||
errUnsupportedCompressionLevel = errors.New("unsupported compression level")
|
||||
@@ -121,7 +94,7 @@ var archiveCmd = &cobra.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -136,21 +109,27 @@ var archiveCmd = &cobra.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
hdrs, err := archive(
|
||||
viper.GetString(driveFlag),
|
||||
hdrs, err := operations.Archive(
|
||||
config.StateConfig{
|
||||
Drive: viper.GetString(driveFlag),
|
||||
Metadata: viper.GetString(metadataFlag),
|
||||
},
|
||||
config.PipeConfig{
|
||||
Compression: viper.GetString(compressionFlag),
|
||||
Encryption: viper.GetString(encryptionFlag),
|
||||
Signature: viper.GetString(signatureFlag),
|
||||
},
|
||||
config.CryptoConfig{
|
||||
Recipient: recipient,
|
||||
Identity: identity,
|
||||
Password: viper.GetString(passwordFlag),
|
||||
},
|
||||
|
||||
viper.GetInt(recordSizeFlag),
|
||||
viper.GetString(fromFlag),
|
||||
viper.GetBool(overwriteFlag),
|
||||
viper.GetString(compressionFlag),
|
||||
viper.GetString(compressionLevelFlag),
|
||||
viper.GetString(encryptionFlag),
|
||||
recipient,
|
||||
viper.GetString(signatureFlag),
|
||||
identity,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return recovery.Index(
|
||||
config.StateConfig{
|
||||
@@ -194,267 +173,6 @@ var archiveCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
func archive(
|
||||
tape string,
|
||||
recordSize int,
|
||||
src string,
|
||||
overwrite bool,
|
||||
compressionFormat string,
|
||||
compressionLevel string,
|
||||
encryptionFormat string,
|
||||
recipient interface{},
|
||||
signatureFormat string,
|
||||
identity interface{},
|
||||
) ([]*tar.Header, error) {
|
||||
dirty := false
|
||||
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, overwrite)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
if overwrite {
|
||||
if isRegular {
|
||||
if err := cleanup(&dirty); err != nil { // dirty will always be false here
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(tape, os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
// Clear the file's content
|
||||
if err := f.Truncate(0); err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
} else {
|
||||
if err := cleanup(&dirty); err != nil { // dirty will always be false here
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(tape, os.O_WRONLY, os.ModeCharDevice)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
// Seek to the start of the tape
|
||||
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
|
||||
tw, isRegular, cleanup, err = openTapeWriter(tape, recordSize, overwrite)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
defer cleanup(&dirty)
|
||||
|
||||
headers := []*tar.Header{}
|
||||
first := true
|
||||
return headers, filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
link := ""
|
||||
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
if link, err = os.Readlink(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
hdr, err := tar.FileInfoHeader(info, link)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := adapters.EnhanceHeader(path, hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hdr.Name = path
|
||||
hdr.Format = tar.FormatPAX
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
// Get the compressed size for the header
|
||||
fileSizeCounter := &counters.CounterWriter{
|
||||
Writer: io.Discard,
|
||||
}
|
||||
|
||||
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressor, err := compress(
|
||||
encryptor,
|
||||
compressionFormat,
|
||||
compressionLevel,
|
||||
isRegular,
|
||||
recordSize,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
signer, sign, err := sign(file, isRegular, signatureFormat, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isRegular {
|
||||
if _, err := io.Copy(compressor, signer); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
buf := make([]byte, controllers.BlockSize*recordSize)
|
||||
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := compressor.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := compressor.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptor.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if hdr.PAXRecords == nil {
|
||||
hdr.PAXRecords = map[string]string{}
|
||||
}
|
||||
hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
|
||||
signature, err := sign()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if signature != "" {
|
||||
hdr.PAXRecords[pax.STFSRecordSignature] = signature
|
||||
}
|
||||
hdr.Size = int64(fileSizeCounter.BytesRead)
|
||||
|
||||
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if first {
|
||||
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
first = false
|
||||
}
|
||||
|
||||
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hdrToAppend := *hdr
|
||||
headers = append(headers, &hdrToAppend)
|
||||
|
||||
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compress and write the file
|
||||
encryptor, err := encrypt(tw, encryptionFormat, recipient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressor, err := compress(
|
||||
encryptor,
|
||||
compressionFormat,
|
||||
compressionLevel,
|
||||
isRegular,
|
||||
recordSize,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isRegular {
|
||||
if _, err := io.Copy(compressor, file); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
buf := make([]byte, controllers.BlockSize*recordSize)
|
||||
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := compressor.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := compressor.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptor.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dirty = true
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func checkKeyAccessible(encryptionFormat string, pathToKey string) error {
|
||||
if encryptionFormat == noneKey {
|
||||
return nil
|
||||
@@ -491,489 +209,11 @@ func checkCompressionLevel(compressionLevel string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func encryptHeader(
|
||||
hdr *tar.Header,
|
||||
encryptionFormat string,
|
||||
recipient interface{},
|
||||
) error {
|
||||
if encryptionFormat == noneKey {
|
||||
return nil
|
||||
}
|
||||
|
||||
newHdr := &tar.Header{
|
||||
Format: tar.FormatPAX,
|
||||
Size: hdr.Size,
|
||||
PAXRecords: map[string]string{},
|
||||
}
|
||||
|
||||
wrappedHeader, err := json.Marshal(hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = encryptString(string(wrappedHeader), encryptionFormat, recipient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*hdr = *newHdr
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func signHeader(
|
||||
hdr *tar.Header,
|
||||
isRegular bool,
|
||||
signatureFormat string,
|
||||
identity interface{},
|
||||
) error {
|
||||
if signatureFormat == noneKey {
|
||||
return nil
|
||||
}
|
||||
|
||||
newHdr := &tar.Header{
|
||||
Format: tar.FormatPAX,
|
||||
Size: hdr.Size,
|
||||
PAXRecords: map[string]string{},
|
||||
}
|
||||
|
||||
wrappedHeader, err := json.Marshal(hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
|
||||
newHdr.PAXRecords[pax.STFSRecordSignature], err = signString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*hdr = *newHdr
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func addSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
|
||||
switch compressionFormat {
|
||||
case compressionFormatGZipKey:
|
||||
fallthrough
|
||||
case compressionFormatParallelGZipKey:
|
||||
name += compressionFormatGZipSuffix
|
||||
case compressionFormatLZ4Key:
|
||||
name += compressionFormatLZ4Suffix
|
||||
case compressionFormatZStandardKey:
|
||||
name += compressionFormatZStandardSuffix
|
||||
case compressionFormatBrotliKey:
|
||||
name += compressionFormatBrotliSuffix
|
||||
case compressionFormatBzip2Key:
|
||||
fallthrough
|
||||
case compressionFormatBzip2ParallelKey:
|
||||
name += compressionFormatBzip2Suffix
|
||||
case noneKey:
|
||||
default:
|
||||
return "", errUnsupportedCompressionFormat
|
||||
}
|
||||
|
||||
switch encryptionFormat {
|
||||
case encryptionFormatAgeKey:
|
||||
name += encryptionFormatAgeSuffix
|
||||
case encryptionFormatPGPKey:
|
||||
name += encryptionFormatPGPSuffix
|
||||
case noneKey:
|
||||
default:
|
||||
return "", errUnsupportedEncryptionFormat
|
||||
}
|
||||
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func parseRecipient(
|
||||
encryptionFormat string,
|
||||
pubkey []byte,
|
||||
) (interface{}, error) {
|
||||
switch encryptionFormat {
|
||||
case encryptionFormatAgeKey:
|
||||
return age.ParseX25519Recipient(string(pubkey))
|
||||
case encryptionFormatPGPKey:
|
||||
return openpgp.ReadKeyRing(bytes.NewBuffer(pubkey))
|
||||
case noneKey:
|
||||
return pubkey, nil
|
||||
default:
|
||||
return nil, errUnsupportedEncryptionFormat
|
||||
}
|
||||
}
|
||||
|
||||
func encrypt(
|
||||
dst io.Writer,
|
||||
encryptionFormat string,
|
||||
recipient interface{},
|
||||
) (io.WriteCloser, error) {
|
||||
switch encryptionFormat {
|
||||
case encryptionFormatAgeKey:
|
||||
recipient, ok := recipient.(*age.X25519Recipient)
|
||||
if !ok {
|
||||
return nil, errRecipientUnparsable
|
||||
}
|
||||
|
||||
return age.Encrypt(dst, recipient)
|
||||
case encryptionFormatPGPKey:
|
||||
recipient, ok := recipient.(openpgp.EntityList)
|
||||
if !ok {
|
||||
return nil, errRecipientUnparsable
|
||||
}
|
||||
|
||||
return openpgp.Encrypt(dst, recipient, nil, nil, nil)
|
||||
case noneKey:
|
||||
return noop.AddClose(dst), nil
|
||||
default:
|
||||
return nil, errUnsupportedEncryptionFormat
|
||||
}
|
||||
}
|
||||
|
||||
func sign(
|
||||
src io.Reader,
|
||||
isRegular bool,
|
||||
signatureFormat string,
|
||||
identity interface{},
|
||||
) (io.Reader, func() (string, error), error) {
|
||||
switch signatureFormat {
|
||||
case signatureFormatMinisignKey:
|
||||
if !isRegular {
|
||||
return nil, nil, errSignatureFormatOnlyRegularSupport
|
||||
}
|
||||
|
||||
identity, ok := identity.(minisign.PrivateKey)
|
||||
if !ok {
|
||||
return nil, nil, errIdentityUnparsable
|
||||
}
|
||||
|
||||
signer := minisign.NewReader(src)
|
||||
|
||||
return signer, func() (string, error) {
|
||||
return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
|
||||
}, nil
|
||||
case signatureFormatPGPKey:
|
||||
identities, ok := identity.(openpgp.EntityList)
|
||||
if !ok {
|
||||
return nil, nil, errIdentityUnparsable
|
||||
}
|
||||
|
||||
if len(identities) < 1 {
|
||||
return nil, nil, errIdentityUnparsable
|
||||
}
|
||||
|
||||
// See openpgp.DetachSign
|
||||
var config *packet.Config
|
||||
signingKey, ok := identities[0].SigningKeyById(config.Now(), config.SigningKey())
|
||||
if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
|
||||
return nil, nil, errIdentityUnparsable
|
||||
}
|
||||
|
||||
sig := new(packet.Signature)
|
||||
sig.SigType = packet.SigTypeBinary
|
||||
sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
|
||||
sig.Hash = config.Hash()
|
||||
sig.CreationTime = config.Now()
|
||||
sigLifetimeSecs := config.SigLifetime()
|
||||
sig.SigLifetimeSecs = &sigLifetimeSecs
|
||||
sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
|
||||
|
||||
hash := sig.Hash.New()
|
||||
|
||||
return io.TeeReader(src, hash), func() (string, error) {
|
||||
if err := sig.Sign(hash, signingKey.PrivateKey, config); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
if err := sig.Serialize(out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
|
||||
}, nil
|
||||
case noneKey:
|
||||
return src, func() (string, error) {
|
||||
return "", nil
|
||||
}, nil
|
||||
default:
|
||||
return nil, nil, errUnsupportedSignatureFormat
|
||||
}
|
||||
}
|
||||
|
||||
func encryptString(
|
||||
src string,
|
||||
encryptionFormat string,
|
||||
recipient interface{},
|
||||
) (string, error) {
|
||||
switch encryptionFormat {
|
||||
case encryptionFormatAgeKey:
|
||||
recipient, ok := recipient.(*age.X25519Recipient)
|
||||
if !ok {
|
||||
return "", errRecipientUnparsable
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
w, err := age.Encrypt(out, recipient)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if _, err := io.WriteString(w, src); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
|
||||
case encryptionFormatPGPKey:
|
||||
recipient, ok := recipient.(openpgp.EntityList)
|
||||
if !ok {
|
||||
return "", errRecipientUnparsable
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if _, err := io.WriteString(w, src); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
|
||||
case noneKey:
|
||||
return src, nil
|
||||
default:
|
||||
return "", errUnsupportedEncryptionFormat
|
||||
}
|
||||
}
|
||||
|
||||
func signString(
|
||||
src string,
|
||||
isRegular bool,
|
||||
signatureFormat string,
|
||||
identity interface{},
|
||||
) (string, error) {
|
||||
switch signatureFormat {
|
||||
case signatureFormatMinisignKey:
|
||||
if !isRegular {
|
||||
return "", errSignatureFormatOnlyRegularSupport
|
||||
}
|
||||
|
||||
identity, ok := identity.(minisign.PrivateKey)
|
||||
if !ok {
|
||||
return "", errIdentityUnparsable
|
||||
}
|
||||
|
||||
return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
|
||||
case signatureFormatPGPKey:
|
||||
identities, ok := identity.(openpgp.EntityList)
|
||||
if !ok {
|
||||
return "", errIdentityUnparsable
|
||||
}
|
||||
|
||||
if len(identities) < 1 {
|
||||
return "", errIdentityUnparsable
|
||||
}
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return base64.StdEncoding.EncodeToString(out.Bytes()), nil
|
||||
case noneKey:
|
||||
return src, nil
|
||||
default:
|
||||
return "", errUnsupportedSignatureFormat
|
||||
}
|
||||
}
|
||||
|
||||
func compress(
|
||||
dst io.Writer,
|
||||
compressionFormat string,
|
||||
compressionLevel string,
|
||||
isRegular bool,
|
||||
recordSize int,
|
||||
) (noop.Flusher, error) {
|
||||
switch compressionFormat {
|
||||
case compressionFormatGZipKey:
|
||||
fallthrough
|
||||
case compressionFormatParallelGZipKey:
|
||||
if compressionFormat == compressionFormatGZipKey {
|
||||
if !isRegular {
|
||||
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
|
||||
|
||||
if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
|
||||
return nil, errCompressionFormatRequiresLargerRecordSize
|
||||
}
|
||||
}
|
||||
|
||||
l := gzip.DefaultCompression
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = gzip.BestSpeed
|
||||
case compressionLevelBalanced:
|
||||
l = gzip.DefaultCompression
|
||||
case compressionLevelSmallest:
|
||||
l = gzip.BestCompression
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
return gzip.NewWriterLevel(dst, l)
|
||||
}
|
||||
|
||||
if !isRegular {
|
||||
return nil, errCompressionFormatOnlyRegularSupport // "device or resource busy"
|
||||
}
|
||||
|
||||
l := pgzip.DefaultCompression
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = pgzip.BestSpeed
|
||||
case compressionLevelBalanced:
|
||||
l = pgzip.DefaultCompression
|
||||
case compressionLevelSmallest:
|
||||
l = pgzip.BestCompression
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
return pgzip.NewWriterLevel(dst, l)
|
||||
case compressionFormatLZ4Key:
|
||||
l := lz4.Level5
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = lz4.Level1
|
||||
case compressionLevelBalanced:
|
||||
l = lz4.Level5
|
||||
case compressionLevelSmallest:
|
||||
l = lz4.Level9
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
|
||||
if !isRegular {
|
||||
maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
|
||||
|
||||
if uint32(maxSize) < uint32(lz4.Block64Kb) {
|
||||
return nil, errCompressionFormatRequiresLargerRecordSize
|
||||
}
|
||||
|
||||
if uint32(maxSize) < uint32(lz4.Block256Kb) {
|
||||
opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
|
||||
} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
|
||||
opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
|
||||
} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
|
||||
opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
|
||||
} else {
|
||||
opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
|
||||
}
|
||||
}
|
||||
|
||||
lz := lz4.NewWriter(dst)
|
||||
if err := lz.Apply(opts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return noop.AddFlush(lz), nil
|
||||
case compressionFormatZStandardKey:
|
||||
l := zstd.SpeedDefault
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = zstd.SpeedFastest
|
||||
case compressionLevelBalanced:
|
||||
l = zstd.SpeedDefault
|
||||
case compressionLevelSmallest:
|
||||
l = zstd.SpeedBestCompression
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
|
||||
if !isRegular {
|
||||
opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
|
||||
}
|
||||
|
||||
zz, err := zstd.NewWriter(dst, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return zz, nil
|
||||
case compressionFormatBrotliKey:
|
||||
if !isRegular {
|
||||
return nil, errCompressionFormatOnlyRegularSupport // "cannot allocate memory"
|
||||
}
|
||||
|
||||
l := brotli.DefaultCompression
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = brotli.BestSpeed
|
||||
case compressionLevelBalanced:
|
||||
l = brotli.DefaultCompression
|
||||
case compressionLevelSmallest:
|
||||
l = brotli.BestCompression
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
br := brotli.NewWriterLevel(dst, l)
|
||||
|
||||
return br, nil
|
||||
case compressionFormatBzip2Key:
|
||||
fallthrough
|
||||
case compressionFormatBzip2ParallelKey:
|
||||
l := bzip2.DefaultCompression
|
||||
switch compressionLevel {
|
||||
case compressionLevelFastest:
|
||||
l = bzip2.BestSpeed
|
||||
case compressionLevelBalanced:
|
||||
l = bzip2.DefaultCompression
|
||||
case compressionLevelSmallest:
|
||||
l = bzip2.BestCompression
|
||||
default:
|
||||
return nil, errUnsupportedCompressionLevel
|
||||
}
|
||||
|
||||
bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
|
||||
Level: l,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return noop.AddFlush(bz), nil
|
||||
case noneKey:
|
||||
return noop.AddFlush(noop.AddClose(dst)), nil
|
||||
default:
|
||||
return nil, errUnsupportedCompressionFormat
|
||||
}
|
||||
}
|
||||
|
||||
func getNearestPowerOf2Lower(n int) int {
|
||||
return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
|
||||
}
|
||||
|
||||
func getNearestLogOf2Lower(n int) int {
|
||||
return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
|
||||
}
|
||||
|
||||
func init() {
|
||||
archiveCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
|
||||
archiveCmd.PersistentFlags().StringP(fromFlag, "f", ".", "File or directory to archive")
|
||||
archiveCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Start writing from the start instead of from the end of the tape or tar file")
|
||||
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
|
||||
archiveCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
|
||||
archiveCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
|
||||
archiveCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
|
||||
archiveCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")
|
||||
|
||||
@@ -2,18 +2,17 @@ package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/pojntfx/stfs/internal/controllers"
|
||||
"github.com/pojntfx/stfs/internal/converters"
|
||||
"github.com/pojntfx/stfs/internal/counters"
|
||||
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
|
||||
"github.com/pojntfx/stfs/internal/encryption"
|
||||
"github.com/pojntfx/stfs/internal/formatting"
|
||||
"github.com/pojntfx/stfs/internal/keys"
|
||||
"github.com/pojntfx/stfs/internal/pax"
|
||||
"github.com/pojntfx/stfs/internal/persisters"
|
||||
"github.com/pojntfx/stfs/internal/signature"
|
||||
"github.com/pojntfx/stfs/internal/tape"
|
||||
"github.com/pojntfx/stfs/pkg/config"
|
||||
"github.com/pojntfx/stfs/pkg/recovery"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -54,7 +53,7 @@ var deleteCmd = &cobra.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -83,7 +82,7 @@ var deleteCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func delete(
|
||||
tape string,
|
||||
drive string,
|
||||
recordSize int,
|
||||
metadata string,
|
||||
name string,
|
||||
@@ -93,7 +92,7 @@ func delete(
|
||||
identity interface{},
|
||||
) error {
|
||||
dirty := false
|
||||
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
|
||||
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -142,11 +141,11 @@ func delete(
|
||||
hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1
|
||||
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete
|
||||
|
||||
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -204,75 +203,6 @@ func delete(
|
||||
)
|
||||
}
|
||||
|
||||
func openTapeWriter(tape string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
|
||||
stat, err := os.Stat(tape)
|
||||
if err == nil {
|
||||
isRegular = stat.Mode().IsRegular()
|
||||
} else {
|
||||
if os.IsNotExist(err) {
|
||||
isRegular = true
|
||||
} else {
|
||||
return nil, false, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var f *os.File
|
||||
if isRegular {
|
||||
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, false, nil, err
|
||||
}
|
||||
|
||||
// No need to go to end manually due to `os.O_APPEND`
|
||||
} else {
|
||||
f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
|
||||
if err != nil {
|
||||
return nil, false, nil, err
|
||||
}
|
||||
|
||||
if !overwrite {
|
||||
// Go to end of tape
|
||||
if err := controllers.GoToEndOfTape(f); err != nil {
|
||||
return nil, false, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var bw *bufio.Writer
|
||||
var counter *counters.CounterWriter
|
||||
if isRegular {
|
||||
tw = tar.NewWriter(f)
|
||||
} else {
|
||||
bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
|
||||
counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
|
||||
tw = tar.NewWriter(counter)
|
||||
}
|
||||
|
||||
return tw, isRegular, func(dirty *bool) error {
|
||||
// Only write the trailer if we wrote to the archive
|
||||
if *dirty {
|
||||
if err := tw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !isRegular {
|
||||
if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
|
||||
// Fill the rest of the record with zeros
|
||||
if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
deleteCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
|
||||
deleteCmd.PersistentFlags().StringP(nameFlag, "n", "", "Name of the file to remove")
|
||||
|
||||
@@ -7,10 +7,13 @@ import (
|
||||
|
||||
"github.com/pojntfx/stfs/internal/converters"
|
||||
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
|
||||
"github.com/pojntfx/stfs/internal/encryption"
|
||||
"github.com/pojntfx/stfs/internal/formatting"
|
||||
"github.com/pojntfx/stfs/internal/keys"
|
||||
"github.com/pojntfx/stfs/internal/pax"
|
||||
"github.com/pojntfx/stfs/internal/persisters"
|
||||
"github.com/pojntfx/stfs/internal/signature"
|
||||
"github.com/pojntfx/stfs/internal/tape"
|
||||
"github.com/pojntfx/stfs/pkg/config"
|
||||
"github.com/pojntfx/stfs/pkg/recovery"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -47,7 +50,7 @@ var moveCmd = &cobra.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -77,7 +80,7 @@ var moveCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func move(
|
||||
tape string,
|
||||
drive string,
|
||||
recordSize int,
|
||||
metadata string,
|
||||
src string,
|
||||
@@ -88,7 +91,7 @@ func move(
|
||||
identity interface{},
|
||||
) error {
|
||||
dirty := false
|
||||
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
|
||||
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -139,11 +142,11 @@ func move(
|
||||
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate
|
||||
hdr.PAXRecords[pax.STFSRecordReplacesName] = dbhdr.Name
|
||||
|
||||
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -11,12 +11,17 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/pojntfx/stfs/internal/adapters"
|
||||
"github.com/pojntfx/stfs/internal/compression"
|
||||
"github.com/pojntfx/stfs/internal/controllers"
|
||||
"github.com/pojntfx/stfs/internal/counters"
|
||||
"github.com/pojntfx/stfs/internal/encryption"
|
||||
"github.com/pojntfx/stfs/internal/formatting"
|
||||
"github.com/pojntfx/stfs/internal/keys"
|
||||
"github.com/pojntfx/stfs/internal/pax"
|
||||
"github.com/pojntfx/stfs/internal/persisters"
|
||||
"github.com/pojntfx/stfs/internal/signature"
|
||||
"github.com/pojntfx/stfs/internal/suffix"
|
||||
"github.com/pojntfx/stfs/internal/tape"
|
||||
"github.com/pojntfx/stfs/pkg/config"
|
||||
"github.com/pojntfx/stfs/pkg/recovery"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -67,7 +72,7 @@ var updateCmd = &cobra.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
recipient, err := parseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
recipient, err := keys.ParseRecipient(viper.GetString(encryptionFlag), pubkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -141,7 +146,7 @@ var updateCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func update(
|
||||
tape string,
|
||||
drive string,
|
||||
recordSize int,
|
||||
src string,
|
||||
replacesContent bool,
|
||||
@@ -153,7 +158,7 @@ func update(
|
||||
identity interface{},
|
||||
) ([]*tar.Header, error) {
|
||||
dirty := false
|
||||
tw, isRegular, cleanup, err := openTapeWriter(tape, recordSize, false)
|
||||
tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false)
|
||||
if err != nil {
|
||||
return []*tar.Header{}, err
|
||||
}
|
||||
@@ -196,12 +201,12 @@ func update(
|
||||
Writer: io.Discard,
|
||||
}
|
||||
|
||||
encryptor, err := encrypt(fileSizeCounter, encryptionFormat, recipient)
|
||||
encryptor, err := encryption.Encrypt(fileSizeCounter, encryptionFormat, recipient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressor, err := compress(
|
||||
compressor, err := compression.Compress(
|
||||
encryptor,
|
||||
compressionFormat,
|
||||
compressionLevel,
|
||||
@@ -217,7 +222,7 @@ func update(
|
||||
return err
|
||||
}
|
||||
|
||||
signer, sign, err := sign(file, isRegular, signatureFormat, identity)
|
||||
signer, sign, err := signature.Sign(file, isRegular, signatureFormat, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -263,7 +268,7 @@ func update(
|
||||
}
|
||||
hdr.Size = int64(fileSizeCounter.BytesRead)
|
||||
|
||||
hdr.Name, err = addSuffix(hdr.Name, compressionFormat, encryptionFormat)
|
||||
hdr.Name, err = suffix.AddSuffix(hdr.Name, compressionFormat, encryptionFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -287,11 +292,11 @@ func update(
|
||||
hdrToAppend := *hdr
|
||||
headers = append(headers, &hdrToAppend)
|
||||
|
||||
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -304,12 +309,12 @@ func update(
|
||||
}
|
||||
|
||||
// Compress and write the file
|
||||
encryptor, err := encrypt(tw, encryptionFormat, recipient)
|
||||
encryptor, err := encryption.Encrypt(tw, encryptionFormat, recipient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressor, err := compress(
|
||||
compressor, err := compression.Compress(
|
||||
encryptor,
|
||||
compressionFormat,
|
||||
compressionLevel,
|
||||
@@ -361,11 +366,11 @@ func update(
|
||||
hdrToAppend := *hdr
|
||||
headers = append(headers, &hdrToAppend)
|
||||
|
||||
if err := signHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := encryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -384,7 +389,7 @@ func init() {
|
||||
updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record")
|
||||
updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update")
|
||||
updateCmd.PersistentFlags().BoolP(overwriteFlag, "o", false, "Replace the content on the tape or tar file")
|
||||
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", compressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", compressionLevelBalanced, knownCompressionLevels))
|
||||
updateCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, knownCompressionLevels))
|
||||
updateCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to public key of recipient to encrypt for")
|
||||
updateCmd.PersistentFlags().StringP(identityFlag, "i", "", "Path to private key to sign with")
|
||||
updateCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")
|
||||
|
||||
Reference in New Issue
Block a user