refactor: Decompose archive func

internal/compression/compress.go

@@ -0,0 +1,188 @@
+package compression
+
+import (
+	"compress/gzip"
+	"io"
+	"math"
+
+	"github.com/andybalholm/brotli"
+	"github.com/dsnet/compress/bzip2"
+	"github.com/klauspost/compress/zstd"
+	"github.com/klauspost/pgzip"
+	"github.com/pierrec/lz4/v4"
+	"github.com/pojntfx/stfs/internal/controllers"
+	"github.com/pojntfx/stfs/internal/noop"
+	"github.com/pojntfx/stfs/pkg/config"
+)
+
+func Compress(
+	dst io.Writer,
+	compressionFormat string,
+	compressionLevel string,
+	isRegular bool,
+	recordSize int,
+) (noop.Flusher, error) {
+	switch compressionFormat {
+	case config.CompressionFormatGZipKey:
+		fallthrough
+	case config.CompressionFormatParallelGZipKey:
+		if compressionFormat == config.CompressionFormatGZipKey {
+			if !isRegular {
+				maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
+
+				if maxSize < 65535 { // See https://www.daylight.com/meetings/mug00/Sayle/gzip.html#:~:text=Stored%20blocks%20are%20allowed%20to,size%20of%20the%20gzip%20header.
+					return nil, config.ErrCompressionFormatRequiresLargerRecordSize
+				}
+			}
+
+			l := gzip.DefaultCompression
+			switch compressionLevel {
+			case config.CompressionLevelFastest:
+				l = gzip.BestSpeed
+			case config.CompressionLevelBalanced:
+				l = gzip.DefaultCompression
+			case config.CompressionLevelSmallest:
+				l = gzip.BestCompression
+			default:
+				return nil, config.ErrCompressionLevelUnsupported
+			}
+
+			return gzip.NewWriterLevel(dst, l)
+		}
+
+		if !isRegular {
+			return nil, config.ErrCompressionFormatOnlyRegularSupport // "device or resource busy"
+		}
+
+		l := pgzip.DefaultCompression
+		switch compressionLevel {
+		case config.CompressionLevelFastest:
+			l = pgzip.BestSpeed
+		case config.CompressionLevelBalanced:
+			l = pgzip.DefaultCompression
+		case config.CompressionLevelSmallest:
+			l = pgzip.BestCompression
+		default:
+			return nil, config.ErrCompressionLevelUnsupported
+		}
+
+		return pgzip.NewWriterLevel(dst, l)
+	case config.CompressionFormatLZ4Key:
+		l := lz4.Level5
+		switch compressionLevel {
+		case config.CompressionLevelFastest:
+			l = lz4.Level1
+		case config.CompressionLevelBalanced:
+			l = lz4.Level5
+		case config.CompressionLevelSmallest:
+			l = lz4.Level9
+		default:
+			return nil, config.ErrCompressionLevelUnsupported
+		}
+
+		opts := []lz4.Option{lz4.CompressionLevelOption(l), lz4.ConcurrencyOption(-1)}
+		if !isRegular {
+			maxSize := getNearestPowerOf2Lower(controllers.BlockSize * recordSize)
+
+			if uint32(maxSize) < uint32(lz4.Block64Kb) {
+				return nil, config.ErrCompressionFormatRequiresLargerRecordSize
+			}
+
+			if uint32(maxSize) < uint32(lz4.Block256Kb) {
+				opts = append(opts, lz4.BlockSizeOption(lz4.Block64Kb))
+			} else if uint32(maxSize) < uint32(lz4.Block1Mb) {
+				opts = append(opts, lz4.BlockSizeOption(lz4.Block256Kb))
+			} else if uint32(maxSize) < uint32(lz4.Block4Mb) {
+				opts = append(opts, lz4.BlockSizeOption(lz4.Block1Mb))
+			} else {
+				opts = append(opts, lz4.BlockSizeOption(lz4.Block4Mb))
+			}
+		}
+
+		lz := lz4.NewWriter(dst)
+		if err := lz.Apply(opts...); err != nil {
+			return nil, err
+		}
+
+		return noop.AddFlush(lz), nil
+	case config.CompressionFormatZStandardKey:
+		l := zstd.SpeedDefault
+		switch compressionLevel {
+		case config.CompressionLevelFastest:
+			l = zstd.SpeedFastest
+		case config.CompressionLevelBalanced:
+			l = zstd.SpeedDefault
+		case config.CompressionLevelSmallest:
+			l = zstd.SpeedBestCompression
+		default:
+			return nil, config.ErrCompressionLevelUnsupported
+		}
+
+		opts := []zstd.EOption{zstd.WithEncoderLevel(l)}
+		if !isRegular {
+			opts = append(opts, zstd.WithWindowSize(getNearestPowerOf2Lower(controllers.BlockSize*recordSize)))
+		}
+
+		zz, err := zstd.NewWriter(dst, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		return zz, nil
+	case config.CompressionFormatBrotliKey:
+		if !isRegular {
+			return nil, config.ErrCompressionFormatOnlyRegularSupport // "cannot allocate memory"
+		}
+
+		l := brotli.DefaultCompression
+		switch compressionLevel {
+		case config.CompressionLevelFastest:
+			l = brotli.BestSpeed
+		case config.CompressionLevelBalanced:
+			l = brotli.DefaultCompression
+		case config.CompressionLevelSmallest:
+			l = brotli.BestCompression
+		default:
+			return nil, config.ErrCompressionLevelUnsupported
+		}
+
+		br := brotli.NewWriterLevel(dst, l)
+
+		return br, nil
+	case config.CompressionFormatBzip2Key:
+		fallthrough
+	case config.CompressionFormatBzip2ParallelKey:
+		l := bzip2.DefaultCompression
+		switch compressionLevel {
+		case config.CompressionLevelFastest:
+			l = bzip2.BestSpeed
+		case config.CompressionLevelBalanced:
+			l = bzip2.DefaultCompression
+		case config.CompressionLevelSmallest:
+			l = bzip2.BestCompression
+		default:
+			return nil, config.ErrCompressionLevelUnsupported
+		}
+
+		bz, err := bzip2.NewWriter(dst, &bzip2.WriterConfig{
+			Level: l,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		return noop.AddFlush(bz), nil
+	case config.NoneKey:
+		return noop.AddFlush(noop.AddClose(dst)), nil
+	default:
+		return nil, config.ErrCompressionFormatUnsupported
+	}
+}
+
+func getNearestPowerOf2Lower(n int) int {
+	return int(math.Pow(2, float64(getNearestLogOf2Lower(n)))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
+}
+
+func getNearestLogOf2Lower(n int) int {
+	return int(math.Log2(float64(n))) // Truncation is intentional, see https://www.geeksforgeeks.org/highest-power-2-less-equal-given-number/
+}

internal/compression/decompress.go

@@ -54,6 +54,6 @@ func Decompress(
 	case config.NoneKey:
 		return io.NopCloser(src), nil
 	default:
-		return nil, config.ErrUnsupportedCompressionFormat
+		return nil, config.ErrCompressionFormatUnsupported
 	}
 }

internal/encryption/decrypt.go

@@ -46,7 +46,7 @@ func Decrypt(
 	case config.NoneKey:
 		return io.NopCloser(src), nil
 	default:
-		return nil, config.ErrUnsupportedEncryptionFormat
+		return nil, config.ErrEncryptionFormatUnsupported
 	}
 }
 
@@ -136,6 +136,6 @@ func DecryptString(
 	case config.NoneKey:
 		return src, nil
 	default:
-		return "", config.ErrUnsupportedEncryptionFormat
+		return "", config.ErrEncryptionFormatUnsupported
 	}
 }

internal/encryption/encrypt.go

@@ -0,0 +1,127 @@
+package encryption
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+
+	"filippo.io/age"
+	"github.com/pojntfx/stfs/internal/noop"
+	"github.com/pojntfx/stfs/internal/pax"
+	"github.com/pojntfx/stfs/pkg/config"
+	"golang.org/x/crypto/openpgp"
+)
+
+func Encrypt(
+	dst io.Writer,
+	encryptionFormat string,
+	recipient interface{},
+) (io.WriteCloser, error) {
+	switch encryptionFormat {
+	case config.EncryptionFormatAgeKey:
+		recipient, ok := recipient.(*age.X25519Recipient)
+		if !ok {
+			return nil, config.ErrRecipientUnparsable
+		}
+
+		return age.Encrypt(dst, recipient)
+	case config.EncryptionFormatPGPKey:
+		recipient, ok := recipient.(openpgp.EntityList)
+		if !ok {
+			return nil, config.ErrRecipientUnparsable
+		}
+
+		return openpgp.Encrypt(dst, recipient, nil, nil, nil)
+	case config.NoneKey:
+		return noop.AddClose(dst), nil
+	default:
+		return nil, config.ErrEncryptionFormatUnsupported
+	}
+}
+
+func EncryptHeader(
+	hdr *tar.Header,
+	encryptionFormat string,
+	recipient interface{},
+) error {
+	if encryptionFormat == config.NoneKey {
+		return nil
+	}
+
+	newHdr := &tar.Header{
+		Format:     tar.FormatPAX,
+		Size:       hdr.Size,
+		PAXRecords: map[string]string{},
+	}
+
+	wrappedHeader, err := json.Marshal(hdr)
+	if err != nil {
+		return err
+	}
+
+	newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], err = EncryptString(string(wrappedHeader), encryptionFormat, recipient)
+	if err != nil {
+		return err
+	}
+
+	*hdr = *newHdr
+
+	return nil
+}
+
+func EncryptString(
+	src string,
+	encryptionFormat string,
+	recipient interface{},
+) (string, error) {
+	switch encryptionFormat {
+	case config.EncryptionFormatAgeKey:
+		recipient, ok := recipient.(*age.X25519Recipient)
+		if !ok {
+			return "", config.ErrRecipientUnparsable
+		}
+
+		out := &bytes.Buffer{}
+		w, err := age.Encrypt(out, recipient)
+		if err != nil {
+			return "", err
+		}
+
+		if _, err := io.WriteString(w, src); err != nil {
+			return "", err
+		}
+
+		if err := w.Close(); err != nil {
+			return "", err
+		}
+
+		return base64.StdEncoding.EncodeToString(out.Bytes()), nil
+	case config.EncryptionFormatPGPKey:
+		recipient, ok := recipient.(openpgp.EntityList)
+		if !ok {
+			return "", config.ErrRecipientUnparsable
+		}
+
+		out := &bytes.Buffer{}
+		w, err := openpgp.Encrypt(out, recipient, nil, nil, nil)
+		if err != nil {
+			return "", err
+		}
+
+		if _, err := io.WriteString(w, src); err != nil {
+			return "", err
+		}
+
+		if err := w.Close(); err != nil {
+			return "", err
+		}
+
+		return base64.StdEncoding.EncodeToString(out.Bytes()), nil
+	case config.NoneKey:
+		return src, nil
+	default:
+		return "", config.ErrEncryptionFormatUnsupported
+	}
+}

internal/keys/identity.go

@@ -65,7 +65,7 @@ func ParseIdentity(
 	case config.NoneKey:
 		return privkey, nil
 	default:
-		return nil, config.ErrUnsupportedEncryptionFormat
+		return nil, config.ErrEncryptionFormatUnsupported
 	}
 }
 
@@ -82,6 +82,6 @@ func ParseSignerIdentity(
 	case config.NoneKey:
 		return privkey, nil
 	default:
-		return nil, config.ErrUnsupportedSignatureFormat
+		return nil, config.ErrSignatureFormatUnsupported
 	}
 }

internal/keys/recipient.go

@@ -21,7 +21,7 @@ func ParseRecipient(
 	case config.NoneKey:
 		return pubkey, nil
 	default:
-		return nil, config.ErrUnsupportedEncryptionFormat
+		return nil, config.ErrEncryptionFormatUnsupported
 	}
 }
 
@@ -42,6 +42,6 @@ func ParseSignerRecipient(
 	case config.NoneKey:
 		return pubkey, nil
 	default:
-		return nil, config.ErrUnsupportedSignatureFormat
+		return nil, config.ErrSignatureFormatUnsupported
 	}
 }

internal/signature/sign.go

@@ -0,0 +1,159 @@
+package signature
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+
+	"aead.dev/minisign"
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+	"github.com/pojntfx/stfs/internal/pax"
+	"github.com/pojntfx/stfs/pkg/config"
+)
+
+func Sign(
+	src io.Reader,
+	isRegular bool,
+	signatureFormat string,
+	identity interface{},
+) (io.Reader, func() (string, error), error) {
+	switch signatureFormat {
+	case config.SignatureFormatMinisignKey:
+		if !isRegular {
+			return nil, nil, config.ErrSignatureFormatOnlyRegularSupport
+		}
+
+		identity, ok := identity.(minisign.PrivateKey)
+		if !ok {
+			return nil, nil, config.ErrIdentityUnparsable
+		}
+
+		signer := minisign.NewReader(src)
+
+		return signer, func() (string, error) {
+			return base64.StdEncoding.EncodeToString(signer.Sign(identity)), nil
+		}, nil
+	case config.SignatureFormatPGPKey:
+		identities, ok := identity.(openpgp.EntityList)
+		if !ok {
+			return nil, nil, config.ErrIdentityUnparsable
+		}
+
+		if len(identities) < 1 {
+			return nil, nil, config.ErrIdentityUnparsable
+		}
+
+		// See openpgp.DetachSign
+		var c *packet.Config
+		signingKey, ok := identities[0].SigningKeyById(c.Now(), c.SigningKey())
+		if !ok || signingKey.PrivateKey == nil || signingKey.PublicKey == nil {
+			return nil, nil, config.ErrIdentityUnparsable
+		}
+
+		sig := new(packet.Signature)
+		sig.SigType = packet.SigTypeBinary
+		sig.PubKeyAlgo = signingKey.PrivateKey.PubKeyAlgo
+		sig.Hash = c.Hash()
+		sig.CreationTime = c.Now()
+		sigLifetimeSecs := c.SigLifetime()
+		sig.SigLifetimeSecs = &sigLifetimeSecs
+		sig.IssuerKeyId = &signingKey.PrivateKey.KeyId
+
+		hash := sig.Hash.New()
+
+		return io.TeeReader(src, hash), func() (string, error) {
+			if err := sig.Sign(hash, signingKey.PrivateKey, c); err != nil {
+				return "", err
+			}
+
+			out := &bytes.Buffer{}
+			if err := sig.Serialize(out); err != nil {
+				return "", err
+			}
+
+			return base64.StdEncoding.EncodeToString(out.Bytes()), nil
+		}, nil
+	case config.NoneKey:
+		return src, func() (string, error) {
+			return "", nil
+		}, nil
+	default:
+		return nil, nil, config.ErrSignatureFormatUnsupported
+	}
+}
+
+func SignHeader(
+	hdr *tar.Header,
+	isRegular bool,
+	signatureFormat string,
+	identity interface{},
+) error {
+	if signatureFormat == config.NoneKey {
+		return nil
+	}
+
+	newHdr := &tar.Header{
+		Format:     tar.FormatPAX,
+		Size:       hdr.Size,
+		PAXRecords: map[string]string{},
+	}
+
+	wrappedHeader, err := json.Marshal(hdr)
+	if err != nil {
+		return err
+	}
+
+	newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader] = string(wrappedHeader)
+	newHdr.PAXRecords[pax.STFSRecordSignature], err = SignString(newHdr.PAXRecords[pax.STFSRecordEmbeddedHeader], isRegular, signatureFormat, identity)
+	if err != nil {
+		return err
+	}
+
+	*hdr = *newHdr
+
+	return nil
+}
+
+func SignString(
+	src string,
+	isRegular bool,
+	signatureFormat string,
+	identity interface{},
+) (string, error) {
+	switch signatureFormat {
+	case config.SignatureFormatMinisignKey:
+		if !isRegular {
+			return "", config.ErrSignatureFormatOnlyRegularSupport
+		}
+
+		identity, ok := identity.(minisign.PrivateKey)
+		if !ok {
+			return "", config.ErrIdentityUnparsable
+		}
+
+		return base64.StdEncoding.EncodeToString(minisign.Sign(identity, []byte(src))), nil
+	case config.SignatureFormatPGPKey:
+		identities, ok := identity.(openpgp.EntityList)
+		if !ok {
+			return "", config.ErrIdentityUnparsable
+		}
+
+		if len(identities) < 1 {
+			return "", config.ErrIdentityUnparsable
+		}
+
+		out := &bytes.Buffer{}
+		if err := openpgp.DetachSign(out, identities[0], bytes.NewBufferString(src), nil); err != nil {
+			return "", err
+		}
+
+		return base64.StdEncoding.EncodeToString(out.Bytes()), nil
+	case config.NoneKey:
+		return src, nil
+	default:
+		return "", config.ErrSignatureFormatUnsupported
+	}
+}

internal/signature/verify.go

@@ -84,7 +84,7 @@ func Verify(
 			return nil
 		}, nil
 	default:
-		return nil, nil, config.ErrUnsupportedSignatureFormat
+		return nil, nil, config.ErrSignatureFormatUnsupported
 	}
 }
 
@@ -190,6 +190,6 @@ func VerifyString(
 	case config.NoneKey:
 		return nil
 	default:
-		return config.ErrUnsupportedSignatureFormat
+		return config.ErrSignatureFormatUnsupported
 	}
 }

internal/suffix/add.go

@@ -0,0 +1,37 @@
+package suffix
+
+import "github.com/pojntfx/stfs/pkg/config"
+
+func AddSuffix(name string, compressionFormat string, encryptionFormat string) (string, error) {
+	switch compressionFormat {
+	case config.CompressionFormatGZipKey:
+		fallthrough
+	case config.CompressionFormatParallelGZipKey:
+		name += CompressionFormatGZipSuffix
+	case config.CompressionFormatLZ4Key:
+		name += CompressionFormatLZ4Suffix
+	case config.CompressionFormatZStandardKey:
+		name += CompressionFormatZStandardSuffix
+	case config.CompressionFormatBrotliKey:
+		name += CompressionFormatBrotliSuffix
+	case config.CompressionFormatBzip2Key:
+		fallthrough
+	case config.CompressionFormatBzip2ParallelKey:
+		name += CompressionFormatBzip2Suffix
+	case config.NoneKey:
+	default:
+		return "", config.ErrCompressionFormatUnsupported
+	}
+
+	switch encryptionFormat {
+	case config.EncryptionFormatAgeKey:
+		name += EncryptionFormatAgeSuffix
+	case config.EncryptionFormatPGPKey:
+		name += EncryptionFormatPGPSuffix
+	case config.NoneKey:
+	default:
+		return "", config.ErrEncryptionFormatUnsupported
+	}
+
+	return name, nil
+}

internal/suffix/remove.go

@@ -14,7 +14,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
 		name = strings.TrimSuffix(name, EncryptionFormatPGPSuffix)
 	case config.NoneKey:
 	default:
-		return "", config.ErrUnsupportedEncryptionFormat
+		return "", config.ErrEncryptionFormatUnsupported
 	}
 
 	switch compressionFormat {
@@ -34,7 +34,7 @@ func RemoveSuffix(name string, compressionFormat string, encryptionFormat string
 		name = strings.TrimSuffix(name, CompressionFormatBzip2Suffix)
 	case config.NoneKey:
 	default:
-		return "", config.ErrUnsupportedCompressionFormat
+		return "", config.ErrCompressionFormatUnsupported
 	}
 
 	return name, nil

internal/tape/read.go

@@ -2,15 +2,15 @@ package tape
 
 import "os"
 
-func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) {
-	fileDescription, err := os.Stat(tape)
+func OpenTapeReadOnly(drive string) (f *os.File, isRegular bool, err error) {
+	fileDescription, err := os.Stat(drive)
 	if err != nil {
 		return nil, false, err
 	}
 
 	isRegular = fileDescription.Mode().IsRegular()
 	if isRegular {
-		f, err = os.Open(tape)
+		f, err = os.Open(drive)
 		if err != nil {
 			return f, isRegular, err
 		}
@@ -18,7 +18,7 @@ func OpenTapeReadOnly(tape string) (f *os.File, isRegular bool, err error) {
 		return f, isRegular, nil
 	}
 
-	f, err = os.OpenFile(tape, os.O_RDONLY, os.ModeCharDevice)
+	f, err = os.OpenFile(drive, os.O_RDONLY, os.ModeCharDevice)
 	if err != nil {
 		return f, isRegular, err
 	}

internal/tape/write.go

@@ -0,0 +1,79 @@
+package tape
+
+import (
+	"archive/tar"
+	"bufio"
+	"os"
+
+	"github.com/pojntfx/stfs/internal/controllers"
+	"github.com/pojntfx/stfs/internal/counters"
+)
+
+func OpenTapeWriteOnly(drive string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {
+	stat, err := os.Stat(drive)
+	if err == nil {
+		isRegular = stat.Mode().IsRegular()
+	} else {
+		if os.IsNotExist(err) {
+			isRegular = true
+		} else {
+			return nil, false, nil, err
+		}
+	}
+
+	var f *os.File
+	if isRegular {
+		f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+		if err != nil {
+			return nil, false, nil, err
+		}
+
+		// No need to go to end manually due to `os.O_APPEND`
+	} else {
+		f, err = os.OpenFile(drive, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
+		if err != nil {
+			return nil, false, nil, err
+		}
+
+		if !overwrite {
+			// Go to end of tape
+			if err := controllers.GoToEndOfTape(f); err != nil {
+				return nil, false, nil, err
+			}
+		}
+	}
+
+	var bw *bufio.Writer
+	var counter *counters.CounterWriter
+	if isRegular {
+		tw = tar.NewWriter(f)
+	} else {
+		bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize)
+		counter = &counters.CounterWriter{Writer: bw, BytesRead: 0}
+		tw = tar.NewWriter(counter)
+	}
+
+	return tw, isRegular, func(dirty *bool) error {
+		// Only write the trailer if we wrote to the archive
+		if *dirty {
+			if err := tw.Close(); err != nil {
+				return err
+			}
+
+			if !isRegular {
+				if controllers.BlockSize*recordSize-counter.BytesRead > 0 {
+					// Fill the rest of the record with zeros
+					if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil {
+						return err
+					}
+				}
+
+				if err := bw.Flush(); err != nil {
+					return err
+				}
+			}
+		}
+
+		return f.Close()
+	}, nil
+}