From 3c13fe84eb3f976eaa0087735d27fac251b866f9 Mon Sep 17 00:00:00 2001 From: Felix Pojtinger Date: Tue, 7 Dec 2021 22:53:59 +0100 Subject: [PATCH] refactor: Decompose `update` func --- cmd/stbak/cmd/update.go | 279 +++-------------------------------- pkg/operations/operations.go | 13 -- pkg/operations/update.go | 260 ++++++++++++++++++++++++++++++++ 3 files changed, 277 insertions(+), 275 deletions(-) create mode 100644 pkg/operations/update.go diff --git a/cmd/stbak/cmd/update.go b/cmd/stbak/cmd/update.go index 9a1e6bd..78e90ab 100644 --- a/cmd/stbak/cmd/update.go +++ b/cmd/stbak/cmd/update.go @@ -4,25 +4,11 @@ import ( "archive/tar" "context" "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strconv" - "github.com/pojntfx/stfs/internal/adapters" - "github.com/pojntfx/stfs/internal/compression" - "github.com/pojntfx/stfs/internal/controllers" - "github.com/pojntfx/stfs/internal/counters" - "github.com/pojntfx/stfs/internal/encryption" - "github.com/pojntfx/stfs/internal/formatting" "github.com/pojntfx/stfs/internal/keys" - "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" - "github.com/pojntfx/stfs/internal/signature" - "github.com/pojntfx/stfs/internal/suffix" - "github.com/pojntfx/stfs/internal/tape" "github.com/pojntfx/stfs/pkg/config" + "github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -87,17 +73,26 @@ var updateCmd = &cobra.Command{ return err } - hdrs, err := update( - viper.GetString(driveFlag), + hdrs, err := operations.Update( + config.StateConfig{ + Drive: viper.GetString(driveFlag), + Metadata: viper.GetString(metadataFlag), + }, + config.PipeConfig{ + Compression: viper.GetString(compressionFlag), + Encryption: viper.GetString(encryptionFlag), + Signature: viper.GetString(signatureFlag), + }, + config.CryptoConfig{ + Recipient: recipient, + Identity: identity, + Password: viper.GetString(passwordFlag), + }, + viper.GetInt(recordSizeFlag), viper.GetString(fromFlag), viper.GetBool(overwriteFlag), - viper.GetString(compressionFlag), viper.GetString(compressionLevelFlag), - viper.GetString(encryptionFlag), - recipient, - viper.GetString(signatureFlag), - identity, ) if err != nil { return err @@ -145,246 +140,6 @@ var updateCmd = &cobra.Command{ }, } -func update( - drive string, - recordSize int, - src string, - replacesContent bool, - compressionFormat string, - compressionLevel string, - encryptionFormat string, - recipient interface{}, - signatureFormat string, - identity interface{}, -) ([]*tar.Header, error) { - dirty := false - tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(drive, recordSize, false) - if err != nil { - return []*tar.Header{}, err - } - defer cleanup(&dirty) - - headers := []*tar.Header{} - first := true - return headers, filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - link := "" - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - if link, err = os.Readlink(path); err != nil { - return err - } - } - - hdr, err := tar.FileInfoHeader(info, link) - if err != nil { - return err - } - - if err := adapters.EnhanceHeader(path, hdr); err != nil { - return err - } - - hdr.Name = path - hdr.Format = tar.FormatPAX - if hdr.PAXRecords == nil { - hdr.PAXRecords = map[string]string{} - } - hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1 - hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate - - if info.Mode().IsRegular() && replacesContent { - // Get the compressed size for the header - fileSizeCounter := &counters.CounterWriter{ - Writer: io.Discard, - } - - encryptor, err := encryption.Encrypt(fileSizeCounter, encryptionFormat, recipient) - if err != nil { - return err - } - - compressor, err := compression.Compress( - encryptor, - compressionFormat, - compressionLevel, - isRegular, - recordSize, - ) - if err != nil { - return err - } - - file, err := os.Open(path) - if err != nil { - return err - } - - signer, sign, err := signature.Sign(file, isRegular, signatureFormat, identity) - if err != nil { - return err - } - - if isRegular { - if _, err := io.Copy(compressor, signer); err != nil { - return err - } - } else { - buf := make([]byte, controllers.BlockSize*recordSize) - if _, err := io.CopyBuffer(compressor, signer, buf); err != nil { - return err - } - } - - if err := file.Close(); err != nil { - return err - } - - if err := compressor.Flush(); err != nil { - return err - } - - if err := compressor.Close(); err != nil { - return err - } - - if err := encryptor.Close(); err != nil { - return err - } - - if hdr.PAXRecords == nil { - hdr.PAXRecords = map[string]string{} - } - hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) - signature, err := sign() - if err != nil { - return err - } - - if signature != "" { - hdr.PAXRecords[pax.STFSRecordSignature] = signature - } - hdr.Size = int64(fileSizeCounter.BytesRead) - - hdr.Name, err = suffix.AddSuffix(hdr.Name, compressionFormat, encryptionFormat) - if err != nil { - return err - } - } - - if first { - if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { - return err - } - - first = false - } - - if replacesContent { - hdr.PAXRecords[pax.STFSRecordReplacesContent] = pax.STFSRecordReplacesContentTrue - - if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { - return err - } - - hdrToAppend := *hdr - headers = append(headers, &hdrToAppend) - - if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { - return err - } - - if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { - return err - } - - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - if !info.Mode().IsRegular() { - return nil - } - - // Compress and write the file - encryptor, err := encryption.Encrypt(tw, encryptionFormat, recipient) - if err != nil { - return err - } - - compressor, err := compression.Compress( - encryptor, - compressionFormat, - compressionLevel, - isRegular, - recordSize, - ) - if err != nil { - return err - } - - file, err := os.Open(path) - if err != nil { - return err - } - - if isRegular { - if _, err := io.Copy(compressor, file); err != nil { - return err - } - } else { - buf := make([]byte, controllers.BlockSize*recordSize) - if _, err := io.CopyBuffer(compressor, file, buf); err != nil { - return err - } - } - - if err := file.Close(); err != nil { - return err - } - - if err := compressor.Flush(); err != nil { - return err - } - - if err := compressor.Close(); err != nil { - return err - } - - if err := encryptor.Close(); err != nil { - return err - } - } else { - hdr.Size = 0 // Don't try to seek after the record - - if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { - return err - } - - hdrToAppend := *hdr - headers = append(headers, &hdrToAppend) - - if err := signature.SignHeader(hdr, isRegular, signatureFormat, identity); err != nil { - return err - } - - if err := encryption.EncryptHeader(hdr, encryptionFormat, recipient); err != nil { - return err - } - - if err := tw.WriteHeader(hdr); err != nil { - return err - } - } - - dirty = true - - return nil - }) -} - func init() { updateCmd.PersistentFlags().IntP(recordSizeFlag, "z", 20, "Amount of 512-bit blocks per record") updateCmd.PersistentFlags().StringP(fromFlag, "f", "", "Path of the file or directory to update") diff --git a/pkg/operations/operations.go b/pkg/operations/operations.go index b5ac592..981b41c 100644 --- a/pkg/operations/operations.go +++ b/pkg/operations/operations.go @@ -4,19 +4,6 @@ import ( "github.com/pojntfx/stfs/pkg/config" ) -func Update( - state config.StateConfig, - pipes config.PipeConfig, - crypto config.CryptoConfig, - - recordSize int, - from string, - overwrite bool, - compressionLevel string, -) error { - return nil -} - func Delete( state config.StateConfig, pipes config.PipeConfig, diff --git a/pkg/operations/update.go b/pkg/operations/update.go new file mode 100644 index 0000000..ca7c2bd --- /dev/null +++ b/pkg/operations/update.go @@ -0,0 +1,260 @@ +package operations + +import ( + "archive/tar" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + + "github.com/pojntfx/stfs/internal/adapters" + "github.com/pojntfx/stfs/internal/compression" + "github.com/pojntfx/stfs/internal/controllers" + "github.com/pojntfx/stfs/internal/counters" + "github.com/pojntfx/stfs/internal/encryption" + "github.com/pojntfx/stfs/internal/formatting" + "github.com/pojntfx/stfs/internal/pax" + "github.com/pojntfx/stfs/internal/signature" + "github.com/pojntfx/stfs/internal/suffix" + "github.com/pojntfx/stfs/internal/tape" + "github.com/pojntfx/stfs/pkg/config" +) + +func Update( + state config.StateConfig, + pipes config.PipeConfig, + crypto config.CryptoConfig, + + recordSize int, + from string, + overwrite bool, + compressionLevel string, +) ([]*tar.Header, error) { + dirty := false + tw, isRegular, cleanup, err := tape.OpenTapeWriteOnly(state.Drive, recordSize, false) + if err != nil { + return []*tar.Header{}, err + } + defer cleanup(&dirty) + + headers := []*tar.Header{} + first := true + return headers, filepath.Walk(from, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + link := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + + if err := adapters.EnhanceHeader(path, hdr); err != nil { + return err + } + + hdr.Name = path + hdr.Format = tar.FormatPAX + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1 + hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionUpdate + + if info.Mode().IsRegular() && overwrite { + // Get the compressed size for the header + fileSizeCounter := &counters.CounterWriter{ + Writer: io.Discard, + } + + encryptor, err := encryption.Encrypt(fileSizeCounter, pipes.Encryption, crypto.Recipient) + if err != nil { + return err + } + + compressor, err := compression.Compress( + encryptor, + pipes.Compression, + compressionLevel, + isRegular, + recordSize, + ) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + + signer, sign, err := signature.Sign(file, isRegular, pipes.Signature, crypto.Identity) + if err != nil { + return err + } + + if isRegular { + if _, err := io.Copy(compressor, signer); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(compressor, signer, buf); err != nil { + return err + } + } + + if err := file.Close(); err != nil { + return err + } + + if err := compressor.Flush(); err != nil { + return err + } + + if err := compressor.Close(); err != nil { + return err + } + + if err := encryptor.Close(); err != nil { + return err + } + + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords[pax.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) + signature, err := sign() + if err != nil { + return err + } + + if signature != "" { + hdr.PAXRecords[pax.STFSRecordSignature] = signature + } + hdr.Size = int64(fileSizeCounter.BytesRead) + + hdr.Name, err = suffix.AddSuffix(hdr.Name, pipes.Compression, pipes.Encryption) + if err != nil { + return err + } + } + + if first { + if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { + return err + } + + first = false + } + + if overwrite { + hdr.PAXRecords[pax.STFSRecordReplacesContent] = pax.STFSRecordReplacesContentTrue + + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { + return err + } + + hdrToAppend := *hdr + headers = append(headers, &hdrToAppend) + + if err := signature.SignHeader(hdr, isRegular, pipes.Signature, crypto.Identity); err != nil { + return err + } + + if err := encryption.EncryptHeader(hdr, pipes.Encryption, crypto.Recipient); err != nil { + return err + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + if !info.Mode().IsRegular() { + return nil + } + + // Compress and write the file + encryptor, err := encryption.Encrypt(tw, pipes.Encryption, crypto.Recipient) + if err != nil { + return err + } + + compressor, err := compression.Compress( + encryptor, + pipes.Compression, + compressionLevel, + isRegular, + recordSize, + ) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + + if isRegular { + if _, err := io.Copy(compressor, file); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(compressor, file, buf); err != nil { + return err + } + } + + if err := file.Close(); err != nil { + return err + } + + if err := compressor.Flush(); err != nil { + return err + } + + if err := compressor.Close(); err != nil { + return err + } + + if err := encryptor.Close(); err != nil { + return err + } + } else { + hdr.Size = 0 // Don't try to seek after the record + + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { + return err + } + + hdrToAppend := *hdr + headers = append(headers, &hdrToAppend) + + if err := signature.SignHeader(hdr, isRegular, pipes.Signature, crypto.Identity); err != nil { + return err + } + + if err := encryption.EncryptHeader(hdr, pipes.Encryption, crypto.Recipient); err != nil { + return err + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + } + + dirty = true + + return nil + }) +}