From 129cfab84a3cf01b7cda0b9d5e337700d6460a4e Mon Sep 17 00:00:00 2001 From: Felicitas Pojtinger Date: Wed, 15 Dec 2021 23:14:22 +0100 Subject: [PATCH] feat: Add `io.Reader`-based API to `archive` cmd --- cmd/stbak/cmd/operation_archive.go | 47 +++++++++++- internal/statext/stat_darwin.go | 22 ------ internal/statext/stat_dragonfly.go | 31 -------- internal/statext/stat_freebsd.go | 22 ------ internal/statext/stat_linux.go | 31 -------- internal/statext/stat_stub.go | 11 --- pkg/config/config.go | 8 +++ pkg/operations/archive.go | 111 ++++++++++++----------------- pkg/operations/update.go | 5 -- 9 files changed, 99 insertions(+), 189 deletions(-) delete mode 100644 internal/statext/stat_darwin.go delete mode 100644 internal/statext/stat_dragonfly.go delete mode 100644 internal/statext/stat_freebsd.go delete mode 100644 internal/statext/stat_linux.go delete mode 100644 internal/statext/stat_stub.go diff --git a/cmd/stbak/cmd/operation_archive.go b/cmd/stbak/cmd/operation_archive.go index 48850aa..f377167 100644 --- a/cmd/stbak/cmd/operation_archive.go +++ b/cmd/stbak/cmd/operation_archive.go @@ -2,6 +2,10 @@ package cmd import ( "fmt" + "io" + "io/fs" + "os" + "path/filepath" "github.com/pojntfx/stfs/internal/compression" "github.com/pojntfx/stfs/internal/keys" @@ -105,8 +109,49 @@ var operationArchiveCmd = &cobra.Command{ logging.NewLogger().PrintHeaderEvent, ) + files := make(chan config.FileConfig) + errs := make(chan error) + go func() { + if err := filepath.Walk(viper.GetString(fromFlag), func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + link := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + files <- config.FileConfig{ + GetFile: func() (io.ReadSeekCloser, error) { + return os.Open(path) + }, + Info: info, + Path: path, + Link: link, + } + + return nil + }); err != nil { + errs <- err + + return + } + + errs <- io.EOF + }() + if _, err := ops.Archive( - viper.GetString(fromFlag), + func() (config.FileConfig, error) { + select { + case file := <-files: + return file, err + case err := <-errs: + return config.FileConfig{}, err + } + }, viper.GetString(compressionLevelFlag), viper.GetBool(overwriteFlag), ); err != nil { diff --git a/internal/statext/stat_darwin.go b/internal/statext/stat_darwin.go deleted file mode 100644 index a03e8b2..0000000 --- a/internal/statext/stat_darwin.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build darwin - -package statext - -import ( - "archive/tar" - "syscall" - - "golang.org/x/sys/unix" -) - -func EnhanceHeader(path string, hdr *tar.Header) error { - var unixStat syscall.Stat_t - if err := syscall.Stat(path, &unixStat); err != nil { - return err - } - - hdr.Devmajor = int64(unix.Major(uint64(unixStat.Dev))) - hdr.Devminor = int64(unix.Minor(uint64(unixStat.Dev))) - - return nil -} diff --git a/internal/statext/stat_dragonfly.go b/internal/statext/stat_dragonfly.go deleted file mode 100644 index 1564cc0..0000000 --- a/internal/statext/stat_dragonfly.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build dragonfly - -package statext - -import ( - "archive/tar" - "syscall" - "time" - - "golang.org/x/sys/unix" -) - -func EnhanceHeader(path string, hdr *tar.Header) error { - var unixStat syscall.Stat_t - if err := syscall.Stat(path, &unixStat); err != nil { - return err - } - - mtimesec, mtimensec := unixStat.Mtim.Unix() - atimesec, atimensec := unixStat.Atim.Unix() - ctimesec, ctimensec := unixStat.Ctim.Unix() - - hdr.ModTime = time.Unix(mtimesec, mtimensec) - hdr.AccessTime = time.Unix(atimesec, atimensec) - hdr.ChangeTime = time.Unix(ctimesec, ctimensec) - - hdr.Devmajor = int64(unix.Major(uint64(unixStat.Dev))) - hdr.Devminor = int64(unix.Minor(uint64(unixStat.Dev))) - - return nil -} diff --git a/internal/statext/stat_freebsd.go b/internal/statext/stat_freebsd.go deleted file mode 100644 index c95ff92..0000000 --- a/internal/statext/stat_freebsd.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build freebsd - -package statext - -import ( - "archive/tar" - "syscall" - - "golang.org/x/sys/unix" -) - -func EnhanceHeader(path string, hdr *tar.Header) error { - var unixStat syscall.Stat_t - if err := syscall.Stat(path, &unixStat); err != nil { - return err - } - - hdr.Devmajor = int64(unix.Major(unixStat.Dev)) - hdr.Devminor = int64(unix.Minor(unixStat.Dev)) - - return nil -} diff --git a/internal/statext/stat_linux.go b/internal/statext/stat_linux.go deleted file mode 100644 index b3d6323..0000000 --- a/internal/statext/stat_linux.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build (linux && amd64) || (linux && 386) || (linux && arm) || (linux && arm64) || (linux && ppc64) || (linux && ppc64le) || (linux && riscv64) || (linux && s390x) - -package statext - -import ( - "archive/tar" - "syscall" - "time" - - "golang.org/x/sys/unix" -) - -func EnhanceHeader(path string, hdr *tar.Header) error { - var unixStat syscall.Stat_t - if err := syscall.Stat(path, &unixStat); err != nil { - return err - } - - mtimesec, mtimensec := unixStat.Mtim.Unix() - atimesec, atimensec := unixStat.Atim.Unix() - ctimesec, ctimensec := unixStat.Ctim.Unix() - - hdr.ModTime = time.Unix(mtimesec, mtimensec) - hdr.AccessTime = time.Unix(atimesec, atimensec) - hdr.ChangeTime = time.Unix(ctimesec, ctimensec) - - hdr.Devmajor = int64(unix.Major(unixStat.Dev)) - hdr.Devminor = int64(unix.Minor(unixStat.Dev)) - - return nil -} diff --git a/internal/statext/stat_stub.go b/internal/statext/stat_stub.go deleted file mode 100644 index 485b7c6..0000000 --- a/internal/statext/stat_stub.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build (linux && mips64) || (linux && mips) || (linux && mipsle) || (linux && mips64le) || !(darwin || dragonfly || freebsd || linux) - -package statext - -import ( - "archive/tar" -) - -func EnhanceHeader(path string, hdr *tar.Header) error { - return nil -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 2a89763..41d3be2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "io" + "io/fs" "os" "github.com/pojntfx/stfs/internal/persisters" @@ -53,3 +54,10 @@ type CryptoConfig struct { type PasswordConfig struct { Password string } + +type FileConfig struct { + GetFile func() (io.ReadSeekCloser, error) + Info fs.FileInfo + Path string + Link string +} diff --git a/pkg/operations/archive.go b/pkg/operations/archive.go index f9a4f2e..348d55b 100644 --- a/pkg/operations/archive.go +++ b/pkg/operations/archive.go @@ -5,9 +5,6 @@ import ( "context" "errors" "io" - "io/fs" - "os" - "path/filepath" "strconv" "strings" @@ -19,7 +16,6 @@ import ( "github.com/pojntfx/stfs/internal/mtio" "github.com/pojntfx/stfs/internal/records" "github.com/pojntfx/stfs/internal/signature" - "github.com/pojntfx/stfs/internal/statext" "github.com/pojntfx/stfs/internal/suffix" "github.com/pojntfx/stfs/internal/tarext" "github.com/pojntfx/stfs/pkg/config" @@ -31,7 +27,7 @@ var ( ) func (o *Operations) Archive( - from string, + getSrc func() (config.FileConfig, error), compressionLevel string, overwrite bool, ) ([]*tar.Header, error) { @@ -59,36 +55,31 @@ func (o *Operations) Archive( } hdrs := []*tar.Header{} - if err := filepath.Walk(from, func(path string, info fs.FileInfo, err error) error { + for { + file, err := getSrc() + if err == io.EOF { + break + } + if err != nil { - return err + return []*tar.Header{}, err } - link := "" - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - if link, err = os.Readlink(path); err != nil { - return err - } - } - - hdr, err := tar.FileInfoHeader(info, link) + hdr, err := tar.FileInfoHeader(file.Info, file.Link) if err != nil { // Skip sockets if strings.Contains(err.Error(), errSocketsNotSupported.Error()) { - return nil + return []*tar.Header{}, nil } - return err + return []*tar.Header{}, err } - if err := statext.EnhanceHeader(path, hdr); err != nil { - return err - } - - hdr.Name = path + hdr.Name = file.Path hdr.Format = tar.FormatPAX - if info.Mode().IsRegular() { + var f io.ReadSeekCloser + if file.Info.Mode().IsRegular() { // Get the compressed size for the header fileSizeCounter := &ioext.CounterWriter{ Writer: io.Discard, @@ -96,7 +87,7 @@ func (o *Operations) Archive( encryptor, err := encryption.Encrypt(fileSizeCounter, o.pipes.Encryption, o.crypto.Recipient) if err != nil { - return err + return []*tar.Header{}, err } compressor, err := compression.Compress( @@ -107,44 +98,41 @@ func (o *Operations) Archive( o.pipes.RecordSize, ) if err != nil { - return err + return []*tar.Header{}, err } - file, err := os.Open(path) + f, err = file.GetFile() if err != nil { - return err + return []*tar.Header{}, err } + defer f.Close() - signer, sign, err := signature.Sign(file, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity) + signer, sign, err := signature.Sign(f, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity) if err != nil { - return err + return []*tar.Header{}, err } if writer.DriveIsRegular { if _, err := io.Copy(compressor, signer); err != nil { - return err + return []*tar.Header{}, err } } else { buf := make([]byte, mtio.BlockSize*o.pipes.RecordSize) if _, err := io.CopyBuffer(compressor, signer, buf); err != nil { - return err + return []*tar.Header{}, err } } - if err := file.Close(); err != nil { - return err - } - if err := compressor.Flush(); err != nil { - return err + return []*tar.Header{}, err } if err := compressor.Close(); err != nil { - return err + return []*tar.Header{}, err } if err := encryptor.Close(); err != nil { - return err + return []*tar.Header{}, err } if hdr.PAXRecords == nil { @@ -153,7 +141,7 @@ func (o *Operations) Archive( hdr.PAXRecords[records.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size)) signature, err := sign() if err != nil { - return err + return []*tar.Header{}, err } if signature != "" { @@ -163,14 +151,14 @@ func (o *Operations) Archive( hdr.Name, err = suffix.AddSuffix(hdr.Name, o.pipes.Compression, o.pipes.Encryption) if err != nil { - return err + return []*tar.Header{}, err } } if o.onHeader != nil { dbhdr, err := converters.TarHeaderToDBHeader(-1, -1, -1, -1, hdr) if err != nil { - return err + return []*tar.Header{}, err } o.onHeader(&config.HeaderEvent{ @@ -184,27 +172,27 @@ func (o *Operations) Archive( hdrs = append(hdrs, &hdrToAppend) if err := signature.SignHeader(hdr, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity); err != nil { - return err + return []*tar.Header{}, err } if err := encryption.EncryptHeader(hdr, o.pipes.Encryption, o.crypto.Recipient); err != nil { - return err + return []*tar.Header{}, err } if err := tw.WriteHeader(hdr); err != nil { - return err + return []*tar.Header{}, err } dirty = true - if !info.Mode().IsRegular() { - return nil + if !file.Info.Mode().IsRegular() { + continue } // Compress and write the file encryptor, err := encryption.Encrypt(tw, o.pipes.Encryption, o.crypto.Recipient) if err != nil { - return err + return []*tar.Header{}, err } compressor, err := compression.Compress( @@ -215,44 +203,35 @@ func (o *Operations) Archive( o.pipes.RecordSize, ) if err != nil { - return err + return []*tar.Header{}, err } - file, err := os.Open(path) - if err != nil { - return err + if _, err := f.Seek(0, io.SeekStart); err != nil { + return []*tar.Header{}, err } if writer.DriveIsRegular { - if _, err := io.Copy(compressor, file); err != nil { - return err + if _, err := io.Copy(compressor, f); err != nil { + return []*tar.Header{}, err } } else { buf := make([]byte, mtio.BlockSize*o.pipes.RecordSize) - if _, err := io.CopyBuffer(compressor, file, buf); err != nil { - return err + if _, err := io.CopyBuffer(compressor, f, buf); err != nil { + return []*tar.Header{}, err } } - if err := file.Close(); err != nil { - return err - } - if err := compressor.Flush(); err != nil { - return err + return []*tar.Header{}, err } if err := compressor.Close(); err != nil { - return err + return []*tar.Header{}, err } if err := encryptor.Close(); err != nil { - return err + return []*tar.Header{}, err } - - return nil - }); err != nil { - return []*tar.Header{}, err } if err := cleanup(&dirty); err != nil { diff --git a/pkg/operations/update.go b/pkg/operations/update.go index ea4e926..f043d87 100644 --- a/pkg/operations/update.go +++ b/pkg/operations/update.go @@ -17,7 +17,6 @@ import ( "github.com/pojntfx/stfs/internal/mtio" "github.com/pojntfx/stfs/internal/records" "github.com/pojntfx/stfs/internal/signature" - "github.com/pojntfx/stfs/internal/statext" "github.com/pojntfx/stfs/internal/suffix" "github.com/pojntfx/stfs/internal/tarext" "github.com/pojntfx/stfs/pkg/config" @@ -66,10 +65,6 @@ func (o *Operations) Update( return err } - if err := statext.EnhanceHeader(path, hdr); err != nil { - return err - } - hdr.Name = path hdr.Format = tar.FormatPAX if hdr.PAXRecords == nil {