mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add io.Reader-based API to archive cmd

Browse Source
This commit is contained in:
Felicitas Pojtinger
2021-12-15 23:14:22 +01:00
parent 3d4ee0d7ed
commit 129cfab84a
9 changed files with 99 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
cmd/stbak/cmd/operation_archive.go
View File

@@ -2,6 +2,10 @@ package cmd
import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/keys"
@@ -105,8 +109,49 @@ var operationArchiveCmd = &cobra.Command{
logging.NewLogger().PrintHeaderEvent,
)
files := make(chan config.FileConfig)
errs := make(chan error)
go func() {
if err := filepath.Walk(viper.GetString(fromFlag), func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
files <- config.FileConfig{
GetFile: func() (io.ReadSeekCloser, error) {
return os.Open(path)
},
Info: info,
Path: path,
Link: link,
}
return nil
}); err != nil {
errs <- err
return
}
errs <- io.EOF
}()
if _, err := ops.Archive(
viper.GetString(fromFlag),
func() (config.FileConfig, error) {
select {
case file := <-files:
return file, err
case err := <-errs:
return config.FileConfig{}, err
}
},
viper.GetString(compressionLevelFlag),
viper.GetBool(overwriteFlag),
); err != nil {

22
internal/statext/stat_darwin.go
View File

@@ -1,22 +0,0 @@
//go:build darwin
package statext
import (
"archive/tar"
"syscall"
"golang.org/x/sys/unix"
)
func EnhanceHeader(path string, hdr *tar.Header) error {
var unixStat syscall.Stat_t
if err := syscall.Stat(path, &unixStat); err != nil {
return err
}
hdr.Devmajor = int64(unix.Major(uint64(unixStat.Dev)))
hdr.Devminor = int64(unix.Minor(uint64(unixStat.Dev)))
return nil
}

31
internal/statext/stat_dragonfly.go
View File

@@ -1,31 +0,0 @@
//go:build dragonfly
package statext
import (
"archive/tar"
"syscall"
"time"
"golang.org/x/sys/unix"
)
func EnhanceHeader(path string, hdr *tar.Header) error {
var unixStat syscall.Stat_t
if err := syscall.Stat(path, &unixStat); err != nil {
return err
}
mtimesec, mtimensec := unixStat.Mtim.Unix()
atimesec, atimensec := unixStat.Atim.Unix()
ctimesec, ctimensec := unixStat.Ctim.Unix()
hdr.ModTime = time.Unix(mtimesec, mtimensec)
hdr.AccessTime = time.Unix(atimesec, atimensec)
hdr.ChangeTime = time.Unix(ctimesec, ctimensec)
hdr.Devmajor = int64(unix.Major(uint64(unixStat.Dev)))
hdr.Devminor = int64(unix.Minor(uint64(unixStat.Dev)))
return nil
}

22
internal/statext/stat_freebsd.go
View File

@@ -1,22 +0,0 @@
//go:build freebsd
package statext
import (
"archive/tar"
"syscall"
"golang.org/x/sys/unix"
)
func EnhanceHeader(path string, hdr *tar.Header) error {
var unixStat syscall.Stat_t
if err := syscall.Stat(path, &unixStat); err != nil {
return err
}
hdr.Devmajor = int64(unix.Major(unixStat.Dev))
hdr.Devminor = int64(unix.Minor(unixStat.Dev))
return nil
}

31
internal/statext/stat_linux.go
View File

@@ -1,31 +0,0 @@
//go:build (linux && amd64) || (linux && 386) || (linux && arm) || (linux && arm64) || (linux && ppc64) || (linux && ppc64le) || (linux && riscv64) || (linux && s390x)
package statext
import (
"archive/tar"
"syscall"
"time"
"golang.org/x/sys/unix"
)
func EnhanceHeader(path string, hdr *tar.Header) error {
var unixStat syscall.Stat_t
if err := syscall.Stat(path, &unixStat); err != nil {
return err
}
mtimesec, mtimensec := unixStat.Mtim.Unix()
atimesec, atimensec := unixStat.Atim.Unix()
ctimesec, ctimensec := unixStat.Ctim.Unix()
hdr.ModTime = time.Unix(mtimesec, mtimensec)
hdr.AccessTime = time.Unix(atimesec, atimensec)
hdr.ChangeTime = time.Unix(ctimesec, ctimensec)
hdr.Devmajor = int64(unix.Major(unixStat.Dev))
hdr.Devminor = int64(unix.Minor(unixStat.Dev))
return nil
}

11
internal/statext/stat_stub.go
View File

@@ -1,11 +0,0 @@
//go:build (linux && mips64) || (linux && mips) || (linux && mipsle) || (linux && mips64le) || !(darwin || dragonfly || freebsd || linux)
package statext
import (
"archive/tar"
)
func EnhanceHeader(path string, hdr *tar.Header) error {
return nil
}

8
pkg/config/config.go
View File

@@ -2,6 +2,7 @@ package config
import (
"io"
"io/fs"
"os"
"github.com/pojntfx/stfs/internal/persisters"
@@ -53,3 +54,10 @@ type CryptoConfig struct {
type PasswordConfig struct {
Password string
}
type FileConfig struct {
GetFile func() (io.ReadSeekCloser, error)
Info fs.FileInfo
Path string
Link string
}

111
pkg/operations/archive.go
View File

@@ -5,9 +5,6 @@ import (
"context"
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
@@ -19,7 +16,6 @@ import (
"github.com/pojntfx/stfs/internal/mtio"
"github.com/pojntfx/stfs/internal/records"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/statext"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tarext"
"github.com/pojntfx/stfs/pkg/config"
@@ -31,7 +27,7 @@ var (
)
func (o *Operations) Archive(
from string,
getSrc func() (config.FileConfig, error),
compressionLevel string,
overwrite bool,
) ([]*tar.Header, error) {
@@ -59,36 +55,31 @@ func (o *Operations) Archive(
}
hdrs := []*tar.Header{}
if err := filepath.Walk(from, func(path string, info fs.FileInfo, err error) error {
for {
file, err := getSrc()
if err == io.EOF {
break
}
if err != nil {
return err
return []*tar.Header{}, err
}
link := ""
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
if link, err = os.Readlink(path); err != nil {
return err
}
}
hdr, err := tar.FileInfoHeader(info, link)
hdr, err := tar.FileInfoHeader(file.Info, file.Link)
if err != nil {
// Skip sockets
if strings.Contains(err.Error(), errSocketsNotSupported.Error()) {
return nil
return []*tar.Header{}, nil
}
return err
return []*tar.Header{}, err
}
if err := statext.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Name = file.Path
hdr.Format = tar.FormatPAX
if info.Mode().IsRegular() {
var f io.ReadSeekCloser
if file.Info.Mode().IsRegular() {
// Get the compressed size for the header
fileSizeCounter := &ioext.CounterWriter{
Writer: io.Discard,
@@ -96,7 +87,7 @@ func (o *Operations) Archive(
encryptor, err := encryption.Encrypt(fileSizeCounter, o.pipes.Encryption, o.crypto.Recipient)
if err != nil {
return err
return []*tar.Header{}, err
}
compressor, err := compression.Compress(
@@ -107,44 +98,41 @@ func (o *Operations) Archive(
o.pipes.RecordSize,
)
if err != nil {
return err
return []*tar.Header{}, err
}
file, err := os.Open(path)
f, err = file.GetFile()
if err != nil {
return err
return []*tar.Header{}, err
}
defer f.Close()
signer, sign, err := signature.Sign(file, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity)
signer, sign, err := signature.Sign(f, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity)
if err != nil {
return err
return []*tar.Header{}, err
}
if writer.DriveIsRegular {
if _, err := io.Copy(compressor, signer); err != nil {
return err
return []*tar.Header{}, err
}
} else {
buf := make([]byte, mtio.BlockSize*o.pipes.RecordSize)
if _, err := io.CopyBuffer(compressor, signer, buf); err != nil {
return err
return []*tar.Header{}, err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
return []*tar.Header{}, err
}
if err := compressor.Close(); err != nil {
return err
return []*tar.Header{}, err
}
if err := encryptor.Close(); err != nil {
return err
return []*tar.Header{}, err
}
if hdr.PAXRecords == nil {
@@ -153,7 +141,7 @@ func (o *Operations) Archive(
hdr.PAXRecords[records.STFSRecordUncompressedSize] = strconv.Itoa(int(hdr.Size))
signature, err := sign()
if err != nil {
return err
return []*tar.Header{}, err
}
if signature != "" {
@@ -163,14 +151,14 @@ func (o *Operations) Archive(
hdr.Name, err = suffix.AddSuffix(hdr.Name, o.pipes.Compression, o.pipes.Encryption)
if err != nil {
return err
return []*tar.Header{}, err
}
}
if o.onHeader != nil {
dbhdr, err := converters.TarHeaderToDBHeader(-1, -1, -1, -1, hdr)
if err != nil {
return err
return []*tar.Header{}, err
}
o.onHeader(&config.HeaderEvent{
@@ -184,27 +172,27 @@ func (o *Operations) Archive(
hdrs = append(hdrs, &hdrToAppend)
if err := signature.SignHeader(hdr, writer.DriveIsRegular, o.pipes.Signature, o.crypto.Identity); err != nil {
return err
return []*tar.Header{}, err
}
if err := encryption.EncryptHeader(hdr, o.pipes.Encryption, o.crypto.Recipient); err != nil {
return err
return []*tar.Header{}, err
}
if err := tw.WriteHeader(hdr); err != nil {
return err
return []*tar.Header{}, err
}
dirty = true
if !info.Mode().IsRegular() {
return nil
if !file.Info.Mode().IsRegular() {
continue
}
// Compress and write the file
encryptor, err := encryption.Encrypt(tw, o.pipes.Encryption, o.crypto.Recipient)
if err != nil {
return err
return []*tar.Header{}, err
}
compressor, err := compression.Compress(
@@ -215,44 +203,35 @@ func (o *Operations) Archive(
o.pipes.RecordSize,
)
if err != nil {
return err
return []*tar.Header{}, err
}
file, err := os.Open(path)
if err != nil {
return err
if _, err := f.Seek(0, io.SeekStart); err != nil {
return []*tar.Header{}, err
}
if writer.DriveIsRegular {
if _, err := io.Copy(compressor, file); err != nil {
return err
if _, err := io.Copy(compressor, f); err != nil {
return []*tar.Header{}, err
}
} else {
buf := make([]byte, mtio.BlockSize*o.pipes.RecordSize)
if _, err := io.CopyBuffer(compressor, file, buf); err != nil {
return err
if _, err := io.CopyBuffer(compressor, f, buf); err != nil {
return []*tar.Header{}, err
}
}
if err := file.Close(); err != nil {
return err
}
if err := compressor.Flush(); err != nil {
return err
return []*tar.Header{}, err
}
if err := compressor.Close(); err != nil {
return err
return []*tar.Header{}, err
}
if err := encryptor.Close(); err != nil {
return err
return []*tar.Header{}, err
}
return nil
}); err != nil {
return []*tar.Header{}, err
}
if err := cleanup(&dirty); err != nil {

5
pkg/operations/update.go
View File

@@ -17,7 +17,6 @@ import (
"github.com/pojntfx/stfs/internal/mtio"
"github.com/pojntfx/stfs/internal/records"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/internal/statext"
"github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/internal/tarext"
"github.com/pojntfx/stfs/pkg/config"
@@ -66,10 +65,6 @@ func (o *Operations) Update(
return err
}
if err := statext.EnhanceHeader(path, hdr); err != nil {
return err
}
hdr.Name = path
hdr.Format = tar.FormatPAX
if hdr.PAXRecords == nil {