mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement deletion based on PAX record meta information

Browse Source
This commit is contained in:
Felix Pojtinger
2021-11-21 15:33:06 +01:00
parent a072e273c9
commit 356b5a399c
10 changed files with 171 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/stbak/cmd/archive.go
View File

@@ -67,7 +67,12 @@ var archiveCmd = &cobra.Command{
return err
}
if !viper.GetBool(overwriteFlag) {
if viper.GetBool(overwriteFlag) {
// Go to start of tape
if err := controllers.SeekToRecordOnTape(f, 0); err != nil {
return err
}
} else {
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return err
@@ -76,6 +81,7 @@ var archiveCmd = &cobra.Command{
}
defer f.Close()
dirty := false
var tw *tar.Writer
if isRegular {
tw = tar.NewWriter(f)
@@ -83,7 +89,14 @@ var archiveCmd = &cobra.Command{
bw := bufio.NewWriterSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag))
tw = tar.NewWriter(bw)
}
defer tw.Close()
defer func() {
// Only write the trailer if we wrote to the archive
if dirty {
if err := tw.Close(); err != nil {
panic(err)
}
}
}()
first := true
return filepath.Walk(viper.GetString(srcFlag), func(path string, info fs.FileInfo, err error) error {
@@ -147,6 +160,8 @@ var archiveCmd = &cobra.Command{
}
}
dirty = true
return nil
})
},

16
cmd/stbak/cmd/index.go
View File

@@ -26,7 +26,7 @@ var indexCmd = &cobra.Command{
}
if viper.GetBool(overwriteFlag) {
f, err := os.OpenFile(viper.GetString(dbFlag), os.O_WRONLY|os.O_CREATE, 0600)
f, err := os.OpenFile(viper.GetString(metadataFlag), os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
@@ -40,7 +40,7 @@ var indexCmd = &cobra.Command{
}
}
metadataPersister := persisters.NewMetadataPersister(viper.GetString(dbFlag))
metadataPersister := persisters.NewMetadataPersister(viper.GetString(metadataFlag))
if err := metadataPersister.Open(); err != nil {
return err
}
@@ -87,13 +87,21 @@ var indexCmd = &cobra.Command{
nextTotalBlocks := math.Ceil(float64((curr)) / float64(controllers.BlockSize))
record = int64(nextTotalBlocks) / int64(viper.GetInt(recordSizeFlag))
block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag)))
block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag))) - 2
if block > int64(viper.GetInt(recordSizeFlag)) {
if block < 0 {
record--
block = int64(viper.GetInt(recordSizeFlag)) - 1
} else if block >= int64(viper.GetInt(recordSizeFlag)) {
record++
block = 0
}
// Seek to record and block
if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*int(record))+int(block)*controllers.BlockSize), io.SeekStart); err != nil {
return err
}
tr = tar.NewReader(f)
hdr, err = tr.Next()

12
cmd/stbak/cmd/list.go
View File

@@ -65,13 +65,21 @@ var listCmd = &cobra.Command{
nextTotalBlocks := math.Ceil(float64((curr)) / float64(controllers.BlockSize))
record = int64(nextTotalBlocks) / int64(viper.GetInt(recordSizeFlag))
block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag)))
block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag))) - 2
if block > int64(viper.GetInt(recordSizeFlag)) {
if block < 0 {
record--
block = int64(viper.GetInt(recordSizeFlag)) - 1
} else if block >= int64(viper.GetInt(recordSizeFlag)) {
record++
block = 0
}
// Seek to record and block
if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*int(record))+int(block)*controllers.BlockSize), io.SeekStart); err != nil {
return err
}
tr = tar.NewReader(f)
hdr, err = tr.Next()

30
cmd/stbak/cmd/query.go
View File

@@ -1,10 +1,9 @@
package cmd
import (
"archive/tar"
"context"
"encoding/json"
"github.com/pojntfx/stfs/pkg/converters"
"github.com/pojntfx/stfs/pkg/formatting"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/spf13/cobra"
@@ -20,7 +19,7 @@ var queryCmd = &cobra.Command{
return err
}
metadataPersister := persisters.NewMetadataPersister(viper.GetString(dbFlag))
metadataPersister := persisters.NewMetadataPersister(viper.GetString(metadataFlag))
if err := metadataPersister.Open(); err != nil {
return err
}
@@ -30,36 +29,19 @@ var queryCmd = &cobra.Command{
return err
}
for i, hdr := range headers {
for i, dbhdr := range headers {
if i == 0 {
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
}
paxRecords := map[string]string{}
if err := json.Unmarshal([]byte(hdr.Paxrecords), &paxRecords); err != nil {
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
if err != nil {
return err
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(hdr.Record, hdr.Block, &tar.Header{
Typeflag: byte(hdr.Typeflag),
Name: hdr.Name,
Linkname: hdr.Linkname,
Size: hdr.Size,
Mode: hdr.Mode,
Uid: int(hdr.UID),
Gid: int(hdr.Gid),
Uname: hdr.Uname,
Gname: hdr.Gname,
ModTime: hdr.Modtime,
AccessTime: hdr.Accesstime,
ChangeTime: hdr.Changetime,
Devmajor: hdr.Devmajor,
Devminor: hdr.Devminor,
PAXRecords: paxRecords,
Format: tar.Format(hdr.Format),
})); err != nil {
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(dbhdr.Record, dbhdr.Block, hdr)); err != nil {
return err
}
}

78
cmd/stbak/cmd/remove.go
View File

@@ -1,9 +1,14 @@
package cmd
import (
"archive/tar"
"context"
"log"
"os"
"github.com/pojntfx/stfs/pkg/controllers"
"github.com/pojntfx/stfs/pkg/converters"
"github.com/pojntfx/stfs/pkg/formatting"
"github.com/pojntfx/stfs/pkg/pax"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -22,17 +27,82 @@ var removeCmd = &cobra.Command{
return err
}
metadataPersister := persisters.NewMetadataPersister(viper.GetString(dbFlag))
isRegular := true
stat, err := os.Stat(viper.GetString(tapeFlag))
if err == nil {
isRegular = stat.Mode().IsRegular()
} else {
if os.IsNotExist(err) {
isRegular = true
} else {
return err
}
}
var f *os.File
if isRegular {
f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
// No need to go to end manually due to `os.O_APPEND`
} else {
f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
return err
}
// Go to end of tape
if err := controllers.GoToEndOfTape(f); err != nil {
return err
}
}
defer f.Close()
dirty := false
tw := tar.NewWriter(f)
defer func() {
// Only write the trailer if we wrote to the archive
if dirty {
if err := tw.Close(); err != nil {
panic(err)
}
}
}()
metadataPersister := persisters.NewMetadataPersister(viper.GetString(metadataFlag))
if err := metadataPersister.Open(); err != nil {
return err
}
hdr, err := metadataPersister.DeleteHeader(context.Background(), viper.GetString(nameFlag))
dbhdr, err := metadataPersister.DeleteHeader(context.Background(), viper.GetString(nameFlag))
if err != nil {
return err
}
log.Println(hdr.Record, hdr.Block)
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
if err != nil {
return err
}
hdr.Size = 0 // Don't try to seek after the record
hdr.PAXRecords[pax.STFSRecordVersion] = pax.STFSRecordVersion1
hdr.PAXRecords[pax.STFSRecordAction] = pax.STFSRecordActionDelete
if err := tw.WriteHeader(hdr); err != nil {
return err
}
dirty = true
if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
return err
}
if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, hdr)); err != nil {
return err
}
return nil
},

2
cmd/stbak/cmd/restore.go
View File

@@ -51,7 +51,7 @@ var restoreCmd = &cobra.Command{
var tr *tar.Reader
if fileDescription.Mode().IsRegular() {
// Seek to record and block
if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*viper.GetInt(recordFlag))+viper.GetInt(blockFlag)*controllers.BlockSize), 0); err != nil {
if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*viper.GetInt(recordFlag))+viper.GetInt(blockFlag)*controllers.BlockSize), io.SeekStart); err != nil {
return err
}

8
cmd/stbak/cmd/root.go
View File

@@ -10,8 +10,8 @@ import (
)
const (
tapeFlag = "tape"
dbFlag = "db"
tapeFlag = "tape"
metadataFlag = "metadata"
)
var rootCmd = &cobra.Command{
@@ -33,10 +33,10 @@ func Execute() {
if err != nil {
panic(err)
}
dbPath := filepath.Join(home, ".local", "share", "stbak", "var", "lib", "stbak", "metadata.sqlite")
metadataPath := filepath.Join(home, ".local", "share", "stbak", "var", "lib", "stbak", "metadata.sqlite")
rootCmd.PersistentFlags().StringP(tapeFlag, "t", "/dev/nst0", "Tape or tar file to use")
rootCmd.PersistentFlags().StringP(dbFlag, "d", dbPath, "Database to use")
rootCmd.PersistentFlags().StringP(metadataFlag, "m", metadataPath, "Metadata database to use")
if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {
panic(err)

36
pkg/converters/header.go Normal file
View File

@@ -0,0 +1,36 @@
package converters
import (
"archive/tar"
"encoding/json"
models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata"
)
func DBHeaderToTarHeader(dbhdr *models.Header) (*tar.Header, error) {
paxRecords := map[string]string{}
if err := json.Unmarshal([]byte(dbhdr.Paxrecords), &paxRecords); err != nil {
return nil, err
}
hdr := &tar.Header{
Typeflag: byte(dbhdr.Typeflag),
Name: dbhdr.Name,
Linkname: dbhdr.Linkname,
Size: dbhdr.Size,
Mode: dbhdr.Mode,
Uid: int(dbhdr.UID),
Gid: int(dbhdr.Gid),
Uname: dbhdr.Uname,
Gname: dbhdr.Gname,
ModTime: dbhdr.Modtime,
AccessTime: dbhdr.Accesstime,
ChangeTime: dbhdr.Changetime,
Devmajor: dbhdr.Devmajor,
Devminor: dbhdr.Devminor,
PAXRecords: paxRecords,
Format: tar.Format(dbhdr.Format),
}
return hdr, nil
}

10
pkg/pax/stfs.go Normal file
View File

@@ -0,0 +1,10 @@
package pax
const (
STFSRecordVersion = "STFS.Version"
STFSRecordVersion1 = "1"
STFSRecordAction = "STFS.Action"
STFSRecordActionCreate = "CREATE"
STFSRecordActionDelete = "DELETE"
)

2
pkg/persisters/metadata.go
View File

@@ -83,7 +83,7 @@ func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice,
}
func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string) (*models.Header, error) {
hdr, err := models.FindHeader(ctx, p.db, name, models.HeaderColumns.Name, models.HeaderColumns.Record, models.HeaderColumns.Block)
hdr, err := models.FindHeader(ctx, p.db, name)
if err != nil {
return nil, err
}