From 7d53509a733fb250cba8a9b1389a88a3b317b5ec Mon Sep 17 00:00:00 2001 From: Felix Pojtinger Date: Sun, 21 Nov 2021 21:20:46 +0100 Subject: [PATCH] feat: Implement automatic indexing after archival --- cmd/stbak/cmd/archive.go | 294 ++++++++++++++++++++--------------- cmd/stbak/cmd/index.go | 308 ++++++++++++++++++++----------------- pkg/persisters/metadata.go | 25 +++ 3 files changed, 357 insertions(+), 270 deletions(-) diff --git a/cmd/stbak/cmd/archive.go b/cmd/stbak/cmd/archive.go index cb7b61b..d3e7bee 100644 --- a/cmd/stbak/cmd/archive.go +++ b/cmd/stbak/cmd/archive.go @@ -3,6 +3,7 @@ package cmd import ( "archive/tar" "bufio" + "context" "io" "io/fs" "os" @@ -12,6 +13,7 @@ import ( "github.com/pojntfx/stfs/pkg/controllers" "github.com/pojntfx/stfs/pkg/counters" "github.com/pojntfx/stfs/pkg/formatting" + "github.com/pojntfx/stfs/pkg/persisters" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -31,157 +33,199 @@ var archiveCmd = &cobra.Command{ return err } - isRegular := true - stat, err := os.Stat(viper.GetString(tapeFlag)) - if err == nil { - isRegular = stat.Mode().IsRegular() - } else { - if os.IsNotExist(err) { - isRegular = true - } else { - return err - } + metadataPersister := persisters.NewMetadataPersister(viper.GetString(metadataFlag)) + if err := metadataPersister.Open(); err != nil { + return err } - var f *os.File - if isRegular { - if viper.GetBool(overwriteFlag) { - f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - - if err := f.Truncate(0); err != nil { - return err - } - } else { - f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - } - - // No need to go to end manually due to `os.O_APPEND` - } else { - f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) + lastIndexedRecord := int64(0) + lastIndexedBlock := int64(0) + if !viper.GetBool(overwriteFlag) { + r, b, err := metadataPersister.GetLastIndexedRecordAndBlock(context.Background(), viper.GetInt(recordSizeFlag)) if err != nil { return err } - if viper.GetBool(overwriteFlag) { - // Go to start of tape - if err := controllers.SeekToRecordOnTape(f, 0); err != nil { - return err - } - } else { - // Go to end of tape - if err := controllers.GoToEndOfTape(f); err != nil { - return err - } + lastIndexedRecord = r + lastIndexedBlock = b + } + + if err := archive( + viper.GetString(tapeFlag), + viper.GetInt(recordSizeFlag), + viper.GetString(srcFlag), + viper.GetBool(overwriteFlag), + ); err != nil { + return err + } + + return index( + viper.GetString(tapeFlag), + viper.GetString(metadataFlag), + viper.GetInt(recordSizeFlag), + int(lastIndexedRecord), + int(lastIndexedBlock), + viper.GetBool(overwriteFlag), + ) + }, +} + +func archive( + tape string, + recordSize int, + src string, + overwrite bool, +) error { + isRegular := true + stat, err := os.Stat(tape) + if err == nil { + isRegular = stat.Mode().IsRegular() + } else { + if os.IsNotExist(err) { + isRegular = true + } else { + return err + } + } + + var f *os.File + if isRegular { + if overwrite { + f, err = os.OpenFile(tape, os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + + if err := f.Truncate(0); err != nil { + return err + } + } else { + f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err } } - defer f.Close() - dirty := false - var tw *tar.Writer - var bw *bufio.Writer - var counter *counters.CounterWriter - if isRegular { - tw = tar.NewWriter(f) - } else { - bw = bufio.NewWriterSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag)) - counter = &counters.CounterWriter{Writer: bw, BytesRead: 0} - tw = tar.NewWriter(counter) + // No need to go to end manually due to `os.O_APPEND` + } else { + f, err = os.OpenFile(tape, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) + if err != nil { + return err } - defer func() { - // Only write the trailer if we wrote to the archive - if dirty { - if err := tw.Close(); err != nil { - panic(err) - } - if !isRegular { - if controllers.BlockSize*viper.GetInt(recordSizeFlag)-counter.BytesRead > 0 { - // Fill the rest of the record with zeros - if _, err := bw.Write(make([]byte, controllers.BlockSize*viper.GetInt(recordSizeFlag)-counter.BytesRead)); err != nil { - panic(err) - } - } + if overwrite { + // Go to start of tape + if err := controllers.SeekToRecordOnTape(f, 0); err != nil { + return err + } + } else { + // Go to end of tape + if err := controllers.GoToEndOfTape(f); err != nil { + return err + } + } + } + defer f.Close() - if err := bw.Flush(); err != nil { + dirty := false + var tw *tar.Writer + var bw *bufio.Writer + var counter *counters.CounterWriter + if isRegular { + tw = tar.NewWriter(f) + } else { + bw = bufio.NewWriterSize(f, controllers.BlockSize*recordSize) + counter = &counters.CounterWriter{Writer: bw, BytesRead: 0} + tw = tar.NewWriter(counter) + } + defer func() { + // Only write the trailer if we wrote to the archive + if dirty { + if err := tw.Close(); err != nil { + panic(err) + } + + if !isRegular { + if controllers.BlockSize*recordSize-counter.BytesRead > 0 { + // Fill the rest of the record with zeros + if _, err := bw.Write(make([]byte, controllers.BlockSize*recordSize-counter.BytesRead)); err != nil { panic(err) } } - } - }() - first := true - return filepath.Walk(viper.GetString(srcFlag), func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - link := "" - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - if link, err = os.Readlink(path); err != nil { - return err + if err := bw.Flush(); err != nil { + panic(err) } } + } + }() - hdr, err := tar.FileInfoHeader(info, link) - if err != nil { + first := true + return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + link := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + + if err := adapters.EnhanceHeader(path, hdr); err != nil { + return err + } + + hdr.Name = path + hdr.Format = tar.FormatPAX + + if first { + if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { return err } - if err := adapters.EnhanceHeader(path, hdr); err != nil { - return err - } + first = false + } - hdr.Name = path - hdr.Format = tar.FormatPAX + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, hdr)); err != nil { + return err + } - if first { - if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { - return err - } - - first = false - } - - if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, hdr)); err != nil { - return err - } - - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - if !info.Mode().IsRegular() { - return nil - } - - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - if isRegular { - if _, err := io.Copy(tw, file); err != nil { - return err - } - } else { - buf := make([]byte, controllers.BlockSize*viper.GetInt(recordSizeFlag)) - if _, err := io.CopyBuffer(tw, file, buf); err != nil { - return err - } - } - - dirty = true + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { return nil - }) - }, + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if isRegular { + if _, err := io.Copy(tw, file); err != nil { + return err + } + } else { + buf := make([]byte, controllers.BlockSize*recordSize) + if _, err := io.CopyBuffer(tw, file, buf); err != nil { + return err + } + } + + dirty = true + + return nil + }) } func init() { diff --git a/cmd/stbak/cmd/index.go b/cmd/stbak/cmd/index.go index 3cefe4f..03c970f 100644 --- a/cmd/stbak/cmd/index.go +++ b/cmd/stbak/cmd/index.go @@ -27,181 +27,199 @@ var indexCmd = &cobra.Command{ return err } - if viper.GetBool(overwriteFlag) { - f, err := os.OpenFile(viper.GetString(metadataFlag), os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } + return index( + viper.GetString(tapeFlag), + viper.GetString(metadataFlag), + viper.GetInt(recordSizeFlag), + viper.GetInt(recordFlag), + viper.GetInt(blockFlag), + viper.GetBool(overwriteFlag), + ) + }, +} - if err := f.Truncate(0); err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - } - - metadataPersister := persisters.NewMetadataPersister(viper.GetString(metadataFlag)) - if err := metadataPersister.Open(); err != nil { - return err - } - - fileDescription, err := os.Stat(viper.GetString(tapeFlag)) +func index( + tape string, + metadata string, + recordSize int, + record int, + block int, + overwrite bool, +) error { + if overwrite { + f, err := os.OpenFile(metadata, os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return err } - var f *os.File - if fileDescription.Mode().IsRegular() { - f, err = os.Open(viper.GetString(tapeFlag)) - if err != nil { - return err - } - } else { - f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_RDONLY, os.ModeCharDevice) - if err != nil { - return err - } + if err := f.Truncate(0); err != nil { + return err } - defer f.Close() - if fileDescription.Mode().IsRegular() { - // Seek to record and block - if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*viper.GetInt(recordFlag))+viper.GetInt(blockFlag)*controllers.BlockSize), 0); err != nil { - return err - } + if err := f.Close(); err != nil { + return err + } + } - tr := tar.NewReader(f) + metadataPersister := persisters.NewMetadataPersister(metadata) + if err := metadataPersister.Open(); err != nil { + return err + } - record := viper.GetInt64(recordFlag) - block := viper.GetInt64(blockFlag) + fileDescription, err := os.Stat(tape) + if err != nil { + return err + } - for { - hdr, err := tr.Next() - if err != nil { - // Seek right after the next two blocks to skip the trailer - if _, err := f.Seek((controllers.BlockSize * 2), io.SeekCurrent); err == nil { - curr, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return err - } + var f *os.File + if fileDescription.Mode().IsRegular() { + f, err = os.Open(tape) + if err != nil { + return err + } + } else { + f, err = os.OpenFile(tape, os.O_RDONLY, os.ModeCharDevice) + if err != nil { + return err + } + } + defer f.Close() - nextTotalBlocks := math.Ceil(float64((curr)) / float64(controllers.BlockSize)) - record = int64(nextTotalBlocks) / int64(viper.GetInt(recordSizeFlag)) - block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag))) - 2 + if fileDescription.Mode().IsRegular() { + // Seek to record and block + if _, err := f.Seek(int64((recordSize*controllers.BlockSize*record)+block*controllers.BlockSize), 0); err != nil { + return err + } - if block < 0 { - record-- - block = int64(viper.GetInt(recordSizeFlag)) - 1 - } else if block >= int64(viper.GetInt(recordSizeFlag)) { - record++ - block = 0 - } + tr := tar.NewReader(f) - // Seek to record and block - if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*int(record))+int(block)*controllers.BlockSize), io.SeekStart); err != nil { - return err - } + record := int64(record) + block := int64(block) - tr = tar.NewReader(f) - - hdr, err = tr.Next() - if err != nil { - if err == io.EOF { - break - } - - return err - } - } else { + for { + hdr, err := tr.Next() + if err != nil { + // Seek right after the next two blocks to skip the trailer + if _, err := f.Seek((controllers.BlockSize * 2), io.SeekCurrent); err == nil { + curr, err := f.Seek(0, io.SeekCurrent) + if err != nil { return err } - } - if err := indexHeader(record, block, hdr, metadataPersister); err != nil { - return nil - } + nextTotalBlocks := math.Ceil(float64((curr)) / float64(controllers.BlockSize)) + record = int64(nextTotalBlocks) / int64(recordSize) + block = int64(nextTotalBlocks) - (record * int64(recordSize)) - 2 - curr, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return err - } + if block < 0 { + record-- + block = int64(recordSize) - 1 + } else if block >= int64(recordSize) { + record++ + block = 0 + } - nextTotalBlocks := math.Ceil(float64((curr + hdr.Size)) / float64(controllers.BlockSize)) - record = int64(nextTotalBlocks) / int64(viper.GetInt(recordSizeFlag)) - block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag))) + // Seek to record and block + if _, err := f.Seek(int64((recordSize*controllers.BlockSize*int(record))+int(block)*controllers.BlockSize), io.SeekStart); err != nil { + return err + } - if block > int64(viper.GetInt(recordSizeFlag)) { - record++ - block = 0 - } - } - } else { - // Seek to record - if err := controllers.SeekToRecordOnTape(f, int32(viper.GetInt(recordFlag))); err != nil { - return err - } - - // Seek to block - br := bufio.NewReaderSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag)) - if _, err := br.Read(make([]byte, viper.GetInt(blockFlag)*controllers.BlockSize)); err != nil { - return err - } - - record := viper.GetInt64(recordFlag) - block := viper.GetInt64(blockFlag) - - curr := int64((viper.GetInt(recordSizeFlag) * controllers.BlockSize * viper.GetInt(recordFlag)) + (viper.GetInt(blockFlag) * controllers.BlockSize)) - counter := &counters.CounterReader{Reader: br, BytesRead: int(curr)} - - tr := tar.NewReader(counter) - for { - hdr, err := tr.Next() - if err != nil { - if err == io.EOF { - if err := controllers.GoToNextFileOnTape(f); err != nil { - // EOD + tr = tar.NewReader(f) + hdr, err = tr.Next() + if err != nil { + if err == io.EOF { break } - record, err = controllers.GetCurrentRecordFromTape(f) - if err != nil { - return err - } - block = 0 - - br = bufio.NewReaderSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag)) - curr := int64(int64(viper.GetInt(recordSizeFlag)) * controllers.BlockSize * record) - counter := &counters.CounterReader{Reader: br, BytesRead: int(curr)} - tr = tar.NewReader(counter) - - continue - } else { return err } - } - - if err := indexHeader(record, block, hdr, metadataPersister); err != nil { - return nil - } - - curr = int64(counter.BytesRead) - - nextTotalBlocks := math.Ceil(float64((curr + hdr.Size)) / float64(controllers.BlockSize)) - record = int64(nextTotalBlocks) / int64(viper.GetInt(recordSizeFlag)) - block = int64(nextTotalBlocks) - (record * int64(viper.GetInt(recordSizeFlag))) - - if block > int64(viper.GetInt(recordSizeFlag)) { - record++ - block = 0 + } else { + return err } } + + if err := indexHeader(record, block, hdr, metadataPersister); err != nil { + return nil + } + + curr, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + + nextTotalBlocks := math.Ceil(float64((curr + hdr.Size)) / float64(controllers.BlockSize)) + record = int64(nextTotalBlocks) / int64(recordSize) + block = int64(nextTotalBlocks) - (record * int64(recordSize)) + + if block > int64(recordSize) { + record++ + block = 0 + } + } + } else { + // Seek to record + if err := controllers.SeekToRecordOnTape(f, int32(record)); err != nil { + return err } - return nil - }, + // Seek to block + br := bufio.NewReaderSize(f, controllers.BlockSize*recordSize) + if _, err := br.Read(make([]byte, block*controllers.BlockSize)); err != nil { + return err + } + + record := int64(record) + block := int64(block) + + curr := int64((recordSize * controllers.BlockSize * int(record)) + (int(block) * controllers.BlockSize)) + counter := &counters.CounterReader{Reader: br, BytesRead: int(curr)} + + tr := tar.NewReader(counter) + for { + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { + if err := controllers.GoToNextFileOnTape(f); err != nil { + // EOD + + break + } + + record, err = controllers.GetCurrentRecordFromTape(f) + if err != nil { + return err + } + block = 0 + + br = bufio.NewReaderSize(f, controllers.BlockSize*recordSize) + curr = int64(int64(recordSize) * controllers.BlockSize * record) + counter = &counters.CounterReader{Reader: br, BytesRead: int(curr)} + tr = tar.NewReader(counter) + + continue + } else { + return err + } + } + + if err := indexHeader(record, block, hdr, metadataPersister); err != nil { + return nil + } + + curr = int64(counter.BytesRead) + + nextTotalBlocks := math.Ceil(float64((curr + hdr.Size)) / float64(controllers.BlockSize)) + record = int64(nextTotalBlocks) / int64(recordSize) + block = int64(nextTotalBlocks) - (record * int64(recordSize)) + + if block > int64(recordSize) { + record++ + block = 0 + } + } + } + + return nil } func init() { diff --git a/pkg/persisters/metadata.go b/pkg/persisters/metadata.go index 426f8b2..3bb4db2 100644 --- a/pkg/persisters/metadata.go +++ b/pkg/persisters/metadata.go @@ -6,11 +6,13 @@ package persisters import ( "context" "database/sql" + "fmt" "github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata" models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata" migrate "github.com/rubenv/sql-migrate" "github.com/volatiletech/sqlboiler/v4/boil" + "github.com/volatiletech/sqlboiler/v4/queries" ) type MetadataPersister struct { @@ -70,3 +72,26 @@ func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, ignor return hdr, nil } + +func (p *MetadataPersister) GetLastIndexedRecordAndBlock(ctx context.Context, recordSize int) (int64, int64, error) { + var header models.Header + if err := queries.Raw( + fmt.Sprintf( + `select %v, %v, ((%v*$1)+%v) as location from %v order by location desc limit 1`, + models.HeaderColumns.Record, + models.HeaderColumns.Block, + models.HeaderColumns.Record, + models.HeaderColumns.Block, + models.TableNames.Headers, + ), + recordSize, + ).Bind(ctx, p.db, &header); err != nil { + if err == sql.ErrNoRows { + return 0, 0, nil + } + + return 0, 0, err + } + + return header.Record, header.Block, nil +}