From 427878126755234d7df0a162508bccd518a38358 Mon Sep 17 00:00:00 2001 From: Felix Pojtinger Date: Tue, 7 Dec 2021 01:37:16 +0100 Subject: [PATCH] fix: Prevent moving headers from corrupting last indexed record & block --- cmd/stbak/cmd/move.go | 55 +++++++++++++++++++++++++++++---- internal/persisters/metadata.go | 29 +++++------------ pkg/recovery/index.go | 37 +++++++++++++--------- 3 files changed, 78 insertions(+), 43 deletions(-) diff --git a/cmd/stbak/cmd/move.go b/cmd/stbak/cmd/move.go index f1dc5ea..4030d08 100644 --- a/cmd/stbak/cmd/move.go +++ b/cmd/stbak/cmd/move.go @@ -11,6 +11,8 @@ import ( "github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/pax" "github.com/pojntfx/stfs/internal/persisters" + "github.com/pojntfx/stfs/pkg/config" + "github.com/pojntfx/stfs/pkg/recovery" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/volatiletech/sqlboiler/v4/boil" @@ -97,6 +99,11 @@ func move( return err } + lastIndexedRecord, lastIndexedBlock, err := metadataPersister.GetLastIndexedRecordAndBlock(context.Background(), viper.GetInt(recordSizeFlag)) + if err != nil { + return err + } + headersToMove := []*models.Header{} dbhdr, err := metadataPersister.GetHeader(context.Background(), src) if err != nil { @@ -114,16 +121,12 @@ func move( headersToMove = append(headersToMove, dbhdrs...) } - // Move the headers in the index - if err := metadataPersister.MoveHeaders(context.Background(), headersToMove, src, dst); err != nil { - return nil - } - if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { return err } // Append move headers to the tape or tar file + hdrs := []*tar.Header{} for _, dbhdr := range headersToMove { hdr, err := converters.DBHeaderToTarHeader(dbhdr) if err != nil { @@ -153,9 +156,49 @@ func move( if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil { return err } + + hdrs = append(hdrs, hdr) } - return nil + return recovery.Index( + config.StateConfig{ + Drive: viper.GetString(driveFlag), + Metadata: viper.GetString(metadataFlag), + }, + config.PipeConfig{ + Compression: viper.GetString(compressionFlag), + Encryption: viper.GetString(encryptionFlag), + Signature: viper.GetString(signatureFlag), + }, + config.CryptoConfig{ + Recipient: recipient, + Identity: identity, + Password: viper.GetString(passwordFlag), + }, + + viper.GetInt(recordSizeFlag), + int(lastIndexedRecord), + int(lastIndexedBlock), + false, + + func(hdr *tar.Header, i int) error { + // Ignore the first header, which is the last header which we already indexed + if i == 0 { + return nil + } + + if len(hdrs) <= i-1 { + return errMissingTarHeader + } + + *hdr = *hdrs[i-1] + + return nil + }, + func(hdr *tar.Header, isRegular bool) error { + return nil // We sign above, no need to verify + }, + ) } func init() { diff --git a/internal/persisters/metadata.go b/internal/persisters/metadata.go index 06bfaa5..84e7ae5 100644 --- a/internal/persisters/metadata.go +++ b/internal/persisters/metadata.go @@ -66,43 +66,28 @@ func (p *MetadataPersister) UpdateHeaderMetadata(ctx context.Context, dbhdr *mod return nil } -func (p *MetadataPersister) moveHeader(ctx context.Context, tx boil.ContextExecutor, oldName string, newName string) error { +func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newName string, lastknownrecord, lastknownblock int64) error { // We can't do this with `dbhdr.Update` because we are renaming the primary key if _, err := queries.Raw( fmt.Sprintf( - ` update %v set %v = ? where %v = ?;`, + ` update %v set %v = ?, %v = ?, %v = ? where %v = ?;`, models.TableNames.Headers, models.HeaderColumns.Name, + models.HeaderColumns.Lastknownrecord, + models.HeaderColumns.Lastknownblock, models.HeaderColumns.Name, ), newName, + lastknownrecord, + lastknownblock, oldName, - ).ExecContext(ctx, tx); err != nil { + ).ExecContext(ctx, p.db); err != nil { return err } return nil } -func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newName string) error { - return p.moveHeader(ctx, p.db, oldName, newName) -} - -func (p *MetadataPersister) MoveHeaders(ctx context.Context, hdrs models.HeaderSlice, oldName string, newName string) error { - tx, err := p.db.BeginTx(ctx, nil) - if err != nil { - return err - } - - for _, hdr := range hdrs { - if err := p.moveHeader(ctx, tx, hdr.Name, strings.TrimSuffix(newName, "/")+strings.TrimPrefix(hdr.Name, strings.TrimSuffix(oldName, "/"))); err != nil { - return err - } - } - - return tx.Commit() -} - func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice, error) { return models.Headers( qm.Where(models.HeaderColumns.Deleted+" != 1"), diff --git a/pkg/recovery/index.go b/pkg/recovery/index.go index f28e849..5ebe5ac 100644 --- a/pkg/recovery/index.go +++ b/pkg/recovery/index.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bufio" "context" + "database/sql" "io" "io/ioutil" "math" @@ -111,7 +112,6 @@ func Index( if err != nil { if err == io.EOF { // EOF - break } @@ -137,7 +137,7 @@ func Index( } if err := indexHeader(record, block, hdr, metadataPersister, pipes.Compression, pipes.Encryption); err != nil { - return nil + return err } curr, err := f.Seek(0, io.SeekCurrent) @@ -221,7 +221,7 @@ func Index( } if err := indexHeader(record, block, hdr, metadataPersister, pipes.Compression, pipes.Encryption); err != nil { - return nil + return err } curr = int64(counter.BytesRead) @@ -326,28 +326,35 @@ func indexHeader( } newHdr = h + + if err := metadataPersister.UpdateHeaderMetadata(context.Background(), newHdr); err != nil { + return err + } } else { // Metadata-only update; use the old record & block oldHdr, err := metadataPersister.GetHeader(context.Background(), oldName) - if err != nil { - return err + if err == nil { + h, err := converters.TarHeaderToDBHeader(oldHdr.Record, record, oldHdr.Block, block, hdr) + if err != nil { + return err + } + + newHdr = h + + if err := metadataPersister.UpdateHeaderMetadata(context.Background(), newHdr); err != nil { + return err + } } - h, err := converters.TarHeaderToDBHeader(oldHdr.Record, record, oldHdr.Block, block, hdr) - if err != nil { + // To support ignoring previous `Move` operations, we need to ignore non-existent headers here, as moving changes the primary keys + if err != nil && err != sql.ErrNoRows { return err } - - newHdr = h - } - - if err := metadataPersister.UpdateHeaderMetadata(context.Background(), newHdr); err != nil { - return err } if moveAfterEdits { - // Move header - if err := metadataPersister.MoveHeader(context.Background(), oldName, hdr.Name); err != nil { + // Move header (will be a no-op if the header has been moved before) + if err := metadataPersister.MoveHeader(context.Background(), oldName, hdr.Name, record, block); err != nil { return err } }