feat: Enable indexing with tape as single source of truth after deletions

cmd/stbak/cmd/index.go

@@ -9,8 +9,10 @@ import (
 	"os"
 
 	"github.com/pojntfx/stfs/pkg/controllers"
+	"github.com/pojntfx/stfs/pkg/converters"
 	"github.com/pojntfx/stfs/pkg/counters"
 	"github.com/pojntfx/stfs/pkg/formatting"
+	"github.com/pojntfx/stfs/pkg/pax"
 	"github.com/pojntfx/stfs/pkg/persisters"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -117,18 +119,8 @@ var indexCmd = &cobra.Command{
 					}
 				}
 
-				if record == 0 && block == 0 {
+				if err := indexHeader(record, block, hdr, metadataPersister); err != nil {
-					if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
+					return nil
-						return err
-					}
-				}
-
-				if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(record, block, hdr)); err != nil {
-					return err
-				}
-
-				if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil {
-					return err
 				}
 
 				curr, err := f.Seek(0, io.SeekCurrent)
@@ -191,18 +183,8 @@ var indexCmd = &cobra.Command{
 					}
 				}
 
-				if record == 0 && block == 0 {
+				if err := indexHeader(record, block, hdr, metadataPersister); err != nil {
-					if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
+					return nil
-						return err
-					}
-				}
-
-				if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(record, block, hdr)); err != nil {
-					return err
-				}
-
-				if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil {
-					return err
 				}
 
 				curr = int64(counter.BytesRead)
@@ -232,3 +214,50 @@ func init() {
 
 	rootCmd.AddCommand(indexCmd)
 }
+
+func indexHeader(record, block int64, hdr *tar.Header, metadataPersister *persisters.MetadataPersister) error {
+	if record == 0 && block == 0 {
+		if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
+			return err
+		}
+	}
+
+	if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(record, block, hdr)); err != nil {
+		return err
+	}
+
+	stfsVersion, ok := hdr.PAXRecords[pax.STFSRecordVersion]
+	if !ok {
+		stfsVersion = pax.STFSRecordVersion1
+	}
+
+	switch stfsVersion {
+	case pax.STFSRecordVersion1:
+		stfsAction, ok := hdr.PAXRecords[pax.STFSRecordAction]
+		if !ok {
+			stfsAction = pax.STFSRecordActionCreate
+		}
+
+		switch stfsAction {
+		case pax.STFSRecordActionCreate:
+			dbhdr, err := converters.TarHeaderToDBHeader(record, block, hdr)
+			if err != nil {
+				return err
+			}
+
+			if err := metadataPersister.UpsertHeader(context.Background(), dbhdr); err != nil {
+				return err
+			}
+		case pax.STFSRecordActionDelete:
+			if _, err := metadataPersister.DeleteHeader(context.Background(), hdr.Name, true); err != nil {
+				return err
+			}
+		default:
+			return pax.ErrUnsupportedAction
+		}
+	default:
+		return pax.ErrUnsupportedVersion
+	}
+
+	return nil
+}

cmd/stbak/cmd/remove.go

@@ -100,7 +100,7 @@ var removeCmd = &cobra.Command{
 			return err
 		}
 
-		dbhdr, err := metadataPersister.DeleteHeader(context.Background(), viper.GetString(nameFlag))
+		dbhdr, err := metadataPersister.DeleteHeader(context.Background(), viper.GetString(nameFlag), false)
 		if err != nil {
 			return err
 		}

pkg/converters/header.go

@@ -34,3 +34,33 @@ func DBHeaderToTarHeader(dbhdr *models.Header) (*tar.Header, error) {
 
 	return hdr, nil
 }
+
+func TarHeaderToDBHeader(record, block int64, tarhdr *tar.Header) (*models.Header, error) {
+	paxRecords, err := json.Marshal(tarhdr.PAXRecords)
+	if err != nil {
+		return nil, err
+	}
+
+	hdr := models.Header{
+		Record:     record,
+		Block:      block,
+		Typeflag:   int64(tarhdr.Typeflag),
+		Name:       tarhdr.Name,
+		Linkname:   tarhdr.Linkname,
+		Size:       tarhdr.Size,
+		Mode:       tarhdr.Mode,
+		UID:        int64(tarhdr.Uid),
+		Gid:        int64(tarhdr.Gid),
+		Uname:      tarhdr.Uname,
+		Gname:      tarhdr.Gname,
+		Modtime:    tarhdr.ModTime,
+		Accesstime: tarhdr.AccessTime,
+		Changetime: tarhdr.ChangeTime,
+		Devmajor:   tarhdr.Devmajor,
+		Devminor:   tarhdr.Devminor,
+		Paxrecords: string(paxRecords),
+		Format:     int64(tarhdr.Format),
+	}
+
+	return &hdr, nil
+}

pkg/pax/stfs.go

@@ -1,5 +1,7 @@
 package pax
 
+import "errors"
+
 const (
 	STFSRecordVersion  = "STFS.Version"
 	STFSRecordVersion1 = "1"
@@ -8,3 +10,8 @@ const (
 	STFSRecordActionCreate = "CREATE"
 	STFSRecordActionDelete = "DELETE"
 )
+
+var (
+	ErrUnsupportedVersion = errors.New("unsupported STFS version")
+	ErrUnsupportedAction  = errors.New("unsupported STFS action")
+)

pkg/persisters/metadata.go

@@ -4,10 +4,8 @@ package persisters
 //go:generate go-bindata -pkg metadata -o ../db/sqlite/migrations/metadata/migrations.go ../../../db/sqlite/migrations/metadata
 
 import (
-	"archive/tar"
 	"context"
 	"database/sql"
-	"encoding/json"
 
 	"github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata"
 	models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata"
@@ -32,33 +30,7 @@ func NewMetadataPersister(dbPath string) *MetadataPersister {
 	}
 }
 
-func (p *MetadataPersister) UpsertHeader(ctx context.Context, record, block int64, hdr *tar.Header) error {
+func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Header) error {
-	paxRecords, err := json.Marshal(hdr.PAXRecords)
-	if err != nil {
-		return err
-	}
-
-	dbhdr := models.Header{
-		Record:     record,
-		Block:      block,
-		Typeflag:   int64(hdr.Typeflag),
-		Name:       hdr.Name,
-		Linkname:   hdr.Linkname,
-		Size:       hdr.Size,
-		Mode:       hdr.Mode,
-		UID:        int64(hdr.Uid),
-		Gid:        int64(hdr.Gid),
-		Uname:      hdr.Uname,
-		Gname:      hdr.Gname,
-		Modtime:    hdr.ModTime,
-		Accesstime: hdr.AccessTime,
-		Changetime: hdr.ChangeTime,
-		Devmajor:   hdr.Devmajor,
-		Devminor:   hdr.Devminor,
-		Paxrecords: string(paxRecords),
-		Format:     int64(hdr.Format),
-	}
-
 	if _, err := models.FindHeader(ctx, p.db, dbhdr.Name, models.HeaderColumns.Name); err != nil {
 		if err == sql.ErrNoRows {
 			if err := dbhdr.Insert(ctx, p.db, boil.Infer()); err != nil {
@@ -82,9 +54,13 @@ func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice,
 	return models.Headers().All(ctx, p.db)
 }
 
-func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string) (*models.Header, error) {
+func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, ignoreNotExists bool) (*models.Header, error) {
 	hdr, err := models.FindHeader(ctx, p.db, name)
 	if err != nil {
+		if err == sql.ErrNoRows && ignoreNotExists {
+			return nil, nil
+		}
+
 		return nil, err
 	}