fix: Prevent deletion of headers from corrupting last indexed record & block

cmd/stbak/cmd/delete.go

@@ -14,6 +14,8 @@ import (
 	"github.com/pojntfx/stfs/internal/keys"
 	"github.com/pojntfx/stfs/internal/pax"
 	"github.com/pojntfx/stfs/internal/persisters"
+	"github.com/pojntfx/stfs/pkg/config"
+	"github.com/pojntfx/stfs/pkg/recovery"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/volatiletech/sqlboiler/v4/boil"
@@ -102,6 +104,11 @@ func delete(
 		return err
 	}
 
+	lastIndexedRecord, lastIndexedBlock, err := metadataPersister.GetLastIndexedRecordAndBlock(context.Background(), viper.GetInt(recordSizeFlag))
+	if err != nil {
+		return err
+	}
+
 	headersToDelete := []*models.Header{}
 	dbhdr, err := metadataPersister.GetHeader(context.Background(), name)
 	if err != nil {
@@ -119,17 +126,18 @@ func delete(
 		headersToDelete = append(headersToDelete, dbhdrs...)
 	}
 
-	// Remove the headers from the index
-	if err := metadataPersister.DeleteHeaders(context.Background(), headersToDelete); err != nil {
-		return nil
-	}
-
 	if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil {
 		return err
 	}
 
-	// Append deletion headers to the tape or tar file
+	// Append deletion hdrs to the tape or tar file
+	hdrs := []*tar.Header{}
 	for _, dbhdr := range headersToDelete {
+		// Check if the header hasn't already been deleted; the records will be corrected in the last index step
+		if _, err := metadataPersister.DeleteHeader(context.Background(), dbhdr.Name, -1, -1, false); err != nil {
+			return err
+		}
+
 		hdr, err := converters.DBHeaderToTarHeader(dbhdr)
 		if err != nil {
 			return err
@@ -156,9 +164,49 @@ func delete(
 		if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(-1, -1, -1, -1, hdr)); err != nil {
 			return err
 		}
+
+		hdrs = append(hdrs, hdr)
 	}
 
-	return nil
+	return recovery.Index(
+		config.StateConfig{
+			Drive:    viper.GetString(driveFlag),
+			Metadata: viper.GetString(metadataFlag),
+		},
+		config.PipeConfig{
+			Compression: viper.GetString(compressionFlag),
+			Encryption:  viper.GetString(encryptionFlag),
+			Signature:   viper.GetString(signatureFlag),
+		},
+		config.CryptoConfig{
+			Recipient: recipient,
+			Identity:  identity,
+			Password:  viper.GetString(passwordFlag),
+		},
+
+		viper.GetInt(recordSizeFlag),
+		int(lastIndexedRecord),
+		int(lastIndexedBlock),
+		false,
+
+		func(hdr *tar.Header, i int) error {
+			// Ignore the first header, which is the last header which we already indexed
+			if i == 0 {
+				return nil
+			}
+
+			if len(hdrs) <= i-1 {
+				return errMissingTarHeader
+			}
+
+			*hdr = *hdrs[i-1]
+
+			return nil
+		},
+		func(hdr *tar.Header, isRegular bool) error {
+			return nil // We sign above, no need to verify
+		},
+	)
 }
 
 func openTapeWriter(tape string, recordSize int, overwrite bool) (tw *tar.Writer, isRegular bool, cleanup func(dirty *bool) error, err error) {

db/sqlite/migrations/metadata/1637447083.sql

@@ -8,6 +8,8 @@ create table headers (
     block integer not null,
     -- Block of the last update header of this header in the record
     lastknownblock integer not null,
+    -- If set, the header has been deleted on tape, but `lastknownrecord` and `lastknownblock` are still of relevance
+    deleted integer not null,
     -- Typeflag is the type of header entry.
     -- The zero value is automatically promoted to either TypeReg or TypeDir
     -- depending on the presence of a trailing slash in Name.

internal/db/sqlite/migrations/metadata/migrations.go

@@ -25,7 +25,7 @@ func bindata_read(data []byte, name string) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-var _db_sqlite_migrations_metadata_1637447083_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x56\xcd\x72\x1b\x37\x0c\x3e\xd7\x4f\x81\xc9\x25\xf6\x54\xab\x73\xa7\x99\x1e\xdc\xb8\x71\x33\x63\x3b\x19\xd9\xaa\x73\xa5\x96\xd8\x5d\x56\x5c\x82\x05\xb9\x92\x37\x4f\xdf\x01\xb9\x54\x64\x2b\x92\xd3\xe9\x49\x2b\x12\xf8\x80\x0f\xc4\x5f\x55\xc1\xcf\xbd\x69\x59\x45\x84\xa5\x3f\xab\x19\xe5\x2b\xaa\x95\x45\xe8\x50\x69\xe4\x00\xe7\x67\x00\x00\x55\x05\x0b\xac\x89\x35\x50\x03\xb1\x33\x61\xba\x07\x72\x10\x3b\xd1\xf1\x98\x04\x39\x4b\x19\x17\xb1\x45\x06\x47\x11\xdc\x60\xed\xec\x7b\x28\x08\x56\x85\x08\x83\xd7\x62\xb6\x00\x9e\xc6\x17\x8d\xb5\xa3\xad\x7b\xcd\xd0\xef\x96\xea\xf5\x4b\x34\x93\xd1\xb2\x6e\x92\x5c\x25\xb1\x1f\x41\xf9\x11\x6f\x0f\xf1\x77\xfe\xbe\x62\xe8\x61\xf4\xd8\x58\xd5\x82\x09\x99\xf0\xe8\x51\xd0\x27\x60\x74\x91\xc7\xf9\x4e\xb8\x43\xf8\x8a\x4c\xb0\x51\x76\x40\x51\x51\x43\xa4\x5e\x45\x53\x2b\x6b\x47\xf0\x4c\x3d\x45\xd4\x10\x09\xd0\xc4\x0e\x39\xe1\x2f\xb0\x05\xca\x9f\x57\x86\x0b\x98\x46\x8f\x4e\x1b\xd7\x96\x58\x7b\xc6\x80\xae\x4e\xe6\x15\x44\x56\xc6\xca\x6d\xb0\x2a\x74\xc2\xf0\x4e\xf5\x98\x5d\x89\x3b\xa7\x8f\xd1\x12\x59\xc1\x69\x8c\xc5\x4c\x22\xdd\x38\x39\x8e\xf8\x14\x77\x1a\xe0\xd9\xf4\x8a\x47\x58\xe3\xf8\x2d\x28\x8a\x5b\x8c\x59\x9a\x1a\xb0\xc6\xad\xe1\x7c\xa3\xac\xd1\xd0\x4c\x44\x6e\xe4\x6c\xfa\xbe\x1f\x7b\x11\xb9\xc8\x81\x37\x6e\x7d\x68\x66\x07\x7d\x43\xad\x04\x2b\x3b\x16\xcc\x57\x14\x66\xab\x31\x62\x48\x12\xd3\xc9\x11\x56\x9f\x91\x7b\x13\x82\x21\x07\xca\x69\xe8\x49\x23\xac\x4c\xcc\xaa\xe9\xdf\x51\xd5\x65\x40\x86\x8f\x57\x42\x87\xb6\x0e\xf3\x2b\x0c\xe6\x44\x22\x5f\x33\x0d\xfe\x40\xa5\x3d\xa5\x92\x8c\x94\xa8\xed\x99\x39\x11\x8f\x6c\xe5\x50\xa7\x3d\xa1\xf3\x31\x97\xc5\x07\xe2\x5e\x45\xc9\xc2\xc1\x05\x8f\xb5\x69\x0c\xea\x99\x5c\x39\x78\x64\x13\x91\xe7\xe9\xe7\xcf\x9c\xca\x4c\x83\xd3\x01\x6e\x49\x3f\x98\x1e\x0b\x56\xa4\x84\xe5\x50\x31\x86\x08\x01\x6b\x72\x3a\x45\xd7\xb4\x8e\x18\x73\x55\x5c\xd6\x35\x86\x20\x7a\xe9\xea\x7d\xa7\x5c\x8b\xe9\x6f\x63\xd0\xea\x50\x2a\x64\x97\x40\x04\x43\x78\xa6\x46\xbc\xa7\x35\x83\xec\xef\xb8\xcf\x43\x05\xf8\x7c\xf9\x45\x04\xaf\xef\x96\xf3\x17\x48\x61\x58\x55\x93\x6f\x8c\x81\xec\x10\x0d\xb9\x13\x30\x3b\xfd\x5b\xd2\xa6\x31\xb5\x12\x79\x88\x85\x78\x4f\x5a\xbe\x21\xb5\x94\x83\xf8\x66\xb7\x93\x34\x9c\x33\xfe\x33\x18\x89\xc3\x54\xd0\xdf\x7c\x84\x30\x78\x4f\x1c\x73\xde\xab\xa4\x74\x02\x35\xd3\xff\x8f\xa8\x75\x52\x3a\x81\x7a\xab\xfe\x26\x06\x8d\x1b\x53\x23\xb8\xa1\x5f\x21\xbf\x2c\xd5\xf7\x9d\xe2\x52\xaa\xa9\xaf\x66\x68\x8d\x9b\x3e\x29\x1f\x4d\xe7\x5b\xe3\xfe\x07\x76\x52\x3e\x5e\xca\x97\x5f\xf2\x48\x0a\xa9\x8d\x42\xaf\xbc\x14\x80\xc4\x01\x9f\x22\x3a\x8d\xba\xf4\xe0\xdc\xd5\x0f\x72\x4c\x8a\xad\xd2\xd8\x18\x87\xba\xc8\x40\xe8\x68\xb0\x1a\x3a\xb5\x41\xe9\x68\xa1\x8c\x90\x86\xac\xa5\xad\xb4\xd3\x86\xb8\xff\x75\xc2\xf8\xe9\xaf\x3f\xee\xae\x3e\x2d\xe6\x6b\x1c\xb7\x65\x70\x54\x15\x3c\x76\xc8\x08\xf9\x4e\xbc\x0b\xd4\x63\x2a\xd1\xe0\x55\x9d\x1a\x96\xb2\x16\x06\xef\x91\x6b\x15\x70\x96\x6a\x62\xc2\x80\x5e\x8d\x05\x47\x48\xd7\xe4\xa2\x9a\xc6\xd3\xdb\xdf\xde\xca\x83\xb2\xaa\xa3\x44\x12\xe7\xed\x7c\x06\x6f\xae\x3f\xdd\x5c\xde\x5d\xcf\xfd\xba\x9d\x6f\x90\xa5\xb9\xbd\xb9\x78\x36\x71\xd6\x38\x26\x0b\x79\xe8\x4c\x0c\x57\x92\x0b\xae\xc2\xde\xc7\x11\x96\x0f\x1f\xaa\x5f\x20\x44\x36\xae\x3d\x88\xd3\xe3\x91\x7e\x60\x02\xc8\xd4\x92\x96\x21\x51\x2f\x11\xd4\xc8\x66\x83\x1a\x1a\xa6\x5e\xbc\x2e\x30\x94\xf2\x34\x97\xbb\x84\x60\x42\x89\x6a\x9d\x26\x57\x8d\x3a\xcf\xae\x4d\xce\xe6\xc5\xfe\xa3\x79\xf5\x54\xe0\xbf\xdf\xd1\xa6\xda\x2d\x5d\x2c\x4c\x6f\x96\x0e\xa7\x17\x8c\x8a\xa7\x84\x38\x68\x36\xb2\x08\xc8\x33\x61\x84\xd5\x08\x8b\x2c\x74\x27\x86\x94\xa4\xd6\x0a\x43\xac\xb0\x69\x88\x23\xb4\x83\x54\xb6\x8a\x7b\x06\x76\xb1\xbe\x37\x42\x40\x2e\x32\x04\x58\xb3\x42\x4e\x83\x9d\x51\xe9\x92\x07\xe4\xaa\x9a\x7a\x6f\x8d\x72\x31\x8d\xb1\xb0\xa3\x61\x52\x37\xf6\x14\x82\x91\x55\x4e\x8a\x24\x6d\x29\x91\xe4\xbd\x32\xc9\xa5\x4b\x6b\xc9\x4b\x12\x1f\x9b\x7d\xce\xcf\x7b\x3a\x6c\x5f\x7d\xc2\x5d\x3b\x17\x49\x13\xa5\x67\x4e\x41\x34\x1c\x62\x81\x3d\x9f\x12\x91\x78\xda\xa1\x96\xf7\x0f\x97\x8b\xf4\xfe\x33\xe9\x3d\x17\x05\xa6\x56\x3e\x2d\xa3\xd4\x00\xba\x9a\xd2\x96\x92\x98\x4c\x86\xcf\x03\x16\x3e\x53\xae\x16\xc7\x5f\xd4\xfb\xd9\xc5\xbb\xb3\xfd\x6d\xf7\x8a\xb6\xee\x4c\x33\xf9\xe7\xdb\xee\xbb\x7f\x03\x00\x00\xff\xff\x4b\x3a\xfd\xed\x12\x0b\x00\x00")
+var _db_sqlite_migrations_metadata_1637447083_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x56\xcd\x72\xdb\x36\x10\x3e\xd7\x4f\xb1\x93\x4b\xec\xa9\xa8\x73\xa7\x99\x1e\xdc\xb8\x71\x3d\x13\x3b\x19\xd9\xaa\x73\x34\x48\x2c\x49\x54\x20\x16\x5d\x80\x92\x99\xa7\xef\x2c\x40\x2a\xb2\x14\xc9\xe9\xf4\x24\x0a\xd8\xfd\xf6\xff\x5b\x14\x05\xfc\xdc\x99\x86\x55\x44\x58\xfa\xb3\x8a\x51\xbe\xa2\x2a\x2d\x42\x8b\x4a\x23\x07\x38\x3f\x03\x00\x28\x0a\x58\x60\x45\xac\x81\x6a\x88\xad\x09\xe3\x3d\x90\x83\xd8\x8a\x8e\xc7\x24\xc8\x59\xca\xb8\x88\x0d\x32\x38\x8a\xe0\x7a\x6b\x67\xdf\x43\x41\xb0\x2a\x44\xe8\xbd\x16\xb3\x13\xe0\x69\x7c\xd1\x58\x39\xda\xb8\xd7\x0c\xfd\x6e\xa9\x5a\xed\xa3\x99\x8c\x96\x75\x93\x64\x99\xc4\x7e\x04\xe5\x47\xbc\x3d\xc4\xdf\xfa\xfb\x8a\xa1\x9b\x1a\x02\xc6\x59\x52\x1f\xc1\x5a\x15\xa0\x44\x74\xa0\xd1\x62\x44\x9d\x72\xa1\x3c\xce\xa0\xec\x23\x3c\xed\x65\xe2\x09\x94\xd3\x3b\xa7\xc9\xde\x13\x28\x46\x08\xd1\x58\x2b\xae\x32\x5a\x5c\x2b\x57\xe5\x54\x4e\xb0\x47\x7d\x7a\x18\x3c\xd6\x56\x35\x60\x42\x2e\xc2\xe0\x51\x60\x46\xff\xd0\x45\x1e\xe6\x5b\xe1\x16\xe1\x2b\x32\xc1\x5a\xd9\x1e\x45\x45\xf5\x91\x3a\x15\x4d\xa5\xac\x1d\xc0\x33\x75\x24\xe6\x22\x01\x9a\xd8\x22\x27\xfc\x05\x36\x40\xf9\xf3\xca\xf0\x04\xa6\xd1\xa3\xd3\xc6\x35\x53\xfd\x3d\x63\x40\x57\x25\xf3\x0a\x22\x2b\x63\xe5\x36\x58\x15\x5a\xc9\xfa\x9d\xea\x30\xbb\x12\xb7\x4e\x1f\x0b\x4b\x64\x05\xa7\x36\x16\x73\x10\xe9\xc6\xc9\x71\xc4\xe7\xb8\xd5\x00\xcf\xa6\x53\x3c\xc0\x0a\x87\x6f\x49\x51\xdc\x60\xcc\xd2\x54\x83\x35\x6e\x05\xe7\x6b\x65\x8d\x86\x7a\x0c\xe4\xa3\x9c\x8d\xdf\xf7\x43\x27\x22\x17\xb9\x19\x8c\x5b\x1d\x9a\xd9\x42\x7f\xa4\x46\x92\x95\x1d\x0b\xe6\x2b\x4a\x64\xe5\x10\x31\x24\x89\xf1\xe4\x48\x54\x9f\x91\x3b\x13\x82\x21\x97\x1a\xa1\x23\x8d\x50\x9a\x98\x55\xd3\xbf\xa3\xaa\xcb\x80\x0c\x37\x57\x12\x0e\x6d\x1c\xe6\x2a\xf4\xe6\x44\x67\x5c\x33\xf5\xfe\x40\xa5\x39\xa5\x92\x8c\x4c\x59\xdb\x31\x73\x22\x1f\xd9\xca\xa1\x4e\x73\x42\xe7\x26\x8f\xea\x07\xe2\x4e\x45\xe9\xc2\xde\x05\x8f\x95\xa9\x0d\xea\x34\x5d\x0e\x1e\xd9\x44\xe4\x79\xfa\xf9\x33\xb7\x32\x53\xef\x74\x80\x5b\xd2\x0f\xa6\xc3\x09\x2b\x52\xc2\x72\xa8\x18\x43\x84\x80\x15\x39\x9d\xb2\x6b\x1a\x47\x8c\x79\x2a\x2e\xab\x0a\x43\x10\xbd\x74\xf5\xbe\x55\xae\xc1\xf4\xb7\x36\x68\x75\x98\x26\x64\xdb\x40\x04\x7d\x78\xa1\x46\xbc\xa3\x35\x83\xec\xef\xb0\x1b\x87\x0a\xf0\xf9\xf2\x8b\x08\x5e\xdf\x2d\xe7\x7b\x48\xa1\x2f\x8b\xd1\x37\xc6\x40\xb6\x8f\x86\xdc\x09\x98\xad\xfe\x2d\x69\x53\x9b\x4a\x89\x3c\xc4\x29\xf0\x8e\xb4\x7c\x43\xa2\xb9\x83\xfc\x66\xb7\x93\x34\x9c\x33\xfe\xd3\x1b\xc9\xc3\x38\xd0\xdf\x7c\x84\xd0\x7b\x4f\x1c\x73\xdf\xab\xa4\x74\x02\x35\x87\xff\x1f\x51\xab\xa4\x74\x02\xf5\x56\xfd\x4d\x0c\x1a\xd7\xa6\x42\x70\x7d\x57\x22\xef\x8f\xea\xfb\x56\xf1\x34\xaa\x89\xeb\x2f\x46\x6a\x5c\x77\x49\xf9\x68\x3b\xdf\x1a\xf7\x3f\xb0\x93\xf2\xf1\x51\xbe\xfc\x92\xd7\x64\x48\x34\x0a\x9d\xf2\x32\x00\x92\x07\x7c\x8e\xe8\x34\xea\x89\x83\x33\xf7\x1f\xf4\x98\x0c\x5b\xa1\xb1\x36\x0e\xf5\x24\x03\xa1\xa5\xde\x6a\x68\xd5\x1a\x85\xd1\xc2\xb4\xd6\x6a\xb2\x96\x36\x42\xa7\x35\x71\xf7\xeb\x88\xf1\xd3\x5f\x7f\xdc\x5d\x7d\x5a\xcc\x57\x38\x6c\xa6\x65\x56\x14\xf0\xd8\x22\x23\xe4\x3b\xf1\x2e\x50\x87\x69\x44\x83\x57\x55\x22\x2c\x65\x2d\xf4\xde\x23\x57\x2a\xe0\x2c\xcd\xc4\x88\x01\x9d\x1a\x26\x1c\x09\xba\x22\x17\xd5\xb8\x32\xdf\xfe\xf6\x56\x0a\xca\xaa\x8a\x92\x49\x9c\x37\xf3\x19\xbc\xb9\xfe\xf4\xf1\xf2\xee\x7a\xee\x57\xcd\x7c\x8d\x2c\xe4\xf6\xe6\xe2\xc5\xc6\x59\xe1\x90\x2c\xe4\xa5\x33\x46\x58\x4a\x2f\xb8\x02\x3b\x1f\x07\x58\x3e\x7c\x28\x7e\x81\x10\xd9\xb8\xe6\x20\x4f\x8f\x47\xf8\xc0\x04\x90\xad\x25\x94\x21\x59\x9f\x32\xa8\x91\xcd\x1a\x35\xd4\x4c\x9d\x78\x3d\xc1\x50\xea\xd3\x3c\xee\x92\x82\x11\x25\xaa\x55\xda\x5c\x15\xea\xbc\xbb\xd6\xb9\x9b\x17\xbb\x45\xf3\xea\x79\x82\xff\x3e\xa3\x8d\xb3\x3b\xb1\x58\x18\x6b\x96\x0e\xc7\x0a\x46\xc5\x63\x43\x1c\x90\x8d\x3c\x4e\xa4\x4c\x18\xa1\x1c\x60\x91\x85\xee\xc4\x90\x92\xd6\x2a\x31\xc4\x02\xeb\x9a\x38\x42\xd3\xcb\x64\xab\xb8\x63\x60\x9b\xeb\x7b\x23\x01\xc8\x45\x86\x00\x6b\x4a\xe4\xb4\xd8\x19\x95\x9e\xfa\x80\x5c\x51\x51\xe7\xad\x51\x2e\xa6\x35\x16\xb6\x61\x98\xc4\xc6\x9e\x42\x30\xf2\xbc\x94\x21\x49\x2f\xa7\x48\x52\xaf\x1c\xe4\xd2\xa5\xa7\xcb\x7e\x10\x37\xf5\x6e\xcc\x2f\x39\x1d\x36\xaf\x96\x70\x4b\xe7\x22\x69\xa2\x70\xe6\x98\x44\xc3\x21\x4e\xb0\xe7\x63\x23\x12\x8f\xef\xba\xe5\xfd\xc3\xe5\x22\xd5\x7f\x26\xdc\x73\x31\xc1\x54\xca\xa7\x07\x32\xd5\x80\xae\xa2\xf4\x4a\x49\x91\x8c\x86\xcf\x03\x4e\xf1\x8c\xbd\x3a\x39\xbe\x37\xef\x67\x17\xef\xce\x76\x5f\xe0\x57\xb4\x71\x67\x9a\xc9\xbf\x7c\x81\xbf\xfb\x37\x00\x00\xff\xff\x7a\x41\x6c\x96\xa6\x0b\x00\x00")
 
 func db_sqlite_migrations_metadata_1637447083_sql() ([]byte, error) {
 	return bindata_read(

internal/db/sqlite/models/metadata/headers.go

@@ -26,6 +26,7 @@ type Header struct {
 	Lastknownrecord int64     `boil:"lastknownrecord" json:"lastknownrecord" toml:"lastknownrecord" yaml:"lastknownrecord"`
 	Block           int64     `boil:"block" json:"block" toml:"block" yaml:"block"`
 	Lastknownblock  int64     `boil:"lastknownblock" json:"lastknownblock" toml:"lastknownblock" yaml:"lastknownblock"`
+	Deleted         int64     `boil:"deleted" json:"deleted" toml:"deleted" yaml:"deleted"`
 	Typeflag        int64     `boil:"typeflag" json:"typeflag" toml:"typeflag" yaml:"typeflag"`
 	Name            string    `boil:"name" json:"name" toml:"name" yaml:"name"`
 	Linkname        string    `boil:"linkname" json:"linkname" toml:"linkname" yaml:"linkname"`
@@ -52,6 +53,7 @@ var HeaderColumns = struct {
 	Lastknownrecord string
 	Block           string
 	Lastknownblock  string
+	Deleted         string
 	Typeflag        string
 	Name            string
 	Linkname        string
@@ -73,6 +75,7 @@ var HeaderColumns = struct {
 	Lastknownrecord: "lastknownrecord",
 	Block:           "block",
 	Lastknownblock:  "lastknownblock",
+	Deleted:         "deleted",
 	Typeflag:        "typeflag",
 	Name:            "name",
 	Linkname:        "linkname",
@@ -96,6 +99,7 @@ var HeaderTableColumns = struct {
 	Lastknownrecord string
 	Block           string
 	Lastknownblock  string
+	Deleted         string
 	Typeflag        string
 	Name            string
 	Linkname        string
@@ -117,6 +121,7 @@ var HeaderTableColumns = struct {
 	Lastknownrecord: "headers.lastknownrecord",
 	Block:           "headers.block",
 	Lastknownblock:  "headers.lastknownblock",
+	Deleted:         "headers.deleted",
 	Typeflag:        "headers.typeflag",
 	Name:            "headers.name",
 	Linkname:        "headers.linkname",
@@ -186,6 +191,7 @@ var HeaderWhere = struct {
 	Lastknownrecord whereHelperint64
 	Block           whereHelperint64
 	Lastknownblock  whereHelperint64
+	Deleted         whereHelperint64
 	Typeflag        whereHelperint64
 	Name            whereHelperstring
 	Linkname        whereHelperstring
@@ -207,6 +213,7 @@ var HeaderWhere = struct {
 	Lastknownrecord: whereHelperint64{field: "\"headers\".\"lastknownrecord\""},
 	Block:           whereHelperint64{field: "\"headers\".\"block\""},
 	Lastknownblock:  whereHelperint64{field: "\"headers\".\"lastknownblock\""},
+	Deleted:         whereHelperint64{field: "\"headers\".\"deleted\""},
 	Typeflag:        whereHelperint64{field: "\"headers\".\"typeflag\""},
 	Name:            whereHelperstring{field: "\"headers\".\"name\""},
 	Linkname:        whereHelperstring{field: "\"headers\".\"linkname\""},
@@ -242,8 +249,8 @@ func (*headerR) NewStruct() *headerR {
 type headerL struct{}
 
 var (
-	headerAllColumns            = []string{"record", "lastknownrecord", "block", "lastknownblock", "typeflag", "name", "linkname", "size", "mode", "uid", "gid", "uname", "gname", "modtime", "accesstime", "changetime", "devmajor", "devminor", "paxrecords", "format"}
+	headerAllColumns            = []string{"record", "lastknownrecord", "block", "lastknownblock", "deleted", "typeflag", "name", "linkname", "size", "mode", "uid", "gid", "uname", "gname", "modtime", "accesstime", "changetime", "devmajor", "devminor", "paxrecords", "format"}
-	headerColumnsWithoutDefault = []string{"record", "lastknownrecord", "block", "lastknownblock", "typeflag", "name", "linkname", "size", "mode", "uid", "gid", "uname", "gname", "modtime", "accesstime", "changetime", "devmajor", "devminor", "paxrecords", "format"}
+	headerColumnsWithoutDefault = []string{"record", "lastknownrecord", "block", "lastknownblock", "deleted", "typeflag", "name", "linkname", "size", "mode", "uid", "gid", "uname", "gname", "modtime", "accesstime", "changetime", "devmajor", "devminor", "paxrecords", "format"}
 	headerColumnsWithDefault    = []string{}
 	headerPrimaryKeyColumns     = []string{"name"}
 )

internal/db/sqlite/models/metadata/headers_test.go

@@ -568,7 +568,7 @@ func testHeadersSelect(t *testing.T) {
 }
 
 var (
-	headerDBTypes = map[string]string{`Record`: `INTEGER`, `Lastknownrecord`: `INTEGER`, `Block`: `INTEGER`, `Lastknownblock`: `INTEGER`, `Typeflag`: `INTEGER`, `Name`: `TEXT`, `Linkname`: `TEXT`, `Size`: `INTEGER`, `Mode`: `INTEGER`, `UID`: `INTEGER`, `Gid`: `INTEGER`, `Uname`: `TEXT`, `Gname`: `TEXT`, `Modtime`: `DATE`, `Accesstime`: `DATE`, `Changetime`: `DATE`, `Devmajor`: `INTEGER`, `Devminor`: `INTEGER`, `Paxrecords`: `TEXT`, `Format`: `INTEGER`}
+	headerDBTypes = map[string]string{`Record`: `INTEGER`, `Lastknownrecord`: `INTEGER`, `Block`: `INTEGER`, `Lastknownblock`: `INTEGER`, `Deleted`: `INTEGER`, `Typeflag`: `INTEGER`, `Name`: `TEXT`, `Linkname`: `TEXT`, `Size`: `INTEGER`, `Mode`: `INTEGER`, `UID`: `INTEGER`, `Gid`: `INTEGER`, `Uname`: `TEXT`, `Gname`: `TEXT`, `Modtime`: `DATE`, `Accesstime`: `DATE`, `Changetime`: `DATE`, `Devmajor`: `INTEGER`, `Devminor`: `INTEGER`, `Paxrecords`: `TEXT`, `Format`: `INTEGER`}
 	_             = bytes.MinRead
 )
 

internal/persisters/metadata.go

@@ -104,7 +104,9 @@ func (p *MetadataPersister) MoveHeaders(ctx context.Context, hdrs models.HeaderS
 }
 
 func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice, error) {
-	return models.Headers().All(ctx, p.db)
+	return models.Headers(
+		qm.Where(models.HeaderColumns.Deleted+" != 1"),
+	).All(ctx, p.db)
 }
 
 func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models.Header, error) {
@@ -114,6 +116,7 @@ func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models
 func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string) (models.HeaderSlice, error) {
 	return models.Headers(
 		qm.Where(models.HeaderColumns.Name+" like ?", strings.TrimSuffix(name, "/")+"/%"), // Prevent double trailing slashes
+		qm.Where(models.HeaderColumns.Deleted+" != 1"),
 	).All(ctx, p.db)
 }
 
@@ -129,10 +132,11 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
 
 		if err := queries.Raw(
 			fmt.Sprintf(
-				`select min(length(%v) - length(replace(%v, "/", ""))) as depth from %v`,
+				`select min(length(%v) - length(replace(%v, "/", ""))) as depth from %v where %v != 1`,
 				models.HeaderColumns.Name,
 				models.HeaderColumns.Name,
 				models.TableNames.Headers,
+				models.HeaderColumns.Deleted,
 			),
 		).Bind(ctx, p.db, &depth); err != nil {
 			if err == sql.ErrNoRows {
@@ -147,7 +151,7 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
 
 	if err := queries.Raw(
 		fmt.Sprintf(
-			`select %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v,
+			`select %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v,
     length(replace(%v, ?, '')) - length(replace(replace(%v, ?, ''), '/', '')) as depth
 from %v
 where %v like ?
@@ -158,11 +162,13 @@ where %v like ?
             and depth = ?
         )
     )
+	and %v != 1
     and not %v in ('', '.', '/', './')`,
 			models.HeaderColumns.Record,
 			models.HeaderColumns.Lastknownrecord,
 			models.HeaderColumns.Block,
 			models.HeaderColumns.Lastknownblock,
+			models.HeaderColumns.Deleted,
 			models.HeaderColumns.Typeflag,
 			models.HeaderColumns.Name,
 			models.HeaderColumns.Linkname,
@@ -184,6 +190,7 @@ where %v like ?
 			models.TableNames.Headers,
 			models.HeaderColumns.Name,
 			models.HeaderColumns.Name,
+			models.HeaderColumns.Deleted,
 			models.HeaderColumns.Name,
 		),
 		prefix,
@@ -202,7 +209,7 @@ where %v like ?
 	return headers, nil
 }
 
-func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, ignoreNotExists bool) (*models.Header, error) {
+func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64, ignoreNotExists bool) (*models.Header, error) {
 	hdr, err := models.FindHeader(ctx, p.db, name)
 	if err != nil {
 		if err == sql.ErrNoRows && ignoreNotExists {
@@ -212,33 +219,26 @@ func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, ignor
 		return nil, err
 	}
 
-	if _, err := hdr.Delete(ctx, p.db); err != nil {
+	if hdr != nil && hdr.Deleted == 1 && !ignoreNotExists {
+		return nil, sql.ErrNoRows
+	}
+
+	hdr.Deleted = 1
+	hdr.Lastknownrecord = lastknownrecord
+	hdr.Lastknownblock = lastknownblock
+
+	if _, err := hdr.Update(ctx, p.db, boil.Infer()); err != nil {
 		return nil, err
 	}
 
 	return hdr, nil
 }
 
-func (p *MetadataPersister) DeleteHeaders(ctx context.Context, hdrs models.HeaderSlice) error {
-	tx, err := p.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-
-	for _, hdr := range hdrs {
-		if _, err := hdr.Delete(ctx, tx); err != nil {
-			return err
-		}
-	}
-
-	return tx.Commit()
-}
-
 func (p *MetadataPersister) GetLastIndexedRecordAndBlock(ctx context.Context, recordSize int) (int64, int64, error) {
 	var header models.Header
 	if err := queries.Raw(
 		fmt.Sprintf(
-			`select %v, %v, ((%v*$1)+%v) as location from %v order by location desc limit 1`,
+			`select %v, %v, ((%v*$1)+%v) as location from %v order by location desc limit 1`, // We include deleted headers here as they are still physically on the tape and have to be considered when re-indexing
 			models.HeaderColumns.Lastknownrecord,
 			models.HeaderColumns.Lastknownblock,
 			models.HeaderColumns.Lastknownrecord,

pkg/recovery/index.go

@@ -306,7 +306,7 @@ func indexHeader(
 				return err
 			}
 		case pax.STFSRecordActionDelete:
-			if _, err := metadataPersister.DeleteHeader(context.Background(), hdr.Name, true); err != nil {
+			if _, err := metadataPersister.DeleteHeader(context.Background(), hdr.Name, record, block, true); err != nil {
 				return err
 			}
 		case pax.STFSRecordActionUpdate: