feat: Add command which writes a tar file's headers into a SQLite database index

cmd/stfs-metadata/main.go

@@ -1,21 +1,43 @@
 package main
 
+//go:generate sqlboiler sqlite3 -o ../../pkg/db/sqlite/models/metadata -c ../../configs/sqlboiler/metadata.toml
+//go:generate go-bindata -pkg metadata -o ../../pkg/db/sqlite/migrations/metadata/migrations.go ../../db/sqlite/migrations/metadata
+
 import (
+	"archive/tar"
+	"bytes"
+	"context"
 	"database/sql"
+	"encoding/json"
 	"flag"
+	"fmt"
+	"io"
+	"log"
 	"os"
 	"path/filepath"
 
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata"
+	models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata"
 	migrate "github.com/rubenv/sql-migrate"
+	"github.com/volatiletech/sqlboiler/v4/boil"
 )
 
-//go:generate sqlboiler sqlite3 -o ../../pkg/db/sqlite/models/metadata -c ../../configs/sqlboiler/metadata.toml
-//go:generate go-bindata -pkg metadata -o ../../pkg/db/sqlite/migrations/metadata/migrations.go ../../db/sqlite/migrations/metadata
+const (
+	blockSize = 512
+)
+
+type HeaderInBlock struct {
+	Record int
+	Block  int
+	Header string
+}
 
 func main() {
 	dbPath := flag.String("db", "/tmp/stfs-metadata.sqlite", "Database file to use")
+	file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open")
+	recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
+	checkpoint := flag.Int("checkpoint", 0, "Log current record after checkpoint kilobytes have been read")
 
 	flag.Parse()
 
@@ -41,4 +63,89 @@ func main() {
 	); err != nil {
 		panic(err)
 	}
+
+	fileDescription, err := os.Stat(*file)
+	if err != nil {
+		panic(err)
+	}
+
+	var f *os.File
+	if fileDescription.Mode().IsRegular() {
+		f, err = os.Open(*file)
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		f, err = os.OpenFile(*file, os.O_RDONLY, os.ModeCharDevice)
+		if err != nil {
+			panic(err)
+		}
+	}
+	defer f.Close()
+
+	record := 0
+	for {
+		// Lock the current record if requested
+		if *checkpoint > 0 && record%*checkpoint == 0 {
+			log.Println("Checkpoint:", record)
+		}
+
+		// Read exactly one record
+		bf := make([]byte, *recordSize*blockSize)
+		if _, err := io.ReadFull(f, bf); err != nil {
+			if err == io.EOF {
+				break
+			}
+
+			panic(err)
+		}
+
+		// Get the headers from the record
+		for i := 0; i < *recordSize; i++ {
+			tr := tar.NewReader(bytes.NewReader(bf[blockSize*i : blockSize*(i+1)]))
+			hdr, err := tr.Next()
+			if err != nil {
+				continue
+			}
+
+			if hdr.Format == tar.FormatUnknown {
+				// EOF
+				break
+			}
+
+			paxRecords, err := json.Marshal(hdr.PAXRecords)
+			if err != nil {
+				panic(err)
+			}
+
+			dbhdr := &models.Header{
+				Typeflag:   int64(hdr.Typeflag),
+				Name:       hdr.Name,
+				Linkname:   hdr.Linkname,
+				Size:       hdr.Size,
+				Mode:       hdr.Mode,
+				UID:        int64(hdr.Uid),
+				Gid:        int64(hdr.Gid),
+				Uname:      hdr.Uname,
+				Gname:      hdr.Gname,
+				Modtime:    hdr.ModTime,
+				Accesstime: hdr.AccessTime,
+				Changetime: hdr.ChangeTime,
+				Devmajor:   hdr.Devmajor,
+				Devminor:   hdr.Devminor,
+				Paxrecords: string(paxRecords),
+				Format:     int64(hdr.Format),
+				Record:     int64(record),
+				Block:      int64(i),
+			}
+
+			if err := dbhdr.Insert(context.Background(), db, boil.Infer()); err != nil {
+				panic(err)
+			}
+
+			fmt.Println(dbhdr)
+		}
+
+		record++
+	}
 }