feat: Add generic persister for metadata

cmd/stcache/cmd/index.go

@@ -1,28 +1,19 @@
 package cmd
 
-//go:generate sqlboiler sqlite3 -o ../../../pkg/db/sqlite/models/metadata -c ../../../configs/sqlboiler/metadata.yaml
-//go:generate go-bindata -pkg metadata -o ../../../pkg/db/sqlite/migrations/metadata/migrations.go ../../../db/sqlite/migrations/metadata
-
 import (
 	"archive/tar"
 	"bufio"
 	"context"
-	"database/sql"
-	"encoding/json"
 	"io"
 	"os"
 	"path/filepath"
 
-	_ "github.com/mattn/go-sqlite3"
 	"github.com/pojntfx/stfs/pkg/controllers"
-	"github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata"
-	models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata"
 	"github.com/pojntfx/stfs/pkg/formatting"
+	"github.com/pojntfx/stfs/pkg/persisters"
 	"github.com/pojntfx/stfs/pkg/readers"
-	migrate "github.com/rubenv/sql-migrate"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/volatiletech/sqlboiler/v4/boil"
 )
 
 const (
@@ -41,27 +32,8 @@ var indexCmd = &cobra.Command{
 			return err
 		}
 
-		leading, _ := filepath.Split(viper.GetString(dbFlag))
-		if err := os.MkdirAll(leading, os.ModePerm); err != nil {
-			return err
-		}
-
-		db, err := sql.Open("sqlite3", viper.GetString(dbFlag))
-		if err != nil {
-			return err
-		}
-		defer db.Close()
-
-		if _, err := migrate.Exec(
-			db,
-			"sqlite3",
-			migrate.AssetMigrationSource{
-				Asset:    metadata.Asset,
-				AssetDir: metadata.AssetDir,
-				Dir:      "../../../db/sqlite/migrations/metadata",
-			},
-			migrate.Up,
-		); err != nil {
+		metadataPersister := persisters.NewMetadataPersister(viper.GetString(dbFlag))
+		if err := metadataPersister.Open(); err != nil {
 			return err
 		}
 
@@ -139,47 +111,10 @@ var indexCmd = &cobra.Command{
 					return err
 				}
 
-				paxHeaders, err := json.Marshal(hdr.PAXRecords)
-				if err != nil {
+				if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil {
 					return err
 				}
 
-				dbhdr := models.Header{
-					Record:     record,
-					Block:      block,
-					Typeflag:   int64(hdr.Typeflag),
-					Name:       hdr.Name,
-					Linkname:   hdr.Linkname,
-					Size:       hdr.Size,
-					Mode:       hdr.Mode,
-					UID:        int64(hdr.Uid),
-					Gid:        int64(hdr.Gid),
-					Uname:      hdr.Uname,
-					Gname:      hdr.Gname,
-					Modtime:    hdr.ModTime,
-					Accesstime: hdr.AccessTime,
-					Changetime: hdr.ChangeTime,
-					Devmajor:   hdr.Devmajor,
-					Devminor:   hdr.Devminor,
-					Paxrecords: string(paxHeaders),
-					Format:     int64(hdr.Format),
-				}
-
-				// TODO: Decompose to persister
-				if _, err := models.FindHeader(context.Background(), db, dbhdr.Name, models.HeaderColumns.Name); err != nil {
-					if err == sql.ErrNoRows {
-						if err := dbhdr.Insert(cmd.Context(), db, boil.Infer()); err != nil {
-							return err
-						}
-					} else {
-						return err
-					}
-				} else {
-					if _, err := dbhdr.Update(cmd.Context(), db, boil.Infer()); err != nil {
-						return err
-					}
-				}
-
 				curr, err := f.Seek(0, io.SeekCurrent)
 				if err != nil {
 					return err
@@ -264,6 +199,10 @@ var indexCmd = &cobra.Command{
 					return err
 				}
 
+				if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil {
+					return err
+				}
+
 				nextBytes := int64(counter.BytesRead) + hdr.Size + controllers.BlockSize - 1
 
 				record = nextBytes / (controllers.BlockSize * int64(viper.GetInt(recordSizeFlag)))

pkg/formatting/csv.go

@@ -20,7 +20,7 @@ func PrintCSV(input []string) error {
 	return w.WriteAll([][]string{input})
 }
 
-func GetTARHeaderAsCSV(record int64, block int64, hdr *tar.Header) []string {
+func GetTARHeaderAsCSV(record, block int64, hdr *tar.Header) []string {
 	return []string{
 		fmt.Sprintf("%v", record), fmt.Sprintf("%v", block), fmt.Sprintf("%v", hdr.Typeflag), hdr.Name, hdr.Linkname, fmt.Sprintf("%v", hdr.Size), fmt.Sprintf("%v", hdr.Mode), fmt.Sprintf("%v", hdr.Uid), fmt.Sprintf("%v", hdr.Gid), fmt.Sprintf("%v", hdr.Uname), fmt.Sprintf("%v", hdr.Gname), hdr.ModTime.Format(time.RFC3339), hdr.AccessTime.Format(time.RFC3339), hdr.ChangeTime.Format(time.RFC3339), fmt.Sprintf("%v", hdr.Devmajor), fmt.Sprintf("%v", hdr.Devminor), fmt.Sprintf("%v", hdr.PAXRecords), fmt.Sprintf("%v", hdr.Format),
 	}

pkg/persisters/metadata.go

@@ -0,0 +1,79 @@
+package persisters
+
+//go:generate sqlboiler sqlite3 -o ../db/sqlite/models/metadata -c ../../../configs/sqlboiler/metadata.yaml
+//go:generate go-bindata -pkg metadata -o ../db/sqlite/migrations/metadata/migrations.go ../../../db/sqlite/migrations/metadata
+
+import (
+	"archive/tar"
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata"
+	models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata"
+	migrate "github.com/rubenv/sql-migrate"
+	"github.com/volatiletech/sqlboiler/v4/boil"
+)
+
+type MetadataPersister struct {
+	*SQLite
+}
+
+func NewMetadataPersister(dbPath string) *MetadataPersister {
+	return &MetadataPersister{
+		&SQLite{
+			DBPath: dbPath,
+			Migrations: migrate.AssetMigrationSource{
+				Asset:    metadata.Asset,
+				AssetDir: metadata.AssetDir,
+				Dir:      "../../../db/sqlite/migrations/metadata",
+			},
+		},
+	}
+}
+
+func (c *MetadataPersister) UpsertHeader(ctx context.Context, record, block int64, hdr *tar.Header) error {
+	paxHeaders, err := json.Marshal(hdr.PAXRecords)
+	if err != nil {
+		return err
+	}
+
+	dbhdr := models.Header{
+		Record:     record,
+		Block:      block,
+		Typeflag:   int64(hdr.Typeflag),
+		Name:       hdr.Name,
+		Linkname:   hdr.Linkname,
+		Size:       hdr.Size,
+		Mode:       hdr.Mode,
+		UID:        int64(hdr.Uid),
+		Gid:        int64(hdr.Gid),
+		Uname:      hdr.Uname,
+		Gname:      hdr.Gname,
+		Modtime:    hdr.ModTime,
+		Accesstime: hdr.AccessTime,
+		Changetime: hdr.ChangeTime,
+		Devmajor:   hdr.Devmajor,
+		Devminor:   hdr.Devminor,
+		Paxrecords: string(paxHeaders),
+		Format:     int64(hdr.Format),
+	}
+
+	if _, err := models.FindHeader(ctx, c.db, dbhdr.Name, models.HeaderColumns.Name); err != nil {
+		if err == sql.ErrNoRows {
+			if err := dbhdr.Insert(ctx, c.db, boil.Infer()); err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		return err
+	}
+
+	if _, err := dbhdr.Update(ctx, c.db, boil.Infer()); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/persisters/sqlite.go

@@ -0,0 +1,44 @@
+package persisters
+
+import (
+	"database/sql"
+	"os"
+	"path/filepath"
+
+	_ "github.com/mattn/go-sqlite3"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+type SQLite struct {
+	DBPath     string
+	Migrations migrate.MigrationSource
+
+	db *sql.DB
+}
+
+func (d *SQLite) Open() error {
+	// Create leading directories for database
+	leadingDir, _ := filepath.Split(d.DBPath)
+	if err := os.MkdirAll(leadingDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	// Open the DB
+	db, err := sql.Open("sqlite3", d.DBPath)
+	if err != nil {
+		return err
+	}
+
+	// Configure the db
+	db.SetMaxOpenConns(1) // Prevent "database locked" errors
+	d.db = db
+
+	// Run migrations if set
+	if d.Migrations != nil {
+		if _, err := migrate.Exec(d.db, "sqlite3", d.Migrations, migrate.Up); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}