diff --git a/cmd/stcache/cmd/index.go b/cmd/stcache/cmd/index.go index 4d41a04..8ea9bbc 100644 --- a/cmd/stcache/cmd/index.go +++ b/cmd/stcache/cmd/index.go @@ -1,28 +1,19 @@ package cmd -//go:generate sqlboiler sqlite3 -o ../../../pkg/db/sqlite/models/metadata -c ../../../configs/sqlboiler/metadata.yaml -//go:generate go-bindata -pkg metadata -o ../../../pkg/db/sqlite/migrations/metadata/migrations.go ../../../db/sqlite/migrations/metadata - import ( "archive/tar" "bufio" "context" - "database/sql" - "encoding/json" "io" "os" "path/filepath" - _ "github.com/mattn/go-sqlite3" "github.com/pojntfx/stfs/pkg/controllers" - "github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata" - models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata" "github.com/pojntfx/stfs/pkg/formatting" + "github.com/pojntfx/stfs/pkg/persisters" "github.com/pojntfx/stfs/pkg/readers" - migrate "github.com/rubenv/sql-migrate" "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/volatiletech/sqlboiler/v4/boil" ) const ( @@ -41,27 +32,8 @@ var indexCmd = &cobra.Command{ return err } - leading, _ := filepath.Split(viper.GetString(dbFlag)) - if err := os.MkdirAll(leading, os.ModePerm); err != nil { - return err - } - - db, err := sql.Open("sqlite3", viper.GetString(dbFlag)) - if err != nil { - return err - } - defer db.Close() - - if _, err := migrate.Exec( - db, - "sqlite3", - migrate.AssetMigrationSource{ - Asset: metadata.Asset, - AssetDir: metadata.AssetDir, - Dir: "../../../db/sqlite/migrations/metadata", - }, - migrate.Up, - ); err != nil { + metadataPersister := persisters.NewMetadataPersister(viper.GetString(dbFlag)) + if err := metadataPersister.Open(); err != nil { return err } @@ -139,47 +111,10 @@ var indexCmd = &cobra.Command{ return err } - paxHeaders, err := json.Marshal(hdr.PAXRecords) - if err != nil { + if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil { return err } - dbhdr := models.Header{ - Record: record, - Block: block, - Typeflag: int64(hdr.Typeflag), - Name: hdr.Name, - Linkname: hdr.Linkname, - Size: hdr.Size, - Mode: hdr.Mode, - UID: int64(hdr.Uid), - Gid: int64(hdr.Gid), - Uname: hdr.Uname, - Gname: hdr.Gname, - Modtime: hdr.ModTime, - Accesstime: hdr.AccessTime, - Changetime: hdr.ChangeTime, - Devmajor: hdr.Devmajor, - Devminor: hdr.Devminor, - Paxrecords: string(paxHeaders), - Format: int64(hdr.Format), - } - - // TODO: Decompose to persister - if _, err := models.FindHeader(context.Background(), db, dbhdr.Name, models.HeaderColumns.Name); err != nil { - if err == sql.ErrNoRows { - if err := dbhdr.Insert(cmd.Context(), db, boil.Infer()); err != nil { - return err - } - } else { - return err - } - } else { - if _, err := dbhdr.Update(cmd.Context(), db, boil.Infer()); err != nil { - return err - } - } - curr, err := f.Seek(0, io.SeekCurrent) if err != nil { return err @@ -264,6 +199,10 @@ var indexCmd = &cobra.Command{ return err } + if err := metadataPersister.UpsertHeader(context.Background(), record, block, hdr); err != nil { + return err + } + nextBytes := int64(counter.BytesRead) + hdr.Size + controllers.BlockSize - 1 record = nextBytes / (controllers.BlockSize * int64(viper.GetInt(recordSizeFlag))) diff --git a/pkg/formatting/csv.go b/pkg/formatting/csv.go index 961cb9e..0efcf74 100644 --- a/pkg/formatting/csv.go +++ b/pkg/formatting/csv.go @@ -20,7 +20,7 @@ func PrintCSV(input []string) error { return w.WriteAll([][]string{input}) } -func GetTARHeaderAsCSV(record int64, block int64, hdr *tar.Header) []string { +func GetTARHeaderAsCSV(record, block int64, hdr *tar.Header) []string { return []string{ fmt.Sprintf("%v", record), fmt.Sprintf("%v", block), fmt.Sprintf("%v", hdr.Typeflag), hdr.Name, hdr.Linkname, fmt.Sprintf("%v", hdr.Size), fmt.Sprintf("%v", hdr.Mode), fmt.Sprintf("%v", hdr.Uid), fmt.Sprintf("%v", hdr.Gid), fmt.Sprintf("%v", hdr.Uname), fmt.Sprintf("%v", hdr.Gname), hdr.ModTime.Format(time.RFC3339), hdr.AccessTime.Format(time.RFC3339), hdr.ChangeTime.Format(time.RFC3339), fmt.Sprintf("%v", hdr.Devmajor), fmt.Sprintf("%v", hdr.Devminor), fmt.Sprintf("%v", hdr.PAXRecords), fmt.Sprintf("%v", hdr.Format), } diff --git a/pkg/persisters/metadata.go b/pkg/persisters/metadata.go new file mode 100644 index 0000000..e9922bc --- /dev/null +++ b/pkg/persisters/metadata.go @@ -0,0 +1,79 @@ +package persisters + +//go:generate sqlboiler sqlite3 -o ../db/sqlite/models/metadata -c ../../../configs/sqlboiler/metadata.yaml +//go:generate go-bindata -pkg metadata -o ../db/sqlite/migrations/metadata/migrations.go ../../../db/sqlite/migrations/metadata + +import ( + "archive/tar" + "context" + "database/sql" + "encoding/json" + + "github.com/pojntfx/stfs/pkg/db/sqlite/migrations/metadata" + models "github.com/pojntfx/stfs/pkg/db/sqlite/models/metadata" + migrate "github.com/rubenv/sql-migrate" + "github.com/volatiletech/sqlboiler/v4/boil" +) + +type MetadataPersister struct { + *SQLite +} + +func NewMetadataPersister(dbPath string) *MetadataPersister { + return &MetadataPersister{ + &SQLite{ + DBPath: dbPath, + Migrations: migrate.AssetMigrationSource{ + Asset: metadata.Asset, + AssetDir: metadata.AssetDir, + Dir: "../../../db/sqlite/migrations/metadata", + }, + }, + } +} + +func (c *MetadataPersister) UpsertHeader(ctx context.Context, record, block int64, hdr *tar.Header) error { + paxHeaders, err := json.Marshal(hdr.PAXRecords) + if err != nil { + return err + } + + dbhdr := models.Header{ + Record: record, + Block: block, + Typeflag: int64(hdr.Typeflag), + Name: hdr.Name, + Linkname: hdr.Linkname, + Size: hdr.Size, + Mode: hdr.Mode, + UID: int64(hdr.Uid), + Gid: int64(hdr.Gid), + Uname: hdr.Uname, + Gname: hdr.Gname, + Modtime: hdr.ModTime, + Accesstime: hdr.AccessTime, + Changetime: hdr.ChangeTime, + Devmajor: hdr.Devmajor, + Devminor: hdr.Devminor, + Paxrecords: string(paxHeaders), + Format: int64(hdr.Format), + } + + if _, err := models.FindHeader(ctx, c.db, dbhdr.Name, models.HeaderColumns.Name); err != nil { + if err == sql.ErrNoRows { + if err := dbhdr.Insert(ctx, c.db, boil.Infer()); err != nil { + return err + } + + return nil + } + + return err + } + + if _, err := dbhdr.Update(ctx, c.db, boil.Infer()); err != nil { + return err + } + + return nil +} diff --git a/pkg/persisters/sqlite.go b/pkg/persisters/sqlite.go new file mode 100644 index 0000000..b31a590 --- /dev/null +++ b/pkg/persisters/sqlite.go @@ -0,0 +1,44 @@ +package persisters + +import ( + "database/sql" + "os" + "path/filepath" + + _ "github.com/mattn/go-sqlite3" + migrate "github.com/rubenv/sql-migrate" +) + +type SQLite struct { + DBPath string + Migrations migrate.MigrationSource + + db *sql.DB +} + +func (d *SQLite) Open() error { + // Create leading directories for database + leadingDir, _ := filepath.Split(d.DBPath) + if err := os.MkdirAll(leadingDir, os.ModePerm); err != nil { + return err + } + + // Open the DB + db, err := sql.Open("sqlite3", d.DBPath) + if err != nil { + return err + } + + // Configure the db + db.SetMaxOpenConns(1) // Prevent "database locked" errors + d.db = db + + // Run migrations if set + if d.Migrations != nil { + if _, err := migrate.Exec(d.db, "sqlite3", d.Migrations, migrate.Up); err != nil { + return err + } + } + + return nil +}