From 7e28ec0aaf3f7a36b4fedd4a43cfbda6c0693c0c Mon Sep 17 00:00:00 2001 From: Felicitas Pojtinger Date: Fri, 19 Nov 2021 16:41:23 +0100 Subject: [PATCH] feat: Start implementation of `stcache index` cmd --- cmd/stcache/cmd/index.go | 220 +++++++++++++++++++++++++++++++++++++++ cmd/stcache/cmd/root.go | 27 +++++ cmd/stcache/main.go | 7 ++ 3 files changed, 254 insertions(+) create mode 100644 cmd/stcache/cmd/index.go create mode 100644 cmd/stcache/cmd/root.go create mode 100644 cmd/stcache/main.go diff --git a/cmd/stcache/cmd/index.go b/cmd/stcache/cmd/index.go new file mode 100644 index 0000000..a3f32bd --- /dev/null +++ b/cmd/stcache/cmd/index.go @@ -0,0 +1,220 @@ +package cmd + +import ( + "archive/tar" + "bufio" + "io" + "os" + "path/filepath" + + "github.com/pojntfx/stfs/pkg/controllers" + "github.com/pojntfx/stfs/pkg/formatting" + "github.com/pojntfx/stfs/pkg/readers" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + dbFlag = "db" + tapeFlag = "tape" + recordSizeFlag = "record-size" + recordFlag = "record" + blockFlag = "block" +) + +var indexCmd = &cobra.Command{ + Use: "index", + Aliases: []string{"i"}, + Short: "Index contents of tape or tar file", + RunE: func(cmd *cobra.Command, args []string) error { + if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil { + return err + } + + fileDescription, err := os.Stat(viper.GetString(tapeFlag)) + if err != nil { + return err + } + + var f *os.File + if fileDescription.Mode().IsRegular() { + f, err = os.Open(viper.GetString(tapeFlag)) + if err != nil { + return err + } + } else { + f, err = os.OpenFile(viper.GetString(tapeFlag), os.O_RDONLY, os.ModeCharDevice) + if err != nil { + return err + } + } + defer f.Close() + + if fileDescription.Mode().IsRegular() { + // Seek to record and block + if _, err := f.Seek(int64((viper.GetInt(recordSizeFlag)*controllers.BlockSize*viper.GetInt(recordFlag))+viper.GetInt(blockFlag)*controllers.BlockSize), 0); err != nil { + return err + } + + tr := tar.NewReader(f) + + record := viper.GetInt64(recordFlag) + block := viper.GetInt64(blockFlag) + firstRecordOfArchive := int64(0) + + for { + hdr, err := tr.Next() + if err != nil { + // Seek right after the next two blocks to skip the trailer + if _, err := f.Seek((int64(viper.GetInt(recordSizeFlag))*controllers.BlockSize*record)+(block+1)*controllers.BlockSize, io.SeekStart); err == nil { + tr = tar.NewReader(f) + + hdr, err = tr.Next() + if err != nil { + if err == io.EOF { + break + } + + return err + } + + block++ + if block > int64(viper.GetInt(recordSizeFlag)) { + record++ + block = 0 + } + + firstRecordOfArchive = record + } else { + return err + } + } + + if record == 0 && block == 0 { + if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { + return err + } + } + + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(record, block, hdr)); err != nil { + return err + } + + curr, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + + nextTotalBlocks := (curr + hdr.Size) / controllers.BlockSize + record = nextTotalBlocks / int64(viper.GetInt(recordSizeFlag)) + + if record == 0 && block == 0 || record == firstRecordOfArchive { + block = nextTotalBlocks - (record * int64(viper.GetInt(recordSizeFlag))) // For the first record of the file or archive, the offset of one is not needed + } else { + block = nextTotalBlocks - (record * int64(viper.GetInt(recordSizeFlag))) + 1 // +1 because we need to start reading right after the last block + } + + if block > int64(viper.GetInt(recordSizeFlag)) { + record++ + block = 0 + } + } + } else { + // Seek to record + if err := controllers.SeekToRecordOnTape(f, int32(viper.GetInt(recordFlag))); err != nil { + return err + } + + // Seek to block + br := bufio.NewReaderSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag)) + if _, err := br.Read(make([]byte, viper.GetInt(blockFlag)*controllers.BlockSize)); err != nil { + return err + } + + record := viper.GetInt64(recordFlag) + block := viper.GetInt64(blockFlag) + + lastBytesRead := (viper.GetInt(recordSizeFlag) * controllers.BlockSize * viper.GetInt(recordFlag)) + (viper.GetInt(blockFlag) * controllers.BlockSize) + counter := &readers.Counter{Reader: br, BytesRead: lastBytesRead} + dirty := false + + for { + tr := tar.NewReader(counter) + hdr, err := tr.Next() + if err != nil { + if lastBytesRead == counter.BytesRead { + if dirty { + // EOD + + break + } + + if err := controllers.GoToNextFileOnTape(f); err != nil { + // EOD + + break + } + + currentRecord, err := controllers.GetCurrentRecordFromTape(f) + if err != nil { + return err + } + + br = bufio.NewReaderSize(f, controllers.BlockSize*viper.GetInt(recordSizeFlag)) + counter = &readers.Counter{Reader: br, BytesRead: (int(currentRecord) * viper.GetInt(recordSizeFlag) * controllers.BlockSize)} // We asume we are at record n, block 0 + + dirty = true + } + + lastBytesRead = counter.BytesRead + + continue + } + + lastBytesRead = counter.BytesRead + + if hdr.Format == tar.FormatUnknown { + continue + } + + dirty = false + + if counter.BytesRead == 0 { + if err := formatting.PrintCSV(formatting.TARHeaderCSV); err != nil { + return err + } + } + + if err := formatting.PrintCSV(formatting.GetTARHeaderAsCSV(record, block, hdr)); err != nil { + return err + } + + nextBytes := int64(counter.BytesRead) + hdr.Size + controllers.BlockSize - 1 + + record = nextBytes / (controllers.BlockSize * int64(viper.GetInt(recordSizeFlag))) + block = (nextBytes - (record * int64(viper.GetInt(recordSizeFlag)) * controllers.BlockSize)) / controllers.BlockSize + } + } + + return nil + }, +} + +func init() { + // Get default working dir + home, err := os.UserHomeDir() + if err != nil { + panic(err) + } + workingDirDefault := filepath.Join(home, ".local", "share", "stcache", "var", "lib", "stcache") + + indexCmd.PersistentFlags().StringP(dbFlag, "d", filepath.Join(workingDirDefault, "index.sqlite"), "Database to use") + indexCmd.PersistentFlags().StringP(tapeFlag, "t", "/dev/nst0", "Tape or tar file to read from") + indexCmd.PersistentFlags().IntP(recordSizeFlag, "e", 20, "Amount of 512-bit blocks per record") + indexCmd.PersistentFlags().IntP(recordFlag, "r", 0, "Record to seek too before counting") + indexCmd.PersistentFlags().IntP(blockFlag, "b", 0, "Block in record to seek too before counting") + + viper.AutomaticEnv() + + rootCmd.AddCommand(indexCmd) +} diff --git a/cmd/stcache/cmd/root.go b/cmd/stcache/cmd/root.go new file mode 100644 index 0000000..f4376f5 --- /dev/null +++ b/cmd/stcache/cmd/root.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var rootCmd = &cobra.Command{ + Use: "stcache", + Short: "Simple Tape Cache", + Long: `Simple Tape Cache (stcache) is a CLI to interact with STFS-managed indexes of tapes or tar files. + +Find more information at: +https://github.com/pojntfx/stfs`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + viper.SetEnvPrefix("stcache") + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_")) + }, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + panic(err) + } +} diff --git a/cmd/stcache/main.go b/cmd/stcache/main.go new file mode 100644 index 0000000..567c8cc --- /dev/null +++ b/cmd/stcache/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/pojntfx/stfs/cmd/stcache/cmd" + +func main() { + cmd.Execute() +}