mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Start implementation of PAX header based metadata

Browse Source
This commit is contained in:
Felix Pojtinger
2021-11-16 01:15:46 +01:00
parent 03087ff783
commit e72854405e
5 changed files with 159 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/stfs-seek-file/main.go
+14 -66
View File
@@ -2,10 +2,7 @@ package main
import (
"archive/tar"
"bytes"
"flag"
"fmt"
"io"
"log"
"os"
)
@@ -14,21 +11,15 @@ const (
blockSize = 512
)
type HeaderInBlock struct {
Record int
Block int
Header string
}
func main() {
file := flag.String("file", "test.tar", "Tar file to open")
recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
checkpoint := flag.Int("checkpoint", 0, "Log current record after checkpoint kilobytes have been read")
seek := flag.Int("seek", 0, "Record to seek too")
record := flag.Int("record", 0, "Record to seek too")
block := flag.Int("block", 0, "Block in record to seek too")
flag.Parse()
bytesToSeek := *recordSize * blockSize * *seek
bytesToSeek := (*recordSize * blockSize * *record) + *block*blockSize
f, err := os.Open(*file)
if err != nil {
@@ -36,59 +27,16 @@ func main() {
}
defer f.Close()
record := 0
for {
// Seek to requested record
if bytesToSeek > 0 && record < *seek {
if _, err := f.Seek(int64(bytesToSeek), 0); err != nil {
panic(err)
}
record = *seek
continue
}
// Lock the current record if requested
if *checkpoint > 0 && record%*checkpoint == 0 {
log.Println("Checkpoint:", record)
}
// Read exactly one record
bf := make([]byte, *recordSize*blockSize)
if _, err := io.ReadFull(f, bf); err != nil {
if err == io.EOF {
break
}
panic(err)
}
// Get the headers from the record
headers := []HeaderInBlock{}
for i := 0; i < *recordSize; i++ {
tr := tar.NewReader(bytes.NewReader(bf[blockSize*i : blockSize*(i+1)]))
hdr, err := tr.Next()
if err != nil {
continue
}
if hdr.Format == tar.FormatUnknown {
// EOF
break
}
headers = append(headers, HeaderInBlock{
Record: record,
Block: i,
Header: fmt.Sprintf("%v", hdr),
})
}
if len(headers) > 0 {
fmt.Println(headers)
}
record++
if _, err := f.Seek(int64(bytesToSeek), 0); err != nil {
panic(err)
}
tr := tar.NewReader(f)
hdr, err := tr.Next()
if err != nil {
panic(err)
}
log.Println(hdr)
}
cmd/stfs-tvf-simple/main.go
+83
View File
@@ -0,0 +1,83 @@
package main
import (
"archive/tar"
"flag"
"io"
"log"
"os"
)
const (
blockSize = 512
)
func main() {
file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open")
recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
flag.Parse()
fileDescription, err := os.Stat(*file)
if err != nil {
panic(err)
}
var f *os.File
if fileDescription.Mode().IsRegular() {
f, err = os.Open(*file)
if err != nil {
panic(err)
}
} else {
f, err = os.OpenFile(*file, os.O_RDONLY, os.ModeCharDevice)
if err != nil {
panic(err)
}
}
defer f.Close()
tr := tar.NewReader(f)
record := int64(0)
block := int64(0)
for {
hdr, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
// TODO: Do `tell` on tape drive instead, which returns the block - but how do we get the current block? Maybe we have to use the old, iterating method and call.Next after we found the correct record & block.
curr, err := f.Seek(0, io.SeekCurrent)
if err != nil {
panic(err)
}
if record == 0 && block == 0 {
log.Println("Record:", 0, "Block:", 0, "Header:", hdr)
} else {
log.Println("Record:", record, "Block:", block, "Header:", hdr)
}
nextTotalBlocks := (curr + hdr.Size) / blockSize
// TODO: This currently returns one block to little on appended tar archives
if record == 0 && block == 0 {
record = nextTotalBlocks / int64(*recordSize)
block = nextTotalBlocks - (record * int64(*recordSize)) // For the first record, the offset of one is not needed
} else {
record = nextTotalBlocks / int64(*recordSize)
block = nextTotalBlocks - (record * int64(*recordSize)) + 1 // +1 because we need to start reading right after the last block
}
if block > int64(*recordSize) {
record++
block = 0
}
}
}
cmd/stfs-uvf-with-header/main.go
+34 -12
View File
@@ -11,6 +11,7 @@ import (
"log"
"os"
"path/filepath"
"strconv"
"syscall"
"time"
"unsafe"
@@ -25,7 +26,9 @@ const (
MTIOCTOP = 0x40086d01 // Do magnetic tape operation
MTEOM = 12 // Goto end of recorded media (for appending files)
STFSVersion = 1
STFSVersion = 1
STFSVersionPAX = "STFS.Version"
STFSHeaderPAX = "STFS.Header"
)
// Operation is struct for MTIOCTOP
@@ -48,6 +51,18 @@ func main() {
} else {
if os.IsNotExist(err) {
isRegular = true
// Create the file
f, err := os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
panic(err)
}
// Create an empty tar archive with a trailer so that we may seek back
tw := tar.NewWriter(f)
if err := tw.Close(); err != nil {
panic(err)
}
} else {
panic(err)
}
@@ -55,12 +70,15 @@ func main() {
var f *os.File
if isRegular {
f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
f, err = os.OpenFile(*file, os.O_RDWR, 0600)
if err != nil {
panic(err)
}
// No need to go to end manually due to `os.O_APPEND`
// Seek backwards two blocks from end (to overwrite the trailer)
if _, err := f.Seek(-1024, io.SeekEnd); err != nil {
panic(err)
}
} else {
// Go to end of file
syscall.Syscall(
@@ -74,6 +92,8 @@ func main() {
)),
)
// TODO: Seek backwards 2 blocks (1024 bytes) with the matching syscall
f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
panic(err)
@@ -81,7 +101,8 @@ func main() {
}
defer f.Close()
tw := tar.NewWriter(f) // We are not closing the tar writer to prevent writing the trailer
tw := tar.NewWriter(f)
defer tw.Close()
if err := filepath.Walk(*dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
@@ -118,20 +139,21 @@ func main() {
hdr.Devmajor = int64(unix.Major(unixStat.Dev))
hdr.Devminor = int64(unix.Minor(unixStat.Dev))
wrapper := &api.Wrapper{
Version: STFSVersion,
Header: &api.Header{
Action: api.Action_CREATE,
Name: path,
},
stfsHeader := &api.Header{
Action: api.Action_CREATE,
}
encodedName, err := proto.Marshal(wrapper)
encodedHeader, err := proto.Marshal(stfsHeader)
if err != nil {
return err
}
hdr.Name = base64.StdEncoding.EncodeToString(encodedName)
hdr.Name = path
hdr.PAXRecords = map[string]string{
STFSVersionPAX: strconv.Itoa(STFSVersion),
STFSHeaderPAX: base64.StdEncoding.EncodeToString(encodedHeader),
}
hdr.Format = tar.FormatPAX
log.Println(hdr)