mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Start implementation of PAX header based metadata

Browse Source
This commit is contained in:
Felicitas Pojtinger
2021-11-16 01:15:46 +01:00
parent deec1feb42
commit 562e733236
5 changed files with 159 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/proto/v1/metadata.proto
View File

@@ -4,11 +4,6 @@ package com.pojtinger.felicitas.stfs;
option go_package = "github.com/pojntfx/stfs/pkg/api/proto/v1";
message Wrapper {
int64 Version = 1;
Header Header = 2;
}
enum Action {
CREATE = 0;
UPDATE = 1;
@@ -17,6 +12,5 @@ enum Action {
message Header {
Action Action = 1;
string Name = 2;
string Replaces = 3;
string Replaces = 2;
}

80
cmd/stfs-seek-file/main.go
View File

@@ -2,10 +2,7 @@ package main
import (
"archive/tar"
"bytes"
"flag"
"fmt"
"io"
"log"
"os"
)
@@ -14,21 +11,15 @@ const (
blockSize = 512
)
type HeaderInBlock struct {
Record int
Block int
Header string
}
func main() {
file := flag.String("file", "test.tar", "Tar file to open")
recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
checkpoint := flag.Int("checkpoint", 0, "Log current record after checkpoint kilobytes have been read")
seek := flag.Int("seek", 0, "Record to seek too")
record := flag.Int("record", 0, "Record to seek too")
block := flag.Int("block", 0, "Block in record to seek too")
flag.Parse()
bytesToSeek := *recordSize * blockSize * *seek
bytesToSeek := (*recordSize * blockSize * *record) + *block*blockSize
f, err := os.Open(*file)
if err != nil {
@@ -36,59 +27,16 @@ func main() {
}
defer f.Close()
record := 0
for {
// Seek to requested record
if bytesToSeek > 0 && record < *seek {
if _, err := f.Seek(int64(bytesToSeek), 0); err != nil {
panic(err)
}
record = *seek
continue
}
// Lock the current record if requested
if *checkpoint > 0 && record%*checkpoint == 0 {
log.Println("Checkpoint:", record)
}
// Read exactly one record
bf := make([]byte, *recordSize*blockSize)
if _, err := io.ReadFull(f, bf); err != nil {
if err == io.EOF {
break
}
panic(err)
}
// Get the headers from the record
headers := []HeaderInBlock{}
for i := 0; i < *recordSize; i++ {
tr := tar.NewReader(bytes.NewReader(bf[blockSize*i : blockSize*(i+1)]))
hdr, err := tr.Next()
if err != nil {
continue
}
if hdr.Format == tar.FormatUnknown {
// EOF
break
}
headers = append(headers, HeaderInBlock{
Record: record,
Block: i,
Header: fmt.Sprintf("%v", hdr),
})
}
if len(headers) > 0 {
fmt.Println(headers)
}
record++
if _, err := f.Seek(int64(bytesToSeek), 0); err != nil {
panic(err)
}
tr := tar.NewReader(f)
hdr, err := tr.Next()
if err != nil {
panic(err)
}
log.Println(hdr)
}

83
cmd/stfs-tvf-simple/main.go Normal file
View File

@@ -0,0 +1,83 @@
package main
import (
"archive/tar"
"flag"
"io"
"log"
"os"
)
const (
blockSize = 512
)
func main() {
file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open")
recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
flag.Parse()
fileDescription, err := os.Stat(*file)
if err != nil {
panic(err)
}
var f *os.File
if fileDescription.Mode().IsRegular() {
f, err = os.Open(*file)
if err != nil {
panic(err)
}
} else {
f, err = os.OpenFile(*file, os.O_RDONLY, os.ModeCharDevice)
if err != nil {
panic(err)
}
}
defer f.Close()
tr := tar.NewReader(f)
record := int64(0)
block := int64(0)
for {
hdr, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
// TODO: Do `tell` on tape drive instead, which returns the block - but how do we get the current block? Maybe we have to use the old, iterating method and call.Next after we found the correct record & block.
curr, err := f.Seek(0, io.SeekCurrent)
if err != nil {
panic(err)
}
if record == 0 && block == 0 {
log.Println("Record:", 0, "Block:", 0, "Header:", hdr)
} else {
log.Println("Record:", record, "Block:", block, "Header:", hdr)
}
nextTotalBlocks := (curr + hdr.Size) / blockSize
// TODO: This currently returns one block to little on appended tar archives
if record == 0 && block == 0 {
record = nextTotalBlocks / int64(*recordSize)
block = nextTotalBlocks - (record * int64(*recordSize)) // For the first record, the offset of one is not needed
} else {
record = nextTotalBlocks / int64(*recordSize)
block = nextTotalBlocks - (record * int64(*recordSize)) + 1 // +1 because we need to start reading right after the last block
}
if block > int64(*recordSize) {
record++
block = 0
}
}
}

46
cmd/stfs-uvf-with-header/main.go
View File

@@ -11,6 +11,7 @@ import (
"log"
"os"
"path/filepath"
"strconv"
"syscall"
"time"
"unsafe"
@@ -25,7 +26,9 @@ const (
MTIOCTOP = 0x40086d01 // Do magnetic tape operation
MTEOM = 12 // Goto end of recorded media (for appending files)
STFSVersion = 1
STFSVersion = 1
STFSVersionPAX = "STFS.Version"
STFSHeaderPAX = "STFS.Header"
)
// Operation is struct for MTIOCTOP
@@ -48,6 +51,18 @@ func main() {
} else {
if os.IsNotExist(err) {
isRegular = true
// Create the file
f, err := os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
panic(err)
}
// Create an empty tar archive with a trailer so that we may seek back
tw := tar.NewWriter(f)
if err := tw.Close(); err != nil {
panic(err)
}
} else {
panic(err)
}
@@ -55,12 +70,15 @@ func main() {
var f *os.File
if isRegular {
f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
f, err = os.OpenFile(*file, os.O_RDWR, 0600)
if err != nil {
panic(err)
}
// No need to go to end manually due to `os.O_APPEND`
// Seek backwards two blocks from end (to overwrite the trailer)
if _, err := f.Seek(-1024, io.SeekEnd); err != nil {
panic(err)
}
} else {
// Go to end of file
syscall.Syscall(
@@ -74,6 +92,8 @@ func main() {
)),
)
// TODO: Seek backwards 2 blocks (1024 bytes) with the matching syscall
f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
if err != nil {
panic(err)
@@ -81,7 +101,8 @@ func main() {
}
defer f.Close()
tw := tar.NewWriter(f) // We are not closing the tar writer to prevent writing the trailer
tw := tar.NewWriter(f)
defer tw.Close()
if err := filepath.Walk(*dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
@@ -118,20 +139,21 @@ func main() {
hdr.Devmajor = int64(unix.Major(unixStat.Dev))
hdr.Devminor = int64(unix.Minor(unixStat.Dev))
wrapper := &api.Wrapper{
Version: STFSVersion,
Header: &api.Header{
Action: api.Action_CREATE,
Name: path,
},
stfsHeader := &api.Header{
Action: api.Action_CREATE,
}
encodedName, err := proto.Marshal(wrapper)
encodedHeader, err := proto.Marshal(stfsHeader)
if err != nil {
return err
}
hdr.Name = base64.StdEncoding.EncodeToString(encodedName)
hdr.Name = path
hdr.PAXRecords = map[string]string{
STFSVersionPAX: strconv.Itoa(STFSVersion),
STFSHeaderPAX: base64.StdEncoding.EncodeToString(encodedHeader),
}
hdr.Format = tar.FormatPAX
log.Println(hdr)

138
pkg/api/proto/v1/metadata.pb.go
View File

@@ -69,75 +69,19 @@ func (Action) EnumDescriptor() ([]byte, []int) {
return file_metadata_proto_rawDescGZIP(), []int{0}
}
type Wrapper struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Version int64 `protobuf:"varint,1,opt,name=Version,proto3" json:"Version,omitempty"`
Header *Header `protobuf:"bytes,2,opt,name=Header,proto3" json:"Header,omitempty"`
}
func (x *Wrapper) Reset() {
*x = Wrapper{}
if protoimpl.UnsafeEnabled {
mi := &file_metadata_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Wrapper) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Wrapper) ProtoMessage() {}
func (x *Wrapper) ProtoReflect() protoreflect.Message {
mi := &file_metadata_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Wrapper.ProtoReflect.Descriptor instead.
func (*Wrapper) Descriptor() ([]byte, []int) {
return file_metadata_proto_rawDescGZIP(), []int{0}
}
func (x *Wrapper) GetVersion() int64 {
if x != nil {
return x.Version
}
return 0
}
func (x *Wrapper) GetHeader() *Header {
if x != nil {
return x.Header
}
return nil
}
type Header struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Action Action `protobuf:"varint,1,opt,name=Action,proto3,enum=com.pojtinger.felicitas.stfs.Action" json:"Action,omitempty"`
Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"`
Replaces string `protobuf:"bytes,3,opt,name=Replaces,proto3" json:"Replaces,omitempty"`
Replaces string `protobuf:"bytes,2,opt,name=Replaces,proto3" json:"Replaces,omitempty"`
}
func (x *Header) Reset() {
*x = Header{}
if protoimpl.UnsafeEnabled {
mi := &file_metadata_proto_msgTypes[1]
mi := &file_metadata_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -150,7 +94,7 @@ func (x *Header) String() string {
func (*Header) ProtoMessage() {}
func (x *Header) ProtoReflect() protoreflect.Message {
mi := &file_metadata_proto_msgTypes[1]
mi := &file_metadata_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -163,7 +107,7 @@ func (x *Header) ProtoReflect() protoreflect.Message {
// Deprecated: Use Header.ProtoReflect.Descriptor instead.
func (*Header) Descriptor() ([]byte, []int) {
return file_metadata_proto_rawDescGZIP(), []int{1}
return file_metadata_proto_rawDescGZIP(), []int{0}
}
func (x *Header) GetAction() Action {
@@ -173,13 +117,6 @@ func (x *Header) GetAction() Action {
return Action_CREATE
}
func (x *Header) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *Header) GetReplaces() string {
if x != nil {
return x.Replaces
@@ -192,26 +129,19 @@ var File_metadata_proto protoreflect.FileDescriptor
var file_metadata_proto_rawDesc = []byte{
0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x12, 0x18, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e,
0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5d, 0x0a, 0x07, 0x57, 0x72,
0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12,
0x38, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e,
0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65,
0x72, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x06, 0x48, 0x65, 0x61,
0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e,
0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x41,
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a,
0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d,
0x65, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20,
0x01, 0x28, 0x09, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a,
0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54,
0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12,
0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66,
0x78, 0x2f, 0x73, 0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5e, 0x0a, 0x06, 0x48, 0x65,
0x61, 0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69,
0x6e, 0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e,
0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a,
0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a, 0x06, 0x41, 0x63,
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x00,
0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06,
0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66, 0x78, 0x2f, 0x73,
0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -227,20 +157,18 @@ func file_metadata_proto_rawDescGZIP() []byte {
}
var file_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_metadata_proto_goTypes = []interface{}{
(Action)(0), // 0: com.pojtinger.felicitas.stfs.Action
(*Wrapper)(nil), // 1: com.pojtinger.felicitas.stfs.Wrapper
(*Header)(nil), // 2: com.pojtinger.felicitas.stfs.Header
(Action)(0), // 0: com.pojtinger.felicitas.stfs.Action
(*Header)(nil), // 1: com.pojtinger.felicitas.stfs.Header
}
var file_metadata_proto_depIdxs = []int32{
2, // 0: com.pojtinger.felicitas.stfs.Wrapper.Header:type_name -> com.pojtinger.felicitas.stfs.Header
0, // 1: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action
2, // [2:2] is the sub-list for method output_type
2, // [2:2] is the sub-list for method input_type
2, // [2:2] is the sub-list for extension type_name
2, // [2:2] is the sub-list for extension extendee
0, // [0:2] is the sub-list for field type_name
0, // 0: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_metadata_proto_init() }
@@ -250,18 +178,6 @@ func file_metadata_proto_init() {
}
if !protoimpl.UnsafeEnabled {
file_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Wrapper); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Header); i {
case 0:
return &v.state
@@ -280,7 +196,7 @@ func file_metadata_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_metadata_proto_rawDesc,
NumEnums: 1,
NumMessages: 2,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},