diff --git a/api/proto/v1/metadata.proto b/api/proto/v1/metadata.proto index 13738e1..49ba950 100644 --- a/api/proto/v1/metadata.proto +++ b/api/proto/v1/metadata.proto @@ -4,11 +4,6 @@ package com.pojtinger.felicitas.stfs; option go_package = "github.com/pojntfx/stfs/pkg/api/proto/v1"; -message Wrapper { - int64 Version = 1; - Header Header = 2; -} - enum Action { CREATE = 0; UPDATE = 1; @@ -17,6 +12,5 @@ enum Action { message Header { Action Action = 1; - string Name = 2; - string Replaces = 3; + string Replaces = 2; } diff --git a/cmd/stfs-seek-file/main.go b/cmd/stfs-seek-file/main.go index 20c5c81..16e682a 100644 --- a/cmd/stfs-seek-file/main.go +++ b/cmd/stfs-seek-file/main.go @@ -2,10 +2,7 @@ package main import ( "archive/tar" - "bytes" "flag" - "fmt" - "io" "log" "os" ) @@ -14,21 +11,15 @@ const ( blockSize = 512 ) -type HeaderInBlock struct { - Record int - Block int - Header string -} - func main() { file := flag.String("file", "test.tar", "Tar file to open") recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record") - checkpoint := flag.Int("checkpoint", 0, "Log current record after checkpoint kilobytes have been read") - seek := flag.Int("seek", 0, "Record to seek too") + record := flag.Int("record", 0, "Record to seek too") + block := flag.Int("block", 0, "Block in record to seek too") flag.Parse() - bytesToSeek := *recordSize * blockSize * *seek + bytesToSeek := (*recordSize * blockSize * *record) + *block*blockSize f, err := os.Open(*file) if err != nil { @@ -36,59 +27,16 @@ func main() { } defer f.Close() - record := 0 - for { - // Seek to requested record - if bytesToSeek > 0 && record < *seek { - if _, err := f.Seek(int64(bytesToSeek), 0); err != nil { - panic(err) - } - - record = *seek - - continue - } - - // Lock the current record if requested - if *checkpoint > 0 && record%*checkpoint == 0 { - log.Println("Checkpoint:", record) - } - - // Read exactly one record - bf := make([]byte, *recordSize*blockSize) - if _, err := io.ReadFull(f, bf); err != nil { - if err == io.EOF { - break - } - - panic(err) - } - - // Get the headers from the record - headers := []HeaderInBlock{} - for i := 0; i < *recordSize; i++ { - tr := tar.NewReader(bytes.NewReader(bf[blockSize*i : blockSize*(i+1)])) - hdr, err := tr.Next() - if err != nil { - continue - } - - if hdr.Format == tar.FormatUnknown { - // EOF - break - } - - headers = append(headers, HeaderInBlock{ - Record: record, - Block: i, - Header: fmt.Sprintf("%v", hdr), - }) - } - - if len(headers) > 0 { - fmt.Println(headers) - } - - record++ + if _, err := f.Seek(int64(bytesToSeek), 0); err != nil { + panic(err) } + + tr := tar.NewReader(f) + + hdr, err := tr.Next() + if err != nil { + panic(err) + } + + log.Println(hdr) } diff --git a/cmd/stfs-tvf-simple/main.go b/cmd/stfs-tvf-simple/main.go new file mode 100644 index 0000000..56be43f --- /dev/null +++ b/cmd/stfs-tvf-simple/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "archive/tar" + "flag" + "io" + "log" + "os" +) + +const ( + blockSize = 512 +) + +func main() { + file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open") + recordSize := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record") + + flag.Parse() + + fileDescription, err := os.Stat(*file) + if err != nil { + panic(err) + } + + var f *os.File + if fileDescription.Mode().IsRegular() { + f, err = os.Open(*file) + if err != nil { + panic(err) + } + } else { + f, err = os.OpenFile(*file, os.O_RDONLY, os.ModeCharDevice) + if err != nil { + panic(err) + } + } + defer f.Close() + + tr := tar.NewReader(f) + + record := int64(0) + block := int64(0) + + for { + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + + panic(err) + } + + // TODO: Do `tell` on tape drive instead, which returns the block - but how do we get the current block? Maybe we have to use the old, iterating method and call.Next after we found the correct record & block. + curr, err := f.Seek(0, io.SeekCurrent) + if err != nil { + panic(err) + } + + if record == 0 && block == 0 { + log.Println("Record:", 0, "Block:", 0, "Header:", hdr) + } else { + log.Println("Record:", record, "Block:", block, "Header:", hdr) + } + + nextTotalBlocks := (curr + hdr.Size) / blockSize + + // TODO: This currently returns one block to little on appended tar archives + if record == 0 && block == 0 { + record = nextTotalBlocks / int64(*recordSize) + block = nextTotalBlocks - (record * int64(*recordSize)) // For the first record, the offset of one is not needed + } else { + record = nextTotalBlocks / int64(*recordSize) + block = nextTotalBlocks - (record * int64(*recordSize)) + 1 // +1 because we need to start reading right after the last block + } + + if block > int64(*recordSize) { + record++ + block = 0 + } + } +} diff --git a/cmd/stfs-uvf-with-header/main.go b/cmd/stfs-uvf-with-header/main.go index 829c606..3124519 100644 --- a/cmd/stfs-uvf-with-header/main.go +++ b/cmd/stfs-uvf-with-header/main.go @@ -11,6 +11,7 @@ import ( "log" "os" "path/filepath" + "strconv" "syscall" "time" "unsafe" @@ -25,7 +26,9 @@ const ( MTIOCTOP = 0x40086d01 // Do magnetic tape operation MTEOM = 12 // Goto end of recorded media (for appending files) - STFSVersion = 1 + STFSVersion = 1 + STFSVersionPAX = "STFS.Version" + STFSHeaderPAX = "STFS.Header" ) // Operation is struct for MTIOCTOP @@ -48,6 +51,18 @@ func main() { } else { if os.IsNotExist(err) { isRegular = true + + // Create the file + f, err := os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + panic(err) + } + + // Create an empty tar archive with a trailer so that we may seek back + tw := tar.NewWriter(f) + if err := tw.Close(); err != nil { + panic(err) + } } else { panic(err) } @@ -55,12 +70,15 @@ func main() { var f *os.File if isRegular { - f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + f, err = os.OpenFile(*file, os.O_RDWR, 0600) if err != nil { panic(err) } - // No need to go to end manually due to `os.O_APPEND` + // Seek backwards two blocks from end (to overwrite the trailer) + if _, err := f.Seek(-1024, io.SeekEnd); err != nil { + panic(err) + } } else { // Go to end of file syscall.Syscall( @@ -74,6 +92,8 @@ func main() { )), ) + // TODO: Seek backwards 2 blocks (1024 bytes) with the matching syscall + f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) if err != nil { panic(err) @@ -81,7 +101,8 @@ func main() { } defer f.Close() - tw := tar.NewWriter(f) // We are not closing the tar writer to prevent writing the trailer + tw := tar.NewWriter(f) + defer tw.Close() if err := filepath.Walk(*dir, func(path string, info fs.FileInfo, err error) error { if err != nil { @@ -118,20 +139,21 @@ func main() { hdr.Devmajor = int64(unix.Major(unixStat.Dev)) hdr.Devminor = int64(unix.Minor(unixStat.Dev)) - wrapper := &api.Wrapper{ - Version: STFSVersion, - Header: &api.Header{ - Action: api.Action_CREATE, - Name: path, - }, + stfsHeader := &api.Header{ + Action: api.Action_CREATE, } - encodedName, err := proto.Marshal(wrapper) + encodedHeader, err := proto.Marshal(stfsHeader) if err != nil { return err } - hdr.Name = base64.StdEncoding.EncodeToString(encodedName) + hdr.Name = path + hdr.PAXRecords = map[string]string{ + STFSVersionPAX: strconv.Itoa(STFSVersion), + STFSHeaderPAX: base64.StdEncoding.EncodeToString(encodedHeader), + } + hdr.Format = tar.FormatPAX log.Println(hdr) diff --git a/pkg/api/proto/v1/metadata.pb.go b/pkg/api/proto/v1/metadata.pb.go index 81ae89f..81f48e0 100644 --- a/pkg/api/proto/v1/metadata.pb.go +++ b/pkg/api/proto/v1/metadata.pb.go @@ -69,75 +69,19 @@ func (Action) EnumDescriptor() ([]byte, []int) { return file_metadata_proto_rawDescGZIP(), []int{0} } -type Wrapper struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Version int64 `protobuf:"varint,1,opt,name=Version,proto3" json:"Version,omitempty"` - Header *Header `protobuf:"bytes,2,opt,name=Header,proto3" json:"Header,omitempty"` -} - -func (x *Wrapper) Reset() { - *x = Wrapper{} - if protoimpl.UnsafeEnabled { - mi := &file_metadata_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Wrapper) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Wrapper) ProtoMessage() {} - -func (x *Wrapper) ProtoReflect() protoreflect.Message { - mi := &file_metadata_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Wrapper.ProtoReflect.Descriptor instead. -func (*Wrapper) Descriptor() ([]byte, []int) { - return file_metadata_proto_rawDescGZIP(), []int{0} -} - -func (x *Wrapper) GetVersion() int64 { - if x != nil { - return x.Version - } - return 0 -} - -func (x *Wrapper) GetHeader() *Header { - if x != nil { - return x.Header - } - return nil -} - type Header struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Action Action `protobuf:"varint,1,opt,name=Action,proto3,enum=com.pojtinger.felicitas.stfs.Action" json:"Action,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` - Replaces string `protobuf:"bytes,3,opt,name=Replaces,proto3" json:"Replaces,omitempty"` + Replaces string `protobuf:"bytes,2,opt,name=Replaces,proto3" json:"Replaces,omitempty"` } func (x *Header) Reset() { *x = Header{} if protoimpl.UnsafeEnabled { - mi := &file_metadata_proto_msgTypes[1] + mi := &file_metadata_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -150,7 +94,7 @@ func (x *Header) String() string { func (*Header) ProtoMessage() {} func (x *Header) ProtoReflect() protoreflect.Message { - mi := &file_metadata_proto_msgTypes[1] + mi := &file_metadata_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -163,7 +107,7 @@ func (x *Header) ProtoReflect() protoreflect.Message { // Deprecated: Use Header.ProtoReflect.Descriptor instead. func (*Header) Descriptor() ([]byte, []int) { - return file_metadata_proto_rawDescGZIP(), []int{1} + return file_metadata_proto_rawDescGZIP(), []int{0} } func (x *Header) GetAction() Action { @@ -173,13 +117,6 @@ func (x *Header) GetAction() Action { return Action_CREATE } -func (x *Header) GetName() string { - if x != nil { - return x.Name - } - return "" -} - func (x *Header) GetReplaces() string { if x != nil { return x.Replaces @@ -192,26 +129,19 @@ var File_metadata_proto protoreflect.FileDescriptor var file_metadata_proto_rawDesc = []byte{ 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x18, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e, - 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5d, 0x0a, 0x07, 0x57, 0x72, - 0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x38, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e, - 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x06, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, - 0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a, - 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, - 0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, - 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66, - 0x78, 0x2f, 0x73, 0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5e, 0x0a, 0x06, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, + 0x6e, 0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, + 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, + 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a, 0x06, 0x41, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x00, + 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, + 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66, 0x78, 0x2f, 0x73, + 0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -227,20 +157,18 @@ func file_metadata_proto_rawDescGZIP() []byte { } var file_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_metadata_proto_goTypes = []interface{}{ - (Action)(0), // 0: com.pojtinger.felicitas.stfs.Action - (*Wrapper)(nil), // 1: com.pojtinger.felicitas.stfs.Wrapper - (*Header)(nil), // 2: com.pojtinger.felicitas.stfs.Header + (Action)(0), // 0: com.pojtinger.felicitas.stfs.Action + (*Header)(nil), // 1: com.pojtinger.felicitas.stfs.Header } var file_metadata_proto_depIdxs = []int32{ - 2, // 0: com.pojtinger.felicitas.stfs.Wrapper.Header:type_name -> com.pojtinger.felicitas.stfs.Header - 0, // 1: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 0, // 0: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_metadata_proto_init() } @@ -250,18 +178,6 @@ func file_metadata_proto_init() { } if !protoimpl.UnsafeEnabled { file_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Wrapper); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Header); i { case 0: return &v.state @@ -280,7 +196,7 @@ func file_metadata_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_metadata_proto_rawDesc, NumEnums: 1, - NumMessages: 2, + NumMessages: 1, NumExtensions: 0, NumServices: 0, },