From deec1feb4287bcb811fd804fc9cd4c3e6729bd09 Mon Sep 17 00:00:00 2001 From: Felicitas Pojtinger Date: Mon, 15 Nov 2021 21:41:42 +0100 Subject: [PATCH] feat: Add archiver with filename-embedded headers --- Makefile | 2 +- api/proto/v1/metadata.proto | 22 +++ cmd/stfs-uvf-with-header/main.go | 160 +++++++++++++++++ go.mod | 3 +- go.sum | 2 + pkg/api/proto/v1/metadata.pb.go | 296 +++++++++++++++++++++++++++++++ 6 files changed, 483 insertions(+), 2 deletions(-) create mode 100644 api/proto/v1/metadata.proto create mode 100644 cmd/stfs-uvf-with-header/main.go create mode 100644 pkg/api/proto/v1/metadata.pb.go diff --git a/Makefile b/Makefile index 0cc1159..13cb4d0 100644 --- a/Makefile +++ b/Makefile @@ -3,4 +3,4 @@ generate: go generate ./... clean: - rm -rf pkg/db \ No newline at end of file + rm -rf pkg/db pkg/api \ No newline at end of file diff --git a/api/proto/v1/metadata.proto b/api/proto/v1/metadata.proto new file mode 100644 index 0000000..13738e1 --- /dev/null +++ b/api/proto/v1/metadata.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package com.pojtinger.felicitas.stfs; + +option go_package = "github.com/pojntfx/stfs/pkg/api/proto/v1"; + +message Wrapper { + int64 Version = 1; + Header Header = 2; +} + +enum Action { + CREATE = 0; + UPDATE = 1; + DELETE = 2; +} + +message Header { + Action Action = 1; + string Name = 2; + string Replaces = 3; +} diff --git a/cmd/stfs-uvf-with-header/main.go b/cmd/stfs-uvf-with-header/main.go new file mode 100644 index 0000000..829c606 --- /dev/null +++ b/cmd/stfs-uvf-with-header/main.go @@ -0,0 +1,160 @@ +package main + +//go:generate sh -c "mkdir -p ../../pkg/api/proto/v1 && protoc --go_out=paths=source_relative,plugins=grpc:../../pkg/api/proto/v1 -I=../../api/proto/v1 ../../api/proto/v1/*.proto" + +import ( + "archive/tar" + "encoding/base64" + "flag" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "syscall" + "time" + "unsafe" + + api "github.com/pojntfx/stfs/pkg/api/proto/v1" + "golang.org/x/sys/unix" + "google.golang.org/protobuf/proto" +) + +// See https://github.com/benmcclelland/mtio +const ( + MTIOCTOP = 0x40086d01 // Do magnetic tape operation + MTEOM = 12 // Goto end of recorded media (for appending files) + + STFSVersion = 1 +) + +// Operation is struct for MTIOCTOP +type Operation struct { + Op int16 // Operation ID + Pad int16 // Padding to match C structures + Count int32 // Operation count +} + +func main() { + file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open") + dir := flag.String("dir", ".", "Directory to add to the file") + + flag.Parse() + + isRegular := true + stat, err := os.Stat(*file) + if err == nil { + isRegular = stat.Mode().IsRegular() + } else { + if os.IsNotExist(err) { + isRegular = true + } else { + panic(err) + } + } + + var f *os.File + if isRegular { + f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + panic(err) + } + + // No need to go to end manually due to `os.O_APPEND` + } else { + // Go to end of file + syscall.Syscall( + syscall.SYS_IOCTL, + f.Fd(), + MTIOCTOP, + uintptr(unsafe.Pointer( + &Operation{ + Op: MTEOM, + }, + )), + ) + + f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice) + if err != nil { + panic(err) + } + } + defer f.Close() + + tw := tar.NewWriter(f) // We are not closing the tar writer to prevent writing the trailer + + if err := filepath.Walk(*dir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + link := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if link, err = os.Readlink(path); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + + hdr.Format = tar.FormatGNU // Required for AccessTime, ChangeTime etc. + + var unixStat syscall.Stat_t + if err := syscall.Stat(path, &unixStat); err != nil { + return err + } + + mtimesec, mtimensec := unixStat.Mtim.Unix() + atimesec, atimensec := unixStat.Atim.Unix() + ctimesec, ctimensec := unixStat.Ctim.Unix() + + hdr.ModTime = time.Unix(mtimesec, mtimensec) + hdr.AccessTime = time.Unix(atimesec, atimensec) + hdr.ChangeTime = time.Unix(ctimesec, ctimensec) + + hdr.Devmajor = int64(unix.Major(unixStat.Dev)) + hdr.Devminor = int64(unix.Minor(unixStat.Dev)) + + wrapper := &api.Wrapper{ + Version: STFSVersion, + Header: &api.Header{ + Action: api.Action_CREATE, + Name: path, + }, + } + + encodedName, err := proto.Marshal(wrapper) + if err != nil { + return err + } + + hdr.Name = base64.StdEncoding.EncodeToString(encodedName) + + log.Println(hdr) + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + if !info.Mode().IsRegular() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(tw, file); err != nil { + return err + } + + return nil + }); err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index e5256fd..27b8cd2 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/volatiletech/randomize v0.0.1 github.com/volatiletech/sqlboiler/v4 v4.7.1 github.com/volatiletech/strmangle v0.0.1 + golang.org/x/sys v0.0.0-20211112193437-faf0a1b62c6b + google.golang.org/protobuf v1.27.1 ) require ( @@ -30,7 +32,6 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect github.com/volatiletech/inflect v0.0.1 // indirect github.com/ziutek/mymysql v1.5.4 // indirect - golang.org/x/sys v0.0.0-20211112193437-faf0a1b62c6b // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect diff --git a/go.sum b/go.sum index 1e3c3a4..6f9f30c 100644 --- a/go.sum +++ b/go.sum @@ -178,6 +178,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -826,6 +827,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/api/proto/v1/metadata.pb.go b/pkg/api/proto/v1/metadata.pb.go new file mode 100644 index 0000000..81ae89f --- /dev/null +++ b/pkg/api/proto/v1/metadata.pb.go @@ -0,0 +1,296 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.14.0 +// source: metadata.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Action int32 + +const ( + Action_CREATE Action = 0 + Action_UPDATE Action = 1 + Action_DELETE Action = 2 +) + +// Enum value maps for Action. +var ( + Action_name = map[int32]string{ + 0: "CREATE", + 1: "UPDATE", + 2: "DELETE", + } + Action_value = map[string]int32{ + "CREATE": 0, + "UPDATE": 1, + "DELETE": 2, + } +) + +func (x Action) Enum() *Action { + p := new(Action) + *p = x + return p +} + +func (x Action) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Action) Descriptor() protoreflect.EnumDescriptor { + return file_metadata_proto_enumTypes[0].Descriptor() +} + +func (Action) Type() protoreflect.EnumType { + return &file_metadata_proto_enumTypes[0] +} + +func (x Action) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Action.Descriptor instead. +func (Action) EnumDescriptor() ([]byte, []int) { + return file_metadata_proto_rawDescGZIP(), []int{0} +} + +type Wrapper struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version int64 `protobuf:"varint,1,opt,name=Version,proto3" json:"Version,omitempty"` + Header *Header `protobuf:"bytes,2,opt,name=Header,proto3" json:"Header,omitempty"` +} + +func (x *Wrapper) Reset() { + *x = Wrapper{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Wrapper) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Wrapper) ProtoMessage() {} + +func (x *Wrapper) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Wrapper.ProtoReflect.Descriptor instead. +func (*Wrapper) Descriptor() ([]byte, []int) { + return file_metadata_proto_rawDescGZIP(), []int{0} +} + +func (x *Wrapper) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *Wrapper) GetHeader() *Header { + if x != nil { + return x.Header + } + return nil +} + +type Header struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Action Action `protobuf:"varint,1,opt,name=Action,proto3,enum=com.pojtinger.felicitas.stfs.Action" json:"Action,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` + Replaces string `protobuf:"bytes,3,opt,name=Replaces,proto3" json:"Replaces,omitempty"` +} + +func (x *Header) Reset() { + *x = Header{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Header) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Header) ProtoMessage() {} + +func (x *Header) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Header.ProtoReflect.Descriptor instead. +func (*Header) Descriptor() ([]byte, []int) { + return file_metadata_proto_rawDescGZIP(), []int{1} +} + +func (x *Header) GetAction() Action { + if x != nil { + return x.Action + } + return Action_CREATE +} + +func (x *Header) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Header) GetReplaces() string { + if x != nil { + return x.Replaces + } + return "" +} + +var File_metadata_proto protoreflect.FileDescriptor + +var file_metadata_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x18, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e, + 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5d, 0x0a, 0x07, 0x57, 0x72, + 0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x38, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e, + 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x06, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, + 0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x41, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, + 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a, + 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, + 0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, + 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66, + 0x78, 0x2f, 0x73, 0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_metadata_proto_rawDescOnce sync.Once + file_metadata_proto_rawDescData = file_metadata_proto_rawDesc +) + +func file_metadata_proto_rawDescGZIP() []byte { + file_metadata_proto_rawDescOnce.Do(func() { + file_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(file_metadata_proto_rawDescData) + }) + return file_metadata_proto_rawDescData +} + +var file_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_metadata_proto_goTypes = []interface{}{ + (Action)(0), // 0: com.pojtinger.felicitas.stfs.Action + (*Wrapper)(nil), // 1: com.pojtinger.felicitas.stfs.Wrapper + (*Header)(nil), // 2: com.pojtinger.felicitas.stfs.Header +} +var file_metadata_proto_depIdxs = []int32{ + 2, // 0: com.pojtinger.felicitas.stfs.Wrapper.Header:type_name -> com.pojtinger.felicitas.stfs.Header + 0, // 1: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_metadata_proto_init() } +func file_metadata_proto_init() { + if File_metadata_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Wrapper); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Header); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_metadata_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_metadata_proto_goTypes, + DependencyIndexes: file_metadata_proto_depIdxs, + EnumInfos: file_metadata_proto_enumTypes, + MessageInfos: file_metadata_proto_msgTypes, + }.Build() + File_metadata_proto = out.File + file_metadata_proto_rawDesc = nil + file_metadata_proto_goTypes = nil + file_metadata_proto_depIdxs = nil +}