feat: Add archiver with filename-embedded headers

Makefile

@@ -3,4 +3,4 @@ generate:
 	go generate ./...
 
 clean:
-	rm -rf pkg/db
+	rm -rf pkg/db pkg/api

api/proto/v1/metadata.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+package com.pojtinger.felicitas.stfs;
+
+option go_package = "github.com/pojntfx/stfs/pkg/api/proto/v1";
+
+message Wrapper {
+    int64 Version = 1;
+    Header Header = 2;
+}
+
+enum Action {
+    CREATE = 0;
+    UPDATE = 1;
+    DELETE = 2;
+}
+
+message Header {
+    Action Action = 1;
+    string Name = 2;
+    string Replaces = 3;
+}

cmd/stfs-uvf-with-header/main.go

@@ -0,0 +1,160 @@
+package main
+
+//go:generate sh -c "mkdir -p ../../pkg/api/proto/v1 && protoc --go_out=paths=source_relative,plugins=grpc:../../pkg/api/proto/v1 -I=../../api/proto/v1 ../../api/proto/v1/*.proto"
+
+import (
+	"archive/tar"
+	"encoding/base64"
+	"flag"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+	"unsafe"
+
+	api "github.com/pojntfx/stfs/pkg/api/proto/v1"
+	"golang.org/x/sys/unix"
+	"google.golang.org/protobuf/proto"
+)
+
+// See https://github.com/benmcclelland/mtio
+const (
+	MTIOCTOP = 0x40086d01 // Do magnetic tape operation
+	MTEOM    = 12         // Goto end of recorded media (for appending files)
+
+	STFSVersion = 1
+)
+
+// Operation is struct for MTIOCTOP
+type Operation struct {
+	Op    int16 // Operation ID
+	Pad   int16 // Padding to match C structures
+	Count int32 // Operation count
+}
+
+func main() {
+	file := flag.String("file", "/dev/nst0", "File (tape drive or tar file) to open")
+	dir := flag.String("dir", ".", "Directory to add to the file")
+
+	flag.Parse()
+
+	isRegular := true
+	stat, err := os.Stat(*file)
+	if err == nil {
+		isRegular = stat.Mode().IsRegular()
+	} else {
+		if os.IsNotExist(err) {
+			isRegular = true
+		} else {
+			panic(err)
+		}
+	}
+
+	var f *os.File
+	if isRegular {
+		f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+		if err != nil {
+			panic(err)
+		}
+
+		// No need to go to end manually due to `os.O_APPEND`
+	} else {
+		// Go to end of file
+		syscall.Syscall(
+			syscall.SYS_IOCTL,
+			f.Fd(),
+			MTIOCTOP,
+			uintptr(unsafe.Pointer(
+				&Operation{
+					Op: MTEOM,
+				},
+			)),
+		)
+
+		f, err = os.OpenFile(*file, os.O_APPEND|os.O_WRONLY, os.ModeCharDevice)
+		if err != nil {
+			panic(err)
+		}
+	}
+	defer f.Close()
+
+	tw := tar.NewWriter(f) // We are not closing the tar writer to prevent writing the trailer
+
+	if err := filepath.Walk(*dir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		link := ""
+		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+			if link, err = os.Readlink(path); err != nil {
+				return err
+			}
+		}
+
+		hdr, err := tar.FileInfoHeader(info, link)
+		if err != nil {
+			return err
+		}
+
+		hdr.Format = tar.FormatGNU // Required for AccessTime, ChangeTime etc.
+
+		var unixStat syscall.Stat_t
+		if err := syscall.Stat(path, &unixStat); err != nil {
+			return err
+		}
+
+		mtimesec, mtimensec := unixStat.Mtim.Unix()
+		atimesec, atimensec := unixStat.Atim.Unix()
+		ctimesec, ctimensec := unixStat.Ctim.Unix()
+
+		hdr.ModTime = time.Unix(mtimesec, mtimensec)
+		hdr.AccessTime = time.Unix(atimesec, atimensec)
+		hdr.ChangeTime = time.Unix(ctimesec, ctimensec)
+
+		hdr.Devmajor = int64(unix.Major(unixStat.Dev))
+		hdr.Devminor = int64(unix.Minor(unixStat.Dev))
+
+		wrapper := &api.Wrapper{
+			Version: STFSVersion,
+			Header: &api.Header{
+				Action: api.Action_CREATE,
+				Name:   path,
+			},
+		}
+
+		encodedName, err := proto.Marshal(wrapper)
+		if err != nil {
+			return err
+		}
+
+		hdr.Name = base64.StdEncoding.EncodeToString(encodedName)
+
+		log.Println(hdr)
+
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		if !info.Mode().IsRegular() {
+			return nil
+		}
+
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		if _, err := io.Copy(tw, file); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		panic(err)
+	}
+}

go.mod

@@ -14,6 +14,8 @@ require (
 	github.com/volatiletech/randomize v0.0.1
 	github.com/volatiletech/sqlboiler/v4 v4.7.1
 	github.com/volatiletech/strmangle v0.0.1
+	golang.org/x/sys v0.0.0-20211112193437-faf0a1b62c6b
+	google.golang.org/protobuf v1.27.1
 )
 
 require (
@@ -30,7 +32,6 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/volatiletech/inflect v0.0.1 // indirect
 	github.com/ziutek/mymysql v1.5.4 // indirect
-	golang.org/x/sys v0.0.0-20211112193437-faf0a1b62c6b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect

go.sum

@@ -178,6 +178,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -826,6 +827,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/api/proto/v1/metadata.pb.go

@@ -0,0 +1,296 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.14.0
+// source: metadata.proto
+
+package v1
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type Action int32
+
+const (
+	Action_CREATE Action = 0
+	Action_UPDATE Action = 1
+	Action_DELETE Action = 2
+)
+
+// Enum value maps for Action.
+var (
+	Action_name = map[int32]string{
+		0: "CREATE",
+		1: "UPDATE",
+		2: "DELETE",
+	}
+	Action_value = map[string]int32{
+		"CREATE": 0,
+		"UPDATE": 1,
+		"DELETE": 2,
+	}
+)
+
+func (x Action) Enum() *Action {
+	p := new(Action)
+	*p = x
+	return p
+}
+
+func (x Action) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Action) Descriptor() protoreflect.EnumDescriptor {
+	return file_metadata_proto_enumTypes[0].Descriptor()
+}
+
+func (Action) Type() protoreflect.EnumType {
+	return &file_metadata_proto_enumTypes[0]
+}
+
+func (x Action) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Action.Descriptor instead.
+func (Action) EnumDescriptor() ([]byte, []int) {
+	return file_metadata_proto_rawDescGZIP(), []int{0}
+}
+
+type Wrapper struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Version int64   `protobuf:"varint,1,opt,name=Version,proto3" json:"Version,omitempty"`
+	Header  *Header `protobuf:"bytes,2,opt,name=Header,proto3" json:"Header,omitempty"`
+}
+
+func (x *Wrapper) Reset() {
+	*x = Wrapper{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_metadata_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Wrapper) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Wrapper) ProtoMessage() {}
+
+func (x *Wrapper) ProtoReflect() protoreflect.Message {
+	mi := &file_metadata_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Wrapper.ProtoReflect.Descriptor instead.
+func (*Wrapper) Descriptor() ([]byte, []int) {
+	return file_metadata_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Wrapper) GetVersion() int64 {
+	if x != nil {
+		return x.Version
+	}
+	return 0
+}
+
+func (x *Wrapper) GetHeader() *Header {
+	if x != nil {
+		return x.Header
+	}
+	return nil
+}
+
+type Header struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Action   Action `protobuf:"varint,1,opt,name=Action,proto3,enum=com.pojtinger.felicitas.stfs.Action" json:"Action,omitempty"`
+	Name     string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"`
+	Replaces string `protobuf:"bytes,3,opt,name=Replaces,proto3" json:"Replaces,omitempty"`
+}
+
+func (x *Header) Reset() {
+	*x = Header{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_metadata_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Header) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Header) ProtoMessage() {}
+
+func (x *Header) ProtoReflect() protoreflect.Message {
+	mi := &file_metadata_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Header.ProtoReflect.Descriptor instead.
+func (*Header) Descriptor() ([]byte, []int) {
+	return file_metadata_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Header) GetAction() Action {
+	if x != nil {
+		return x.Action
+	}
+	return Action_CREATE
+}
+
+func (x *Header) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *Header) GetReplaces() string {
+	if x != nil {
+		return x.Replaces
+	}
+	return ""
+}
+
+var File_metadata_proto protoreflect.FileDescriptor
+
+var file_metadata_proto_rawDesc = []byte{
+	0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x18, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e,
+	0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x22, 0x5d, 0x0a, 0x07, 0x57, 0x72,
+	0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12,
+	0x38, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x2e,
+	0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x52, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x06, 0x48, 0x65, 0x61,
+	0x64, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x6f, 0x6a, 0x74, 0x69, 0x6e,
+	0x67, 0x65, 0x72, 0x2e, 0x66, 0x65, 0x6c, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x41,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a,
+	0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d,
+	0x65, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2a, 0x2c, 0x0a,
+	0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54,
+	0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12,
+	0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6a, 0x6e, 0x74, 0x66,
+	0x78, 0x2f, 0x73, 0x74, 0x66, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_metadata_proto_rawDescOnce sync.Once
+	file_metadata_proto_rawDescData = file_metadata_proto_rawDesc
+)
+
+func file_metadata_proto_rawDescGZIP() []byte {
+	file_metadata_proto_rawDescOnce.Do(func() {
+		file_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(file_metadata_proto_rawDescData)
+	})
+	return file_metadata_proto_rawDescData
+}
+
+var file_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_metadata_proto_goTypes = []interface{}{
+	(Action)(0),     // 0: com.pojtinger.felicitas.stfs.Action
+	(*Wrapper)(nil), // 1: com.pojtinger.felicitas.stfs.Wrapper
+	(*Header)(nil),  // 2: com.pojtinger.felicitas.stfs.Header
+}
+var file_metadata_proto_depIdxs = []int32{
+	2, // 0: com.pojtinger.felicitas.stfs.Wrapper.Header:type_name -> com.pojtinger.felicitas.stfs.Header
+	0, // 1: com.pojtinger.felicitas.stfs.Header.Action:type_name -> com.pojtinger.felicitas.stfs.Action
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_metadata_proto_init() }
+func file_metadata_proto_init() {
+	if File_metadata_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Wrapper); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Header); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_metadata_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_metadata_proto_goTypes,
+		DependencyIndexes: file_metadata_proto_depIdxs,
+		EnumInfos:         file_metadata_proto_enumTypes,
+		MessageInfos:      file_metadata_proto_msgTypes,
+	}.Build()
+	File_metadata_proto = out.File
+	file_metadata_proto_rawDesc = nil
+	file_metadata_proto_goTypes = nil
+	file_metadata_proto_depIdxs = nil
+}