diff --git a/cmd/stfs/cmd/serve_ftp.go b/cmd/stfs/cmd/serve_ftp.go index f8f7cfb..e0831d8 100644 --- a/cmd/stfs/cmd/serve_ftp.go +++ b/cmd/stfs/cmd/serve_ftp.go @@ -13,7 +13,7 @@ import ( "github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/pkg/cache" "github.com/pojntfx/stfs/pkg/config" - sfs "github.com/pojntfx/stfs/pkg/fs" + "github.com/pojntfx/stfs/pkg/fs" "github.com/pojntfx/stfs/pkg/keys" "github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/persisters" @@ -177,7 +177,7 @@ var serveFTPCmd = &cobra.Command{ }, ) - stfs := sfs.NewSTFS( + stfs := fs.NewSTFS( readOps, writeOps, diff --git a/cmd/stfs/cmd/serve_http.go b/cmd/stfs/cmd/serve_http.go index e03e740..ac43362 100644 --- a/cmd/stfs/cmd/serve_http.go +++ b/cmd/stfs/cmd/serve_http.go @@ -13,7 +13,7 @@ import ( "github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/pkg/cache" "github.com/pojntfx/stfs/pkg/config" - sfs "github.com/pojntfx/stfs/pkg/fs" + "github.com/pojntfx/stfs/pkg/fs" "github.com/pojntfx/stfs/pkg/keys" "github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/persisters" @@ -120,7 +120,7 @@ var serveHTTPCmd = &cobra.Command{ }, ) - stfs := sfs.NewSTFS( + stfs := fs.NewSTFS( readOps, nil, diff --git a/examples/full/main.go b/examples/full/main.go new file mode 100644 index 0000000..a2e73d6 --- /dev/null +++ b/examples/full/main.go @@ -0,0 +1,262 @@ +package main + +import ( + "log" + "os" + "path/filepath" + "time" + + "github.com/pojntfx/stfs/examples" + "github.com/pojntfx/stfs/pkg/cache" + "github.com/pojntfx/stfs/pkg/config" + "github.com/pojntfx/stfs/pkg/fs" + "github.com/pojntfx/stfs/pkg/keys" + "github.com/pojntfx/stfs/pkg/operations" + "github.com/pojntfx/stfs/pkg/persisters" + "github.com/pojntfx/stfs/pkg/tape" + "github.com/pojntfx/stfs/pkg/utility" + "github.com/spf13/afero" +) + +func createFs( + drive string, + metadata string, + + recordSize int, + readOnly bool, + verbose bool, + + signature string, + signaturePassword string, + + encryption string, + encryptionPassword string, + + compression string, + compressionLevel string, + + writeCache string, + writeCacheDir string, + + fileSystemCache string, + fileSystemCacheDir string, + fileSystemCacheDuration time.Duration, +) (afero.Fs, error) { + signaturePrivkey, signaturePubkey, err := utility.Keygen( + config.PipeConfig{ + Signature: signature, + Encryption: config.NoneKey, + }, + config.PasswordConfig{ + Password: signaturePassword, + }, + ) + if err != nil { + return nil, err + } + + signatureRecipient, err := keys.ParseSignerRecipient(signature, signaturePubkey) + if err != nil { + return nil, err + } + + signatureIdentity, err := keys.ParseSignerIdentity(signature, signaturePrivkey, signaturePassword) + if err != nil { + return nil, err + } + + encryptionPrivkey, encryptionPubkey, err := utility.Keygen( + config.PipeConfig{ + Signature: config.NoneKey, + Encryption: encryption, + }, + config.PasswordConfig{ + Password: encryptionPassword, + }, + ) + if err != nil { + return nil, err + } + + encryptionRecipient, err := keys.ParseRecipient(encryption, encryptionPubkey) + if err != nil { + return nil, err + } + + encryptionIdentity, err := keys.ParseIdentity(encryption, encryptionPrivkey, encryptionPassword) + if err != nil { + return nil, err + } + + tm := tape.NewTapeManager( + drive, + recordSize, + false, + ) + + metadataPersister := persisters.NewMetadataPersister(metadata) + if err := metadataPersister.Open(); err != nil { + return nil, err + } + + jsonLogger := &examples.Logger{ + Verbose: verbose, + } + + metadataConfig := config.MetadataConfig{ + Metadata: metadataPersister, + } + pipeConfig := config.PipeConfig{ + Compression: compression, + Encryption: encryption, + Signature: signature, + RecordSize: recordSize, + } + backendConfig := config.BackendConfig{ + GetWriter: tm.GetWriter, + CloseWriter: tm.Close, + + GetReader: tm.GetReader, + CloseReader: tm.Close, + + GetDrive: tm.GetDrive, + CloseDrive: tm.Close, + } + readCryptoConfig := config.CryptoConfig{ + Recipient: signatureRecipient, + Identity: encryptionIdentity, + Password: encryptionPassword, + } + + readOps := operations.NewOperations( + backendConfig, + metadataConfig, + + pipeConfig, + readCryptoConfig, + + func(event *config.HeaderEvent) { + jsonLogger.Debug("Header read", event) + }, + ) + + writeOps := operations.NewOperations( + backendConfig, + metadataConfig, + + pipeConfig, + config.CryptoConfig{ + Recipient: encryptionRecipient, + Identity: signatureIdentity, + Password: signaturePassword, + }, + + func(event *config.HeaderEvent) { + jsonLogger.Debug("Header write", event) + }, + ) + + stfs := fs.NewSTFS( + readOps, + writeOps, + + config.MetadataConfig{ + Metadata: metadataPersister, + }, + + compressionLevel, + func() (cache.WriteCache, func() error, error) { + return cache.NewCacheWrite( + writeCacheDir, + writeCache, + ) + }, + false, + readOnly, + + func(hdr *config.Header) { + jsonLogger.Trace("Header transform", hdr) + }, + jsonLogger, + ) + + root, err := stfs.Initialize("/", os.ModePerm) + if err != nil { + return nil, err + } + + return cache.NewCacheFilesystem( + stfs, + root, + fileSystemCache, + fileSystemCacheDuration, + fileSystemCacheDir, + ) +} + +func main() { + tmp, err := os.MkdirTemp(os.TempDir(), "stfs-test-*") + if err != nil { + panic(err) + } + + drive := filepath.Join(tmp, "drive.tar") + metadata := filepath.Join(tmp, "metadata.sqlite") + + recordSize := 20 + readOnly := false + verbose := true + + signature := config.SignatureFormatPGPKey + signaturePassword := "testSignaturePassword" + + encryption := config.EncryptionFormatAgeKey + encryptionPassword := "testEncryptionPassword" + + compression := config.CompressionFormatZStandardKey + compressionLevel := config.CompressionLevelFastest + + writeCacheDir := filepath.Join(tmp, "write-cache") + writeCache := config.WriteCacheTypeFile + + fileSystemCache := config.FileSystemCacheTypeDir + fileSystemCacheDir := filepath.Join(tmp, "filesystem-cache") + fileSystemCacheDuration := time.Hour + + fs, err := createFs( + drive, + metadata, + + recordSize, + readOnly, + verbose, + + signature, + signaturePassword, + + encryption, + encryptionPassword, + + compression, + compressionLevel, + + writeCache, + writeCacheDir, + + fileSystemCache, + fileSystemCacheDir, + fileSystemCacheDuration, + ) + if err != nil { + panic(err) + } + + log.Println("stat /") + + stat, err := fs.Stat("/") + if err != nil { + panic(err) + } + + log.Println("Result of stat /:", stat) +} diff --git a/examples/logger.go b/examples/logger.go new file mode 100644 index 0000000..34ee99f --- /dev/null +++ b/examples/logger.go @@ -0,0 +1,46 @@ +package examples + +import ( + "encoding/json" + "log" + + golog "github.com/fclairamb/go-log" +) + +type Logger struct { + Verbose bool +} + +func (l Logger) log(level, event string, keyvals ...interface{}) { + k, _ := json.Marshal(keyvals) + + log.Println(level, event, string(k)) +} + +func (l Logger) Trace(event string, keyvals ...interface{}) { + if l.Verbose { + l.log("TRACE", event, keyvals) + } +} + +func (l Logger) Debug(event string, keyvals ...interface{}) { + if l.Verbose { + l.log("DEBUG", event, keyvals) + } +} + +func (l Logger) Info(event string, keyvals ...interface{}) { + l.log("INFO", event, keyvals) +} + +func (l Logger) Warn(event string, keyvals ...interface{}) { + l.log("WARN", event, keyvals) +} + +func (l Logger) Error(event string, keyvals ...interface{}) { + l.log("ERROR", event, keyvals) +} + +func (l Logger) With(keyvals ...interface{}) golog.Logger { + return l +} diff --git a/examples/fs/main.go b/examples/simple/main.go similarity index 78% rename from examples/fs/main.go rename to examples/simple/main.go index 2bf5579..d64147a 100644 --- a/examples/fs/main.go +++ b/examples/simple/main.go @@ -1,60 +1,20 @@ package main import ( - "encoding/json" "flag" "log" "os" "path/filepath" + "github.com/pojntfx/stfs/examples" "github.com/pojntfx/stfs/pkg/cache" "github.com/pojntfx/stfs/pkg/config" - sfs "github.com/pojntfx/stfs/pkg/fs" + "github.com/pojntfx/stfs/pkg/fs" "github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/persisters" "github.com/pojntfx/stfs/pkg/tape" - - golog "github.com/fclairamb/go-log" ) -type logger struct { - verbose bool -} - -func (l logger) log(level, event string, keyvals ...interface{}) { - k, _ := json.Marshal(keyvals) - - log.Println(level, event, string(k)) -} - -func (l logger) Trace(event string, keyvals ...interface{}) { - if l.verbose { - l.log("TRACE", event, keyvals) - } -} - -func (l logger) Debug(event string, keyvals ...interface{}) { - if l.verbose { - l.log("DEBUG", event, keyvals) - } -} - -func (l logger) Info(event string, keyvals ...interface{}) { - l.log("INFO", event, keyvals) -} - -func (l logger) Warn(event string, keyvals ...interface{}) { - l.log("WARN", event, keyvals) -} - -func (l logger) Error(event string, keyvals ...interface{}) { - l.log("ERROR", event, keyvals) -} - -func (l logger) With(keyvals ...interface{}) golog.Logger { - return l -} - func main() { driveFlag := flag.String("drive", "/dev/nst0", "Tape or tar file to use") recordSizeFlag := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record") @@ -75,8 +35,8 @@ func main() { panic(err) } - l := &logger{ - verbose: *verboseFlag, + l := &examples.Logger{ + Verbose: *verboseFlag, } metadataConfig := config.MetadataConfig{ @@ -123,7 +83,7 @@ func main() { }, ) - stfs := sfs.NewSTFS( + stfs := fs.NewSTFS( readOps, writeOps, diff --git a/pkg/config/constants.go b/pkg/config/constants.go index 3473c3b..2cf7e00 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -1,7 +1,7 @@ package config const ( - NoneKey = "none" + NoneKey = "" CompressionFormatGZipKey = "gzip" CompressionFormatParallelGZipKey = "parallelgzip"