mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add full example with encryption, compression and signatures

Browse Source
This commit is contained in:
Felix Pojtinger
2022-01-08 23:14:55 +01:00
parent a15c7e7cac
commit fc05ffd532
6 changed files with 318 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/stfs/cmd/serve_ftp.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config"
sfs "github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/keys"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
@@ -177,7 +177,7 @@ var serveFTPCmd = &cobra.Command{
},
)
stfs := sfs.NewSTFS(
stfs := fs.NewSTFS(
readOps,
writeOps,

4
cmd/stfs/cmd/serve_http.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config"
sfs "github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/keys"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
@@ -120,7 +120,7 @@ var serveHTTPCmd = &cobra.Command{
},
)
stfs := sfs.NewSTFS(
stfs := fs.NewSTFS(
readOps,
nil,

262
examples/full/main.go Normal file
View File

@@ -0,0 +1,262 @@
package main
import (
"log"
"os"
"path/filepath"
"time"
"github.com/pojntfx/stfs/examples"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/keys"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape"
"github.com/pojntfx/stfs/pkg/utility"
"github.com/spf13/afero"
)
func createFs(
drive string,
metadata string,
recordSize int,
readOnly bool,
verbose bool,
signature string,
signaturePassword string,
encryption string,
encryptionPassword string,
compression string,
compressionLevel string,
writeCache string,
writeCacheDir string,
fileSystemCache string,
fileSystemCacheDir string,
fileSystemCacheDuration time.Duration,
) (afero.Fs, error) {
signaturePrivkey, signaturePubkey, err := utility.Keygen(
config.PipeConfig{
Signature: signature,
Encryption: config.NoneKey,
},
config.PasswordConfig{
Password: signaturePassword,
},
)
if err != nil {
return nil, err
}
signatureRecipient, err := keys.ParseSignerRecipient(signature, signaturePubkey)
if err != nil {
return nil, err
}
signatureIdentity, err := keys.ParseSignerIdentity(signature, signaturePrivkey, signaturePassword)
if err != nil {
return nil, err
}
encryptionPrivkey, encryptionPubkey, err := utility.Keygen(
config.PipeConfig{
Signature: config.NoneKey,
Encryption: encryption,
},
config.PasswordConfig{
Password: encryptionPassword,
},
)
if err != nil {
return nil, err
}
encryptionRecipient, err := keys.ParseRecipient(encryption, encryptionPubkey)
if err != nil {
return nil, err
}
encryptionIdentity, err := keys.ParseIdentity(encryption, encryptionPrivkey, encryptionPassword)
if err != nil {
return nil, err
}
tm := tape.NewTapeManager(
drive,
recordSize,
false,
)
metadataPersister := persisters.NewMetadataPersister(metadata)
if err := metadataPersister.Open(); err != nil {
return nil, err
}
jsonLogger := &examples.Logger{
Verbose: verbose,
}
metadataConfig := config.MetadataConfig{
Metadata: metadataPersister,
}
pipeConfig := config.PipeConfig{
Compression: compression,
Encryption: encryption,
Signature: signature,
RecordSize: recordSize,
}
backendConfig := config.BackendConfig{
GetWriter: tm.GetWriter,
CloseWriter: tm.Close,
GetReader: tm.GetReader,
CloseReader: tm.Close,
GetDrive: tm.GetDrive,
CloseDrive: tm.Close,
}
readCryptoConfig := config.CryptoConfig{
Recipient: signatureRecipient,
Identity: encryptionIdentity,
Password: encryptionPassword,
}
readOps := operations.NewOperations(
backendConfig,
metadataConfig,
pipeConfig,
readCryptoConfig,
func(event *config.HeaderEvent) {
jsonLogger.Debug("Header read", event)
},
)
writeOps := operations.NewOperations(
backendConfig,
metadataConfig,
pipeConfig,
config.CryptoConfig{
Recipient: encryptionRecipient,
Identity: signatureIdentity,
Password: signaturePassword,
},
func(event *config.HeaderEvent) {
jsonLogger.Debug("Header write", event)
},
)
stfs := fs.NewSTFS(
readOps,
writeOps,
config.MetadataConfig{
Metadata: metadataPersister,
},
compressionLevel,
func() (cache.WriteCache, func() error, error) {
return cache.NewCacheWrite(
writeCacheDir,
writeCache,
)
},
false,
readOnly,
func(hdr *config.Header) {
jsonLogger.Trace("Header transform", hdr)
},
jsonLogger,
)
root, err := stfs.Initialize("/", os.ModePerm)
if err != nil {
return nil, err
}
return cache.NewCacheFilesystem(
stfs,
root,
fileSystemCache,
fileSystemCacheDuration,
fileSystemCacheDir,
)
}
func main() {
tmp, err := os.MkdirTemp(os.TempDir(), "stfs-test-*")
if err != nil {
panic(err)
}
drive := filepath.Join(tmp, "drive.tar")
metadata := filepath.Join(tmp, "metadata.sqlite")
recordSize := 20
readOnly := false
verbose := true
signature := config.SignatureFormatPGPKey
signaturePassword := "testSignaturePassword"
encryption := config.EncryptionFormatAgeKey
encryptionPassword := "testEncryptionPassword"
compression := config.CompressionFormatZStandardKey
compressionLevel := config.CompressionLevelFastest
writeCacheDir := filepath.Join(tmp, "write-cache")
writeCache := config.WriteCacheTypeFile
fileSystemCache := config.FileSystemCacheTypeDir
fileSystemCacheDir := filepath.Join(tmp, "filesystem-cache")
fileSystemCacheDuration := time.Hour
fs, err := createFs(
drive,
metadata,
recordSize,
readOnly,
verbose,
signature,
signaturePassword,
encryption,
encryptionPassword,
compression,
compressionLevel,
writeCache,
writeCacheDir,
fileSystemCache,
fileSystemCacheDir,
fileSystemCacheDuration,
)
if err != nil {
panic(err)
}
log.Println("stat /")
stat, err := fs.Stat("/")
if err != nil {
panic(err)
}
log.Println("Result of stat /:", stat)
}

46
examples/logger.go Normal file
View File

@@ -0,0 +1,46 @@
package examples
import (
"encoding/json"
"log"
golog "github.com/fclairamb/go-log"
)
type Logger struct {
Verbose bool
}
func (l Logger) log(level, event string, keyvals ...interface{}) {
k, _ := json.Marshal(keyvals)
log.Println(level, event, string(k))
}
func (l Logger) Trace(event string, keyvals ...interface{}) {
if l.Verbose {
l.log("TRACE", event, keyvals)
}
}
func (l Logger) Debug(event string, keyvals ...interface{}) {
if l.Verbose {
l.log("DEBUG", event, keyvals)
}
}
func (l Logger) Info(event string, keyvals ...interface{}) {
l.log("INFO", event, keyvals)
}
func (l Logger) Warn(event string, keyvals ...interface{}) {
l.log("WARN", event, keyvals)
}
func (l Logger) Error(event string, keyvals ...interface{}) {
l.log("ERROR", event, keyvals)
}
func (l Logger) With(keyvals ...interface{}) golog.Logger {
return l
}

50
examples/fs/main.go → examples/simple/main.go
View File

@@ -1,60 +1,20 @@
package main
import (
"encoding/json"
"flag"
"log"
"os"
"path/filepath"
"github.com/pojntfx/stfs/examples"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config"
sfs "github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape"
golog "github.com/fclairamb/go-log"
)
type logger struct {
verbose bool
}
func (l logger) log(level, event string, keyvals ...interface{}) {
k, _ := json.Marshal(keyvals)
log.Println(level, event, string(k))
}
func (l logger) Trace(event string, keyvals ...interface{}) {
if l.verbose {
l.log("TRACE", event, keyvals)
}
}
func (l logger) Debug(event string, keyvals ...interface{}) {
if l.verbose {
l.log("DEBUG", event, keyvals)
}
}
func (l logger) Info(event string, keyvals ...interface{}) {
l.log("INFO", event, keyvals)
}
func (l logger) Warn(event string, keyvals ...interface{}) {
l.log("WARN", event, keyvals)
}
func (l logger) Error(event string, keyvals ...interface{}) {
l.log("ERROR", event, keyvals)
}
func (l logger) With(keyvals ...interface{}) golog.Logger {
return l
}
func main() {
driveFlag := flag.String("drive", "/dev/nst0", "Tape or tar file to use")
recordSizeFlag := flag.Int("recordSize", 20, "Amount of 512-bit blocks per record")
@@ -75,8 +35,8 @@ func main() {
panic(err)
}
l := &logger{
verbose: *verboseFlag,
l := &examples.Logger{
Verbose: *verboseFlag,
}
metadataConfig := config.MetadataConfig{
@@ -123,7 +83,7 @@ func main() {
},
)
stfs := sfs.NewSTFS(
stfs := fs.NewSTFS(
readOps,
writeOps,

2
pkg/config/constants.go
View File

@@ -1,7 +1,7 @@
package config
const (
NoneKey = "none"
NoneKey = ""
CompressionFormatGZipKey = "gzip"
CompressionFormatParallelGZipKey = "parallelgzip"