feat: Make STFS API public

Also: Happy new year :)
This commit is contained in:
Felicitas Pojtinger
2021-12-31 23:57:21 +01:00
parent d52c45c603
commit 5089333d13
32 changed files with 214 additions and 222 deletions

View File

@@ -2,9 +2,9 @@ package cmd
import ( import (
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/inventory" "github.com/pojntfx/stfs/pkg/inventory"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
) )

View File

@@ -2,9 +2,9 @@ package cmd
import ( import (
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/inventory" "github.com/pojntfx/stfs/pkg/inventory"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
) )

View File

@@ -2,9 +2,9 @@ package cmd
import ( import (
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/inventory" "github.com/pojntfx/stfs/pkg/inventory"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
) )

View File

@@ -7,12 +7,12 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/pojntfx/stfs/internal/compression" "github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -37,15 +37,15 @@ var operationArchiveCmd = &cobra.Command{
return err return err
} }
if err := compression.CheckCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil { if err := check.CheckCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag))

View File

@@ -1,11 +1,12 @@
package cmd package cmd
import ( import (
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -24,11 +25,11 @@ var operationDeleteCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag))

View File

@@ -1,11 +1,12 @@
package cmd package cmd
import ( import (
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -20,11 +21,11 @@ var operationMoveCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag))

View File

@@ -5,11 +5,12 @@ import (
"io/fs" "io/fs"
"os" "os"
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -28,11 +29,11 @@ var operationRestoreCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag))

View File

@@ -7,12 +7,12 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/pojntfx/stfs/internal/compression" "github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -27,15 +27,15 @@ var operationUpdateCmd = &cobra.Command{
return err return err
} }
if err := compression.CheckCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil { if err := check.CheckCompressionLevel(viper.GetString(compressionLevelFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(identityFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(encryptionFlag), viper.GetString(recipientFlag))

View File

@@ -5,6 +5,7 @@ import (
"io/fs" "io/fs"
"os" "os"
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
@@ -29,11 +30,11 @@ var recoveryFetchCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag))

View File

@@ -3,12 +3,13 @@ package cmd
import ( import (
"archive/tar" "archive/tar"
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/encryption" "github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/signature" "github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/recovery" "github.com/pojntfx/stfs/pkg/recovery"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -23,11 +24,11 @@ var recoveryIndexCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag))

View File

@@ -1,6 +1,7 @@
package cmd package cmd
import ( import (
"github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
@@ -18,11 +19,11 @@ var recoveryQueryCmd = &cobra.Command{
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag))

View File

@@ -6,10 +6,8 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/pojntfx/stfs/internal/compression" "github.com/pojntfx/stfs/internal/check"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/signature"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -45,15 +43,15 @@ https://github.com/pojntfx/stfs`,
boil.DebugWriter = logging.NewJSONLoggerWriter(verbosity, "SQL Query", "query") boil.DebugWriter = logging.NewJSONLoggerWriter(verbosity, "SQL Query", "query")
} }
if err := compression.CheckCompressionFormat(viper.GetString(compressionFlag)); err != nil { if err := check.CheckCompressionFormat(viper.GetString(compressionFlag)); err != nil {
return err return err
} }
if err := encryption.CheckEncryptionFormat(viper.GetString(encryptionFlag)); err != nil { if err := check.CheckEncryptionFormat(viper.GetString(encryptionFlag)); err != nil {
return err return err
} }
return signature.CheckSignatureFormat(viper.GetString(signatureFlag)) return check.CheckSignatureFormat(viper.GetString(signatureFlag))
}, },
} }

View File

@@ -8,15 +8,16 @@ import (
"time" "time"
ftpserver "github.com/fclairamb/ftpserverlib" ftpserver "github.com/fclairamb/ftpserverlib"
"github.com/pojntfx/stfs/internal/cache" "github.com/pojntfx/stfs/internal/check"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
sfs "github.com/pojntfx/stfs/internal/fs"
"github.com/pojntfx/stfs/internal/ftp" "github.com/pojntfx/stfs/internal/ftp"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
sfs "github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -47,27 +48,27 @@ var serveFTPCmd = &cobra.Command{
return err return err
} }
if err := cache.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil { if err := check.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil {
return err return err
} }
if err := cache.CheckWriteCacheType(viper.GetString(cacheWriteFlag)); err != nil { if err := check.CheckWriteCacheType(viper.GetString(cacheWriteFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(encryptionIdentityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(encryptionIdentityFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(encryptionRecipientFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(encryptionRecipientFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(signatureIdentityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(signatureIdentityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(signatureRecipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(signatureRecipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
signaturePubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(signatureRecipientFlag)) signaturePubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(signatureRecipientFlag))
@@ -192,7 +193,7 @@ var serveFTPCmd = &cobra.Command{
}, },
) )
stfs := sfs.NewFileSystem( stfs := sfs.NewSTFS(
readOps, readOps,
writeOps, writeOps,
@@ -201,7 +202,7 @@ var serveFTPCmd = &cobra.Command{
}, },
viper.GetString(compressionLevelFlag), viper.GetString(compressionLevelFlag),
func() (sfs.WriteCache, func() error, error) { func() (cache.WriteCache, func() error, error) {
return cache.NewCacheWrite( return cache.NewCacheWrite(
filepath.Join(viper.GetString(cacheDirFlag), "write"), filepath.Join(viper.GetString(cacheDirFlag), "write"),
viper.GetString(cacheWriteFlag), viper.GetString(cacheWriteFlag),
@@ -258,8 +259,8 @@ func init() {
serveFTPCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, config.KnownCompressionLevels)) serveFTPCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, config.KnownCompressionLevels))
serveFTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address") serveFTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address")
serveFTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, cache.KnownFileSystemCacheTypes)) serveFTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, config.KnownFileSystemCacheTypes))
serveFTPCmd.PersistentFlags().StringP(cacheWriteFlag, "q", cache.WriteCacheTypeFile, fmt.Sprintf("Write cache to use (default %v, available are %v)", cache.WriteCacheTypeFile, cache.KnownWriteCacheTypes)) serveFTPCmd.PersistentFlags().StringP(cacheWriteFlag, "q", config.WriteCacheTypeFile, fmt.Sprintf("Write cache to use (default %v, available are %v)", config.WriteCacheTypeFile, config.KnownWriteCacheTypes))
serveFTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated") serveFTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated")
serveFTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled") serveFTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled")

View File

@@ -7,15 +7,16 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"github.com/pojntfx/stfs/internal/cache" "github.com/pojntfx/stfs/internal/check"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
sfs "github.com/pojntfx/stfs/internal/fs"
"github.com/pojntfx/stfs/internal/handlers" "github.com/pojntfx/stfs/internal/handlers"
"github.com/pojntfx/stfs/internal/keys" "github.com/pojntfx/stfs/internal/keys"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
sfs "github.com/pojntfx/stfs/pkg/fs"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/persisters"
"github.com/pojntfx/stfs/pkg/tape" "github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -38,15 +39,15 @@ var serveHTTPCmd = &cobra.Command{
return err return err
} }
if err := cache.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil { if err := check.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil {
return err return err
} }
if err := keys.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil { if err := check.CheckKeyAccessible(viper.GetString(encryptionFlag), viper.GetString(identityFlag)); err != nil {
return err return err
} }
return keys.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) return check.CheckKeyAccessible(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag)) pubkey, err := keys.ReadKey(viper.GetString(signatureFlag), viper.GetString(recipientFlag))
@@ -119,7 +120,7 @@ var serveHTTPCmd = &cobra.Command{
}, },
) )
stfs := sfs.NewFileSystem( stfs := sfs.NewSTFS(
readOps, readOps,
nil, nil,
@@ -170,7 +171,7 @@ func init() {
serveHTTPCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key") serveHTTPCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")
serveHTTPCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to the public key to verify with") serveHTTPCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to the public key to verify with")
serveHTTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address") serveHTTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address")
serveHTTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, cache.KnownFileSystemCacheTypes)) serveHTTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, config.KnownFileSystemCacheTypes))
serveHTTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated") serveHTTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated")
serveHTTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled") serveHTTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled")

View File

@@ -1,19 +0,0 @@
package cache
import (
"github.com/pojntfx/stfs/pkg/config"
)
const (
FileSystemCacheTypeMemory = "memory"
FileSystemCacheTypeDir = "dir"
WriteCacheTypeMemory = "memory"
WriteCacheTypeFile = "file"
)
var (
KnownFileSystemCacheTypes = []string{config.NoneKey, FileSystemCacheTypeMemory, FileSystemCacheTypeDir}
KnownWriteCacheTypes = []string{WriteCacheTypeMemory, WriteCacheTypeFile}
)

View File

@@ -1,11 +0,0 @@
package cache
import "errors"
var (
ErrFileSystemCacheTypeUnsupported = errors.New("file system cache type unsupported")
ErrFileSystemCacheTypeUnknown = errors.New("file system cache type unknown")
ErrWriteCacheTypeUnsupported = errors.New("write cache type unsupported")
ErrWriteCacheTypeUnknown = errors.New("write cache type unknown")
)

View File

@@ -1,4 +1,4 @@
package compression package check
import ( import (
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"

View File

@@ -1,4 +1,4 @@
package encryption package check
import "github.com/pojntfx/stfs/pkg/config" import "github.com/pojntfx/stfs/pkg/config"

View File

@@ -1,16 +1,18 @@
package cache package check
import "github.com/pojntfx/stfs/pkg/config"
func CheckFileSystemCacheType(cacheType string) error { func CheckFileSystemCacheType(cacheType string) error {
cacheTypeIsKnown := false cacheTypeIsKnown := false
for _, candidate := range KnownFileSystemCacheTypes { for _, candidate := range config.KnownFileSystemCacheTypes {
if cacheType == candidate { if cacheType == candidate {
cacheTypeIsKnown = true cacheTypeIsKnown = true
} }
} }
if !cacheTypeIsKnown { if !cacheTypeIsKnown {
return ErrFileSystemCacheTypeUnknown return config.ErrFileSystemCacheTypeUnknown
} }
return nil return nil
@@ -19,14 +21,14 @@ func CheckFileSystemCacheType(cacheType string) error {
func CheckWriteCacheType(cacheType string) error { func CheckWriteCacheType(cacheType string) error {
cacheTypeIsKnown := false cacheTypeIsKnown := false
for _, candidate := range KnownWriteCacheTypes { for _, candidate := range config.KnownWriteCacheTypes {
if cacheType == candidate { if cacheType == candidate {
cacheTypeIsKnown = true cacheTypeIsKnown = true
} }
} }
if !cacheTypeIsKnown { if !cacheTypeIsKnown {
return ErrWriteCacheTypeUnknown return config.ErrWriteCacheTypeUnknown
} }
return nil return nil

View File

@@ -1,4 +1,4 @@
package keys package check
import ( import (
"errors" "errors"

View File

@@ -1,4 +1,4 @@
package signature package check
import "github.com/pojntfx/stfs/pkg/config" import "github.com/pojntfx/stfs/pkg/config"

View File

@@ -3,7 +3,6 @@ package fs
import ( import (
"bytes" "bytes"
"database/sql" "database/sql"
"errors"
"io" "io"
"io/fs" "io/fs"
"log" "log"
@@ -13,33 +12,19 @@ import (
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/ioext" "github.com/pojntfx/stfs/internal/ioext"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/inventory" "github.com/pojntfx/stfs/pkg/inventory"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var (
ErrIsDirectory = errors.New("is a directory")
)
type WriteCache interface {
io.Closer
io.Reader
io.Seeker
io.Writer
Truncate(size int64) error
Size() (int64, error)
Sync() error
}
type FileFlags struct { type FileFlags struct {
read bool Read bool
write bool Write bool
append bool Append bool
truncate bool Truncate bool
} }
type File struct { type File struct {
@@ -55,7 +40,7 @@ type File struct {
flags *FileFlags flags *FileFlags
compressionLevel string compressionLevel string
getFileBuffer func() (WriteCache, func() error, error) getFileBuffer func() (cache.WriteCache, func() error, error)
name string name string
info os.FileInfo info os.FileInfo
@@ -65,7 +50,7 @@ type File struct {
readOpReader *ioext.CounterReadCloser readOpReader *ioext.CounterReadCloser
readOpWriter io.WriteCloser readOpWriter io.WriteCloser
writeBuf WriteCache writeBuf cache.WriteCache
cleanWriteBuf func() error cleanWriteBuf func() error
onHeader func(hdr *models.Header) onHeader func(hdr *models.Header)
@@ -83,7 +68,7 @@ func NewFile(
flags *FileFlags, flags *FileFlags,
compressionLevel string, compressionLevel string,
getFileBuffer func() (WriteCache, func() error, error), getFileBuffer func() (cache.WriteCache, func() error, error),
name string, name string,
info os.FileInfo, info os.FileInfo,
@@ -118,7 +103,7 @@ func (f *File) syncWithoutLocking() error {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return ErrIsDirectory return config.ErrIsDirectory
} }
if f.writeBuf != nil { if f.writeBuf != nil {
@@ -261,13 +246,13 @@ func (f *File) enterWriteMode() error {
} }
} }
if f.flags.truncate { if f.flags.Truncate {
if err := f.writeBuf.Truncate(0); err != nil { if err := f.writeBuf.Truncate(0); err != nil {
return err return err
} }
} }
if !f.flags.append { if !f.flags.Append {
if _, err := f.writeBuf.Seek(0, io.SeekStart); err != nil { if _, err := f.writeBuf.Seek(0, io.SeekStart); err != nil {
return err return err
} }
@@ -285,7 +270,7 @@ func (f *File) seekWithoutLocking(offset int64, whence int) (int64, error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if f.writeBuf != nil { if f.writeBuf != nil {
@@ -305,7 +290,7 @@ func (f *File) seekWithoutLocking(offset int64, whence int) (int64, error) {
case io.SeekEnd: case io.SeekEnd:
dst = f.info.Size() - offset dst = f.info.Size() - offset
default: default:
return -1, ErrNotImplemented return -1, config.ErrNotImplemented
} }
if f.readOpReader == nil || f.readOpWriter == nil || dst < int64(f.readOpReader.BytesRead) { // We have to re-open as we can't seek backwards if f.readOpReader == nil || f.readOpWriter == nil || dst < int64(f.readOpReader.BytesRead) { // We have to re-open as we can't seek backwards
@@ -426,10 +411,10 @@ func (f *File) Read(p []byte) (n int, err error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if !f.flags.read { if !f.flags.Read {
return -1, os.ErrPermission return -1, os.ErrPermission
} }
@@ -495,10 +480,10 @@ func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if !f.flags.read { if !f.flags.Read {
return -1, os.ErrPermission return -1, os.ErrPermission
} }
@@ -531,10 +516,10 @@ func (f *File) Write(p []byte) (n int, err error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if !f.flags.write { if !f.flags.Write {
return -1, os.ErrPermission return -1, os.ErrPermission
} }
@@ -563,10 +548,10 @@ func (f *File) WriteAt(p []byte, off int64) (n int, err error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if !f.flags.write { if !f.flags.Write {
return -1, os.ErrPermission return -1, os.ErrPermission
} }
@@ -591,10 +576,10 @@ func (f *File) WriteString(s string) (ret int, err error) {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return -1, ErrIsDirectory return -1, config.ErrIsDirectory
} }
if !f.flags.write { if !f.flags.Write {
return -1, os.ErrPermission return -1, os.ErrPermission
} }
@@ -615,10 +600,10 @@ func (f *File) Truncate(size int64) error {
}) })
if f.info.IsDir() { if f.info.IsDir() {
return ErrIsDirectory return config.ErrIsDirectory
} }
if !f.flags.write { if !f.flags.Write {
return os.ErrPermission return os.ErrPermission
} }

View File

@@ -13,7 +13,7 @@ type SQLite struct {
DBPath string DBPath string
Migrations migrate.MigrationSource Migrations migrate.MigrationSource
db *sql.DB DB *sql.DB
} }
func (s *SQLite) Open() error { func (s *SQLite) Open() error {
@@ -31,11 +31,11 @@ func (s *SQLite) Open() error {
// Configure the db // Configure the db
db.SetMaxOpenConns(1) // Prevent "database locked" errors db.SetMaxOpenConns(1) // Prevent "database locked" errors
s.db = db s.DB = db
// Run migrations if set // Run migrations if set
if s.Migrations != nil { if s.Migrations != nil {
if _, err := migrate.Exec(s.db, "sqlite3", s.Migrations, migrate.Up); err != nil { if _, err := migrate.Exec(s.DB, "sqlite3", s.Migrations, migrate.Up); err != nil {
return err return err
} }
} }

14
pkg/cache/cache.go vendored Normal file
View File

@@ -0,0 +1,14 @@
package cache
import "io"
type WriteCache interface {
io.Closer
io.Reader
io.Seeker
io.Writer
Truncate(size int64) error
Size() (int64, error)
Sync() error
}

View File

@@ -17,13 +17,13 @@ func NewCacheFilesystem(
cacheDir string, cacheDir string,
) (afero.Fs, error) { ) (afero.Fs, error) {
switch cacheType { switch cacheType {
case FileSystemCacheTypeMemory: case config.FileSystemCacheTypeMemory:
if pathext.IsRoot(root) { if pathext.IsRoot(root) {
return afero.NewCacheOnReadFs(base, afero.NewMemMapFs(), ttl), nil return afero.NewCacheOnReadFs(base, afero.NewMemMapFs(), ttl), nil
} }
return afero.NewCacheOnReadFs(afero.NewBasePathFs(base, root), afero.NewMemMapFs(), ttl), nil return afero.NewCacheOnReadFs(afero.NewBasePathFs(base, root), afero.NewMemMapFs(), ttl), nil
case FileSystemCacheTypeDir: case config.FileSystemCacheTypeDir:
if err := os.RemoveAll(cacheDir); err != nil { if err := os.RemoveAll(cacheDir); err != nil {
return nil, err return nil, err
} }
@@ -44,6 +44,6 @@ func NewCacheFilesystem(
return afero.NewBasePathFs(base, root), nil return afero.NewBasePathFs(base, root), nil
default: default:
return nil, ErrFileSystemCacheTypeUnsupported return nil, config.ErrFileSystemCacheTypeUnsupported
} }
} }

View File

@@ -6,7 +6,7 @@ import (
"path/filepath" "path/filepath"
"github.com/mattetti/filebuffer" "github.com/mattetti/filebuffer"
"github.com/pojntfx/stfs/internal/fs" "github.com/pojntfx/stfs/pkg/config"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
@@ -45,9 +45,9 @@ func (f filebufferWithSize) Truncate(size int64) error {
func NewCacheWrite( func NewCacheWrite(
root string, root string,
cacheType string, cacheType string,
) (cache fs.WriteCache, cleanup func() error, err error) { ) (cache WriteCache, cleanup func() error, err error) {
switch cacheType { switch cacheType {
case WriteCacheTypeMemory: case config.WriteCacheTypeMemory:
buff := &filebufferWithSize{filebuffer.New([]byte{})} buff := &filebufferWithSize{filebuffer.New([]byte{})}
return buff, func() error { return buff, func() error {
@@ -55,7 +55,7 @@ func NewCacheWrite(
return nil return nil
}, nil }, nil
case WriteCacheTypeFile: case config.WriteCacheTypeFile:
tmpdir := filepath.Join(root, "io") tmpdir := filepath.Join(root, "io")
if err := os.MkdirAll(tmpdir, os.ModePerm); err != nil { if err := os.MkdirAll(tmpdir, os.ModePerm); err != nil {
@@ -71,6 +71,6 @@ func NewCacheWrite(
return os.Remove(f.Name()) return os.Remove(f.Name())
}, nil }, nil
default: default:
return nil, nil, ErrWriteCacheTypeUnsupported return nil, nil, config.ErrWriteCacheTypeUnsupported
} }
} }

View File

@@ -5,7 +5,7 @@ import (
"io/fs" "io/fs"
"os" "os"
"github.com/pojntfx/stfs/internal/persisters" "github.com/pojntfx/stfs/pkg/persisters"
) )
type DriveReaderConfig struct { type DriveReaderConfig struct {

View File

@@ -26,6 +26,14 @@ const (
HeaderEventTypeMove = "move" HeaderEventTypeMove = "move"
HeaderEventTypeRestore = "restore" HeaderEventTypeRestore = "restore"
HeaderEventTypeUpdate = "update" HeaderEventTypeUpdate = "update"
FileSystemNameSTFS = "STFS"
FileSystemCacheTypeMemory = "memory"
FileSystemCacheTypeDir = "dir"
WriteCacheTypeMemory = "memory"
WriteCacheTypeFile = "file"
) )
var ( var (
@@ -36,4 +44,8 @@ var (
KnownEncryptionFormats = []string{NoneKey, EncryptionFormatAgeKey, EncryptionFormatPGPKey} KnownEncryptionFormats = []string{NoneKey, EncryptionFormatAgeKey, EncryptionFormatPGPKey}
KnownSignatureFormats = []string{NoneKey, SignatureFormatMinisignKey, SignatureFormatPGPKey} KnownSignatureFormats = []string{NoneKey, SignatureFormatMinisignKey, SignatureFormatPGPKey}
KnownFileSystemCacheTypes = []string{NoneKey, FileSystemCacheTypeMemory, FileSystemCacheTypeDir}
KnownWriteCacheTypes = []string{WriteCacheTypeMemory, WriteCacheTypeFile}
) )

View File

@@ -32,4 +32,13 @@ var (
ErrSTFSVersionUnsupported = errors.New("STFS version unsupported") ErrSTFSVersionUnsupported = errors.New("STFS version unsupported")
ErrSTFSActionUnsupported = errors.New("STFS action unsupported") ErrSTFSActionUnsupported = errors.New("STFS action unsupported")
ErrNotImplemented = errors.New("not implemented")
ErrIsDirectory = errors.New("is a directory")
ErrFileSystemCacheTypeUnsupported = errors.New("file system cache type unsupported")
ErrFileSystemCacheTypeUnknown = errors.New("file system cache type unknown")
ErrWriteCacheTypeUnsupported = errors.New("write cache type unsupported")
ErrWriteCacheTypeUnknown = errors.New("write cache type unknown")
) )

View File

@@ -3,7 +3,6 @@ package fs
import ( import (
"archive/tar" "archive/tar"
"database/sql" "database/sql"
"errors"
"io" "io"
"os" "os"
"os/user" "os/user"
@@ -13,49 +12,43 @@ import (
"time" "time"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
ifs "github.com/pojntfx/stfs/internal/fs"
"github.com/pojntfx/stfs/internal/logging" "github.com/pojntfx/stfs/internal/logging"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/inventory" "github.com/pojntfx/stfs/pkg/inventory"
"github.com/pojntfx/stfs/pkg/operations" "github.com/pojntfx/stfs/pkg/operations"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var ( type STFS struct {
ErrNotImplemented = errors.New("not implemented")
)
const (
FileSystemNameSTFS = "STFS"
)
type FileSystem struct {
readOps *operations.Operations readOps *operations.Operations
writeOps *operations.Operations writeOps *operations.Operations
metadata config.MetadataConfig metadata config.MetadataConfig
compressionLevel string compressionLevel string
getFileBuffer func() (WriteCache, func() error, error) getFileBuffer func() (cache.WriteCache, func() error, error)
ignoreReadWritePermissions bool ignoreReadWritePermissions bool
onHeader func(hdr *models.Header) onHeader func(hdr *models.Header)
log *logging.JSONLogger log *logging.JSONLogger
} }
func NewFileSystem( func NewSTFS(
readOps *operations.Operations, readOps *operations.Operations,
writeOps *operations.Operations, writeOps *operations.Operations,
metadata config.MetadataConfig, metadata config.MetadataConfig,
compressionLevel string, compressionLevel string,
getFileBuffer func() (WriteCache, func() error, error), getFileBuffer func() (cache.WriteCache, func() error, error),
ignorePermissionFlags bool, ignorePermissionFlags bool,
onHeader func(hdr *models.Header), onHeader func(hdr *models.Header),
log *logging.JSONLogger, log *logging.JSONLogger,
) afero.Fs { ) afero.Fs {
return &FileSystem{ return &STFS{
readOps: readOps, readOps: readOps,
writeOps: writeOps, writeOps: writeOps,
@@ -70,15 +63,15 @@ func NewFileSystem(
} }
} }
func (f *FileSystem) Name() string { func (f *STFS) Name() string {
f.log.Debug("FileSystem.Name", map[string]interface{}{ f.log.Debug("FileSystem.Name", map[string]interface{}{
"name": FileSystemNameSTFS, "name": config.FileSystemNameSTFS,
}) })
return FileSystemNameSTFS return config.FileSystemNameSTFS
} }
func (f *FileSystem) Create(name string) (afero.File, error) { func (f *STFS) Create(name string) (afero.File, error) {
f.log.Debug("FileSystem.Name", map[string]interface{}{ f.log.Debug("FileSystem.Name", map[string]interface{}{
"name": name, "name": name,
}) })
@@ -86,7 +79,7 @@ func (f *FileSystem) Create(name string) (afero.File, error) {
return os.OpenFile(name, os.O_CREATE, 0666) return os.OpenFile(name, os.O_CREATE, 0666)
} }
func (f *FileSystem) mknode(dir bool, name string, perm os.FileMode) error { func (f *STFS) mknode(dir bool, name string, perm os.FileMode) error {
f.log.Trace("FileSystem.mknode", map[string]interface{}{ f.log.Trace("FileSystem.mknode", map[string]interface{}{
"name": name, "name": name,
"perm": perm, "perm": perm,
@@ -163,7 +156,7 @@ func (f *FileSystem) mknode(dir bool, name string, perm os.FileMode) error {
return nil return nil
} }
func (f *FileSystem) Mkdir(name string, perm os.FileMode) error { func (f *STFS) Mkdir(name string, perm os.FileMode) error {
f.log.Debug("FileSystem.Mkdir", map[string]interface{}{ f.log.Debug("FileSystem.Mkdir", map[string]interface{}{
"name": name, "name": name,
"perm": perm, "perm": perm,
@@ -172,7 +165,7 @@ func (f *FileSystem) Mkdir(name string, perm os.FileMode) error {
return f.mknode(true, name, perm) return f.mknode(true, name, perm)
} }
func (f *FileSystem) MkdirAll(path string, perm os.FileMode) error { func (f *STFS) MkdirAll(path string, perm os.FileMode) error {
f.log.Debug("FileSystem.MkdirAll", map[string]interface{}{ f.log.Debug("FileSystem.MkdirAll", map[string]interface{}{
"path": path, "path": path,
"perm": perm, "perm": perm,
@@ -196,7 +189,7 @@ func (f *FileSystem) MkdirAll(path string, perm os.FileMode) error {
return nil return nil
} }
func (f *FileSystem) Open(name string) (afero.File, error) { func (f *STFS) Open(name string) (afero.File, error) {
f.log.Debug("FileSystem.Open", map[string]interface{}{ f.log.Debug("FileSystem.Open", map[string]interface{}{
"name": name, "name": name,
}) })
@@ -204,38 +197,38 @@ func (f *FileSystem) Open(name string) (afero.File, error) {
return f.OpenFile(name, os.O_RDWR, os.ModePerm) return f.OpenFile(name, os.O_RDWR, os.ModePerm)
} }
func (f *FileSystem) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { func (f *STFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
f.log.Debug("FileSystem.OpenFile", map[string]interface{}{ f.log.Debug("FileSystem.OpenFile", map[string]interface{}{
"name": name, "name": name,
"flag": flag, "flag": flag,
"perm": perm, "perm": perm,
}) })
flags := &FileFlags{} flags := &ifs.FileFlags{}
if flag&os.O_RDONLY != 0 { if flag&os.O_RDONLY != 0 {
flags.read = true flags.Read = true
} }
if flag&os.O_WRONLY != 0 { if flag&os.O_WRONLY != 0 {
flags.write = true flags.Write = true
} }
if flag&os.O_RDWR != 0 { if flag&os.O_RDWR != 0 {
flags.read = true flags.Read = true
flags.write = true flags.Write = true
} }
if f.ignoreReadWritePermissions { if f.ignoreReadWritePermissions {
flags.read = true flags.Read = true
flags.write = true flags.Write = true
} }
if flag&os.O_APPEND != 0 { if flag&os.O_APPEND != 0 {
flags.append = true flags.Append = true
} }
if flag&os.O_TRUNC != 0 { if flag&os.O_TRUNC != 0 {
flags.truncate = true flags.Truncate = true
} }
hdr, err := inventory.Stat( hdr, err := inventory.Stat(
@@ -271,7 +264,7 @@ func (f *FileSystem) OpenFile(name string, flag int, perm os.FileMode) (afero.Fi
} }
} }
return NewFile( return ifs.NewFile(
f.readOps, f.readOps,
f.writeOps, f.writeOps,
@@ -285,14 +278,14 @@ func (f *FileSystem) OpenFile(name string, flag int, perm os.FileMode) (afero.Fi
f.getFileBuffer, f.getFileBuffer,
path.Base(hdr.Name), path.Base(hdr.Name),
NewFileInfoFromTarHeader(hdr, f.log), ifs.NewFileInfoFromTarHeader(hdr, f.log),
f.onHeader, f.onHeader,
f.log, f.log,
), nil ), nil
} }
func (f *FileSystem) Remove(name string) error { func (f *STFS) Remove(name string) error {
f.log.Debug("FileSystem.Remove", map[string]interface{}{ f.log.Debug("FileSystem.Remove", map[string]interface{}{
"name": name, "name": name,
}) })
@@ -300,7 +293,7 @@ func (f *FileSystem) Remove(name string) error {
return f.writeOps.Delete(name) return f.writeOps.Delete(name)
} }
func (f *FileSystem) RemoveAll(path string) error { func (f *STFS) RemoveAll(path string) error {
f.log.Debug("FileSystem.RemoveAll", map[string]interface{}{ f.log.Debug("FileSystem.RemoveAll", map[string]interface{}{
"path": path, "path": path,
}) })
@@ -308,7 +301,7 @@ func (f *FileSystem) RemoveAll(path string) error {
return f.writeOps.Delete(path) return f.writeOps.Delete(path)
} }
func (f *FileSystem) Rename(oldname, newname string) error { func (f *STFS) Rename(oldname, newname string) error {
f.log.Debug("FileSystem.Rename", map[string]interface{}{ f.log.Debug("FileSystem.Rename", map[string]interface{}{
"oldname": oldname, "oldname": oldname,
"newname": newname, "newname": newname,
@@ -317,7 +310,7 @@ func (f *FileSystem) Rename(oldname, newname string) error {
return f.writeOps.Move(oldname, newname) return f.writeOps.Move(oldname, newname)
} }
func (f *FileSystem) Stat(name string) (os.FileInfo, error) { func (f *STFS) Stat(name string) (os.FileInfo, error) {
f.log.Debug("FileSystem.Stat", map[string]interface{}{ f.log.Debug("FileSystem.Stat", map[string]interface{}{
"name": name, "name": name,
}) })
@@ -337,10 +330,10 @@ func (f *FileSystem) Stat(name string) (os.FileInfo, error) {
return nil, err return nil, err
} }
return NewFileInfoFromTarHeader(hdr, f.log), nil return ifs.NewFileInfoFromTarHeader(hdr, f.log), nil
} }
func (f *FileSystem) updateMetadata(hdr *tar.Header) error { func (f *STFS) updateMetadata(hdr *tar.Header) error {
done := false done := false
if _, err := f.writeOps.Update( if _, err := f.writeOps.Update(
func() (config.FileConfig, error) { func() (config.FileConfig, error) {
@@ -367,7 +360,7 @@ func (f *FileSystem) updateMetadata(hdr *tar.Header) error {
return nil return nil
} }
func (f *FileSystem) Chmod(name string, mode os.FileMode) error { func (f *STFS) Chmod(name string, mode os.FileMode) error {
f.log.Debug("FileSystem.Chmod", map[string]interface{}{ f.log.Debug("FileSystem.Chmod", map[string]interface{}{
"name": mode, "name": mode,
}) })
@@ -392,7 +385,7 @@ func (f *FileSystem) Chmod(name string, mode os.FileMode) error {
return f.updateMetadata(hdr) return f.updateMetadata(hdr)
} }
func (f *FileSystem) Chown(name string, uid, gid int) error { func (f *STFS) Chown(name string, uid, gid int) error {
f.log.Debug("FileSystem.Chown", map[string]interface{}{ f.log.Debug("FileSystem.Chown", map[string]interface{}{
"name": name, "name": name,
"uid": uid, "uid": uid,
@@ -420,7 +413,7 @@ func (f *FileSystem) Chown(name string, uid, gid int) error {
return f.updateMetadata(hdr) return f.updateMetadata(hdr)
} }
func (f *FileSystem) Chtimes(name string, atime time.Time, mtime time.Time) error { func (f *STFS) Chtimes(name string, atime time.Time, mtime time.Time) error {
f.log.Debug("FileSystem.Chtimes", map[string]interface{}{ f.log.Debug("FileSystem.Chtimes", map[string]interface{}{
"name": name, "name": name,
"atime": atime, "atime": atime,
@@ -448,27 +441,27 @@ func (f *FileSystem) Chtimes(name string, atime time.Time, mtime time.Time) erro
return f.updateMetadata(hdr) return f.updateMetadata(hdr)
} }
func (f *FileSystem) LstatIfPossible(name string) (os.FileInfo, bool, error) { func (f *STFS) LstatIfPossible(name string) (os.FileInfo, bool, error) {
f.log.Debug("FileSystem.LstatIfPossible", map[string]interface{}{ f.log.Debug("FileSystem.LstatIfPossible", map[string]interface{}{
"name": name, "name": name,
}) })
return nil, false, ErrNotImplemented return nil, false, config.ErrNotImplemented
} }
func (f *FileSystem) SymlinkIfPossible(oldname, newname string) error { func (f *STFS) SymlinkIfPossible(oldname, newname string) error {
f.log.Debug("FileSystem.SymlinkIfPossible", map[string]interface{}{ f.log.Debug("FileSystem.SymlinkIfPossible", map[string]interface{}{
"oldname": oldname, "oldname": oldname,
"newname": newname, "newname": newname,
}) })
return ErrNotImplemented return config.ErrNotImplemented
} }
func (f *FileSystem) ReadlinkIfPossible(name string) (string, error) { func (f *STFS) ReadlinkIfPossible(name string) (string, error) {
f.log.Debug("FileSystem.ReadlinkIfPossible", map[string]interface{}{ f.log.Debug("FileSystem.ReadlinkIfPossible", map[string]interface{}{
"name": name, "name": name,
}) })
return "", ErrNotImplemented return "", config.ErrNotImplemented
} }

View File

@@ -14,6 +14,7 @@ import (
"github.com/pojntfx/stfs/internal/db/sqlite/migrations/metadata" "github.com/pojntfx/stfs/internal/db/sqlite/migrations/metadata"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/pathext" "github.com/pojntfx/stfs/internal/pathext"
ipersisters "github.com/pojntfx/stfs/internal/persisters"
migrate "github.com/rubenv/sql-migrate" migrate "github.com/rubenv/sql-migrate"
"github.com/volatiletech/sqlboiler/v4/boil" "github.com/volatiletech/sqlboiler/v4/boil"
"github.com/volatiletech/sqlboiler/v4/queries" "github.com/volatiletech/sqlboiler/v4/queries"
@@ -25,12 +26,12 @@ type depth struct {
} }
type MetadataPersister struct { type MetadataPersister struct {
*SQLite *ipersisters.SQLite
} }
func NewMetadataPersister(dbPath string) *MetadataPersister { func NewMetadataPersister(dbPath string) *MetadataPersister {
return &MetadataPersister{ return &MetadataPersister{
&SQLite{ &ipersisters.SQLite{
DBPath: dbPath, DBPath: dbPath,
Migrations: migrate.AssetMigrationSource{ Migrations: migrate.AssetMigrationSource{
Asset: metadata.Asset, Asset: metadata.Asset,
@@ -44,12 +45,12 @@ func NewMetadataPersister(dbPath string) *MetadataPersister {
func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Header) error { func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Header) error {
hdr := *dbhdr hdr := *dbhdr
if _, err := models.FindHeader(ctx, p.db, hdr.Name, models.HeaderColumns.Name); err != nil { if _, err := models.FindHeader(ctx, p.DB, hdr.Name, models.HeaderColumns.Name); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
if _, err := models.FindHeader(ctx, p.db, p.withRelativeRoot(ctx, hdr.Name), models.HeaderColumns.Name); err == nil { if _, err := models.FindHeader(ctx, p.DB, p.withRelativeRoot(ctx, hdr.Name), models.HeaderColumns.Name); err == nil {
hdr.Name = p.withRelativeRoot(ctx, hdr.Name) hdr.Name = p.withRelativeRoot(ctx, hdr.Name)
} else { } else {
if err := hdr.Insert(ctx, p.db, boil.Infer()); err != nil { if err := hdr.Insert(ctx, p.DB, boil.Infer()); err != nil {
return err return err
} }
@@ -61,7 +62,7 @@ func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Head
} }
} }
if _, err := hdr.Update(ctx, p.db, boil.Infer()); err != nil { if _, err := hdr.Update(ctx, p.DB, boil.Infer()); err != nil {
return err return err
} }
@@ -69,12 +70,12 @@ func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Head
} }
func (p *MetadataPersister) UpdateHeaderMetadata(ctx context.Context, dbhdr *models.Header) error { func (p *MetadataPersister) UpdateHeaderMetadata(ctx context.Context, dbhdr *models.Header) error {
if _, err := dbhdr.Update(ctx, p.db, boil.Infer()); err != nil { if _, err := dbhdr.Update(ctx, p.DB, boil.Infer()); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
hdr := *dbhdr hdr := *dbhdr
hdr.Name = p.withRelativeRoot(ctx, dbhdr.Name) hdr.Name = p.withRelativeRoot(ctx, dbhdr.Name)
if _, err := hdr.Update(ctx, p.db, boil.Infer()); err != nil { if _, err := hdr.Update(ctx, p.DB, boil.Infer()); err != nil {
return err return err
} }
} else { } else {
@@ -100,7 +101,7 @@ func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newN
lastknownrecord, lastknownrecord,
lastknownblock, lastknownblock,
oldName, oldName,
).ExecContext(ctx, p.db) ).ExecContext(ctx, p.DB)
if err != nil { if err != nil {
return err return err
} }
@@ -124,7 +125,7 @@ func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newN
lastknownrecord, lastknownrecord,
lastknownblock, lastknownblock,
oldName, oldName,
).ExecContext(ctx, p.db); err != nil { ).ExecContext(ctx, p.DB); err != nil {
return err return err
} }
} }
@@ -135,20 +136,20 @@ func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newN
func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice, error) { func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice, error) {
return models.Headers( return models.Headers(
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).All(ctx, p.db) ).All(ctx, p.DB)
} }
func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models.Header, error) { func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models.Header, error) {
hdr, err := models.Headers( hdr, err := models.Headers(
qm.Where(models.HeaderColumns.Name+" = ?", name), qm.Where(models.HeaderColumns.Name+" = ?", name),
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).One(ctx, p.db) ).One(ctx, p.DB)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
hdr, err = models.Headers( hdr, err = models.Headers(
qm.Where(models.HeaderColumns.Name+" = ?", p.withRelativeRoot(ctx, name)), qm.Where(models.HeaderColumns.Name+" = ?", p.withRelativeRoot(ctx, name)),
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).One(ctx, p.db) ).One(ctx, p.DB)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -164,7 +165,7 @@ func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string)
headers, err := models.Headers( headers, err := models.Headers(
qm.Where(models.HeaderColumns.Name+" like ?", strings.TrimSuffix(name, "/")+"/%"), // Prevent double trailing slashes qm.Where(models.HeaderColumns.Name+" like ?", strings.TrimSuffix(name, "/")+"/%"), // Prevent double trailing slashes
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).All(ctx, p.db) ).All(ctx, p.DB)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -173,7 +174,7 @@ func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string)
headers, err = models.Headers( headers, err = models.Headers(
qm.Where(models.HeaderColumns.Name+" like ?", p.withRelativeRoot(ctx, strings.TrimSuffix(name, "/")+"/%")), // Prevent double trailing slashes qm.Where(models.HeaderColumns.Name+" like ?", p.withRelativeRoot(ctx, strings.TrimSuffix(name, "/")+"/%")), // Prevent double trailing slashes
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).All(ctx, p.db) ).All(ctx, p.DB)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -201,7 +202,7 @@ func (p *MetadataPersister) GetRootPath(ctx context.Context) (string, error) {
models.TableNames.Headers, models.TableNames.Headers,
models.HeaderColumns.Deleted, models.HeaderColumns.Deleted,
), ),
).Bind(ctx, p.db, &root); err != nil { ).Bind(ctx, p.DB, &root); err != nil {
return "", err return "", err
} }
@@ -226,7 +227,7 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
models.TableNames.Headers, models.TableNames.Headers,
models.HeaderColumns.Deleted, models.HeaderColumns.Deleted,
), ),
).Bind(ctx, p.db, &depth); err != nil { ).Bind(ctx, p.DB, &depth); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return headers, nil return headers, nil
} }
@@ -291,7 +292,7 @@ where %v like ?
rootDepth, rootDepth,
rootDepth+1, rootDepth+1,
limit+1, // +1 to accomodate the parent directory if it exists limit+1, // +1 to accomodate the parent directory if it exists
).Bind(ctx, p.db, &headers); err != nil { ).Bind(ctx, p.DB, &headers); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return headers, nil return headers, nil
} }
@@ -307,7 +308,7 @@ where %v like ?
prefix+"%", prefix+"%",
rootDepth, rootDepth,
rootDepth+1, rootDepth+1,
).Bind(ctx, p.db, &headers); err != nil { ).Bind(ctx, p.DB, &headers); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return headers, nil return headers, nil
} }
@@ -346,10 +347,10 @@ where %v like ?
} }
func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*models.Header, error) { func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*models.Header, error) {
hdr, err := models.FindHeader(ctx, p.db, name) hdr, err := models.FindHeader(ctx, p.DB, name)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
hdr, err = models.FindHeader(ctx, p.db, p.withRelativeRoot(ctx, name)) hdr, err = models.FindHeader(ctx, p.DB, p.withRelativeRoot(ctx, name))
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, err return nil, err
} }
@@ -366,7 +367,7 @@ func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastk
hdr.Lastknownrecord = lastknownrecord hdr.Lastknownrecord = lastknownrecord
hdr.Lastknownblock = lastknownblock hdr.Lastknownblock = lastknownblock
if _, err := hdr.Update(ctx, p.db, boil.Infer()); err != nil { if _, err := hdr.Update(ctx, p.DB, boil.Infer()); err != nil {
return nil, err return nil, err
} }
@@ -385,7 +386,7 @@ func (p *MetadataPersister) GetLastIndexedRecordAndBlock(ctx context.Context, re
models.TableNames.Headers, models.TableNames.Headers,
), ),
recordSize, recordSize,
).Bind(ctx, p.db, &header); err != nil { ).Bind(ctx, p.DB, &header); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return 0, 0, nil return 0, 0, nil
} }
@@ -397,7 +398,7 @@ func (p *MetadataPersister) GetLastIndexedRecordAndBlock(ctx context.Context, re
} }
func (p *MetadataPersister) PurgeAllHeaders(ctx context.Context) error { func (p *MetadataPersister) PurgeAllHeaders(ctx context.Context) error {
if _, err := models.Headers().DeleteAll(ctx, p.db); err != nil { if _, err := models.Headers().DeleteAll(ctx, p.DB); err != nil {
return err return err
} }
@@ -408,7 +409,7 @@ func (p *MetadataPersister) headerExistsExact(ctx context.Context, name string)
exists, err := models.Headers( exists, err := models.Headers(
qm.Where(models.HeaderColumns.Name+" = ?", name), qm.Where(models.HeaderColumns.Name+" = ?", name),
qm.Where(models.HeaderColumns.Deleted+" != 1"), qm.Where(models.HeaderColumns.Deleted+" != 1"),
).Exists(ctx, p.db) ).Exists(ctx, p.DB)
if err != nil { if err != nil {
return err return err
} }

View File

@@ -14,10 +14,10 @@ import (
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata" models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/ioext" "github.com/pojntfx/stfs/internal/ioext"
"github.com/pojntfx/stfs/internal/mtio" "github.com/pojntfx/stfs/internal/mtio"
"github.com/pojntfx/stfs/internal/persisters"
"github.com/pojntfx/stfs/internal/records" "github.com/pojntfx/stfs/internal/records"
"github.com/pojntfx/stfs/internal/suffix" "github.com/pojntfx/stfs/internal/suffix"
"github.com/pojntfx/stfs/pkg/config" "github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/persisters"
) )
func Index( func Index(