mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add interchangeable write buffers based on io.File and filebuffer.Buffer

Browse Source
This commit is contained in:
Felix Pojtinger
2021-12-24 16:55:14 +01:00
parent 35ebd268a2
commit c8ff51d2c4
9 changed files with 159 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/stfs/cmd/serve_ftp.go
+17 -22
View File
@@ -3,7 +3,6 @@ package cmd
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
@@ -19,7 +18,6 @@ import (
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/operations"
"github.com/pojntfx/stfs/pkg/tape"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@@ -32,6 +30,8 @@ const (
signatureIdentityFlag = "signature-identity"
signaturePasswordFlag = "signature-password"
signatureRecipientFlag = "signature-recipient"
cacheWriteFlag = "cache-write-type"
)
var (
@@ -47,7 +47,11 @@ var serveFTPCmd = &cobra.Command{
return err
}
if err := cache.CheckCacheType(viper.GetString(cacheFlag)); err != nil {
if err := cache.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil {
return err
}
if err := cache.CheckWriteCacheType(viper.GetString(cacheWriteFlag)); err != nil {
return err
}
@@ -193,32 +197,22 @@ var serveFTPCmd = &cobra.Command{
},
viper.GetString(compressionLevelFlag),
func() (afero.File, func() error, error) {
tmpdir := filepath.Join(viper.GetString(cacheDirFlag), "io")
if err := os.MkdirAll(tmpdir, os.ModePerm); err != nil {
return nil, nil, err
}
f, err := ioutil.TempFile(tmpdir, "*")
if err != nil {
return nil, nil, err
}
return f, func() error {
return os.Remove(filepath.Join(tmpdir, f.Name()))
}, nil
func() (sfs.WriteCache, func() error, error) {
return cache.NewCacheWrite(
filepath.Join(viper.GetString(cacheDirFlag), "write"),
viper.GetString(cacheWriteFlag),
)
},
logger.PrintHeader,
)
fs, err := cache.Cache(
fs, err := cache.NewCacheFilesystem(
stfs,
root,
viper.GetString(cacheFlag),
viper.GetString(cacheFileSystemFlag),
viper.GetDuration(cacheDurationFlag),
filepath.Join(viper.GetString(cacheDirFlag), "cache"),
filepath.Join(viper.GetString(cacheDirFlag), "filesystem"),
)
if err != nil {
return err
@@ -256,7 +250,8 @@ func init() {
serveFTPCmd.PersistentFlags().StringP(compressionLevelFlag, "l", config.CompressionLevelBalanced, fmt.Sprintf("Compression level to use (default %v, available are %v)", config.CompressionLevelBalanced, config.KnownCompressionLevels))
serveFTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address")
serveFTPCmd.PersistentFlags().StringP(cacheFlag, "n", config.NoneKey, fmt.Sprintf("Cache to use (default %v, available are %v)", config.NoneKey, cache.KnownCacheTypes))
serveFTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, cache.KnownFileSystemCacheTypes))
serveFTPCmd.PersistentFlags().StringP(cacheWriteFlag, "q", cache.WriteCacheTypeFile, fmt.Sprintf("Write cache to use (default %v, available are %v)", cache.WriteCacheTypeFile, cache.KnownWriteCacheTypes))
serveFTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated")
serveFTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled")
cmd/stfs/cmd/serve_http.go
+11 -28
View File
@@ -3,10 +3,8 @@ package cmd
import (
"context"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"time"
@@ -25,10 +23,10 @@ import (
)
const (
laddrFlag = "laddr"
cacheFlag = "cache"
cacheDirFlag = "cache-dir"
cacheDurationFlag = "cache-duration"
laddrFlag = "laddr"
cacheFileSystemFlag = "cache-filesystem-type"
cacheDirFlag = "cache-dir"
cacheDurationFlag = "cache-duration"
)
var serveHTTPCmd = &cobra.Command{
@@ -40,7 +38,7 @@ var serveHTTPCmd = &cobra.Command{
return err
}
if err := cache.CheckCacheType(viper.GetString(cacheFlag)); err != nil {
if err := cache.CheckFileSystemCacheType(viper.GetString(cacheFileSystemFlag)); err != nil {
return err
}
@@ -127,33 +125,18 @@ var serveHTTPCmd = &cobra.Command{
Metadata: metadataPersister,
},
"", // We never write
func() (afero.File, func() error, error) {
tmpdir := filepath.Join(viper.GetString(cacheDirFlag), "io")
if err := os.MkdirAll(tmpdir, os.ModePerm); err != nil {
return nil, nil, err
}
f, err := ioutil.TempFile(tmpdir, "*")
if err != nil {
return nil, nil, err
}
return f, func() error {
return os.Remove(filepath.Join(tmpdir, f.Name()))
}, nil
},
"", // We never write
nil, // We never write
logger.PrintHeader,
)
fs, err := cache.Cache(
fs, err := cache.NewCacheFilesystem(
stfs,
root,
viper.GetString(cacheFlag),
viper.GetString(cacheFileSystemFlag),
viper.GetDuration(cacheDurationFlag),
filepath.Join(viper.GetString(cacheDirFlag), "cache"),
filepath.Join(viper.GetString(cacheDirFlag), "filesystem"),
)
if err != nil {
return err
@@ -178,7 +161,7 @@ func init() {
serveHTTPCmd.PersistentFlags().StringP(passwordFlag, "p", "", "Password for the private key")
serveHTTPCmd.PersistentFlags().StringP(recipientFlag, "r", "", "Path to the public key to verify with")
serveHTTPCmd.PersistentFlags().StringP(laddrFlag, "a", "localhost:1337", "Listen address")
serveHTTPCmd.PersistentFlags().StringP(cacheFlag, "n", config.NoneKey, fmt.Sprintf("Cache to use (default %v, available are %v)", config.NoneKey, cache.KnownCacheTypes))
serveHTTPCmd.PersistentFlags().StringP(cacheFileSystemFlag, "n", config.NoneKey, fmt.Sprintf("File system cache to use (default %v, available are %v)", config.NoneKey, cache.KnownFileSystemCacheTypes))
serveHTTPCmd.PersistentFlags().DurationP(cacheDurationFlag, "u", time.Hour, "Duration until cache is invalidated")
serveHTTPCmd.PersistentFlags().StringP(cacheDirFlag, "w", cacheDir, "Directory to use if dir cache is enabled")