feat: Add interchangeable write buffers based on io.File and filebuffer.Buffer

internal/cache/check.go

@@ -1,16 +1,32 @@
 package cache
 
-func CheckCacheType(cacheType string) error {
+func CheckFileSystemCacheType(cacheType string) error {
 	cacheTypeIsKnown := false
 
-	for _, candidate := range KnownCacheTypes {
+	for _, candidate := range KnownFileSystemCacheTypes {
 		if cacheType == candidate {
 			cacheTypeIsKnown = true
 		}
 	}
 
 	if !cacheTypeIsKnown {
-		return ErrCacheTypeUnknown
+		return ErrFileSystemCacheTypeUnknown
+	}
+
+	return nil
+}
+
+func CheckWriteCacheType(cacheType string) error {
+	cacheTypeIsKnown := false
+
+	for _, candidate := range KnownWriteCacheTypes {
+		if cacheType == candidate {
+			cacheTypeIsKnown = true
+		}
+	}
+
+	if !cacheTypeIsKnown {
+		return ErrWriteCacheTypeUnknown
 	}
 
 	return nil

internal/cache/constants.go

@@ -5,10 +5,15 @@ import (
 )
 
 const (
-	CacheTypeMemory = "memory"
-	CacheTypeDir    = "dir"
+	FileSystemCacheTypeMemory = "memory"
+	FileSystemCacheTypeDir    = "dir"
+
+	WriteCacheTypeMemory = "memory"
+	WriteCacheTypeFile   = "file"
 )
 
 var (
-	KnownCacheTypes = []string{config.NoneKey, CacheTypeMemory, CacheTypeDir}
+	KnownFileSystemCacheTypes = []string{config.NoneKey, FileSystemCacheTypeMemory, FileSystemCacheTypeDir}
+
+	KnownWriteCacheTypes = []string{WriteCacheTypeMemory, WriteCacheTypeFile}
 )

internal/cache/error.go

@@ -3,6 +3,9 @@ package cache
 import "errors"
 
 var (
-	ErrCacheTypeUnsupported = errors.New("cache type unsupported")
-	ErrCacheTypeUnknown     = errors.New("cache type unknown")
+	ErrFileSystemCacheTypeUnsupported = errors.New("file system cache type unsupported")
+	ErrFileSystemCacheTypeUnknown     = errors.New("file system cache type unknown")
+
+	ErrWriteCacheTypeUnsupported = errors.New("write cache type unsupported")
+	ErrWriteCacheTypeUnknown     = errors.New("write cache type unknown")
 )

internal/cache/filesystem.go

@@ -9,7 +9,7 @@ import (
 	"github.com/spf13/afero"
 )
 
-func Cache(
+func NewCacheFilesystem(
 	base afero.Fs,
 	root string,
 	cacheType string,
@@ -17,13 +17,13 @@ func Cache(
 	cacheDir string,
 ) (afero.Fs, error) {
 	switch cacheType {
-	case CacheTypeMemory:
+	case FileSystemCacheTypeMemory:
 		if pathext.IsRoot(root) {
 			return afero.NewCacheOnReadFs(base, afero.NewMemMapFs(), ttl), nil
 		}
 
 		return afero.NewCacheOnReadFs(afero.NewBasePathFs(base, root), afero.NewMemMapFs(), ttl), nil
-	case CacheTypeDir:
+	case FileSystemCacheTypeDir:
 		if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil {
 			return nil, err
 		}
@@ -40,6 +40,6 @@ func Cache(
 
 		return afero.NewBasePathFs(base, root), nil
 	default:
-		return nil, ErrCacheTypeUnsupported
+		return nil, ErrFileSystemCacheTypeUnsupported
 	}
 }

internal/cache/write.go

@@ -0,0 +1,76 @@
+package cache
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mattetti/filebuffer"
+	"github.com/pojntfx/stfs/internal/fs"
+	"github.com/spf13/afero"
+)
+
+type fileWithSize struct {
+	afero.File
+}
+
+func (f fileWithSize) Size() (int64, error) {
+	info, err := f.Stat()
+	if err != nil {
+		return -1, err
+	}
+
+	return info.Size(), nil
+}
+
+type filebufferWithSize struct {
+	*filebuffer.Buffer
+}
+
+func (f filebufferWithSize) Size() (int64, error) {
+	return int64(f.Buff.Len()), nil
+}
+
+func (f filebufferWithSize) Sync() error {
+	// No need to sync a in-memory buffer
+	return nil
+}
+
+func (f filebufferWithSize) Truncate(size int64) error {
+	f.Buff.Truncate(int(size))
+
+	return nil
+}
+
+func NewCacheWrite(
+	root string,
+	cacheType string,
+) (cache fs.WriteCache, cleanup func() error, err error) {
+	switch cacheType {
+	case WriteCacheTypeMemory:
+		buff := &filebufferWithSize{filebuffer.New([]byte{})}
+
+		return buff, func() error {
+			buff = nil
+
+			return nil
+		}, nil
+	case WriteCacheTypeFile:
+		tmpdir := filepath.Join(root, "io")
+
+		if err := os.MkdirAll(tmpdir, os.ModePerm); err != nil {
+			return nil, nil, err
+		}
+
+		f, err := ioutil.TempFile(tmpdir, "*")
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return fileWithSize{f}, func() error {
+			return os.Remove(filepath.Join(tmpdir, f.Name()))
+		}, nil
+	default:
+		return nil, nil, ErrWriteCacheTypeUnsupported
+	}
+}

internal/fs/file.go

@@ -21,6 +21,17 @@ var (
 	ErrIsDirectory = errors.New("is a directory")
 )
 
+type WriteCache interface {
+	io.Closer
+	io.Reader
+	io.Seeker
+	io.Writer
+
+	Truncate(size int64) error
+	Size() (int64, error)
+	Sync() error
+}
+
 type File struct {
 	afero.File
 
@@ -33,7 +44,7 @@ type File struct {
 	link string
 
 	compressionLevel string
-	getFileBuffer    func() (afero.File, func() error, error)
+	getFileBuffer    func() (WriteCache, func() error, error)
 
 	name string
 	info os.FileInfo
@@ -44,7 +55,8 @@ type File struct {
 	readOpWriter io.WriteCloser
 
 	// TODO: Find a non-in-memory method to do this
-	writeBuf      afero.File
+	writeBuf      WriteCache
+	asdf          afero.File
 	cleanWriteBuf func() error
 
 	onHeader func(hdr *models.Header)
@@ -60,7 +72,7 @@ func NewFile(
 	link string,
 
 	compressionLevel string,
-	getFileBuffer func() (afero.File, func() error, error),
+	getFileBuffer func() (WriteCache, func() error, error),
 
 	name string,
 	info os.FileInfo,
@@ -154,14 +166,14 @@ func (f *File) syncWithoutLocking() error {
 				}
 				done = true
 
-				stat, err := f.writeBuf.Stat()
+				size, err := f.writeBuf.Size()
 				if err != nil {
 					return config.FileConfig{}, err
 				}
 
 				f.info = &FileInfo{
 					name:    f.info.Name(),
-					size:    stat.Size(),
+					size:    size,
 					mode:    f.info.Mode(),
 					modTime: f.info.ModTime(),
 					isDir:   f.info.IsDir(),

internal/fs/filesystem.go

@@ -31,7 +31,7 @@ type FileSystem struct {
 	metadata config.MetadataConfig
 
 	compressionLevel string
-	getFileBuffer    func() (afero.File, func() error, error)
+	getFileBuffer    func() (WriteCache, func() error, error)
 
 	onHeader func(hdr *models.Header)
 }
@@ -43,7 +43,7 @@ func NewFileSystem(
 	metadata config.MetadataConfig,
 
 	compressionLevel string,
-	getFileBuffer func() (afero.File, func() error, error),
+	getFileBuffer func() (WriteCache, func() error, error),
 
 	onHeader func(hdr *models.Header),
 ) afero.Fs {