mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Prevent cleaning initialization path before creating root directory, add initialization tests

Browse Source
This commit is contained in:
Felicitas Pojtinger
2022-01-10 23:28:45 +01:00
parent 7b83d3bc20
commit 2487479433
11 changed files with 231 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/stfs/cmd/operation_archive.go
View File

@@ -157,6 +157,7 @@ var operationArchiveCmd = &cobra.Command{
},
viper.GetString(compressionLevelFlag),
viper.GetBool(overwriteFlag),
false,
); err != nil {
return err
}

1
cmd/stfs/cmd/recovery_index.go
View File

@@ -92,6 +92,7 @@ var recoveryIndexCmd = &cobra.Command{
viper.GetInt(recordFlag),
viper.GetInt(blockFlag),
viper.GetBool(overwriteFlag),
false,
0,
func(hdr *tar.Header, i int) error {

2
pkg/config/config.go
View File

@@ -63,7 +63,7 @@ type Header struct {
}
type MetadataPersister interface {
UpsertHeader(ctx context.Context, dbhdr *Header) error
UpsertHeader(ctx context.Context, dbhdr *Header, initializing bool) error
UpdateHeaderMetadata(ctx context.Context, dbhdr *Header) error
MoveHeader(ctx context.Context, oldName string, newName string, lastknownrecord, lastknownblock int64) error
GetHeaders(ctx context.Context) ([]*Header, error)

14
pkg/fs/filesystem.go
View File

@@ -92,7 +92,7 @@ func (f *STFS) Create(name string) (afero.File, error) {
return f.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666)
}
func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string) error {
func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string, initializing bool) error {
f.log.Trace("FileSystem.mknodeWithoutLocking", map[string]interface{}{
"name": name,
"perm": perm,
@@ -169,6 +169,7 @@ func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, ove
},
f.compressionLevel,
overwrite,
initializing,
); err != nil {
return err
}
@@ -198,7 +199,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin
return "", os.ErrPermission
}
if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, ""); err != nil {
if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, "", true); err != nil {
return "", err
}
@@ -223,6 +224,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin
0,
0,
true,
true,
0,
func(hdr *tar.Header, i int) error {
@@ -259,7 +261,7 @@ func (f *STFS) Mkdir(name string, perm os.FileMode) error {
f.ioLock.Lock()
defer f.ioLock.Unlock()
return f.mknodeWithoutLocking(true, name, perm, false, "")
return f.mknodeWithoutLocking(true, name, perm, false, "", false)
}
func (f *STFS) MkdirAll(path string, perm os.FileMode) error {
@@ -285,7 +287,7 @@ func (f *STFS) MkdirAll(path string, perm os.FileMode) error {
currentPath = filepath.Join(currentPath, part)
}
if err := f.mknodeWithoutLocking(true, currentPath, perm, false, ""); err != nil {
if err := f.mknodeWithoutLocking(true, currentPath, perm, false, "", false); err != nil {
return err
}
}
@@ -346,7 +348,7 @@ func (f *STFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, er
if err != nil {
if err == sql.ErrNoRows {
if !f.readOnly && flag&os.O_CREATE != 0 && flag&os.O_EXCL == 0 {
if err := f.mknodeWithoutLocking(false, name, perm, false, ""); err != nil {
if err := f.mknodeWithoutLocking(false, name, perm, false, "", false); err != nil {
return nil, err
}
@@ -648,7 +650,7 @@ func (f *STFS) SymlinkIfPossible(oldname, newname string) error {
f.ioLock.Lock()
defer f.ioLock.Unlock()
return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname)
return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname, false)
}
func (f *STFS) ReadlinkIfPossible(name string) (string, error) {

284
pkg/fs/filesystem_test.go
View File

@@ -319,6 +319,8 @@ func createSTFS(
fileSystemCache string,
fileSystemCacheDir string,
fileSystemCacheDuration time.Duration,
initialize bool,
) (afero.Fs, error) {
tm := tape.NewTapeManager(
drive,
@@ -411,10 +413,14 @@ func createSTFS(
jsonLogger,
)
root, err := stfs.Initialize("/", os.ModePerm)
root := ""
if initialize {
var err error
root, err = stfs.Initialize("/", os.ModePerm)
if err != nil {
return nil, err
}
}
return cache.NewCacheFilesystem(
stfs,
@@ -425,7 +431,7 @@ func createSTFS(
)
}
func createFss() ([]fsConfig, error) {
func createFss(initialize bool) ([]fsConfig, error) {
fss := []fsConfig{}
baseTmp, err := os.MkdirTemp(os.TempDir(), "stfs-test-*")
@@ -484,6 +490,8 @@ func createFss() ([]fsConfig, error) {
config.fileSystemCache,
fileSystemCacheDir,
config.fileSystemCacheDuration,
initialize,
)
if err != nil {
return nil, err
@@ -501,8 +509,8 @@ func createFss() ([]fsConfig, error) {
return fss, nil
}
func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fsConfig)) {
fss, err := createFss()
func runTestForAllFss(t *testing.T, name string, initialize bool, action func(t *testing.T, fs fsConfig)) {
fss, err := createFss(initialize)
if err != nil {
t.Fatal(err)
@@ -539,8 +547,8 @@ func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fs
}
}
func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B, fs fsConfig)) {
fss, err := createFss()
func runBenchmarkForAllFss(b *testing.B, name string, initialize bool, action func(b *testing.B, fs fsConfig)) {
fss, err := createFss(initialize)
if err != nil {
b.Fatal(err)
@@ -575,81 +583,8 @@ func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B,
}
}
type createArgs struct {
name string
}
var createTests = []struct {
name string
args createArgs
wantErr bool
}{
{
"Can create /test.txt",
createArgs{"/test.txt"},
false,
},
// FIXME: STFS can create file in non-existent directory, which should not be possible
// {
// "Can not create /nonexistent/test.txt",
// args{"/nonexistent/test.txt"},
// true,
// },
// FIXME: STFS can create `/` file even if / exists
// {
// "Can create /",
// args{"/"},
// true,
// },
}
func TestSTFS_Create(t *testing.T) {
for _, tt := range createTests {
runTestForAllFss(t, tt.name, func(t *testing.T, fs fsConfig) {
file, err := fs.fs.Create(tt.args.name)
if (err != nil) != tt.wantErr {
t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
want, err := fs.fs.Stat(tt.args.name)
if err != nil {
t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
got, err := fs.fs.Stat(file.Name())
if err != nil {
t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, want) {
t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want)
return
}
})
}
}
func BenchmarkSTFS_Create(b *testing.B) {
for _, tt := range createTests {
runBenchmarkForAllFss(b, tt.name, func(b *testing.B, fs fsConfig) {
if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr {
b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
})
}
}
func TestSTFS_Name(t *testing.T) {
fss, err := createFss()
fss, err := createFss(true)
if err != nil {
t.Fatal(err)
@@ -685,3 +620,192 @@ func TestSTFS_Name(t *testing.T) {
})
}
}
type createArgs struct {
name string
}
var createTests = []struct {
name string
args createArgs
wantErr bool
}{
{
"Can create /test.txt",
createArgs{"/test.txt"},
false,
},
// FIXME: STFS can create file in non-existent directory, which should not be possible
// {
// "Can not create /nonexistent/test.txt",
// createArgs{"/nonexistent/test.txt"},
// true,
// },
// FIXME: STFS can create `/` file even if / exists
// {
// "Can create /",
// createArgs{"/"},
// true,
// },
}
func TestSTFS_Create(t *testing.T) {
for _, tt := range createTests {
runTestForAllFss(t, tt.name, true, func(t *testing.T, fs fsConfig) {
file, err := fs.fs.Create(tt.args.name)
if (err != nil) != tt.wantErr {
t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
want, err := fs.fs.Stat(tt.args.name)
if err != nil {
t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
if file == nil {
if (err != nil) != tt.wantErr {
t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
return
}
got, err := fs.fs.Stat(file.Name())
if err != nil {
t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, want) {
t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want)
return
}
})
}
}
func BenchmarkSTFS_Create(b *testing.B) {
for _, tt := range createTests {
runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) {
if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr {
b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
return
}
})
}
}
type initializeArgs struct {
rootProposal string
rootPerm os.FileMode
}
var initializeTests = []struct {
name string
args initializeArgs
wantRoot string
wantErr bool
}{
{
"Can create absolute root directory /",
initializeArgs{"/", os.ModePerm},
"/",
false,
},
{
"Can create absolute root directory /test",
initializeArgs{"/test", os.ModePerm},
"/test",
false,
},
{
"Can create absolute root directory /test/yes",
initializeArgs{"/test/yes", os.ModePerm},
"/test/yes",
false,
},
{
"Can create relative root directory ' '",
initializeArgs{" ", os.ModePerm},
" ",
false,
},
{
"Can create relative root directory ''",
initializeArgs{"", os.ModePerm},
"",
false,
},
{
"Can create relative root directory .",
initializeArgs{".", os.ModePerm},
".",
false,
},
{
"Can create relative root directory test",
initializeArgs{"test", os.ModePerm},
"test",
false,
},
{
"Can create absolute root directory test/yes",
initializeArgs{"test/yes", os.ModePerm},
"test/yes",
false,
},
}
func TestSTFS_Initialize(t *testing.T) {
for _, tt := range initializeTests {
runTestForAllFss(t, tt.name, false, func(t *testing.T, fs fsConfig) {
f, ok := fs.fs.(*STFS)
if !ok {
if fs.fs.Name() == config.FileSystemNameSTFS {
t.Fatal("Initialize function missing from filesystem")
return
}
// Skip non-STFS filesystems
return
}
gotRoot, err := f.Initialize(tt.args.rootProposal, tt.args.rootPerm)
if (err != nil) != tt.wantErr {
t.Errorf("STFS.Initialize() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotRoot != tt.wantRoot {
t.Errorf("STFS.Initialize() = %v, want %v", gotRoot, tt.wantRoot)
}
})
}
}
func BenchmarkSTFS_Initialize(b *testing.B) {
for _, tt := range initializeTests {
runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) {
_, ok := fs.fs.(*STFS)
if !ok {
if fs.fs.Name() == config.FileSystemNameSTFS {
b.Fatal("Initialize function missing from filesystem")
return
}
// Skip non-STFS filesystems
return
}
})
}
}

2
pkg/operations/archive.go
View File

@@ -29,6 +29,7 @@ func (o *Operations) Archive(
getSrc func() (config.FileConfig, error),
compressionLevel string,
overwrite bool,
initializing bool,
) ([]*tar.Header, error) {
o.diskOperationLock.Lock()
defer o.diskOperationLock.Unlock()
@@ -277,6 +278,7 @@ func (o *Operations) Archive(
int(lastIndexedRecord),
int(lastIndexedBlock),
overwrite,
initializing,
index,
func(hdr *tar.Header, i int) error {

1
pkg/operations/delete.go
View File

@@ -125,6 +125,7 @@ func (o *Operations) Delete(name string) error {
int(lastIndexedRecord),
int(lastIndexedBlock),
false,
false,
1, // Ignore the first header, which is the last header which we already indexed
func(hdr *tar.Header, i int) error {

1
pkg/operations/move.go
View File

@@ -144,6 +144,7 @@ func (o *Operations) Move(from string, to string) error {
int(lastIndexedRecord),
int(lastIndexedBlock),
false,
false,
1, // Ignore the first header, which is the last header which we already indexed
func(hdr *tar.Header, i int) error {

1
pkg/operations/update.go
View File

@@ -305,6 +305,7 @@ func (o *Operations) Update(
int(lastIndexedRecord),
int(lastIndexedBlock),
false,
false,
1, // Ignore the first header, which is the last header which we already indexed
func(hdr *tar.Header, i int) error {

4
pkg/persisters/metadata.go
View File

@@ -99,11 +99,13 @@ func (p *MetadataPersister) GetRootPath(ctx context.Context) (string, error) {
return root.Name, nil
}
func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header) error {
func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header, initializing bool) error {
idbhdr := converters.ConfigHeaderToDBHeader(dbhdr)
hdr := *idbhdr
if !initializing {
hdr.Name = p.getSanitizedPath(ctx, idbhdr.Name)
}
if _, err := models.FindHeader(ctx, p.sqlite.DB, hdr.Name, models.HeaderColumns.Name); err != nil {
if err == sql.ErrNoRows {

8
pkg/recovery/index.go
View File

@@ -29,6 +29,7 @@ func Index(
record int,
block int,
overwrite bool,
initializing bool,
offset int,
decryptHeader func(
@@ -117,7 +118,7 @@ func Index(
return err
}
if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil {
if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil {
return err
}
}
@@ -203,7 +204,7 @@ func Index(
return err
}
if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil {
if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil {
return err
}
}
@@ -238,6 +239,7 @@ func indexHeader(
metadataPersister config.MetadataPersister,
compressionFormat string,
encryptionFormat string,
initializing bool,
onHeader func(hdr *config.Header),
) error {
uncompressedSize, ok := hdr.PAXRecords[records.STFSRecordUncompressedSize]
@@ -286,7 +288,7 @@ func indexHeader(
return err
}
if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr)); err != nil {
if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr), initializing); err != nil {
return err
}
case records.STFSRecordActionDelete: