fix: Prevent cleaning initialization path before creating root directory, add initialization tests

cmd/stfs/cmd/operation_archive.go

@@ -157,6 +157,7 @@ var operationArchiveCmd = &cobra.Command{
 			},
 			viper.GetString(compressionLevelFlag),
 			viper.GetBool(overwriteFlag),
+			false,
 		); err != nil {
 			return err
 		}

cmd/stfs/cmd/recovery_index.go

@@ -92,6 +92,7 @@ var recoveryIndexCmd = &cobra.Command{
 			viper.GetInt(recordFlag),
 			viper.GetInt(blockFlag),
 			viper.GetBool(overwriteFlag),
+			false,
 			0,
 
 			func(hdr *tar.Header, i int) error {

pkg/config/config.go

@@ -63,7 +63,7 @@ type Header struct {
 }
 
 type MetadataPersister interface {
-	UpsertHeader(ctx context.Context, dbhdr *Header) error
+	UpsertHeader(ctx context.Context, dbhdr *Header, initializing bool) error
 	UpdateHeaderMetadata(ctx context.Context, dbhdr *Header) error
 	MoveHeader(ctx context.Context, oldName string, newName string, lastknownrecord, lastknownblock int64) error
 	GetHeaders(ctx context.Context) ([]*Header, error)

pkg/fs/filesystem.go

@@ -92,7 +92,7 @@ func (f *STFS) Create(name string) (afero.File, error) {
 	return f.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666)
 }
 
-func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string) error {
+func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string, initializing bool) error {
 	f.log.Trace("FileSystem.mknodeWithoutLocking", map[string]interface{}{
 		"name": name,
 		"perm": perm,
@@ -169,6 +169,7 @@ func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, ove
 		},
 		f.compressionLevel,
 		overwrite,
+		initializing,
 	); err != nil {
 		return err
 	}
@@ -198,7 +199,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin
 				return "", os.ErrPermission
 			}
 
-			if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, ""); err != nil {
+			if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, "", true); err != nil {
 				return "", err
 			}
 
@@ -223,6 +224,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin
 			0,
 			0,
 			true,
+			true,
 			0,
 
 			func(hdr *tar.Header, i int) error {
@@ -259,7 +261,7 @@ func (f *STFS) Mkdir(name string, perm os.FileMode) error {
 	f.ioLock.Lock()
 	defer f.ioLock.Unlock()
 
-	return f.mknodeWithoutLocking(true, name, perm, false, "")
+	return f.mknodeWithoutLocking(true, name, perm, false, "", false)
 }
 
 func (f *STFS) MkdirAll(path string, perm os.FileMode) error {
@@ -285,7 +287,7 @@ func (f *STFS) MkdirAll(path string, perm os.FileMode) error {
 			currentPath = filepath.Join(currentPath, part)
 		}
 
-		if err := f.mknodeWithoutLocking(true, currentPath, perm, false, ""); err != nil {
+		if err := f.mknodeWithoutLocking(true, currentPath, perm, false, "", false); err != nil {
 			return err
 		}
 	}
@@ -346,7 +348,7 @@ func (f *STFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, er
 	if err != nil {
 		if err == sql.ErrNoRows {
 			if !f.readOnly && flag&os.O_CREATE != 0 && flag&os.O_EXCL == 0 {
-				if err := f.mknodeWithoutLocking(false, name, perm, false, ""); err != nil {
+				if err := f.mknodeWithoutLocking(false, name, perm, false, "", false); err != nil {
 					return nil, err
 				}
 
@@ -648,7 +650,7 @@ func (f *STFS) SymlinkIfPossible(oldname, newname string) error {
 	f.ioLock.Lock()
 	defer f.ioLock.Unlock()
 
-	return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname)
+	return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname, false)
 }
 
 func (f *STFS) ReadlinkIfPossible(name string) (string, error) {

pkg/fs/filesystem_test.go

@@ -319,6 +319,8 @@ func createSTFS(
 	fileSystemCache string,
 	fileSystemCacheDir string,
 	fileSystemCacheDuration time.Duration,
+
+	initialize bool,
 ) (afero.Fs, error) {
 	tm := tape.NewTapeManager(
 		drive,
@@ -411,9 +413,13 @@ func createSTFS(
 		jsonLogger,
 	)
 
-	root, err := stfs.Initialize("/", os.ModePerm)
+	root := ""
-	if err != nil {
+	if initialize {
-		return nil, err
+		var err error
+		root, err = stfs.Initialize("/", os.ModePerm)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return cache.NewCacheFilesystem(
@@ -425,7 +431,7 @@ func createSTFS(
 	)
 }
 
-func createFss() ([]fsConfig, error) {
+func createFss(initialize bool) ([]fsConfig, error) {
 	fss := []fsConfig{}
 
 	baseTmp, err := os.MkdirTemp(os.TempDir(), "stfs-test-*")
@@ -484,6 +490,8 @@ func createFss() ([]fsConfig, error) {
 			config.fileSystemCache,
 			fileSystemCacheDir,
 			config.fileSystemCacheDuration,
+
+			initialize,
 		)
 		if err != nil {
 			return nil, err
@@ -501,8 +509,8 @@ func createFss() ([]fsConfig, error) {
 	return fss, nil
 }
 
-func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fsConfig)) {
+func runTestForAllFss(t *testing.T, name string, initialize bool, action func(t *testing.T, fs fsConfig)) {
-	fss, err := createFss()
+	fss, err := createFss(initialize)
 	if err != nil {
 		t.Fatal(err)
 
@@ -539,8 +547,8 @@ func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fs
 	}
 }
 
-func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B, fs fsConfig)) {
+func runBenchmarkForAllFss(b *testing.B, name string, initialize bool, action func(b *testing.B, fs fsConfig)) {
-	fss, err := createFss()
+	fss, err := createFss(initialize)
 	if err != nil {
 		b.Fatal(err)
 
@@ -575,81 +583,8 @@ func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B,
 	}
 }
 
-type createArgs struct {
-	name string
-}
-
-var createTests = []struct {
-	name    string
-	args    createArgs
-	wantErr bool
-}{
-	{
-		"Can create /test.txt",
-		createArgs{"/test.txt"},
-		false,
-	},
-	// FIXME: STFS can create file in non-existent directory, which should not be possible
-	// {
-	// 	"Can not create /nonexistent/test.txt",
-	// 	args{"/nonexistent/test.txt"},
-	// 	true,
-	// },
-	// FIXME: STFS can create `/` file even if / exists
-	// {
-	// 	"Can create /",
-	// 	args{"/"},
-	// 	true,
-	// },
-}
-
-func TestSTFS_Create(t *testing.T) {
-	for _, tt := range createTests {
-		runTestForAllFss(t, tt.name, func(t *testing.T, fs fsConfig) {
-			file, err := fs.fs.Create(tt.args.name)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
-
-				return
-			}
-
-			want, err := fs.fs.Stat(tt.args.name)
-			if err != nil {
-				t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
-
-				return
-			}
-
-			got, err := fs.fs.Stat(file.Name())
-			if err != nil {
-				t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
-
-				return
-			}
-
-			if !reflect.DeepEqual(got, want) {
-				t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want)
-
-				return
-			}
-		})
-	}
-}
-
-func BenchmarkSTFS_Create(b *testing.B) {
-	for _, tt := range createTests {
-		runBenchmarkForAllFss(b, tt.name, func(b *testing.B, fs fsConfig) {
-			if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr {
-				b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
-
-				return
-			}
-		})
-	}
-}
-
 func TestSTFS_Name(t *testing.T) {
-	fss, err := createFss()
+	fss, err := createFss(true)
 	if err != nil {
 		t.Fatal(err)
 
@@ -685,3 +620,192 @@ func TestSTFS_Name(t *testing.T) {
 		})
 	}
 }
+
+type createArgs struct {
+	name string
+}
+
+var createTests = []struct {
+	name    string
+	args    createArgs
+	wantErr bool
+}{
+	{
+		"Can create /test.txt",
+		createArgs{"/test.txt"},
+		false,
+	},
+	// FIXME: STFS can create file in non-existent directory, which should not be possible
+	// {
+	// 	"Can not create /nonexistent/test.txt",
+	// 	createArgs{"/nonexistent/test.txt"},
+	// 	true,
+	// },
+	// FIXME: STFS can create `/` file even if / exists
+	// {
+	// 	"Can create /",
+	// 	createArgs{"/"},
+	// 	true,
+	// },
+}
+
+func TestSTFS_Create(t *testing.T) {
+	for _, tt := range createTests {
+		runTestForAllFss(t, tt.name, true, func(t *testing.T, fs fsConfig) {
+			file, err := fs.fs.Create(tt.args.name)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
+
+				return
+			}
+
+			want, err := fs.fs.Stat(tt.args.name)
+			if err != nil {
+				t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
+
+				return
+			}
+
+			if file == nil {
+				if (err != nil) != tt.wantErr {
+					t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
+
+					return
+				}
+
+				return
+			}
+
+			got, err := fs.fs.Stat(file.Name())
+			if err != nil {
+				t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
+
+				return
+			}
+
+			if !reflect.DeepEqual(got, want) {
+				t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want)
+
+				return
+			}
+		})
+	}
+}
+
+func BenchmarkSTFS_Create(b *testing.B) {
+	for _, tt := range createTests {
+		runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) {
+			if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr {
+				b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr)
+
+				return
+			}
+		})
+	}
+}
+
+type initializeArgs struct {
+	rootProposal string
+	rootPerm     os.FileMode
+}
+
+var initializeTests = []struct {
+	name     string
+	args     initializeArgs
+	wantRoot string
+	wantErr  bool
+}{
+	{
+		"Can create absolute root directory /",
+		initializeArgs{"/", os.ModePerm},
+		"/",
+		false,
+	},
+	{
+		"Can create absolute root directory /test",
+		initializeArgs{"/test", os.ModePerm},
+		"/test",
+		false,
+	},
+	{
+		"Can create absolute root directory /test/yes",
+		initializeArgs{"/test/yes", os.ModePerm},
+		"/test/yes",
+		false,
+	},
+	{
+		"Can create relative root directory ' '",
+		initializeArgs{" ", os.ModePerm},
+		" ",
+		false,
+	},
+	{
+		"Can create relative root directory ''",
+		initializeArgs{"", os.ModePerm},
+		"",
+		false,
+	},
+	{
+		"Can create relative root directory .",
+		initializeArgs{".", os.ModePerm},
+		".",
+		false,
+	},
+	{
+		"Can create relative root directory test",
+		initializeArgs{"test", os.ModePerm},
+		"test",
+		false,
+	},
+	{
+		"Can create absolute root directory test/yes",
+		initializeArgs{"test/yes", os.ModePerm},
+		"test/yes",
+		false,
+	},
+}
+
+func TestSTFS_Initialize(t *testing.T) {
+	for _, tt := range initializeTests {
+		runTestForAllFss(t, tt.name, false, func(t *testing.T, fs fsConfig) {
+			f, ok := fs.fs.(*STFS)
+			if !ok {
+				if fs.fs.Name() == config.FileSystemNameSTFS {
+					t.Fatal("Initialize function missing from filesystem")
+
+					return
+				}
+
+				// Skip non-STFS filesystems
+				return
+			}
+
+			gotRoot, err := f.Initialize(tt.args.rootProposal, tt.args.rootPerm)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("STFS.Initialize() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotRoot != tt.wantRoot {
+				t.Errorf("STFS.Initialize() = %v, want %v", gotRoot, tt.wantRoot)
+			}
+		})
+	}
+}
+
+func BenchmarkSTFS_Initialize(b *testing.B) {
+	for _, tt := range initializeTests {
+		runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) {
+			_, ok := fs.fs.(*STFS)
+			if !ok {
+				if fs.fs.Name() == config.FileSystemNameSTFS {
+					b.Fatal("Initialize function missing from filesystem")
+
+					return
+				}
+
+				// Skip non-STFS filesystems
+				return
+			}
+		})
+	}
+}

pkg/operations/archive.go

@@ -29,6 +29,7 @@ func (o *Operations) Archive(
 	getSrc func() (config.FileConfig, error),
 	compressionLevel string,
 	overwrite bool,
+	initializing bool,
 ) ([]*tar.Header, error) {
 	o.diskOperationLock.Lock()
 	defer o.diskOperationLock.Unlock()
@@ -277,6 +278,7 @@ func (o *Operations) Archive(
 		int(lastIndexedRecord),
 		int(lastIndexedBlock),
 		overwrite,
+		initializing,
 		index,
 
 		func(hdr *tar.Header, i int) error {

pkg/operations/delete.go

@@ -125,6 +125,7 @@ func (o *Operations) Delete(name string) error {
 		int(lastIndexedRecord),
 		int(lastIndexedBlock),
 		false,
+		false,
 		1, // Ignore the first header, which is the last header which we already indexed
 
 		func(hdr *tar.Header, i int) error {

pkg/operations/move.go

@@ -144,6 +144,7 @@ func (o *Operations) Move(from string, to string) error {
 		int(lastIndexedRecord),
 		int(lastIndexedBlock),
 		false,
+		false,
 		1, // Ignore the first header, which is the last header which we already indexed
 
 		func(hdr *tar.Header, i int) error {

pkg/operations/update.go

@@ -305,6 +305,7 @@ func (o *Operations) Update(
 		int(lastIndexedRecord),
 		int(lastIndexedBlock),
 		false,
+		false,
 		1, // Ignore the first header, which is the last header which we already indexed
 
 		func(hdr *tar.Header, i int) error {

pkg/persisters/metadata.go

@@ -99,11 +99,13 @@ func (p *MetadataPersister) GetRootPath(ctx context.Context) (string, error) {
 	return root.Name, nil
 }
 
-func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header) error {
+func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header, initializing bool) error {
 	idbhdr := converters.ConfigHeaderToDBHeader(dbhdr)
 
 	hdr := *idbhdr
-	hdr.Name = p.getSanitizedPath(ctx, idbhdr.Name)
+	if !initializing {
+		hdr.Name = p.getSanitizedPath(ctx, idbhdr.Name)
+	}
 
 	if _, err := models.FindHeader(ctx, p.sqlite.DB, hdr.Name, models.HeaderColumns.Name); err != nil {
 		if err == sql.ErrNoRows {

pkg/recovery/index.go

@@ -29,6 +29,7 @@ func Index(
 	record int,
 	block int,
 	overwrite bool,
+	initializing bool,
 	offset int,
 
 	decryptHeader func(
@@ -117,7 +118,7 @@ func Index(
 					return err
 				}
 
-				if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil {
+				if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil {
 					return err
 				}
 			}
@@ -203,7 +204,7 @@ func Index(
 					return err
 				}
 
-				if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil {
+				if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil {
 					return err
 				}
 			}
@@ -238,6 +239,7 @@ func indexHeader(
 	metadataPersister config.MetadataPersister,
 	compressionFormat string,
 	encryptionFormat string,
+	initializing bool,
 	onHeader func(hdr *config.Header),
 ) error {
 	uncompressedSize, ok := hdr.PAXRecords[records.STFSRecordUncompressedSize]
@@ -286,7 +288,7 @@ func indexHeader(
 				return err
 			}
 
-			if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr)); err != nil {
+			if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr), initializing); err != nil {
 				return err
 			}
 		case records.STFSRecordActionDelete: