diff --git a/cmd/stfs/cmd/operation_archive.go b/cmd/stfs/cmd/operation_archive.go index 5f8f024..b552bc6 100644 --- a/cmd/stfs/cmd/operation_archive.go +++ b/cmd/stfs/cmd/operation_archive.go @@ -157,6 +157,7 @@ var operationArchiveCmd = &cobra.Command{ }, viper.GetString(compressionLevelFlag), viper.GetBool(overwriteFlag), + false, ); err != nil { return err } diff --git a/cmd/stfs/cmd/recovery_index.go b/cmd/stfs/cmd/recovery_index.go index d2f9734..7c8a3a3 100644 --- a/cmd/stfs/cmd/recovery_index.go +++ b/cmd/stfs/cmd/recovery_index.go @@ -92,6 +92,7 @@ var recoveryIndexCmd = &cobra.Command{ viper.GetInt(recordFlag), viper.GetInt(blockFlag), viper.GetBool(overwriteFlag), + false, 0, func(hdr *tar.Header, i int) error { diff --git a/pkg/config/config.go b/pkg/config/config.go index 7618b36..aace373 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -63,7 +63,7 @@ type Header struct { } type MetadataPersister interface { - UpsertHeader(ctx context.Context, dbhdr *Header) error + UpsertHeader(ctx context.Context, dbhdr *Header, initializing bool) error UpdateHeaderMetadata(ctx context.Context, dbhdr *Header) error MoveHeader(ctx context.Context, oldName string, newName string, lastknownrecord, lastknownblock int64) error GetHeaders(ctx context.Context) ([]*Header, error) diff --git a/pkg/fs/filesystem.go b/pkg/fs/filesystem.go index 0ad096b..a8cff01 100644 --- a/pkg/fs/filesystem.go +++ b/pkg/fs/filesystem.go @@ -92,7 +92,7 @@ func (f *STFS) Create(name string) (afero.File, error) { return f.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666) } -func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string) error { +func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, overwrite bool, linkname string, initializing bool) error { f.log.Trace("FileSystem.mknodeWithoutLocking", map[string]interface{}{ "name": name, "perm": perm, @@ -169,6 +169,7 @@ func (f *STFS) mknodeWithoutLocking(dir bool, name string, perm os.FileMode, ove }, f.compressionLevel, overwrite, + initializing, ); err != nil { return err } @@ -198,7 +199,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin return "", os.ErrPermission } - if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, ""); err != nil { + if err := f.mknodeWithoutLocking(true, rootProposal, rootPerm, true, "", true); err != nil { return "", err } @@ -223,6 +224,7 @@ func (f *STFS) Initialize(rootProposal string, rootPerm os.FileMode) (root strin 0, 0, true, + true, 0, func(hdr *tar.Header, i int) error { @@ -259,7 +261,7 @@ func (f *STFS) Mkdir(name string, perm os.FileMode) error { f.ioLock.Lock() defer f.ioLock.Unlock() - return f.mknodeWithoutLocking(true, name, perm, false, "") + return f.mknodeWithoutLocking(true, name, perm, false, "", false) } func (f *STFS) MkdirAll(path string, perm os.FileMode) error { @@ -285,7 +287,7 @@ func (f *STFS) MkdirAll(path string, perm os.FileMode) error { currentPath = filepath.Join(currentPath, part) } - if err := f.mknodeWithoutLocking(true, currentPath, perm, false, ""); err != nil { + if err := f.mknodeWithoutLocking(true, currentPath, perm, false, "", false); err != nil { return err } } @@ -346,7 +348,7 @@ func (f *STFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, er if err != nil { if err == sql.ErrNoRows { if !f.readOnly && flag&os.O_CREATE != 0 && flag&os.O_EXCL == 0 { - if err := f.mknodeWithoutLocking(false, name, perm, false, ""); err != nil { + if err := f.mknodeWithoutLocking(false, name, perm, false, "", false); err != nil { return nil, err } @@ -648,7 +650,7 @@ func (f *STFS) SymlinkIfPossible(oldname, newname string) error { f.ioLock.Lock() defer f.ioLock.Unlock() - return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname) + return f.mknodeWithoutLocking(false, oldname, os.ModePerm, false, newname, false) } func (f *STFS) ReadlinkIfPossible(name string) (string, error) { diff --git a/pkg/fs/filesystem_test.go b/pkg/fs/filesystem_test.go index 005e235..6c92b05 100644 --- a/pkg/fs/filesystem_test.go +++ b/pkg/fs/filesystem_test.go @@ -319,6 +319,8 @@ func createSTFS( fileSystemCache string, fileSystemCacheDir string, fileSystemCacheDuration time.Duration, + + initialize bool, ) (afero.Fs, error) { tm := tape.NewTapeManager( drive, @@ -411,9 +413,13 @@ func createSTFS( jsonLogger, ) - root, err := stfs.Initialize("/", os.ModePerm) - if err != nil { - return nil, err + root := "" + if initialize { + var err error + root, err = stfs.Initialize("/", os.ModePerm) + if err != nil { + return nil, err + } } return cache.NewCacheFilesystem( @@ -425,7 +431,7 @@ func createSTFS( ) } -func createFss() ([]fsConfig, error) { +func createFss(initialize bool) ([]fsConfig, error) { fss := []fsConfig{} baseTmp, err := os.MkdirTemp(os.TempDir(), "stfs-test-*") @@ -484,6 +490,8 @@ func createFss() ([]fsConfig, error) { config.fileSystemCache, fileSystemCacheDir, config.fileSystemCacheDuration, + + initialize, ) if err != nil { return nil, err @@ -501,8 +509,8 @@ func createFss() ([]fsConfig, error) { return fss, nil } -func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fsConfig)) { - fss, err := createFss() +func runTestForAllFss(t *testing.T, name string, initialize bool, action func(t *testing.T, fs fsConfig)) { + fss, err := createFss(initialize) if err != nil { t.Fatal(err) @@ -539,8 +547,8 @@ func runTestForAllFss(t *testing.T, name string, action func(t *testing.T, fs fs } } -func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B, fs fsConfig)) { - fss, err := createFss() +func runBenchmarkForAllFss(b *testing.B, name string, initialize bool, action func(b *testing.B, fs fsConfig)) { + fss, err := createFss(initialize) if err != nil { b.Fatal(err) @@ -575,81 +583,8 @@ func runBenchmarkForAllFss(b *testing.B, name string, action func(b *testing.B, } } -type createArgs struct { - name string -} - -var createTests = []struct { - name string - args createArgs - wantErr bool -}{ - { - "Can create /test.txt", - createArgs{"/test.txt"}, - false, - }, - // FIXME: STFS can create file in non-existent directory, which should not be possible - // { - // "Can not create /nonexistent/test.txt", - // args{"/nonexistent/test.txt"}, - // true, - // }, - // FIXME: STFS can create `/` file even if / exists - // { - // "Can create /", - // args{"/"}, - // true, - // }, -} - -func TestSTFS_Create(t *testing.T) { - for _, tt := range createTests { - runTestForAllFss(t, tt.name, func(t *testing.T, fs fsConfig) { - file, err := fs.fs.Create(tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) - - return - } - - want, err := fs.fs.Stat(tt.args.name) - if err != nil { - t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) - - return - } - - got, err := fs.fs.Stat(file.Name()) - if err != nil { - t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) - - return - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want) - - return - } - }) - } -} - -func BenchmarkSTFS_Create(b *testing.B) { - for _, tt := range createTests { - runBenchmarkForAllFss(b, tt.name, func(b *testing.B, fs fsConfig) { - if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr { - b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) - - return - } - }) - } -} - func TestSTFS_Name(t *testing.T) { - fss, err := createFss() + fss, err := createFss(true) if err != nil { t.Fatal(err) @@ -685,3 +620,192 @@ func TestSTFS_Name(t *testing.T) { }) } } + +type createArgs struct { + name string +} + +var createTests = []struct { + name string + args createArgs + wantErr bool +}{ + { + "Can create /test.txt", + createArgs{"/test.txt"}, + false, + }, + // FIXME: STFS can create file in non-existent directory, which should not be possible + // { + // "Can not create /nonexistent/test.txt", + // createArgs{"/nonexistent/test.txt"}, + // true, + // }, + // FIXME: STFS can create `/` file even if / exists + // { + // "Can create /", + // createArgs{"/"}, + // true, + // }, +} + +func TestSTFS_Create(t *testing.T) { + for _, tt := range createTests { + runTestForAllFss(t, tt.name, true, func(t *testing.T, fs fsConfig) { + file, err := fs.fs.Create(tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) + + return + } + + want, err := fs.fs.Stat(tt.args.name) + if err != nil { + t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) + + return + } + + if file == nil { + if (err != nil) != tt.wantErr { + t.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) + + return + } + + return + } + + got, err := fs.fs.Stat(file.Name()) + if err != nil { + t.Errorf("%v.Stat() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) + + return + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("%v.Create().Name() = %v, want %v", fs.fs.Name(), got, want) + + return + } + }) + } +} + +func BenchmarkSTFS_Create(b *testing.B) { + for _, tt := range createTests { + runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) { + if _, err := fs.fs.Create(tt.args.name); (err != nil) != tt.wantErr { + b.Errorf("%v.Create() error = %v, wantErr %v", fs.fs.Name(), err, tt.wantErr) + + return + } + }) + } +} + +type initializeArgs struct { + rootProposal string + rootPerm os.FileMode +} + +var initializeTests = []struct { + name string + args initializeArgs + wantRoot string + wantErr bool +}{ + { + "Can create absolute root directory /", + initializeArgs{"/", os.ModePerm}, + "/", + false, + }, + { + "Can create absolute root directory /test", + initializeArgs{"/test", os.ModePerm}, + "/test", + false, + }, + { + "Can create absolute root directory /test/yes", + initializeArgs{"/test/yes", os.ModePerm}, + "/test/yes", + false, + }, + { + "Can create relative root directory ' '", + initializeArgs{" ", os.ModePerm}, + " ", + false, + }, + { + "Can create relative root directory ''", + initializeArgs{"", os.ModePerm}, + "", + false, + }, + { + "Can create relative root directory .", + initializeArgs{".", os.ModePerm}, + ".", + false, + }, + { + "Can create relative root directory test", + initializeArgs{"test", os.ModePerm}, + "test", + false, + }, + { + "Can create absolute root directory test/yes", + initializeArgs{"test/yes", os.ModePerm}, + "test/yes", + false, + }, +} + +func TestSTFS_Initialize(t *testing.T) { + for _, tt := range initializeTests { + runTestForAllFss(t, tt.name, false, func(t *testing.T, fs fsConfig) { + f, ok := fs.fs.(*STFS) + if !ok { + if fs.fs.Name() == config.FileSystemNameSTFS { + t.Fatal("Initialize function missing from filesystem") + + return + } + + // Skip non-STFS filesystems + return + } + + gotRoot, err := f.Initialize(tt.args.rootProposal, tt.args.rootPerm) + if (err != nil) != tt.wantErr { + t.Errorf("STFS.Initialize() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotRoot != tt.wantRoot { + t.Errorf("STFS.Initialize() = %v, want %v", gotRoot, tt.wantRoot) + } + }) + } +} + +func BenchmarkSTFS_Initialize(b *testing.B) { + for _, tt := range initializeTests { + runBenchmarkForAllFss(b, tt.name, true, func(b *testing.B, fs fsConfig) { + _, ok := fs.fs.(*STFS) + if !ok { + if fs.fs.Name() == config.FileSystemNameSTFS { + b.Fatal("Initialize function missing from filesystem") + + return + } + + // Skip non-STFS filesystems + return + } + }) + } +} diff --git a/pkg/operations/archive.go b/pkg/operations/archive.go index c5363ab..61e35fc 100644 --- a/pkg/operations/archive.go +++ b/pkg/operations/archive.go @@ -29,6 +29,7 @@ func (o *Operations) Archive( getSrc func() (config.FileConfig, error), compressionLevel string, overwrite bool, + initializing bool, ) ([]*tar.Header, error) { o.diskOperationLock.Lock() defer o.diskOperationLock.Unlock() @@ -277,6 +278,7 @@ func (o *Operations) Archive( int(lastIndexedRecord), int(lastIndexedBlock), overwrite, + initializing, index, func(hdr *tar.Header, i int) error { diff --git a/pkg/operations/delete.go b/pkg/operations/delete.go index 4d547a0..bdc4bbb 100644 --- a/pkg/operations/delete.go +++ b/pkg/operations/delete.go @@ -125,6 +125,7 @@ func (o *Operations) Delete(name string) error { int(lastIndexedRecord), int(lastIndexedBlock), false, + false, 1, // Ignore the first header, which is the last header which we already indexed func(hdr *tar.Header, i int) error { diff --git a/pkg/operations/move.go b/pkg/operations/move.go index eb744aa..91fa99f 100644 --- a/pkg/operations/move.go +++ b/pkg/operations/move.go @@ -144,6 +144,7 @@ func (o *Operations) Move(from string, to string) error { int(lastIndexedRecord), int(lastIndexedBlock), false, + false, 1, // Ignore the first header, which is the last header which we already indexed func(hdr *tar.Header, i int) error { diff --git a/pkg/operations/update.go b/pkg/operations/update.go index 9f091cb..5a406c6 100644 --- a/pkg/operations/update.go +++ b/pkg/operations/update.go @@ -305,6 +305,7 @@ func (o *Operations) Update( int(lastIndexedRecord), int(lastIndexedBlock), false, + false, 1, // Ignore the first header, which is the last header which we already indexed func(hdr *tar.Header, i int) error { diff --git a/pkg/persisters/metadata.go b/pkg/persisters/metadata.go index 86ccb30..646b94f 100644 --- a/pkg/persisters/metadata.go +++ b/pkg/persisters/metadata.go @@ -99,11 +99,13 @@ func (p *MetadataPersister) GetRootPath(ctx context.Context) (string, error) { return root.Name, nil } -func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header) error { +func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header, initializing bool) error { idbhdr := converters.ConfigHeaderToDBHeader(dbhdr) hdr := *idbhdr - hdr.Name = p.getSanitizedPath(ctx, idbhdr.Name) + if !initializing { + hdr.Name = p.getSanitizedPath(ctx, idbhdr.Name) + } if _, err := models.FindHeader(ctx, p.sqlite.DB, hdr.Name, models.HeaderColumns.Name); err != nil { if err == sql.ErrNoRows { diff --git a/pkg/recovery/index.go b/pkg/recovery/index.go index 87f9c41..69d3b7d 100644 --- a/pkg/recovery/index.go +++ b/pkg/recovery/index.go @@ -29,6 +29,7 @@ func Index( record int, block int, overwrite bool, + initializing bool, offset int, decryptHeader func( @@ -117,7 +118,7 @@ func Index( return err } - if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil { + if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil { return err } } @@ -203,7 +204,7 @@ func Index( return err } - if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, onHeader); err != nil { + if err := indexHeader(record, block, hdr, metadata.Metadata, pipes.Compression, pipes.Encryption, initializing, onHeader); err != nil { return err } } @@ -238,6 +239,7 @@ func indexHeader( metadataPersister config.MetadataPersister, compressionFormat string, encryptionFormat string, + initializing bool, onHeader func(hdr *config.Header), ) error { uncompressedSize, ok := hdr.PAXRecords[records.STFSRecordUncompressedSize] @@ -286,7 +288,7 @@ func indexHeader( return err } - if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr)); err != nil { + if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr), initializing); err != nil { return err } case records.STFSRecordActionDelete: