mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Start implementation of symlink support to readdir

Browse Source
This commit is contained in:
Felicitas Pojtinger
2022-01-24 23:51:32 +01:00
parent d5d0738909
commit c4ba80416a
2 changed files with 231 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
pkg/fs/file_test.go
View File

@@ -632,7 +632,7 @@ var readdirTests = []struct {
open string open string
args readdirArgs args readdirArgs
wantErr bool wantErr bool
prepare func(afero.Fs) error prepare func(symFs) error
check func([]os.FileInfo) error check func([]os.FileInfo) error
withCache bool withCache bool
withOsFs bool withOsFs bool
@@ -642,7 +642,7 @@ var readdirTests = []struct {
"/", "/",
readdirArgs{-1}, readdirArgs{-1},
false, false,
func(f afero.Fs) error { return nil }, func(f symFs) error { return nil },
func(f []os.FileInfo) error { func(f []os.FileInfo) error {
if len(f) > 0 { if len(f) > 0 {
return errors.New("found unexpected children in empty directory") return errors.New("found unexpected children in empty directory")
@@ -658,7 +658,7 @@ var readdirTests = []struct {
"", "",
readdirArgs{-1}, readdirArgs{-1},
false, false,
func(f afero.Fs) error { return nil }, func(f symFs) error { return nil },
func(f []os.FileInfo) error { func(f []os.FileInfo) error {
if len(f) > 0 { if len(f) > 0 {
return errors.New("found unexpected children in empty directory") return errors.New("found unexpected children in empty directory")
@@ -674,7 +674,7 @@ var readdirTests = []struct {
"/", "/",
readdirArgs{-1}, readdirArgs{-1},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -693,8 +693,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := len(wantNames)
gotLength := 1 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -709,7 +709,7 @@ var readdirTests = []struct {
"/", "/",
readdirArgs{-1}, readdirArgs{-1},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -738,8 +738,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := len(wantNames)
gotLength := 3 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -754,7 +754,7 @@ var readdirTests = []struct {
"/", "/",
readdirArgs{0}, readdirArgs{0},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -783,8 +783,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := len(wantNames)
gotLength := 3 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -799,7 +799,7 @@ var readdirTests = []struct {
"/", "/",
readdirArgs{2}, readdirArgs{2},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -828,8 +828,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := 2
gotLength := 2 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -844,7 +844,7 @@ var readdirTests = []struct {
"/mydir", "/mydir",
readdirArgs{2}, readdirArgs{2},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -876,8 +876,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := 2
gotLength := 2 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -892,7 +892,7 @@ var readdirTests = []struct {
"/mydir/nested", "/mydir/nested",
readdirArgs{3}, readdirArgs{3},
false, false,
func(f afero.Fs) error { func(f symFs) error {
if _, err := f.Create("/test.txt"); err != nil { if _, err := f.Create("/test.txt"); err != nil {
return err return err
} }
@@ -929,8 +929,8 @@ var readdirTests = []struct {
} }
} }
wantLength := len(f) wantLength := len(wantNames)
gotLength := 3 gotLength := len(f)
if wantLength != gotLength { if wantLength != gotLength {
return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength) return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
} }
@@ -940,6 +940,186 @@ var readdirTests = []struct {
true, true,
true, true,
}, },
// {
// "Can readdir 5 in /mydir/nested if there are multiple children and non-broken symlinks",
// "/mydir/nested",
// readdirArgs{5},
// false,
// func(f symFs) error {
// if _, err := f.Create("/test.txt"); err != nil {
// return err
// }
// if err := f.MkdirAll("/mydir/nested", os.ModePerm); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/asdf.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm2.txt"); err != nil {
// return err
// }
// if err := f.SymlinkIfPossible("/test.txt", "/mydir/nested/existingsymlink"); err != nil {
// return nil
// }
// if err := f.SymlinkIfPossible("/mydir/nested/hmm.txt", "/mydir/nested/existingsymlink2"); err != nil {
// return nil
// }
// return nil
// },
// func(f []os.FileInfo) error {
// wantNames := map[string]struct{}{
// "asdf.txt": {},
// "hmm.txt": {},
// "hmm2.txt": {},
// "existingsymlink": {},
// "existingsymlink2": {},
// }
// for _, info := range f {
// name, ok := wantNames[info.Name()]
// if !ok {
// return fmt.Errorf("could not find file or directory with name %v", name)
// }
// }
// wantLength := len(wantNames)
// gotLength := len(f)
// if wantLength != gotLength {
// return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
// }
// return nil
// },
// true,
// true,
// },
// {
// "Can readdir 5 in /mydir/nested if there are multiple children and broken symlinks",
// "/mydir/nested",
// readdirArgs{5},
// false,
// func(f symFs) error {
// if err := f.MkdirAll("/mydir/nested", os.ModePerm); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/asdf.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm2.txt"); err != nil {
// return err
// }
// if err := f.SymlinkIfPossible("/mydir/nested/hmm2.txt", "/mydir/nested/existingsymlink"); err != nil {
// return nil
// }
// if err := f.SymlinkIfPossible("/test.txt", "/mydir/nested/brokensymlink"); err != nil {
// return nil
// }
// return nil
// },
// func(f []os.FileInfo) error {
// wantNames := map[string]struct{}{
// "asdf.txt": {},
// "hmm.txt": {},
// "hmm2.txt": {},
// "existingsymlink": {},
// "brokensymlink": {},
// }
// for _, info := range f {
// name, ok := wantNames[info.Name()]
// if !ok {
// return fmt.Errorf("could not find file or directory with name %v", name)
// }
// }
// wantLength := len(wantNames)
// gotLength := len(f)
// if wantLength != gotLength {
// return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
// }
// return nil
// },
// true,
// true,
// },
// {
// "Can readdir 4 in /existingsymlink if there are multiple children and symlinks",
// "/existingsymlink",
// readdirArgs{4},
// false,
// func(f symFs) error {
// if err := f.MkdirAll("/mydir/nested", os.ModePerm); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/asdf.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm.txt"); err != nil {
// return err
// }
// if _, err := f.Create("/mydir/nested/hmm2.txt"); err != nil {
// return err
// }
// if err := f.SymlinkIfPossible("/test.txt", "/mydir/nested/brokensymlink"); err != nil {
// return nil
// }
// if err := f.SymlinkIfPossible("/mydir/nested", "/existingsymlink"); err != nil {
// return nil
// }
// return nil
// },
// func(f []os.FileInfo) error {
// wantNames := map[string]struct{}{
// "asdf.txt": {},
// "hmm.txt": {},
// "hmm2.txt": {},
// "brokensymlink": {},
// }
// for _, info := range f {
// name, ok := wantNames[info.Name()]
// if !ok {
// return fmt.Errorf("could not find file or directory with name %v", name)
// }
// }
// wantLength := len(f)
// gotLength := 3
// if wantLength != gotLength {
// return fmt.Errorf("invalid amount of children, got %v, want %v", gotLength, wantLength)
// }
// return nil
// },
// true,
// true,
// },
} }
func TestFile_Readdir(t *testing.T) { func TestFile_Readdir(t *testing.T) {

41
pkg/persisters/metadata.go
View File

@@ -261,7 +261,6 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
name = p.getSanitizedPath(ctx, name) name = p.getSanitizedPath(ctx, name)
prefix := strings.TrimSuffix(name, "/") + "/" prefix := strings.TrimSuffix(name, "/") + "/"
rootDepth := 0 rootDepth := 0
headers := []*config.Header{}
// We want <=, not < // We want <=, not <
if limit > 0 { if limit > 0 {
@@ -283,7 +282,7 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
), ),
).Bind(ctx, p.sqlite.DB, &depth); err != nil { ).Bind(ctx, p.sqlite.DB, &depth); err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return headers, nil return []*config.Header{}, nil
} }
return nil, err return nil, err
@@ -292,7 +291,15 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
rootDepth = int(depth.Depth) rootDepth = int(depth.Depth)
} }
getHeaders := func(prefix string) ([]*config.Header, error) { getHeaders := func(prefix string, useLinkname bool) ([]*config.Header, error) {
pk := models.HeaderColumns.Name
exclude := models.HeaderColumns.Linkname
if useLinkname {
pk = models.HeaderColumns.Linkname
exclude = models.HeaderColumns.Name
}
headers := []*config.Header{}
query := fmt.Sprintf( query := fmt.Sprintf(
`select %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, `select %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v,
length(replace(%v, ?, '')) - length(replace(replace(%v, ?, ''), '/', '')) as depth length(replace(%v, ?, '')) - length(replace(replace(%v, ?, ''), '/', '')) as depth
@@ -306,6 +313,7 @@ where %v like ?
) )
) )
and %v != 1 and %v != 1
and %v = ""
and not %v in ('', '.', '/', './')`, and not %v in ('', '.', '/', './')`,
models.HeaderColumns.Record, models.HeaderColumns.Record,
models.HeaderColumns.Lastknownrecord, models.HeaderColumns.Lastknownrecord,
@@ -313,8 +321,8 @@ where %v like ?
models.HeaderColumns.Lastknownblock, models.HeaderColumns.Lastknownblock,
models.HeaderColumns.Deleted, models.HeaderColumns.Deleted,
models.HeaderColumns.Typeflag, models.HeaderColumns.Typeflag,
models.HeaderColumns.Name, pk,
models.HeaderColumns.Linkname, exclude,
models.HeaderColumns.Size, models.HeaderColumns.Size,
models.HeaderColumns.Mode, models.HeaderColumns.Mode,
models.HeaderColumns.UID, models.HeaderColumns.UID,
@@ -328,13 +336,14 @@ where %v like ?
models.HeaderColumns.Devminor, models.HeaderColumns.Devminor,
models.HeaderColumns.Paxrecords, models.HeaderColumns.Paxrecords,
models.HeaderColumns.Format, models.HeaderColumns.Format,
models.HeaderColumns.Name, pk,
models.HeaderColumns.Name, pk,
models.TableNames.Headers, models.TableNames.Headers,
models.HeaderColumns.Name, pk,
models.HeaderColumns.Name, pk,
models.HeaderColumns.Deleted, models.HeaderColumns.Deleted,
models.HeaderColumns.Name, exclude,
pk,
) )
if limit > 0 { if limit > 0 {
@@ -373,11 +382,21 @@ where %v like ?
return headers, nil return headers, nil
} }
headers, err := getHeaders(prefix) headers := []*config.Header{}
nameHeaders, err := getHeaders(prefix, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
linknameHeaders, err := getHeaders(prefix, true)
if err != nil {
return nil, err
}
headers = append(headers, nameHeaders...)
headers = append(headers, linknameHeaders...)
outhdrs := []*config.Header{} outhdrs := []*config.Header{}
for _, hdr := range headers { for _, hdr := range headers {
prefix := strings.TrimSuffix(hdr.Name, "/") prefix := strings.TrimSuffix(hdr.Name, "/")