package shell import ( "reflect" "testing" "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" ) func newTestFileFilter(include, exclude string) *FileFilter { defaultInt64 := int64(-1) return &FileFilter{ include: &include, exclude: &exclude, minSize: &defaultInt64, maxSize: &defaultInt64, minAge: &defaultInt64, maxAge: &defaultInt64, minCacheAge: &defaultInt64, now: time.Now().Unix(), } } func testFileEntry(name string) *filer_pb.Entry { return &filer_pb.Entry{ Name: name, Attributes: &filer_pb.FuseAttributes{}, } } func testDirEntry(name string) *filer_pb.Entry { return &filer_pb.Entry{ Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, } } func TestPlanLocalToRemoteSync(t *testing.T) { tests := []struct { name string dirToCopy string localFiles map[string]*filer_pb.Entry remoteFiles map[string]bool // path -> isDirectory forceUpdate bool deleteExtraneous bool fileFilter *FileFilter wantFilesToCopy []string wantFilesToDelete []string }{ { name: "copy local-only files, no delete flag", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), "/mnt/b.txt": testFileEntry("b.txt"), }, remoteFiles: map[string]bool{ "/mnt/b.txt": false, "/mnt/orphan.txt": false, }, fileFilter: newTestFileFilter("", ""), wantFilesToCopy: []string{"/mnt/a.txt"}, }, { name: "delete flag removes remote-only files", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, "/mnt/orphan.txt": false, "/mnt/orphan2.txt": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", ""), wantFilesToDelete: []string{"/mnt/orphan.txt", "/mnt/orphan2.txt"}, }, { name: "delete with empty local storage removes all remote files", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{}, remoteFiles: map[string]bool{ "/mnt/orphan.txt": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", ""), wantFilesToDelete: []string{"/mnt/orphan.txt"}, }, { name: "delete respects include pattern", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.pdf": testFileEntry("a.pdf"), }, remoteFiles: map[string]bool{ "/mnt/a.pdf": false, "/mnt/orphan.pdf": false, "/mnt/keep.txt": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("*.pdf", ""), wantFilesToDelete: []string{"/mnt/orphan.pdf"}, }, { name: "delete respects exclude pattern", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, "/mnt/orphan.txt": false, "/mnt/keep.bak": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", "*.bak"), wantFilesToDelete: []string{"/mnt/orphan.txt"}, }, { name: "delete ignores size/age filters", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, "/mnt/orphan.txt": false, }, deleteExtraneous: true, fileFilter: func() *FileFilter { ff := newTestFileFilter("", "") minSize := int64(1000000) // would exclude orphan.txt from copying if it applied to deletion ff.minSize = &minSize return ff }(), wantFilesToDelete: []string{"/mnt/orphan.txt"}, }, { name: "delete removes orphaned files, directory entries left alone", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, "/mnt/old": true, "/mnt/old/sub": true, "/mnt/old/sub/orphan.txt": false, "/mnt/old/sub2": true, "/mnt/old/sub2/orphan.txt": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", ""), wantFilesToDelete: []string{ "/mnt/old/sub/orphan.txt", "/mnt/old/sub2/orphan.txt", }, }, { name: "exclude pattern protects remote files under nested directory", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, "/mnt/old": true, "/mnt/old/orphan.txt": false, "/mnt/old/keep.bak": false, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", "*.bak"), wantFilesToDelete: []string{"/mnt/old/orphan.txt"}, }, { name: "remote files outside -dir are never deleted", dirToCopy: "/mnt/foo", localFiles: map[string]*filer_pb.Entry{ "/mnt/foo/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/foo/a.txt": false, "/mnt/foo/orphan.txt": false, "/mnt/foobar/b.txt": false, // sibling sharing the "foo" key prefix, must be left alone }, deleteExtraneous: true, fileFilter: newTestFileFilter("", ""), wantFilesToDelete: []string{"/mnt/foo/orphan.txt"}, }, { name: "directory present locally is not deleted", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/dir": testDirEntry("dir"), }, remoteFiles: map[string]bool{ "/mnt/dir": true, }, deleteExtraneous: true, fileFilter: newTestFileFilter("", ""), }, { name: "forceUpdate copies files that exist on remote", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{ "/mnt/a.txt": false, }, forceUpdate: true, fileFilter: newTestFileFilter("", ""), wantFilesToCopy: []string{"/mnt/a.txt"}, }, { name: "local directories are not copied", dirToCopy: "/mnt", localFiles: map[string]*filer_pb.Entry{ "/mnt/dir": testDirEntry("dir"), "/mnt/a.txt": testFileEntry("a.txt"), }, remoteFiles: map[string]bool{}, fileFilter: newTestFileFilter("", ""), wantFilesToCopy: []string{"/mnt/a.txt"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { plan := planLocalToRemoteSync(tt.localFiles, tt.remoteFiles, util.FullPath(tt.dirToCopy), tt.forceUpdate, tt.deleteExtraneous, tt.fileFilter) if !reflect.DeepEqual(plan.filesToCopy, tt.wantFilesToCopy) { t.Errorf("filesToCopy = %v, want %v", plan.filesToCopy, tt.wantFilesToCopy) } if !reflect.DeepEqual(plan.filesToDelete, tt.wantFilesToDelete) { t.Errorf("filesToDelete = %v, want %v", plan.filesToDelete, tt.wantFilesToDelete) } }) } } func TestFileFilter_matchesName(t *testing.T) { tests := []struct { name string include string exclude string fileName string want bool }{ {"no filters", "", "", "a.txt", true}, {"include match", "*.pdf", "", "a.pdf", true}, {"include mismatch", "*.pdf", "", "a.txt", false}, {"exclude match", "", "*.tmp", "a.tmp", false}, {"exclude mismatch", "", "*.tmp", "a.txt", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ff := newTestFileFilter(tt.include, tt.exclude) if got := ff.matchesName(tt.fileName); got != tt.want { t.Errorf("matchesName(%q) = %v, want %v", tt.fileName, got, tt.want) } }) } }