From 167c114dae408b6768879b4f8f9703d64f13cd9a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2026 13:32:35 -0700 Subject: [PATCH] ci: fix FUSE mounts against the new runner image (#10484) * ci: restore the setuid bit on a shadowed fusermount3 Newer ubuntu-22.04 runner images carry a source-built fusermount3 in /usr/local/bin that shadows the distro one in PATH and is not setuid root. go-fuse looks the helper up through PATH, so every unprivileged mount fails with "mount failed: Operation not permitted". * test: fail a fuse test as soon as its mount process dies A mount that cannot mount at all exits within a second, but the harness still waited out the 30s readiness timeout and then reported "mount point not ready within timeout", leaving the real cause buried in the log tail. Watch the child processes and report their exit instead. * mount: report a failed mount without a goroutine dump A mount failure is an environment problem - no /dev/fuse, fusermount not setuid, stale mount point - and the all-goroutine stack dump Fatalf adds buries the one line that says so. --- .github/workflows/fuse-dlm-integration.yml | 12 ++++ .github/workflows/fuse-integration.yml | 12 ++++ .github/workflows/fuse-p2p-integration.yml | 12 ++++ test/fuse_integration/framework_test.go | 74 +++++++++++++++++----- weed/command/mount_std.go | 4 +- 5 files changed, 96 insertions(+), 18 deletions(-) diff --git a/.github/workflows/fuse-dlm-integration.yml b/.github/workflows/fuse-dlm-integration.yml index b64681694..18026be60 100644 --- a/.github/workflows/fuse-dlm-integration.yml +++ b/.github/workflows/fuse-dlm-integration.yml @@ -44,6 +44,18 @@ jobs: sudo apt-get install -y libfuse3-dev echo 'user_allow_other' | sudo tee -a /etc/fuse.conf sudo chmod 644 /etc/fuse.conf + # Some runner images carry a second, source-built fusermount3 in + # /usr/local/bin that shadows the distro one in PATH without its setuid + # bit, and every unprivileged mount then fails with EPERM. go-fuse takes + # the first fusermount3 in PATH, so repair that one. + fusermount_bin=$(command -v fusermount3 || true) + if [ -n "$fusermount_bin" ]; then + if [ ! -u "$fusermount_bin" ]; then + sudo chown root:root "$fusermount_bin" + sudo chmod u+s "$fusermount_bin" + fi + ls -l "$fusermount_bin" + fi - name: Build SeaweedFS run: go build -o weed/weed -buildvcs=false ./weed diff --git a/.github/workflows/fuse-integration.yml b/.github/workflows/fuse-integration.yml index 6e78ea53f..c1dbe5d05 100644 --- a/.github/workflows/fuse-integration.yml +++ b/.github/workflows/fuse-integration.yml @@ -45,6 +45,18 @@ jobs: # Allow non-root FUSE mounts with allow_other echo 'user_allow_other' | sudo tee -a /etc/fuse.conf sudo chmod 644 /etc/fuse.conf + # Some runner images carry a second, source-built fusermount3 in + # /usr/local/bin that shadows the distro one in PATH without its setuid + # bit, and every unprivileged mount then fails with EPERM. go-fuse takes + # the first fusermount3 in PATH, so repair that one. + fusermount_bin=$(command -v fusermount3 || true) + if [ -n "$fusermount_bin" ]; then + if [ ! -u "$fusermount_bin" ]; then + sudo chown root:root "$fusermount_bin" + sudo chmod u+s "$fusermount_bin" + fi + ls -l "$fusermount_bin" + fi # Verify FUSE installation fusermount3 --version || fusermount --version || true ls -la /dev/fuse diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 8e0784692..b289c65bc 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -50,6 +50,18 @@ jobs: sudo apt-get install -y libfuse3-dev echo 'user_allow_other' | sudo tee -a /etc/fuse.conf sudo chmod 644 /etc/fuse.conf + # Some runner images carry a second, source-built fusermount3 in + # /usr/local/bin that shadows the distro one in PATH without its setuid + # bit, and every unprivileged mount then fails with EPERM. go-fuse takes + # the first fusermount3 in PATH, so repair that one. + fusermount_bin=$(command -v fusermount3 || true) + if [ -n "$fusermount_bin" ]; then + if [ ! -u "$fusermount_bin" ]; then + sudo chown root:root "$fusermount_bin" + sudo chmod u+s "$fusermount_bin" + fi + ls -l "$fusermount_bin" + fi - name: Build SeaweedFS run: go build -o weed/weed -buildvcs=false ./weed diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index 0c8b5ab39..90cbb980f 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -24,8 +24,8 @@ type FuseTestFramework struct { mountPoint string dataDir string logDir string - miniProcess *os.Process - mountProcess *os.Process + miniProcess *managedProcess + mountProcess *managedProcess filerAddr string filerPort int weedBinary string @@ -146,7 +146,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Wait for filer to be ready (mini starts all services on filerPort) - if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil { + if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil { f.dumpLog("mini") return fmt.Errorf("weed mini not ready: %v", err) } @@ -173,16 +173,11 @@ func (f *FuseTestFramework) Cleanup() { f.DumpLogs() } - if f.mountProcess != nil { - f.unmountFuse() - } - // Stop processes in reverse order - for _, proc := range []*os.Process{f.mountProcess, f.miniProcess} { - if proc != nil { - proc.Signal(syscall.SIGTERM) - proc.Wait() - } + f.unmountFuse() + if f.miniProcess != nil { + f.miniProcess.stop() + f.miniProcess = nil } f.copyLogsForCI() @@ -209,9 +204,41 @@ func (f *FuseTestFramework) GetFilerAddr() string { return f.filerAddr } +// managedProcess is a started weed sub-command whose exit is watched, so a wait +// for it to come up ends the moment it dies instead of burning its full timeout. +type managedProcess struct { + cmd *exec.Cmd + done chan struct{} + err error // exit error, read only after done is closed +} + +// exited returns the exit error once the process is gone, nil while it runs. +func (p *managedProcess) exited() error { + select { + case <-p.done: + if p.err != nil { + return p.err + } + return fmt.Errorf("exited with status 0") + default: + return nil + } +} + +// stop asks the process to terminate and waits for it to go away. +func (p *managedProcess) stop() { + p.cmd.Process.Signal(syscall.SIGTERM) + select { + case <-p.done: + case <-time.After(10 * time.Second): + p.cmd.Process.Kill() + <-p.done + } +} + // startProcess is a helper that starts a weed sub-command with output captured // to a log file in f.logDir. -func (f *FuseTestFramework) startProcess(name string, args []string) (*os.Process, error) { +func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) { logFile, err := os.Create(filepath.Join(f.logDir, name+".log")) if err != nil { return nil, fmt.Errorf("create log file: %v", err) @@ -226,7 +253,13 @@ func (f *FuseTestFramework) startProcess(name string, args []string) (*os.Proces } // Close the file handle — the child process inherited it. logFile.Close() - return cmd.Process, nil + + p := &managedProcess{cmd: cmd, done: make(chan struct{})} + go func() { + p.err = cmd.Wait() + close(p.done) + }() + return p, nil } // dumpLog prints the last lines of a process log file to the test output @@ -326,8 +359,7 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error { // unmountFuse unmounts the FUSE filesystem func (f *FuseTestFramework) unmountFuse() error { if f.mountProcess != nil { - f.mountProcess.Signal(syscall.SIGTERM) - f.mountProcess.Wait() + f.mountProcess.stop() f.mountProcess = nil } @@ -338,7 +370,7 @@ func (f *FuseTestFramework) unmountFuse() error { } // waitForService waits for a service to be available -func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error { +func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { conn, err := net.DialTimeout("tcp", addr, 1*time.Second) @@ -346,6 +378,9 @@ func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) e conn.Close() return nil } + if exitErr := proc.exited(); exitErr != nil { + return fmt.Errorf("process %v before %s accepted connections", exitErr, addr) + } time.Sleep(100 * time.Millisecond) } return fmt.Errorf("service at %s not ready within timeout", addr) @@ -368,6 +403,11 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { return nil } } + // A mount that cannot mount at all (no /dev/fuse, fusermount not setuid) + // dies within a second; reporting that beats waiting out the timeout. + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process %v", exitErr) + } time.Sleep(100 * time.Millisecond) } return fmt.Errorf("mount point not ready within timeout") diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index b7a981421..3f357cedc 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -397,7 +397,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions) if err != nil { - glog.Fatalf("Mount fail: %v", err) + // A failed mount is an environment problem (no /dev/fuse, fusermount not + // setuid, stale mount point); the goroutine dump Fatalf adds buries it. + glog.Exitf("Mount fail: %v", err) } grace.OnInterrupt(func() { if err := unmount.Unmount(dir); err != nil {