ci: fix FUSE mounts against the new runner image (#10484)

* ci: restore the setuid bit on a shadowed fusermount3

Newer ubuntu-22.04 runner images carry a source-built fusermount3 in
/usr/local/bin that shadows the distro one in PATH and is not setuid
root. go-fuse looks the helper up through PATH, so every unprivileged
mount fails with "mount failed: Operation not permitted".

* test: fail a fuse test as soon as its mount process dies

A mount that cannot mount at all exits within a second, but the harness
still waited out the 30s readiness timeout and then reported "mount
point not ready within timeout", leaving the real cause buried in the
log tail. Watch the child processes and report their exit instead.

* mount: report a failed mount without a goroutine dump

A mount failure is an environment problem - no /dev/fuse, fusermount not
setuid, stale mount point - and the all-goroutine stack dump Fatalf adds
buries the one line that says so.

.github/workflows/fuse-dlm-integration.yml

@@ -44,6 +44,18 @@ jobs:
           sudo apt-get install -y libfuse3-dev
           echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
           sudo chmod 644 /etc/fuse.conf
+          # Some runner images carry a second, source-built fusermount3 in
+          # /usr/local/bin that shadows the distro one in PATH without its setuid
+          # bit, and every unprivileged mount then fails with EPERM. go-fuse takes
+          # the first fusermount3 in PATH, so repair that one.
+          fusermount_bin=$(command -v fusermount3 || true)
+          if [ -n "$fusermount_bin" ]; then
+            if [ ! -u "$fusermount_bin" ]; then
+              sudo chown root:root "$fusermount_bin"
+              sudo chmod u+s "$fusermount_bin"
+            fi
+            ls -l "$fusermount_bin"
+          fi
 
       - name: Build SeaweedFS
         run: go build -o weed/weed -buildvcs=false ./weed

.github/workflows/fuse-integration.yml

@@ -45,6 +45,18 @@ jobs:
         # Allow non-root FUSE mounts with allow_other
         echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
         sudo chmod 644 /etc/fuse.conf
+        # Some runner images carry a second, source-built fusermount3 in
+        # /usr/local/bin that shadows the distro one in PATH without its setuid
+        # bit, and every unprivileged mount then fails with EPERM. go-fuse takes
+        # the first fusermount3 in PATH, so repair that one.
+        fusermount_bin=$(command -v fusermount3 || true)
+        if [ -n "$fusermount_bin" ]; then
+          if [ ! -u "$fusermount_bin" ]; then
+            sudo chown root:root "$fusermount_bin"
+            sudo chmod u+s "$fusermount_bin"
+          fi
+          ls -l "$fusermount_bin"
+        fi
         # Verify FUSE installation
         fusermount3 --version || fusermount --version || true
         ls -la /dev/fuse

.github/workflows/fuse-p2p-integration.yml

@@ -50,6 +50,18 @@ jobs:
           sudo apt-get install -y libfuse3-dev
           echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
           sudo chmod 644 /etc/fuse.conf
+          # Some runner images carry a second, source-built fusermount3 in
+          # /usr/local/bin that shadows the distro one in PATH without its setuid
+          # bit, and every unprivileged mount then fails with EPERM. go-fuse takes
+          # the first fusermount3 in PATH, so repair that one.
+          fusermount_bin=$(command -v fusermount3 || true)
+          if [ -n "$fusermount_bin" ]; then
+            if [ ! -u "$fusermount_bin" ]; then
+              sudo chown root:root "$fusermount_bin"
+              sudo chmod u+s "$fusermount_bin"
+            fi
+            ls -l "$fusermount_bin"
+          fi
 
       - name: Build SeaweedFS
         run: go build -o weed/weed -buildvcs=false ./weed

test/fuse_integration/framework_test.go

@@ -24,8 +24,8 @@ type FuseTestFramework struct {
 	mountPoint   string
 	dataDir      string
 	logDir       string
-	miniProcess  *os.Process
-	mountProcess *os.Process
+	miniProcess  *managedProcess
+	mountProcess *managedProcess
 	filerAddr    string
 	filerPort    int
 	weedBinary   string
@@ -146,7 +146,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	}
 
 	// Wait for filer to be ready (mini starts all services on filerPort)
-	if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
 		f.dumpLog("mini")
 		return fmt.Errorf("weed mini not ready: %v", err)
 	}
@@ -173,16 +173,11 @@ func (f *FuseTestFramework) Cleanup() {
 		f.DumpLogs()
 	}
 
-	if f.mountProcess != nil {
-		f.unmountFuse()
-	}
-
 	// Stop processes in reverse order
-	for _, proc := range []*os.Process{f.mountProcess, f.miniProcess} {
-		if proc != nil {
-			proc.Signal(syscall.SIGTERM)
-			proc.Wait()
-		}
+	f.unmountFuse()
+	if f.miniProcess != nil {
+		f.miniProcess.stop()
+		f.miniProcess = nil
 	}
 
 	f.copyLogsForCI()
@@ -209,9 +204,41 @@ func (f *FuseTestFramework) GetFilerAddr() string {
 	return f.filerAddr
 }
 
+// managedProcess is a started weed sub-command whose exit is watched, so a wait
+// for it to come up ends the moment it dies instead of burning its full timeout.
+type managedProcess struct {
+	cmd  *exec.Cmd
+	done chan struct{}
+	err  error // exit error, read only after done is closed
+}
+
+// exited returns the exit error once the process is gone, nil while it runs.
+func (p *managedProcess) exited() error {
+	select {
+	case <-p.done:
+		if p.err != nil {
+			return p.err
+		}
+		return fmt.Errorf("exited with status 0")
+	default:
+		return nil
+	}
+}
+
+// stop asks the process to terminate and waits for it to go away.
+func (p *managedProcess) stop() {
+	p.cmd.Process.Signal(syscall.SIGTERM)
+	select {
+	case <-p.done:
+	case <-time.After(10 * time.Second):
+		p.cmd.Process.Kill()
+		<-p.done
+	}
+}
+
 // startProcess is a helper that starts a weed sub-command with output captured
 // to a log file in f.logDir.
-func (f *FuseTestFramework) startProcess(name string, args []string) (*os.Process, error) {
+func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
 	logFile, err := os.Create(filepath.Join(f.logDir, name+".log"))
 	if err != nil {
 		return nil, fmt.Errorf("create log file: %v", err)
@@ -226,7 +253,13 @@ func (f *FuseTestFramework) startProcess(name string, args []string) (*os.Proces
 	}
 	// Close the file handle — the child process inherited it.
 	logFile.Close()
-	return cmd.Process, nil
+
+	p := &managedProcess{cmd: cmd, done: make(chan struct{})}
+	go func() {
+		p.err = cmd.Wait()
+		close(p.done)
+	}()
+	return p, nil
 }
 
 // dumpLog prints the last lines of a process log file to the test output
@@ -326,8 +359,7 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
 // unmountFuse unmounts the FUSE filesystem
 func (f *FuseTestFramework) unmountFuse() error {
 	if f.mountProcess != nil {
-		f.mountProcess.Signal(syscall.SIGTERM)
-		f.mountProcess.Wait()
+		f.mountProcess.stop()
 		f.mountProcess = nil
 	}
 
@@ -338,7 +370,7 @@ func (f *FuseTestFramework) unmountFuse() error {
 }
 
 // waitForService waits for a service to be available
-func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
+func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
 		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
@@ -346,6 +378,9 @@ func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) e
 			conn.Close()
 			return nil
 		}
+		if exitErr := proc.exited(); exitErr != nil {
+			return fmt.Errorf("process %v before %s accepted connections", exitErr, addr)
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
 	return fmt.Errorf("service at %s not ready within timeout", addr)
@@ -368,6 +403,11 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
 				return nil
 			}
 		}
+		// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
+		// dies within a second; reporting that beats waiting out the timeout.
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process %v", exitErr)
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
 	return fmt.Errorf("mount point not ready within timeout")

weed/command/mount_std.go

@@ -397,7 +397,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
 	if err != nil {
-		glog.Fatalf("Mount fail: %v", err)
+		// A failed mount is an environment problem (no /dev/fuse, fusermount not
+		// setuid, stale mount point); the goroutine dump Fatalf adds buries it.
+		glog.Exitf("Mount fail: %v", err)
 	}
 	grace.OnInterrupt(func() {
 		if err := unmount.Unmount(dir); err != nil {