diff --git a/README.md b/README.md index 9ce2153..51ee399 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # scoutfs-notify Observer-only file access notifications for [ScoutFS](https://github.com/versity/scoutfs), -distributed as a rebasable `git format-patch` series plus a small userspace -relay daemon. Layers onto each upstream release with minimal maintenance. +distributed as a rebasable `git format-patch` series plus a small Go +userspace relay daemon. Layers onto each upstream release with minimal +maintenance. ## What the series adds @@ -18,11 +19,11 @@ Three patches against the scoutfs source tree: `scoutfs_file_aio_read` / `scoutfs_file_read_iter`. Every hook is a single predicted-false branch when no reader is attached. Nothing in the data-waiter state machine is touched. -3. **scoutfs-notifyd** — userspace daemon that binds +3. **scoutfs-notifyd (Go)** — userspace daemon that binds `/run/scoutfs//notify.sock` (AF_UNIX SOCK_SEQPACKET, mode 0600, root-only), drains the ring, and broadcasts each record to connected - clients. Shipped with a systemd template unit - `scoutfs-notifyd@.service`. + clients. Pure-stdlib Go; no external module dependencies. Shipped + with a systemd template unit `scoutfs-notifyd@.service`. ## Base @@ -32,6 +33,17 @@ Currently rebased against: See [base.txt](./base.txt). +## Build dependencies added + +On top of the stock scoutfs build requirements, patch 3 adds: + +* `golang >= 1.21` on the build host (RHEL/EL9: `golang`, EL8: `go-toolset`). + Debian 12 / Ubuntu 22.04+: `golang-go`. + +The Go build is offline (`GOPROXY=off`) — no network access required at +build time. `CGO_ENABLED=0` so the produced binary is a pure-Go static +ELF. + ## Applying ```sh @@ -39,7 +51,7 @@ See [base.txt](./base.txt). ``` Runs `git am --3way` on each patch. For a tarball instead of a git tree, -use `patch -p1` in a loop (see script). +loop `patch -p1 < patches/*.patch`. ## Rebasing onto a new upstream release @@ -52,17 +64,17 @@ git format-patch v1.30..notify -o patches/ # update base.txt and commit ``` -## Tag pinning +## Tag history -Each release of this patch set is tagged as `v-notify-`: - - v1.29-notify-1 (retired — included mount options) - v1.29-notify-2 (current — daemon-driven, no knobs) + v1.29-notify-3 (current — Go daemon, struct alignment fix) + v1.29-notify-2 (retired — C daemon; had latent ring-size compile bug) + v1.29-notify-1 (retired — shipped mount options) ## Quick smoke test after installation ```sh systemctl enable --now scoutfs-notifyd@mnt-scoutfs.service -socat - UNIX-CONNECT:/run/scoutfs/$(stat -c %d /mnt/scoutfs)/notify.sock \ - | xxd | head # watch events stream as you touch files +FSID=$(stat -f -c %i /mnt/scoutfs) # or use your scoutfs cli +socat - UNIX-CONNECT:/run/scoutfs/${FSID}/notify.sock | xxd | head +# touch some files in /mnt/scoutfs in another terminal ``` diff --git a/patches/0001-notify-core-file-access-notification-infrastructure.patch b/patches/0001-notify-core-file-access-notification-infrastructure.patch index 730f126..d5464d0 100644 --- a/patches/0001-notify-core-file-access-notification-infrastructure.patch +++ b/patches/0001-notify-core-file-access-notification-infrastructure.patch @@ -1,18 +1,20 @@ -From 7564fa7095161816c28c774af97636a8b9ad6f17 Mon Sep 17 00:00:00 2001 +From c73736df8b8206e66a72a432781034c56765a64a Mon Sep 17 00:00:00 2001 From: William Gill -Date: Wed, 22 Apr 2026 13:46:39 -0500 +Date: Wed, 22 Apr 2026 14:21:34 -0500 Subject: [PATCH 1/3] notify: core file-access notification infrastructure Adds a per-mount observer-only notification ring and an ioctl to drain it. The feature has no user-facing configuration: the ring -is always allocated at mount and emits are enabled automatically +is always allocated at mount, and emits are enabled automatically while a userspace reader is attached. What this patch adds: - New files kmod/src/notify.{h,c}. A fixed-size 64 KiB ring of 64-byte records, a single-reader drain ioctl, and setup/destroy - hooks for scoutfs_sb_info. + hooks for scoutfs_sb_info. The 64-byte record layout pads the + struct so NOTIFY_RING_BYTES / sizeof(record) is a power of two + (1024) and each record lands on a cache line. - New ioctl SCOUTFS_IOC_READ_NOTIFY (nr 25). Fills the caller's array of scoutfs_ioctl_notify_event; supports a timeout_ms wait policy. CAP_SYS_ADMIN required. atomic_cmpxchg on @@ -32,7 +34,8 @@ Design properties: - Emit is non-blocking: one leaf spinlock, no allocations, no sleeping locks, drop-on-full with seq still advancing so readers see gaps. -- Ring size (64 KiB = 1024 events) is a compile-time constant. +- Ring size (64 KiB = 1024 64-byte events) is a compile-time + constant. - No mount option, no sysfs toggle. The watcher daemon's attach/detach is the only gate. - Event type values 1..2 are defined for file OPEN and READ. @@ -45,12 +48,12 @@ follow-up patch. kmod/src/Makefile | 1 + kmod/src/counters.h | 3 + kmod/src/ioctl.c | 3 + - kmod/src/ioctl.h | 78 +++++++++ + kmod/src/ioctl.h | 84 ++++++++++ kmod/src/notify.c | 373 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/notify.h | 39 +++++ kmod/src/super.c | 3 + kmod/src/super.h | 5 + - 8 files changed, 505 insertions(+) + 8 files changed, 511 insertions(+) create mode 100644 kmod/src/notify.c create mode 100644 kmod/src/notify.h @@ -102,10 +105,10 @@ index 0a5fc4c..b3a4d9a 100644 return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h -index c0d2285..fc58f8c 100644 +index c0d2285..d468d4c 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h -@@ -876,4 +876,82 @@ struct scoutfs_ioctl_punch_offline { +@@ -876,4 +876,88 @@ struct scoutfs_ioctl_punch_offline { #define SCOUTFS_IOC_PUNCH_OFFLINE \ _IOW(SCOUTFS_IOCTL_MAGIC, 24, struct scoutfs_ioctl_punch_offline) @@ -156,7 +159,13 @@ index c0d2285..fc58f8c 100644 + __u32 uid; + __u8 type; + __u8 flags; -+ __u8 _pad[6]; ++ /* ++ * Pads the record to 64 bytes for cache-line alignment and to ++ * make ring capacity a power of two for a 64 KiB ring. Also ++ * reserves space for future event-type fields without breaking ++ * the wire size. ++ */ ++ __u8 _pad[14]; +}; + +/* diff --git a/patches/0002-notify-file-open-read-hook-sites.patch b/patches/0002-notify-file-open-read-hook-sites.patch index 89a5b31..ed2c4a7 100644 --- a/patches/0002-notify-file-open-read-hook-sites.patch +++ b/patches/0002-notify-file-open-read-hook-sites.patch @@ -1,6 +1,6 @@ -From 1b6569c33f7337c6af16933441241c84874a6f5d Mon Sep 17 00:00:00 2001 +From 34a879b1f6e77915df8930c173eef9f17887da2f Mon Sep 17 00:00:00 2001 From: William Gill -Date: Wed, 22 Apr 2026 13:47:13 -0500 +Date: Wed, 22 Apr 2026 14:21:50 -0500 Subject: [PATCH 2/3] notify: file open/read hook sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 @@ -12,9 +12,9 @@ operations that carry user-visible activity we want to observe. OPEN (data.c): A new scoutfs_file_open() wrapper is installed as ->open in scoutfs_file_fops. The wrapper calls generic_file_open() to - preserve the existing VFS default semantics for regular files, - then — only on success, and only when notifications are enabled - — emits a SCOUTFS_NOTIFY_TYPE_OPEN record. The file mode is + preserve existing VFS default semantics for regular files, + then — only on success and only when notifications are enabled + — emits a SCOUTFS_NOTIFY_TYPE_OPEN record. File mode is inspected for FMODE_WRITE to set SCOUTFS_NOTIFY_F_WRITE_OPEN. READ (file.c): @@ -26,14 +26,11 @@ READ (file.c): iocb->ki_pos. Every hook is behind unlikely(READ_ONCE(sbi->notify_enabled)), so -when no userspace reader is attached the hook is a single +when no userspace reader is attached the hook reduces to a single predicted-false branch. No scoutfs state is mutated, no error is propagated, and no existing control flow is altered. -Nothing in the data-waiter state machine (scoutfs_data_wait_check, -scoutfs_data_wait, scoutfs_data_wait_changed, the waiter rbtree, -or SCOUTFS_IOC_DATA_WAITING / DATA_WAIT_ERR) is touched. That -work is deferred to a future series. +Nothing in the data-waiter state machine is touched. --- kmod/src/data.c | 28 ++++++++++++++++++++++++++++ kmod/src/file.c | 11 +++++++++++ diff --git a/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon-Go.patch b/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon-Go.patch new file mode 100644 index 0000000..d6f0e9b --- /dev/null +++ b/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon-Go.patch @@ -0,0 +1,797 @@ +From 4286c88487c986db0d468be540aca3ad67d32c25 Mon Sep 17 00:00:00 2001 +From: William Gill +Date: Wed, 22 Apr 2026 14:31:15 -0500 +Subject: [PATCH 3/3] notify: scoutfs-notifyd userspace relay daemon (Go) + +Adds the userspace side of the file access notification feature. +The kmod provides a ring and a drain ioctl; this daemon binds the +well-known per-mount socket, drains the ring, and broadcasts each +record to connected clients. + +Written in Go using the standard library only. The go.mod has no +external module requirements, so builds work offline and the +packaging path sets GOPROXY=off to make that guarantee explicit. + +Socket: + + /run/scoutfs//notify.sock + AF_UNIX / SOCK_SEQPACKET ("unixpacket" network in Go) + mode 0600, owner root:root + + The fsid is discovered at startup via SCOUTFS_IOC_STATFS_MORE on + the given mountpoint, so the daemon never consults its + environment or a config file. The shipped systemd unit creates + /run/scoutfs via RuntimeDirectory=scoutfs. + +Protocol: + + Each broadcast is one SOCK_SEQPACKET message carrying exactly one + 64-byte struct scoutfs_ioctl_notify_event record. Clients issue + recv(2) in a loop; each returned message is one event. + +Concurrency: + + Three goroutines: a reader blocking on the drain ioctl, an + acceptLoop blocking on the listener, and a broadcaster that owns + the client set and does all sends. Communication via channels; + no shared mutable state across goroutines. + + Slow-client protection: writes are issued with an in-the-past + deadline so a full kernel buffer returns a timeout error + immediately, dropping the client. + + Reader exclusivity is owned by the kmod ioctl (returns -EBUSY to + a second attached reader); starting a second instance against + the same mount logs and backs off. + + Graceful shutdown on SIGTERM / SIGINT / SIGQUIT via a cancelled + context plus listener close; all goroutines rendezvous via a + sync.WaitGroup before the socket inode is unlinked. + +Packaging: + + - utils/Makefile: new target notifyd/scoutfs-notifyd built by + "go build" with CGO_ENABLED=0, GOPROXY=off, -trimpath, and + -ldflags="-s -w" for a small, reproducible static binary. + - utils/scoutfs-utils.spec.in: adds BuildRequires on golang + (>= 1.21); installs /usr/sbin/scoutfs-notifyd and + /usr/lib/systemd/system/scoutfs-notifyd@.service. + - utils/man/scoutfs-notifyd.8 documents the tool, protocol and + drop policy. + +Activation: + + systemctl enable --now scoutfs-notifyd@mnt-scoutfs.service + +Watchers then connect to /run/scoutfs//notify.sock and +recv() events. +--- + utils/Makefile | 28 +- + utils/man/scoutfs-notifyd.8 | 120 +++++++ + utils/notifyd/go.mod | 3 + + utils/notifyd/main.go | 434 +++++++++++++++++++++++++ + utils/notifyd/scoutfs-notifyd@.service | 40 +++ + utils/scoutfs-utils.spec.in | 6 + + 6 files changed, 628 insertions(+), 3 deletions(-) + create mode 100644 utils/man/scoutfs-notifyd.8 + create mode 100644 utils/notifyd/go.mod + create mode 100644 utils/notifyd/main.go + create mode 100644 utils/notifyd/scoutfs-notifyd@.service + +diff --git a/utils/Makefile b/utils/Makefile +index e0f7614..d882e30 100644 +--- a/utils/Makefile ++++ b/utils/Makefile +@@ -18,7 +18,13 @@ BIN := src/scoutfs + OBJ := $(patsubst %.c,%.o,$(wildcard src/*.c)) + DEPS := $(wildcard */*.d) + +-all: $(BIN) ++# scoutfs-notifyd is a Go program. Go manages its own dependency and ++# build graph, so we just treat the binary as a single target with a ++# coarse source-dependency on every .go file under notifyd/. ++NOTIFYD := notifyd/scoutfs-notifyd ++NOTIFYD_SRC := $(wildcard notifyd/*.go) notifyd/go.mod ++ ++all: $(BIN) $(NOTIFYD) + + ifneq ($(DEPS),) + -include $(DEPS) +@@ -29,13 +35,28 @@ QU = @echo + VE = @ + else + QU = @: +-VE = ++VE = + endif + + $(BIN): $(OBJ) + $(QU) [BIN $@] + $(VE)gcc -o $@ $^ -luuid -lm -lcrypto -lblkid + ++# Build the Go daemon. No external module dependencies, so the build ++# works offline (as in rpmbuild). We force GOPROXY=off to make that ++# guarantee explicit; GOCACHE is redirected into the source dir so a ++# read-only HOME doesn't break anything. ++$(NOTIFYD): $(NOTIFYD_SRC) ++ $(QU) [GO $@] ++ $(VE)cd notifyd && \ ++ GOCACHE=$${GOCACHE:-$$(pwd)/.gocache} \ ++ GOFLAGS=-mod=mod \ ++ GOPROXY=off \ ++ CGO_ENABLED=0 \ ++ go build -trimpath -buildvcs=false \ ++ -ldflags "-s -w" \ ++ -o scoutfs-notifyd . ++ + %.o %.d: %.c Makefile sparse.sh + $(QU) [CC $<] + $(VE)gcc $(CFLAGS) -MD -MP -MF $*.d -c $< -o $*.o +@@ -65,4 +86,5 @@ dist: $(RPM_DIR) scoutfs-utils.spec + tar rf $(TARFILE) --transform="s@.*\(src/.*\)@scoutfs-utils-$(RPM_VERSION)/\1@" $(FMTIOC_KMOD) + + clean: +- @rm -f $(BIN) $(OBJ) $(DEPS) .sparse.* ++ @rm -f $(BIN) $(OBJ) $(NOTIFYD) $(DEPS) .sparse.* ++ @rm -rf notifyd/.gocache +diff --git a/utils/man/scoutfs-notifyd.8 b/utils/man/scoutfs-notifyd.8 +new file mode 100644 +index 0000000..a0dcd46 +--- /dev/null ++++ b/utils/man/scoutfs-notifyd.8 +@@ -0,0 +1,120 @@ ++.TH scoutfs-notifyd 8 ++.SH NAME ++scoutfs-notifyd \- scoutfs file access notification relay daemon ++ ++.SH SYNOPSIS ++.B scoutfs-notifyd ++.I mountpoint ++ ++.SH DESCRIPTION ++The ++.B scoutfs-notifyd ++daemon drains the file access notification ring of a scoutfs mount ++and broadcasts each event to every process connected to ++.I /run/scoutfs//notify.sock. ++ ++Events are observer-only records of file ++.B open ++and ++.B read ++activity, produced by the scoutfs kernel module for the mount ++identified by ++.IR mountpoint . ++The daemon is the single privileged reader of the ring and is expected ++to be started by the systemd template unit ++.I scoutfs-notifyd@.service ++with the instance name set to the escaped mountpoint path. ++ ++.SH OPTIONS ++ ++.TP ++.I mountpoint ++An absolute path to a directory on the scoutfs volume to monitor. ++Typically this is the mountpoint itself. The daemon opens the path, ++reads the volume's fsid via the ++.B SCOUTFS_IOC_STATFS_MORE ++ioctl, and builds the listener socket path from that fsid. ++ ++.SH SOCKET ++ ++The listener is an ++.B AF_UNIX ++.B SOCK_SEQPACKET ++socket at ++.I /run/scoutfs//notify.sock ++with mode ++.B 0600 ++and owner ++.BR root:root . ++ ++Each broadcast is one atomic message carrying a single packed 64-byte ++record matching ++.BR "struct scoutfs_ioctl_notify_event" . ++Clients read exactly one record per ++.BR recv (2) ++call. The fields are: ++ ++.nf ++.RS 4 ++__u64 seq; /* monotonic; gap = dropped events */ ++__u64 ino; /* scoutfs inode number */ ++__u64 offset; /* byte offset of the read, 0 for OPEN */ ++__u64 length; /* byte length of the read, 0 for OPEN */ ++__u64 time_ns; /* CLOCK_REALTIME at the event */ ++__u32 pid; /* task group id */ ++__u32 uid; /* effective uid (init user namespace) */ ++__u8 type; /* 1 = OPEN, 2 = READ */ ++__u8 flags; /* bit 0 = opened writable */ ++__u8 _pad[6]; ++.RE ++.fi ++ ++Numeric values 3 and higher in ++.B type ++are reserved for future scoutfs events (data-waiter observation). ++ ++.SH DROP POLICY ++ ++The in-kernel ring is lossy. When it fills, records are discarded but ++the monotonic ++.B seq ++field still advances, so consumers that track seq will detect gaps ++rather than silent loss. ++ ++The daemon forwards to clients with ++.B MSG_DONTWAIT ++and ++.BR MSG_NOSIGNAL . ++A client that cannot keep up is disconnected immediately; it is ++expected to reconnect. ++ ++.SH PERMISSIONS ++ ++The daemon requires ++.B CAP_SYS_ADMIN ++to call ++.BR SCOUTFS_IOC_READ_NOTIFY . ++The shipped systemd unit grants that capability and strips everything ++else. Only one reader may be attached to a scoutfs mount's ring at a ++time; starting a second daemon against the same mount returns ++.BR EBUSY . ++ ++.SH EXIT STATUS ++ ++The daemon exits 0 on clean shutdown (SIGTERM, SIGINT, SIGQUIT) and ++non-zero on error. ++ ++.SH FILES ++.TP ++.I /run/scoutfs//notify.sock ++Listener socket, per scoutfs fsid. ++ ++.TP ++.I /usr/lib/systemd/system/scoutfs-notifyd@.service ++Template unit. Enable and start a specific mount with, e.g., ++.BR "systemctl enable --now scoutfs-notifyd@mnt-scoutfs.service" . ++ ++.SH SEE ALSO ++.BR scoutfs (5), ++.BR scoutfs (8), ++.BR systemd.unit (5) +diff --git a/utils/notifyd/go.mod b/utils/notifyd/go.mod +new file mode 100644 +index 0000000..2d5fd0e +--- /dev/null ++++ b/utils/notifyd/go.mod +@@ -0,0 +1,3 @@ ++module scoutfs.org/notifyd ++ ++go 1.21 +diff --git a/utils/notifyd/main.go b/utils/notifyd/main.go +new file mode 100644 +index 0000000..9478bae +--- /dev/null ++++ b/utils/notifyd/main.go +@@ -0,0 +1,434 @@ ++// Copyright (C) 2026 Versity Software, Inc. All rights reserved. ++// ++// This program is free software; you can redistribute it and/or ++// modify it under the terms of the GNU General Public ++// License v2 as published by the Free Software Foundation. ++ ++// scoutfs-notifyd drains the scoutfs file access notification ring for ++// one mount and broadcasts each record to clients connected to ++// /run/scoutfs//notify.sock. ++// ++// Usage: scoutfs-notifyd ++// ++// Design: ++// ++// - Three goroutines: reader (blocks on the drain ioctl), accept ++// (blocks on listener), broadcaster (owns the client set and does ++// all sends). Communication via unbuffered / small-buffered ++// channels; no shared mutable state across goroutines. ++// ++// - Socket type is AF_UNIX / SOCK_SEQPACKET ("unixpacket" network in ++// Go's net package). Each Write is one atomic 64-byte message. ++// ++// - Slow-client protection: writes are done with an in-the-past ++// deadline so a full kernel send buffer returns a timeout error ++// immediately; the offending client is closed and dropped. ++// ++// - Requires CAP_SYS_ADMIN for SCOUTFS_IOC_READ_NOTIFY. The socket ++// file is chmod'd to 0600. ++// ++// - Clean shutdown on SIGTERM / SIGINT / SIGQUIT via a cancelled ++// context plus listener close. ++package main ++ ++import ( ++ "context" ++ "errors" ++ "fmt" ++ "log" ++ "net" ++ "os" ++ "os/signal" ++ "path/filepath" ++ "sync" ++ "syscall" ++ "time" ++ "unsafe" ++) ++ ++// --------------------------------------------------------------------------- ++// ABI — matches kmod/src/ioctl.h ++// --------------------------------------------------------------------------- ++ ++// Event mirrors struct scoutfs_ioctl_notify_event. The in-memory ++// layout must match the kernel record exactly so the daemon can memcpy ++// events from the drain buffer directly onto the wire. ++type Event struct { ++ Seq uint64 ++ Ino uint64 ++ Offset uint64 ++ Length uint64 ++ TimeNs uint64 ++ Pid uint32 ++ Uid uint32 ++ Type uint8 ++ Flags uint8 ++ _pad [14]byte ++} ++ ++const eventSize = 64 ++ ++// ReadNotifyArgs mirrors struct scoutfs_ioctl_read_notify. ++type ReadNotifyArgs struct { ++ EventsPtr uint64 ++ EventsNr uint32 ++ TimeoutMs uint32 ++ Flags uint64 ++} ++ ++// StatfsMore mirrors struct scoutfs_ioctl_statfs_more. ++type StatfsMore struct { ++ Fsid uint64 ++ Rid uint64 ++ CommittedSeq uint64 ++ TotalMetaBlocks uint64 ++ TotalDataBlocks uint64 ++ ReservedMetaBlocks uint64 ++} ++ ++// Linux _IOC encoding. The values below are the generic ABI used on ++// x86, ARM, ARM64, RISC-V, and all architectures scoutfs currently ++// targets. PowerPC/Alpha/MIPS/SPARC would need different constants. ++const ( ++ iocNrShift = 0 ++ iocTypeShift = 8 ++ iocSizeShift = 16 ++ iocDirShift = 30 ++ ++ iocDirRead = 2 ++ iocDirWrite = 1 ++ ++ scoutfsIoctlMagic = 0xE8 ++ ++ // sizeof(struct scoutfs_ioctl_statfs_more) ++ statfsMoreSize = 48 ++ // sizeof(struct scoutfs_ioctl_read_notify) ++ readNotifySize = 24 ++) ++ ++func iocEncode(dir, typ, nr, size uintptr) uintptr { ++ return (dir << iocDirShift) | ++ (size << iocSizeShift) | ++ (typ << iocTypeShift) | ++ (nr << iocNrShift) ++} ++ ++var ( ++ scoutfsIocStatfsMore = iocEncode(iocDirRead, ++ scoutfsIoctlMagic, 10, statfsMoreSize) ++ scoutfsIocReadNotify = iocEncode(iocDirRead|iocDirWrite, ++ scoutfsIoctlMagic, 25, readNotifySize) ++) ++ ++func init() { ++ // Hard fail at startup if the Go struct layout doesn't match the ++ // ABI. Cheap insurance against a compiler or platform that pads ++ // differently from what we expect. ++ if sz := unsafe.Sizeof(Event{}); sz != eventSize { ++ panic(fmt.Sprintf("Event struct is %d bytes, expected %d", ++ sz, eventSize)) ++ } ++ if sz := unsafe.Sizeof(ReadNotifyArgs{}); sz != readNotifySize { ++ panic(fmt.Sprintf("ReadNotifyArgs is %d bytes, expected %d", ++ sz, readNotifySize)) ++ } ++ if sz := unsafe.Sizeof(StatfsMore{}); sz != statfsMoreSize { ++ panic(fmt.Sprintf("StatfsMore is %d bytes, expected %d", ++ sz, statfsMoreSize)) ++ } ++} ++ ++// --------------------------------------------------------------------------- ++// Syscall wrappers ++// --------------------------------------------------------------------------- ++ ++// getFsid opens the mountpoint and resolves the volume's fsid. ++// Returns (fsid, mountFd, nil) on success. The caller owns mountFd. ++func getFsid(mountpoint string) (uint64, int, error) { ++ fd, err := syscall.Open(mountpoint, ++ syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_CLOEXEC, 0) ++ if err != nil { ++ return 0, 0, fmt.Errorf("open %s: %w", mountpoint, err) ++ } ++ var sm StatfsMore ++ _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, ++ uintptr(fd), ++ scoutfsIocStatfsMore, ++ uintptr(unsafe.Pointer(&sm))) ++ if errno != 0 { ++ syscall.Close(fd) ++ return 0, 0, fmt.Errorf( ++ "SCOUTFS_IOC_STATFS_MORE on %s: %w "+ ++ "(is this a scoutfs mount?)", mountpoint, errno) ++ } ++ return sm.Fsid, fd, nil ++} ++ ++// drainNotify blocks for up to timeoutMs and fills buf with events. ++// Returns the number of events actually filled. errno ETIMEDOUT is ++// expected every timeout window with no activity. ++func drainNotify(fd int, buf []Event, timeoutMs uint32) (int, error) { ++ if len(buf) == 0 { ++ return 0, errors.New("empty batch") ++ } ++ args := ReadNotifyArgs{ ++ EventsPtr: uint64(uintptr(unsafe.Pointer(&buf[0]))), ++ EventsNr: uint32(len(buf)), ++ TimeoutMs: timeoutMs, ++ } ++ ret, _, errno := syscall.Syscall(syscall.SYS_IOCTL, ++ uintptr(fd), ++ scoutfsIocReadNotify, ++ uintptr(unsafe.Pointer(&args))) ++ if errno != 0 { ++ return 0, errno ++ } ++ return int(ret), nil ++} ++ ++// eventBytes returns a byte slice aliasing ev's memory. Safe to pass ++// to Write; the returned slice must not outlive ev. ++func eventBytes(ev *Event) []byte { ++ return (*[eventSize]byte)(unsafe.Pointer(ev))[:] ++} ++ ++// --------------------------------------------------------------------------- ++// Socket setup ++// --------------------------------------------------------------------------- ++ ++const ( ++ ioctlBatchEvents = 64 ++ ioctlTimeoutMs = 100 ++ maxClients = 128 ++ runDirMode = 0o755 ++ sockMode = 0o600 ++) ++ ++// ensureRunDir creates /run/scoutfs and /run/scoutfs//. ++func ensureRunDir(dir string) error { ++ if err := os.MkdirAll(dir, runDirMode); err != nil { ++ return fmt.Errorf("mkdir %s: %w", dir, err) ++ } ++ return nil ++} ++ ++// createListener binds the AF_UNIX SOCK_SEQPACKET listener at path and ++// restricts it to root via chmod 0600. ++func createListener(path string) (*net.UnixListener, error) { ++ // Remove stale socket from a prior crashed instance. ++ _ = os.Remove(path) ++ ++ addr := &net.UnixAddr{Net: "unixpacket", Name: path} ++ l, err := net.ListenUnix("unixpacket", addr) ++ if err != nil { ++ return nil, fmt.Errorf("ListenUnix %s: %w", path, err) ++ } ++ ++ if err := os.Chmod(path, sockMode); err != nil { ++ l.Close() ++ os.Remove(path) ++ return nil, fmt.Errorf("chmod %s: %w", path, err) ++ } ++ ++ return l, nil ++} ++ ++// --------------------------------------------------------------------------- ++// Goroutines ++// --------------------------------------------------------------------------- ++ ++// reader blocks on the drain ioctl and feeds events into evCh. It ++// exits when ctx is cancelled, when the ioctl returns a fatal error, ++// or when the mount goes away. evCh is closed on exit so the ++// broadcaster can unblock. ++func reader(ctx context.Context, fd int, evCh chan<- Event) { ++ defer close(evCh) ++ batch := make([]Event, ioctlBatchEvents) ++ ++ for { ++ if err := ctx.Err(); err != nil { ++ return ++ } ++ n, err := drainNotify(fd, batch, ioctlTimeoutMs) ++ if err != nil { ++ switch { ++ case errors.Is(err, syscall.ETIMEDOUT), ++ errors.Is(err, syscall.EINTR): ++ continue ++ case errors.Is(err, syscall.EBUSY): ++ log.Printf("another reader is attached; " + ++ "backing off 1s") ++ select { ++ case <-ctx.Done(): ++ return ++ case <-time.After(time.Second): ++ } ++ continue ++ default: ++ log.Printf("SCOUTFS_IOC_READ_NOTIFY: %v", err) ++ return ++ } ++ } ++ for i := 0; i < n; i++ { ++ select { ++ case <-ctx.Done(): ++ return ++ case evCh <- batch[i]: ++ } ++ } ++ } ++} ++ ++// acceptLoop hands each accepted connection off on acCh and exits when ++// the listener is closed. acCh is closed on exit. ++func acceptLoop(l *net.UnixListener, acCh chan<- *net.UnixConn) { ++ defer close(acCh) ++ for { ++ c, err := l.AcceptUnix() ++ if err != nil { ++ if !errors.Is(err, net.ErrClosed) { ++ log.Printf("accept: %v", err) ++ } ++ return ++ } ++ acCh <- c ++ } ++} ++ ++// broadcaster owns the client set. It consumes events from evCh and ++// new connections from acCh; it exits when ctx is cancelled or both ++// input channels are closed. On exit it closes every remaining ++// client connection. ++func broadcaster(ctx context.Context, evCh <-chan Event, acCh <-chan *net.UnixConn) { ++ clients := make(map[*net.UnixConn]struct{}) ++ defer func() { ++ for c := range clients { ++ c.Close() ++ } ++ }() ++ ++ for { ++ select { ++ case <-ctx.Done(): ++ return ++ ++ case c, ok := <-acCh: ++ if !ok { ++ acCh = nil ++ if evCh == nil { ++ return ++ } ++ continue ++ } ++ if len(clients) >= maxClients { ++ log.Printf("client limit (%d) reached, "+ ++ "rejecting", maxClients) ++ c.Close() ++ continue ++ } ++ clients[c] = struct{}{} ++ ++ case ev, ok := <-evCh: ++ if !ok { ++ evCh = nil ++ if acCh == nil { ++ return ++ } ++ continue ++ } ++ buf := eventBytes(&ev) ++ for c := range clients { ++ // Make Write non-blocking: a deadline in the ++ // past causes a full kernel buffer to return ++ // a timeout error immediately. ++ _ = c.SetWriteDeadline(time.Now()) ++ if _, err := c.Write(buf); err != nil { ++ delete(clients, c) ++ c.Close() ++ } ++ } ++ } ++ } ++} ++ ++// --------------------------------------------------------------------------- ++// Main ++// --------------------------------------------------------------------------- ++ ++func usage(out *os.File) { ++ fmt.Fprintf(out, ++ "usage: %s \n\n"+ ++ " Drains the scoutfs file access notification ring "+ ++ "for the given\n"+ ++ " mount and broadcasts events to clients "+ ++ "connected to\n"+ ++ " /run/scoutfs//notify.sock.\n\n"+ ++ " Requires CAP_SYS_ADMIN. Socket is root-only "+ ++ "(mode 0600).\n", ++ filepath.Base(os.Args[0])) ++} ++ ++func main() { ++ log.SetFlags(log.LstdFlags) ++ log.SetPrefix("scoutfs-notifyd: ") ++ ++ if len(os.Args) != 2 || ++ os.Args[1] == "" || ++ os.Args[1] == "-h" || ++ os.Args[1] == "--help" { ++ usage(os.Stderr) ++ os.Exit(2) ++ } ++ mountpoint := os.Args[1] ++ ++ fsid, mntFd, err := getFsid(mountpoint) ++ if err != nil { ++ log.Fatalf("%v", err) ++ } ++ defer syscall.Close(mntFd) ++ ++ runDir := fmt.Sprintf("/run/scoutfs/%d", fsid) ++ sockPath := filepath.Join(runDir, "notify.sock") ++ ++ if err := ensureRunDir(runDir); err != nil { ++ log.Fatalf("%v", err) ++ } ++ ++ listener, err := createListener(sockPath) ++ if err != nil { ++ log.Fatalf("%v", err) ++ } ++ ++ log.Printf("listening on %s", sockPath) ++ ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ ++ sigCh := make(chan os.Signal, 1) ++ signal.Notify(sigCh, ++ syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) ++ ++ evCh := make(chan Event, 512) ++ acCh := make(chan *net.UnixConn, 16) ++ ++ var wg sync.WaitGroup ++ wg.Add(3) ++ go func() { defer wg.Done(); reader(ctx, mntFd, evCh) }() ++ go func() { defer wg.Done(); acceptLoop(listener, acCh) }() ++ go func() { defer wg.Done(); broadcaster(ctx, evCh, acCh) }() ++ ++ sig := <-sigCh ++ log.Printf("caught %v, shutting down", sig) ++ ++ // Cancel the context (stops reader + broadcaster at next check), ++ // close the listener (unblocks acceptLoop), then wait. ++ cancel() ++ listener.Close() ++ wg.Wait() ++ ++ // Remove the socket inode we created. systemd's RuntimeDirectory ++ // cleans the enclosing dir on service stop. ++ _ = os.Remove(sockPath) ++ ++ log.Printf("exited cleanly") ++} +diff --git a/utils/notifyd/scoutfs-notifyd@.service b/utils/notifyd/scoutfs-notifyd@.service +new file mode 100644 +index 0000000..2e87f17 +--- /dev/null ++++ b/utils/notifyd/scoutfs-notifyd@.service +@@ -0,0 +1,40 @@ ++[Unit] ++Description=ScoutFS file access notification daemon for %I ++Documentation=man:scoutfs-notifyd(8) ++# The mount must exist before we can open it and read its fsid. ++RequiresMountsFor=/%I ++ ++[Service] ++Type=simple ++User=root ++Group=root ++UMask=0077 ++# scoutfs-notifyd creates /run/scoutfs// and binds the socket ++# inside it; the top-level /run/scoutfs is created here so the daemon ++# never has to. ++RuntimeDirectory=scoutfs ++RuntimeDirectoryMode=0755 ++ExecStart=/usr/sbin/scoutfs-notifyd /%I ++Restart=on-failure ++RestartSec=5s ++StartLimitBurst=5 ++ ++# The daemon needs CAP_SYS_ADMIN to drive the read-notify ioctl. ++# Everything else can be stripped. ++CapabilityBoundingSet=CAP_SYS_ADMIN ++AmbientCapabilities=CAP_SYS_ADMIN ++NoNewPrivileges=yes ++PrivateTmp=yes ++ProtectSystem=strict ++ProtectHome=yes ++ProtectKernelTunables=yes ++ProtectKernelModules=yes ++ProtectControlGroups=yes ++RestrictAddressFamilies=AF_UNIX ++LockPersonality=yes ++MemoryDenyWriteExecute=yes ++RestrictRealtime=yes ++RestrictSUIDSGID=yes ++ ++[Install] ++WantedBy=multi-user.target +diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in +index fb24b81..2c8a5d8 100644 +--- a/utils/scoutfs-utils.spec.in ++++ b/utils/scoutfs-utils.spec.in +@@ -17,6 +17,8 @@ BuildRequires: gzip + BuildRequires: libuuid-devel + BuildRequires: openssl-devel + BuildRequires: libblkid-devel ++# scoutfs-notifyd is a Go program (pure stdlib, no external modules). ++BuildRequires: golang >= 1.21 + + #Requires: kmod-scoutfs = %{version} + +@@ -52,19 +54,23 @@ cp man/*.5.gz $RPM_BUILD_ROOT%{_mandir}/man5/. + cp man/*.7.gz $RPM_BUILD_ROOT%{_mandir}/man7/. + cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. + install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs ++install -m 755 -D notifyd/scoutfs-notifyd $RPM_BUILD_ROOT%{_sbindir}/scoutfs-notifyd + install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h + install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h + install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced + install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service + install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example ++install -m 644 -D notifyd/scoutfs-notifyd@.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-notifyd@.service + + %files + %defattr(644,root,root,755) + %{_mandir}/man*/scoutfs*.gz + /%{_unitdir}/scoutfs-fenced.service ++/%{_unitdir}/scoutfs-notifyd@.service + %{_sysconfdir}/scoutfs + %defattr(755,root,root,755) + %{_sbindir}/scoutfs ++%{_sbindir}/scoutfs-notifyd + %{_libexecdir}/scoutfs-fenced + + %files -n scoutfs-devel +-- +2.49.0.windows.1 + diff --git a/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon.patch b/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon.patch deleted file mode 100644 index 7288f71..0000000 --- a/patches/0003-notify-scoutfs-notifyd-userspace-relay-daemon.patch +++ /dev/null @@ -1,828 +0,0 @@ -From 4a2b2f062c4d733444f1fa40257725625d9bebe3 Mon Sep 17 00:00:00 2001 -From: William Gill -Date: Wed, 22 Apr 2026 13:47:45 -0500 -Subject: [PATCH 3/3] notify: scoutfs-notifyd userspace relay daemon - -Adds the userspace side of the file access notification feature. -The kmod provides a ring and a drain ioctl; this daemon binds the -well-known socket, drains the ring, and broadcasts each record to -connected clients. - -Socket: - - /run/scoutfs//notify.sock - AF_UNIX / SOCK_SEQPACKET - mode 0600, owner root:root - - The fsid is discovered at startup via SCOUTFS_IOC_STATFS_MORE on - the given mountpoint, so the daemon never consults its - environment or a config file for the path. The shipped systemd - unit creates /run/scoutfs/ via RuntimeDirectory=scoutfs. - -Protocol: - - Each broadcast is one SOCK_SEQPACKET message carrying exactly one - 64-byte struct scoutfs_ioctl_notify_event record. Clients issue - recv(2) in a loop; each returned message is one event. - -Behavior: - - - Single-threaded with epoll. The drain ioctl uses a 100 ms - timeout so the loop paces itself; new clients, client - disconnects, and shutdown signals are handled between ioctl - calls non-blocking. - - Slow-client protection: sends use MSG_DONTWAIT | MSG_NOSIGNAL. - Any send error drops the client immediately. - - MAX_CLIENTS is 128; extras are rejected with a log warning. - - Reader exclusivity is owned by the kmod ioctl (returns -EBUSY - to a second attached reader); starting a second instance of - the daemon against the same mount logs the error and exits. - - Graceful shutdown on SIGTERM / SIGINT / SIGQUIT via signalfd. - -Packaging: - - - utils/Makefile: new target notifyd/scoutfs-notifyd built from - utils/notifyd/scoutfs-notifyd.c with -I../kmod/src for the - shared ioctl.h. No extra libs beyond libc. - - utils/scoutfs-utils.spec.in: installs the binary at - /usr/sbin/scoutfs-notifyd and the template unit at - /usr/lib/systemd/system/scoutfs-notifyd@.service. - - utils/man/scoutfs-notifyd.8 documents the tool, socket - protocol, and drop policy. - -Activation: - - systemctl enable --now scoutfs-notifyd@mnt-scoutfs.service - -where the instance name is the escaped mountpoint path. Watchers -then connect to /run/scoutfs//notify.sock and recv() events. ---- - utils/Makefile | 21 +- - utils/man/scoutfs-notifyd.8 | 120 ++++++ - utils/notifyd/scoutfs-notifyd.c | 501 +++++++++++++++++++++++++ - utils/notifyd/scoutfs-notifyd@.service | 40 ++ - utils/scoutfs-utils.spec.in | 4 + - 5 files changed, 683 insertions(+), 3 deletions(-) - create mode 100644 utils/man/scoutfs-notifyd.8 - create mode 100644 utils/notifyd/scoutfs-notifyd.c - create mode 100644 utils/notifyd/scoutfs-notifyd@.service - -diff --git a/utils/Makefile b/utils/Makefile -index e0f7614..3535928 100644 ---- a/utils/Makefile -+++ b/utils/Makefile -@@ -18,7 +18,10 @@ BIN := src/scoutfs - OBJ := $(patsubst %.c,%.o,$(wildcard src/*.c)) - DEPS := $(wildcard */*.d) - --all: $(BIN) -+NOTIFYD := notifyd/scoutfs-notifyd -+NOTIFYD_OBJ := $(patsubst %.c,%.o,$(wildcard notifyd/*.c)) -+ -+all: $(BIN) $(NOTIFYD) - - ifneq ($(DEPS),) - -include $(DEPS) -@@ -29,13 +32,25 @@ QU = @echo - VE = @ - else - QU = @: --VE = -+VE = - endif - - $(BIN): $(OBJ) - $(QU) [BIN $@] - $(VE)gcc -o $@ $^ -luuid -lm -lcrypto -lblkid - -+# scoutfs-notifyd is a single-source daemon; it needs -I../kmod/src for -+# ioctl.h but no external libraries beyond libc. -+$(NOTIFYD): $(NOTIFYD_OBJ) -+ $(QU) [BIN $@] -+ $(VE)gcc -o $@ $^ -+ -+notifyd/%.o notifyd/%.d: notifyd/%.c Makefile sparse.sh -+ $(QU) [CC $<] -+ $(VE)gcc $(CFLAGS) -I../kmod/src -MD -MP -MF notifyd/$*.d -c $< -o notifyd/$*.o -+ $(QU) [SP $<] -+ $(VE)./sparse.sh -Wbitwise -D__CHECKER__ $(CFLAGS) -I../kmod/src $< -+ - %.o %.d: %.c Makefile sparse.sh - $(QU) [CC $<] - $(VE)gcc $(CFLAGS) -MD -MP -MF $*.d -c $< -o $*.o -@@ -65,4 +80,4 @@ dist: $(RPM_DIR) scoutfs-utils.spec - tar rf $(TARFILE) --transform="s@.*\(src/.*\)@scoutfs-utils-$(RPM_VERSION)/\1@" $(FMTIOC_KMOD) - - clean: -- @rm -f $(BIN) $(OBJ) $(DEPS) .sparse.* -+ @rm -f $(BIN) $(OBJ) $(NOTIFYD) $(NOTIFYD_OBJ) $(DEPS) .sparse.* -diff --git a/utils/man/scoutfs-notifyd.8 b/utils/man/scoutfs-notifyd.8 -new file mode 100644 -index 0000000..a0dcd46 ---- /dev/null -+++ b/utils/man/scoutfs-notifyd.8 -@@ -0,0 +1,120 @@ -+.TH scoutfs-notifyd 8 -+.SH NAME -+scoutfs-notifyd \- scoutfs file access notification relay daemon -+ -+.SH SYNOPSIS -+.B scoutfs-notifyd -+.I mountpoint -+ -+.SH DESCRIPTION -+The -+.B scoutfs-notifyd -+daemon drains the file access notification ring of a scoutfs mount -+and broadcasts each event to every process connected to -+.I /run/scoutfs//notify.sock. -+ -+Events are observer-only records of file -+.B open -+and -+.B read -+activity, produced by the scoutfs kernel module for the mount -+identified by -+.IR mountpoint . -+The daemon is the single privileged reader of the ring and is expected -+to be started by the systemd template unit -+.I scoutfs-notifyd@.service -+with the instance name set to the escaped mountpoint path. -+ -+.SH OPTIONS -+ -+.TP -+.I mountpoint -+An absolute path to a directory on the scoutfs volume to monitor. -+Typically this is the mountpoint itself. The daemon opens the path, -+reads the volume's fsid via the -+.B SCOUTFS_IOC_STATFS_MORE -+ioctl, and builds the listener socket path from that fsid. -+ -+.SH SOCKET -+ -+The listener is an -+.B AF_UNIX -+.B SOCK_SEQPACKET -+socket at -+.I /run/scoutfs//notify.sock -+with mode -+.B 0600 -+and owner -+.BR root:root . -+ -+Each broadcast is one atomic message carrying a single packed 64-byte -+record matching -+.BR "struct scoutfs_ioctl_notify_event" . -+Clients read exactly one record per -+.BR recv (2) -+call. The fields are: -+ -+.nf -+.RS 4 -+__u64 seq; /* monotonic; gap = dropped events */ -+__u64 ino; /* scoutfs inode number */ -+__u64 offset; /* byte offset of the read, 0 for OPEN */ -+__u64 length; /* byte length of the read, 0 for OPEN */ -+__u64 time_ns; /* CLOCK_REALTIME at the event */ -+__u32 pid; /* task group id */ -+__u32 uid; /* effective uid (init user namespace) */ -+__u8 type; /* 1 = OPEN, 2 = READ */ -+__u8 flags; /* bit 0 = opened writable */ -+__u8 _pad[6]; -+.RE -+.fi -+ -+Numeric values 3 and higher in -+.B type -+are reserved for future scoutfs events (data-waiter observation). -+ -+.SH DROP POLICY -+ -+The in-kernel ring is lossy. When it fills, records are discarded but -+the monotonic -+.B seq -+field still advances, so consumers that track seq will detect gaps -+rather than silent loss. -+ -+The daemon forwards to clients with -+.B MSG_DONTWAIT -+and -+.BR MSG_NOSIGNAL . -+A client that cannot keep up is disconnected immediately; it is -+expected to reconnect. -+ -+.SH PERMISSIONS -+ -+The daemon requires -+.B CAP_SYS_ADMIN -+to call -+.BR SCOUTFS_IOC_READ_NOTIFY . -+The shipped systemd unit grants that capability and strips everything -+else. Only one reader may be attached to a scoutfs mount's ring at a -+time; starting a second daemon against the same mount returns -+.BR EBUSY . -+ -+.SH EXIT STATUS -+ -+The daemon exits 0 on clean shutdown (SIGTERM, SIGINT, SIGQUIT) and -+non-zero on error. -+ -+.SH FILES -+.TP -+.I /run/scoutfs//notify.sock -+Listener socket, per scoutfs fsid. -+ -+.TP -+.I /usr/lib/systemd/system/scoutfs-notifyd@.service -+Template unit. Enable and start a specific mount with, e.g., -+.BR "systemctl enable --now scoutfs-notifyd@mnt-scoutfs.service" . -+ -+.SH SEE ALSO -+.BR scoutfs (5), -+.BR scoutfs (8), -+.BR systemd.unit (5) -diff --git a/utils/notifyd/scoutfs-notifyd.c b/utils/notifyd/scoutfs-notifyd.c -new file mode 100644 -index 0000000..d81c74e ---- /dev/null -+++ b/utils/notifyd/scoutfs-notifyd.c -@@ -0,0 +1,501 @@ -+/* -+ * Copyright (C) 2026 Versity Software, Inc. All rights reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public -+ * License v2 as published by the Free Software Foundation. -+ */ -+ -+/* -+ * scoutfs-notifyd -+ * -+ * Drains the scoutfs file access notification ring (via -+ * SCOUTFS_IOC_READ_NOTIFY) and fans each record out to every client -+ * connected to /run/scoutfs//notify.sock. -+ * -+ * Usage: scoutfs-notifyd -+ * -+ * Design notes: -+ * -+ * - One process, one thread. A short blocking ioctl (100 ms) drives -+ * the loop; new clients and shutdown signals are handled between -+ * ioctl calls via epoll with timeout 0. No cross-thread state. -+ * -+ * - Socket type is SOCK_SEQPACKET so each broadcast is one atomic -+ * 64-byte message on the wire. Clients read exactly one -+ * struct scoutfs_ioctl_notify_event per recv(). -+ * -+ * - Permissions: socket owner root, mode 0600. The listener directory -+ * /run/scoutfs// is created with mode 0755. -+ * -+ * - Slow-client protection: sends use MSG_DONTWAIT | MSG_NOSIGNAL. -+ * Any send error (EAGAIN, EPIPE, ECONNRESET, ...) drops the client. -+ * The kmod-side ring absorbs the brief lag. -+ * -+ * - Exit cleanly on SIGTERM/SIGINT/SIGQUIT; systemd restarts on -+ * failure per the shipped unit file. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "ioctl.h" -+ -+#define PROGNAME "scoutfs-notifyd" -+ -+#define RUN_DIR_FMT "/run/scoutfs/%llu" -+#define SOCK_PATH_FMT "/run/scoutfs/%llu/notify.sock" -+ -+#define IOCTL_BATCH_EVENTS 64 -+#define IOCTL_TIMEOUT_MS 100 -+#define MAX_CLIENTS 128 -+#define SOCK_BACKLOG 16 -+ -+static volatile sig_atomic_t g_shutdown; -+ -+struct ctx { -+ int mnt_fd; -+ int listen_fd; -+ int epoll_fd; -+ int signal_fd; -+ char sock_path[128]; -+ char run_dir[128]; -+ int clients[MAX_CLIENTS]; -+ int nclients; -+}; -+ -+static void logmsg(int prio, const char *fmt, ...) -+ __attribute__((format(printf, 2, 3))); -+ -+static void logmsg(int prio, const char *fmt, ...) -+{ -+ va_list ap; -+ -+ va_start(ap, fmt); -+ vsyslog(prio, fmt, ap); -+ va_end(ap); -+ -+ /* also to stderr when not yet daemonized (systemd captures it) */ -+ va_start(ap, fmt); -+ vfprintf(stderr, fmt, ap); -+ fputc('\n', stderr); -+ va_end(ap); -+} -+ -+/* -+ * Open the mountpoint and resolve its fsid via SCOUTFS_IOC_STATFS_MORE. -+ * The fsid is unique per scoutfs mount and forms the per-mount socket -+ * path component. -+ */ -+static int get_fsid(const char *mountpoint, uint64_t *fsid_out, int *mnt_fd_out) -+{ -+ struct scoutfs_ioctl_statfs_more sm = { 0 }; -+ int fd; -+ -+ fd = open(mountpoint, O_RDONLY | O_DIRECTORY); -+ if (fd < 0) { -+ logmsg(LOG_ERR, "open(%s): %s", mountpoint, strerror(errno)); -+ return -1; -+ } -+ -+ if (ioctl(fd, SCOUTFS_IOC_STATFS_MORE, &sm) < 0) { -+ logmsg(LOG_ERR, "SCOUTFS_IOC_STATFS_MORE on %s: %s (is this a scoutfs mount?)", -+ mountpoint, strerror(errno)); -+ close(fd); -+ return -1; -+ } -+ -+ *fsid_out = sm.fsid; -+ *mnt_fd_out = fd; -+ return 0; -+} -+ -+/* -+ * mkdir -p for a single-level run dir. We only have to create the -+ * leaf; systemd (or the shipped tmpfiles.d) creates /run/scoutfs. -+ */ -+static int ensure_run_dir(const char *dir) -+{ -+ struct stat st; -+ -+ if (mkdir("/run/scoutfs", 0755) < 0 && errno != EEXIST) { -+ logmsg(LOG_ERR, "mkdir /run/scoutfs: %s", strerror(errno)); -+ return -1; -+ } -+ if (mkdir(dir, 0755) < 0 && errno != EEXIST) { -+ logmsg(LOG_ERR, "mkdir %s: %s", dir, strerror(errno)); -+ return -1; -+ } -+ if (stat(dir, &st) < 0 || !S_ISDIR(st.st_mode)) { -+ logmsg(LOG_ERR, "%s: not a directory", dir); -+ return -1; -+ } -+ return 0; -+} -+ -+static int create_listener(const char *path) -+{ -+ struct sockaddr_un addr = { .sun_family = AF_UNIX }; -+ int fd; -+ -+ if (strlen(path) >= sizeof(addr.sun_path)) { -+ logmsg(LOG_ERR, "socket path too long: %s", path); -+ return -1; -+ } -+ strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); -+ -+ fd = socket(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC | SOCK_NONBLOCK, 0); -+ if (fd < 0) { -+ logmsg(LOG_ERR, "socket(): %s", strerror(errno)); -+ return -1; -+ } -+ -+ /* Remove any stale socket left by a previous instance. */ -+ (void)unlink(path); -+ -+ if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { -+ logmsg(LOG_ERR, "bind(%s): %s", path, strerror(errno)); -+ close(fd); -+ return -1; -+ } -+ -+ /* Root-only access. chmod after bind so there's no brief window. */ -+ if (chmod(path, 0600) < 0) { -+ logmsg(LOG_ERR, "chmod(%s, 0600): %s", path, strerror(errno)); -+ close(fd); -+ unlink(path); -+ return -1; -+ } -+ -+ if (listen(fd, SOCK_BACKLOG) < 0) { -+ logmsg(LOG_ERR, "listen(): %s", strerror(errno)); -+ close(fd); -+ unlink(path); -+ return -1; -+ } -+ -+ return fd; -+} -+ -+static int setup_signalfd(void) -+{ -+ sigset_t mask; -+ int fd; -+ -+ sigemptyset(&mask); -+ sigaddset(&mask, SIGTERM); -+ sigaddset(&mask, SIGINT); -+ sigaddset(&mask, SIGQUIT); -+ -+ if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) { -+ logmsg(LOG_ERR, "sigprocmask: %s", strerror(errno)); -+ return -1; -+ } -+ -+ fd = signalfd(-1, &mask, SFD_CLOEXEC | SFD_NONBLOCK); -+ if (fd < 0) -+ logmsg(LOG_ERR, "signalfd: %s", strerror(errno)); -+ return fd; -+} -+ -+static int epoll_add(int epfd, int fd, uint32_t events, uint64_t tag) -+{ -+ struct epoll_event ev = { .events = events, .data.u64 = tag }; -+ -+ return epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev); -+} -+ -+/* -+ * Tag encoding so epoll events route cleanly without a per-fd lookup. -+ * tag[63..56] = kind -+ * tag[ 7.. 0] = client index (when kind == KIND_CLIENT) -+ */ -+#define KIND_LISTEN 1 -+#define KIND_SIGNAL 2 -+#define KIND_CLIENT 3 -+ -+#define TAG_KIND(t) ((int)((t) >> 56)) -+#define TAG_INDEX(t) ((int)((t) & 0xff)) -+#define TAG_MAKE(k, i) (((uint64_t)(k) << 56) | (uint32_t)(i)) -+ -+static void client_close(struct ctx *c, int idx) -+{ -+ if (idx < 0 || idx >= c->nclients) -+ return; -+ (void)epoll_ctl(c->epoll_fd, EPOLL_CTL_DEL, c->clients[idx], NULL); -+ close(c->clients[idx]); -+ c->clients[idx] = c->clients[c->nclients - 1]; -+ c->nclients--; -+ /* re-tag the swapped-in client to its new index */ -+ if (idx < c->nclients) { -+ struct epoll_event ev = { -+ .events = EPOLLIN | EPOLLHUP | EPOLLERR, -+ .data.u64 = TAG_MAKE(KIND_CLIENT, idx), -+ }; -+ (void)epoll_ctl(c->epoll_fd, EPOLL_CTL_MOD, -+ c->clients[idx], &ev); -+ } -+} -+ -+static void client_accept(struct ctx *c) -+{ -+ int fd; -+ -+ for (;;) { -+ fd = accept4(c->listen_fd, NULL, NULL, -+ SOCK_CLOEXEC | SOCK_NONBLOCK); -+ if (fd < 0) { -+ if (errno == EAGAIN || errno == EWOULDBLOCK) -+ return; -+ logmsg(LOG_WARNING, "accept: %s", strerror(errno)); -+ return; -+ } -+ -+ if (c->nclients >= MAX_CLIENTS) { -+ logmsg(LOG_WARNING, "client limit (%d) reached, rejecting", -+ MAX_CLIENTS); -+ close(fd); -+ continue; -+ } -+ -+ if (epoll_add(c->epoll_fd, fd, -+ EPOLLIN | EPOLLHUP | EPOLLERR, -+ TAG_MAKE(KIND_CLIENT, c->nclients)) < 0) { -+ logmsg(LOG_WARNING, "epoll_add client: %s", -+ strerror(errno)); -+ close(fd); -+ continue; -+ } -+ -+ c->clients[c->nclients++] = fd; -+ } -+} -+ -+/* -+ * Send one event to all connected clients. Errors drop the client. -+ * Iterate backwards so client_close's swap-remove doesn't skip entries. -+ */ -+static void broadcast_event(struct ctx *c, -+ const struct scoutfs_ioctl_notify_event *ev) -+{ -+ int i; -+ -+ for (i = c->nclients - 1; i >= 0; i--) { -+ ssize_t sent = send(c->clients[i], ev, sizeof(*ev), -+ MSG_DONTWAIT | MSG_NOSIGNAL); -+ if (sent == (ssize_t)sizeof(*ev)) -+ continue; -+ /* -+ * SOCK_SEQPACKET is atomic per-message; partial send -+ * shouldn't happen, but treat anything less as a drop. -+ */ -+ client_close(c, i); -+ } -+} -+ -+/* -+ * Drain kernel events with a short timeout. A positive return means -+ * we should keep polling without delay; zero means go to epoll_wait -+ * with a longer sleep. -+ */ -+static int drain_kernel(struct ctx *c) -+{ -+ struct scoutfs_ioctl_notify_event events[IOCTL_BATCH_EVENTS]; -+ struct scoutfs_ioctl_read_notify req = { -+ .events_ptr = (uint64_t)(uintptr_t)events, -+ .events_nr = IOCTL_BATCH_EVENTS, -+ .timeout_ms = IOCTL_TIMEOUT_MS, -+ .flags = 0, -+ }; -+ int ret; -+ int i; -+ -+ ret = ioctl(c->mnt_fd, SCOUTFS_IOC_READ_NOTIFY, &req); -+ if (ret < 0) { -+ if (errno == ETIMEDOUT || errno == EINTR) -+ return 0; -+ if (errno == EBUSY) { -+ logmsg(LOG_ERR, -+ "another reader is attached; backing off"); -+ sleep(1); -+ return 0; -+ } -+ logmsg(LOG_ERR, "SCOUTFS_IOC_READ_NOTIFY: %s", -+ strerror(errno)); -+ g_shutdown = 1; -+ return -1; -+ } -+ -+ for (i = 0; i < ret; i++) -+ broadcast_event(c, &events[i]); -+ -+ return ret; -+} -+ -+static int handle_epoll(struct ctx *c, int timeout_ms) -+{ -+ struct epoll_event evs[16]; -+ int n; -+ int i; -+ -+ n = epoll_wait(c->epoll_fd, evs, -+ (int)(sizeof(evs) / sizeof(evs[0])), timeout_ms); -+ if (n < 0) { -+ if (errno == EINTR) -+ return 0; -+ logmsg(LOG_ERR, "epoll_wait: %s", strerror(errno)); -+ return -1; -+ } -+ -+ for (i = 0; i < n; i++) { -+ uint64_t tag = evs[i].data.u64; -+ int kind = TAG_KIND(tag); -+ -+ switch (kind) { -+ case KIND_LISTEN: -+ client_accept(c); -+ break; -+ -+ case KIND_SIGNAL: -+ g_shutdown = 1; -+ return 0; -+ -+ case KIND_CLIENT: -+ /* any readability / hangup → drop client */ -+ client_close(c, TAG_INDEX(tag)); -+ break; -+ -+ default: -+ logmsg(LOG_WARNING, "unexpected epoll tag kind %d", -+ kind); -+ break; -+ } -+ } -+ -+ return 0; -+} -+ -+static void cleanup(struct ctx *c) -+{ -+ int i; -+ -+ for (i = 0; i < c->nclients; i++) -+ close(c->clients[i]); -+ c->nclients = 0; -+ -+ if (c->listen_fd >= 0) -+ close(c->listen_fd); -+ if (c->sock_path[0]) -+ unlink(c->sock_path); -+ if (c->signal_fd >= 0) -+ close(c->signal_fd); -+ if (c->epoll_fd >= 0) -+ close(c->epoll_fd); -+ if (c->mnt_fd >= 0) -+ close(c->mnt_fd); -+} -+ -+static void usage(FILE *f) -+{ -+ fprintf(f, -+ "usage: %s \n" -+ "\n" -+ " Drains the scoutfs file access notification ring for the\n" -+ " given mount and broadcasts events to clients connected to\n" -+ " /run/scoutfs//notify.sock.\n" -+ "\n" -+ " Requires CAP_SYS_ADMIN. Socket is root-only (mode 0600).\n", -+ PROGNAME); -+} -+ -+int main(int argc, char **argv) -+{ -+ struct ctx c = { -+ .mnt_fd = -1, -+ .listen_fd = -1, -+ .epoll_fd = -1, -+ .signal_fd = -1, -+ }; -+ uint64_t fsid; -+ int ret = EXIT_FAILURE; -+ -+ openlog(PROGNAME, LOG_PID | LOG_PERROR, LOG_DAEMON); -+ -+ if (argc != 2 || argv[1][0] == '-') { -+ usage(stderr); -+ goto out; -+ } -+ -+ if (get_fsid(argv[1], &fsid, &c.mnt_fd) < 0) -+ goto out; -+ -+ snprintf(c.run_dir, sizeof(c.run_dir), -+ RUN_DIR_FMT, (unsigned long long)fsid); -+ snprintf(c.sock_path, sizeof(c.sock_path), -+ SOCK_PATH_FMT, (unsigned long long)fsid); -+ -+ if (ensure_run_dir(c.run_dir) < 0) -+ goto out; -+ -+ c.listen_fd = create_listener(c.sock_path); -+ if (c.listen_fd < 0) -+ goto out; -+ -+ c.signal_fd = setup_signalfd(); -+ if (c.signal_fd < 0) -+ goto out; -+ -+ c.epoll_fd = epoll_create1(EPOLL_CLOEXEC); -+ if (c.epoll_fd < 0) { -+ logmsg(LOG_ERR, "epoll_create1: %s", strerror(errno)); -+ goto out; -+ } -+ -+ if (epoll_add(c.epoll_fd, c.listen_fd, EPOLLIN, -+ TAG_MAKE(KIND_LISTEN, 0)) < 0 || -+ epoll_add(c.epoll_fd, c.signal_fd, EPOLLIN, -+ TAG_MAKE(KIND_SIGNAL, 0)) < 0) { -+ logmsg(LOG_ERR, "epoll_ctl: %s", strerror(errno)); -+ goto out; -+ } -+ -+ logmsg(LOG_INFO, "listening on %s", c.sock_path); -+ -+ while (!g_shutdown) { -+ /* -+ * The ioctl blocks up to IOCTL_TIMEOUT_MS when the ring -+ * is empty, so this loop naturally paces itself and the -+ * daemon is idle between events. Client / signal fds -+ * are drained non-blocking between ioctl calls. -+ */ -+ if (drain_kernel(&c) < 0) -+ break; -+ if (handle_epoll(&c, 0) < 0) -+ break; -+ } -+ -+ logmsg(LOG_INFO, "shutting down"); -+ ret = EXIT_SUCCESS; -+ -+out: -+ cleanup(&c); -+ closelog(); -+ return ret; -+} -diff --git a/utils/notifyd/scoutfs-notifyd@.service b/utils/notifyd/scoutfs-notifyd@.service -new file mode 100644 -index 0000000..2e87f17 ---- /dev/null -+++ b/utils/notifyd/scoutfs-notifyd@.service -@@ -0,0 +1,40 @@ -+[Unit] -+Description=ScoutFS file access notification daemon for %I -+Documentation=man:scoutfs-notifyd(8) -+# The mount must exist before we can open it and read its fsid. -+RequiresMountsFor=/%I -+ -+[Service] -+Type=simple -+User=root -+Group=root -+UMask=0077 -+# scoutfs-notifyd creates /run/scoutfs// and binds the socket -+# inside it; the top-level /run/scoutfs is created here so the daemon -+# never has to. -+RuntimeDirectory=scoutfs -+RuntimeDirectoryMode=0755 -+ExecStart=/usr/sbin/scoutfs-notifyd /%I -+Restart=on-failure -+RestartSec=5s -+StartLimitBurst=5 -+ -+# The daemon needs CAP_SYS_ADMIN to drive the read-notify ioctl. -+# Everything else can be stripped. -+CapabilityBoundingSet=CAP_SYS_ADMIN -+AmbientCapabilities=CAP_SYS_ADMIN -+NoNewPrivileges=yes -+PrivateTmp=yes -+ProtectSystem=strict -+ProtectHome=yes -+ProtectKernelTunables=yes -+ProtectKernelModules=yes -+ProtectControlGroups=yes -+RestrictAddressFamilies=AF_UNIX -+LockPersonality=yes -+MemoryDenyWriteExecute=yes -+RestrictRealtime=yes -+RestrictSUIDSGID=yes -+ -+[Install] -+WantedBy=multi-user.target -diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in -index fb24b81..0aa91f5 100644 ---- a/utils/scoutfs-utils.spec.in -+++ b/utils/scoutfs-utils.spec.in -@@ -52,19 +52,23 @@ cp man/*.5.gz $RPM_BUILD_ROOT%{_mandir}/man5/. - cp man/*.7.gz $RPM_BUILD_ROOT%{_mandir}/man7/. - cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. - install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs -+install -m 755 -D notifyd/scoutfs-notifyd $RPM_BUILD_ROOT%{_sbindir}/scoutfs-notifyd - install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h - install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h - install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced - install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service - install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example -+install -m 644 -D notifyd/scoutfs-notifyd@.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-notifyd@.service - - %files - %defattr(644,root,root,755) - %{_mandir}/man*/scoutfs*.gz - /%{_unitdir}/scoutfs-fenced.service -+/%{_unitdir}/scoutfs-notifyd@.service - %{_sysconfdir}/scoutfs - %defattr(755,root,root,755) - %{_sbindir}/scoutfs -+%{_sbindir}/scoutfs-notifyd - %{_libexecdir}/scoutfs-fenced - - %files -n scoutfs-devel --- -2.49.0.windows.1 -