fix(mount): add -posixDirNlink option for correct directory nlink

Add optional POSIX-compliant directory nlink counting
(nlink = 2 + number_of_subdirectories) behind the -posixDirNlink
flag. This requires listing cached directory entries on every stat,
which has a performance cost, so it's off by default.

When disabled (default), directories report nlink=2, which is the
POSIX-compliant baseline for empty directories and a common default
in distributed filesystems.

- Set nlink baseline to 2 for all directories (was 1)
- Add applyDirNlink() gated behind option.PosixDirNlink
- Add -posixDirNlink CLI flag

.github/workflows/binaries_dev.yml

@@ -18,6 +18,7 @@ jobs:
 
       - name: Delete old release assets
         uses: mknejp/delete-release-assets@v1
+        continue-on-error: true
         with:
           token: ${{ github.token }}
           tag: dev

.github/workflows/fuse-dlm-integration.yml

@@ -0,0 +1,63 @@
+name: "FUSE DLM Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/command/mount*.go'
+      - 'weed/mount/**'
+      - 'weed/cluster/**'
+      - 'test/fuse_dlm/**'
+      - '.github/workflows/fuse-dlm-integration.yml'
+  push:
+    branches: [master]
+    paths:
+      - 'weed/command/mount*.go'
+      - 'weed/mount/**'
+      - 'weed/cluster/**'
+      - 'test/fuse_dlm/**'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  fuse-dlm-integration:
+    name: FUSE DLM Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install FUSE dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libfuse3-dev
+          echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
+          sudo chmod 644 /etc/fuse.conf
+
+      - name: Build SeaweedFS
+        run: go build -o weed/weed -buildvcs=false ./weed
+
+      - name: Run DLM integration tests
+        timeout-minutes: 25
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+        run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: fuse-dlm-test-logs
+          path: /tmp/seaweedfs-fuse-dlm-logs/
+          retention-days: 3

.github/workflows/pjdfstest.yml

@@ -0,0 +1,87 @@
+name: "pjdfstest POSIX Compliance"
+
+on:
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'test/pjdfstest/**'
+      - '.github/workflows/pjdfstest.yml'
+  pull_request:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'test/pjdfstest/**'
+      - '.github/workflows/pjdfstest.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: pjdfstest/${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  pjdfstest:
+    name: pjdfstest
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6
+
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Build SeaweedFS e2e image
+      run: |
+        cd docker
+        make build_e2e || {
+          echo "Retrying without buildx cache..."
+          make binary_race
+          docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
+        }
+
+    - name: Build pjdfstest image
+      run: |
+        docker build -t chrislusf/seaweedfs:pjdfstest test/pjdfstest/
+
+    - name: Start SeaweedFS cluster
+      run: |
+        docker compose -f test/pjdfstest/docker-compose.yml up --wait
+
+    - name: Run pjdfstest
+      run: |
+        set -o pipefail
+        docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
+          /run.sh 2>&1 | tee /tmp/pjdfstest-output.log
+
+    - name: Collect logs
+      if: always()
+      run: |
+        mkdir -p /tmp/pjdfstest-docker-logs
+        for svc in master volume filer mount; do
+          docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
+            > "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
+        done
+
+    - name: Tear down
+      if: always()
+      run: |
+        docker compose -f test/pjdfstest/docker-compose.yml down -v
+
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: pjdfstest-results
+        path: |
+          /tmp/pjdfstest-output.log
+          /tmp/pjdfstest-docker-logs/
+        retention-days: 7

.github/workflows/rust_binaries_dev.yml

@@ -19,6 +19,7 @@ jobs:
     steps:
       - name: Delete old Rust volume dev assets
         uses: mknejp/delete-release-assets@v1
+        continue-on-error: true
         with:
           token: ${{ github.token }}
           tag: dev

Makefile

@@ -42,8 +42,7 @@ test: admin-generate
 
 # Admin component targets
 admin-generate:
-	@echo "Generating admin component templates..."
-	@cd $(ADMIN_DIR) && templ generate ./view
+	@cd $(ADMIN_DIR) && $(MAKE) generate
 
 admin-build: admin-generate
 	@echo "Building admin component..."

go.mod

@@ -489,14 +489,14 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.42.0 // indirect
-	go.opentelemetry.io/otel/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/trace v1.42.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
-	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/term v0.41.0
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect

go.sum

@@ -2128,8 +2128,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
-go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
+go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
@@ -2144,14 +2144,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
-go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
-go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
-go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
-go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
-go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
-go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
-go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
+go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
+go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
+go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
+go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
+go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
+go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
+go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=

k8s/charts/seaweedfs/values.yaml

@@ -1339,9 +1339,14 @@ worker:
   # Admin server to connect to
   adminServer: ""
 
-  # Worker job types - comma-separated list
-  # Available: vacuum, volume_balance, erasure_coding
-  jobType: "vacuum,volume_balance,erasure_coding"
+  # Worker job types - categories or comma-separated list of names/aliases
+  # Categories: all, default, heavy
+  #   default: vacuum, volume_balance, ec_balance, admin_script
+  #   heavy: erasure_coding, iceberg_maintenance
+  #   all: every registered job type
+  # Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
+  # Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
+  jobType: "all"
 
   # Maximum number of concurrent detection requests
   maxDetect: 1

seaweed-volume/src/remote_storage/s3_tier.rs

@@ -486,6 +486,11 @@ impl S3TierRegistry {
         self.backends.keys().cloned().collect()
     }
 
+    /// Remove a backend by name.
+    pub fn remove(&mut self, name: &str) {
+        self.backends.remove(name);
+    }
+
     pub fn clear(&mut self) {
         self.backends.clear();
     }

seaweed-volume/src/server/grpc_server.rs

@@ -4192,6 +4192,7 @@ mod tests {
         let addr = listener.local_addr().unwrap();
         listener.set_nonblocking(true).unwrap();
         let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+        let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
 
         std::thread::spawn(move || {
             let runtime = tokio::runtime::Builder::new_current_thread()
@@ -4253,6 +4254,7 @@ mod tests {
                 }));
 
                 let listener = tokio::net::TcpListener::from_std(listener).unwrap();
+                let _ = ready_tx.send(());
                 axum::serve(listener, app)
                     .with_graceful_shutdown(async move {
                         let _ = shutdown_rx.await;
@@ -4262,6 +4264,8 @@ mod tests {
             });
         });
 
+        // Wait for the server thread to be ready before returning.
+        ready_rx.recv().unwrap();
         (format!("http://{}", addr), shutdown_tx)
     }
 
@@ -4307,7 +4311,9 @@ mod tests {
         std::fs::remove_file(&dat_path).unwrap();
 
         let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
-        global_s3_tier_registry().write().unwrap().clear();
+        // Use a test-specific backend_id to avoid racing with other tests
+        // that share the global registry.  Never call clear() — only
+        // register/remove our own entries.
         let tier_config = S3TierConfig {
             access_key: "access".to_string(),
             secret_key: "secret".to_string(),
@@ -4319,14 +4325,16 @@ mod tests {
         };
         {
             let mut registry = global_s3_tier_registry().write().unwrap();
-            registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
-            registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
+            registry.register(
+                "s3.incr_copy_test".to_string(),
+                S3TierBackend::new(&tier_config),
+            );
         }
 
         let vif = crate::storage::volume::VifVolumeInfo {
             files: vec![crate::storage::volume::VifRemoteFile {
                 backend_type: "s3".to_string(),
-                backend_id: "default".to_string(),
+                backend_id: "incr_copy_test".to_string(),
                 key: "remote-key".to_string(),
                 offset: 0,
                 file_size: dat_bytes.len() as u64,
@@ -4535,7 +4543,10 @@ mod tests {
         assert_eq!(copied, dat_bytes[super_block_size as usize..]);
 
         let _ = shutdown_tx.send(());
-        global_s3_tier_registry().write().unwrap().clear();
+        global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.incr_copy_test");
     }
 
     #[tokio::test]

seaweed-volume/src/server/heartbeat.rs

@@ -1408,10 +1408,8 @@ mod tests {
     #[test]
     fn test_apply_storage_backends_registers_s3_default_aliases() {
         let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Do not call clear() on the global registry — other tests may be
+        // running concurrently.  Just register our entries and verify them.
 
         apply_storage_backends(
             &state,
@@ -1443,10 +1441,8 @@ mod tests {
     #[test]
     fn test_apply_storage_backends_ignores_unsupported_types() {
         let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Do not call clear() on the global registry — other tests may be
+        // running concurrently.
 
         apply_storage_backends(
             &state,
@@ -1457,12 +1453,17 @@ mod tests {
             }],
         );
 
+        // The per-state registry is freshly created and should have no entries
+        // since "rclone" is unsupported.
         let registry = state.s3_tier_registry.read().unwrap();
         assert!(registry.names().is_empty());
+        // Only check that the unsupported type was not added to the global
+        // registry.  Other tests may have their own entries present.
         let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
             .read()
             .unwrap();
-        assert!(global_registry.names().is_empty());
+        assert!(global_registry.get("rclone.default").is_none());
+        assert!(global_registry.get("rclone").is_none());
     }
 
     #[test]

seaweed-volume/src/storage/store.rs

@@ -1158,7 +1158,13 @@ mod tests {
                 Vec::new(),
             )
             .unwrap();
-        store.volume_size_limit.store(1024, Ordering::Relaxed);
+        // Use a large volume_size_limit so the unused-space difference between
+        // preallocate=true (0) and preallocate=false (~2 × limit) is big enough
+        // that integer-division rounding and disk-free fluctuations between the
+        // two get_disk_stats calls cannot make the quotients equal.
+        store
+            .volume_size_limit
+            .store(100 * 1024 * 1024, Ordering::Relaxed);
         store
             .add_volume(
                 VolumeId(61),

seaweed-volume/src/storage/volume.rs

@@ -3225,6 +3225,7 @@ mod tests {
         let addr = listener.local_addr().unwrap();
         listener.set_nonblocking(true).unwrap();
         let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+        let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
 
         std::thread::spawn(move || {
             let runtime = tokio::runtime::Builder::new_current_thread()
@@ -3279,6 +3280,7 @@ mod tests {
                 }));
 
                 let listener = tokio::net::TcpListener::from_std(listener).unwrap();
+                let _ = ready_tx.send(());
                 axum::serve(listener, app)
                     .with_graceful_shutdown(async move {
                         let _ = shutdown_rx.await;
@@ -3288,6 +3290,8 @@ mod tests {
             });
         });
 
+        // Wait for the server thread to be ready before returning.
+        ready_rx.recv().unwrap();
         (format!("http://{}", addr), shutdown_tx)
     }
 
@@ -3833,7 +3837,7 @@ mod tests {
             let vif = VifVolumeInfo {
                 files: vec![VifRemoteFile {
                     backend_type: "s3".to_string(),
-                    backend_id: "default".to_string(),
+                    backend_id: "vif_rw_test".to_string(),
                     key: "remote-key".to_string(),
                     offset: 0,
                     file_size: v.dat_file_size().unwrap(),
@@ -3849,6 +3853,27 @@ mod tests {
             )
             .unwrap();
 
+            // Register a dummy S3 backend so load_remote_dat_file can look it
+            // up.  This test never reads remote data, so a dead endpoint is fine.
+            // Use a test-specific backend_id to avoid racing with other tests
+            // that share the global registry.
+            let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                region: "us-east-1".to_string(),
+                bucket: "bucket-a".to_string(),
+                endpoint: "http://127.0.0.1:1".to_string(),
+                storage_class: "STANDARD".to_string(),
+                force_path_style: true,
+            };
+            crate::remote_storage::s3_tier::global_s3_tier_registry()
+                .write()
+                .unwrap()
+                .register(
+                    "s3.vif_rw_test".to_string(),
+                    crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
+                );
+
             v.dat_file_size().unwrap()
         };
 
@@ -3892,6 +3917,11 @@ mod tests {
             .unwrap();
         assert!(deleted_size.0 > 0);
         assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
+
+        crate::remote_storage::s3_tier::global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.vif_rw_test");
     }
 
     #[test]
@@ -4054,10 +4084,9 @@ mod tests {
         std::fs::remove_file(&dat_path).unwrap();
 
         let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Use a test-specific backend_id to avoid racing with other tests
+        // that share the global registry.  Never call clear() — only
+        // register/remove our own entries.
         let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
             access_key: "access".to_string(),
             secret_key: "secret".to_string(),
@@ -4072,11 +4101,7 @@ mod tests {
                 .write()
                 .unwrap();
             registry.register(
-                "s3.default".to_string(),
-                crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
-            );
-            registry.register(
-                "s3".to_string(),
+                "s3.remote_only_rw".to_string(),
                 crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
             );
         }
@@ -4084,7 +4109,7 @@ mod tests {
         let vif = VifVolumeInfo {
             files: vec![VifRemoteFile {
                 backend_type: "s3".to_string(),
-                backend_id: "default".to_string(),
+                backend_id: "remote_only_rw".to_string(),
                 key: "remote-key".to_string(),
                 offset: 0,
                 file_size: dat_bytes.len() as u64,
@@ -4141,6 +4166,10 @@ mod tests {
         assert_eq!(meta.data_size, 11);
 
         let _ = shutdown_tx.send(());
+        crate::remote_storage::s3_tier::global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.remote_only_rw");
     }
 
     /// Volume destroy removes .vif alongside the primary data files.

seaweedfs-rdma-sidecar/go.mod

@@ -33,11 +33,11 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/image v0.36.0 // indirect
-	golang.org/x/net v0.50.0 // indirect
+	golang.org/x/image v0.38.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 )

seaweedfs-rdma-sidecar/go.sum

@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
 golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

test/fuse_dlm/dlm_concurrent_write_test.go

@@ -0,0 +1,277 @@
+package fuse_dlm
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
+// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
+// writes are serialized — one blocks until the other completes.
+//
+// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
+// and filer metadata subscription, which are asynchronous. This test verifies
+// write integrity, not instant read convergence.
+func TestDLMConcurrentWritersSameFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const iterations = 5
+	for iter := 0; iter < iterations; iter++ {
+		fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
+		payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
+		payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
+			assert.NoError(t, err, "mount0 write iteration %d", iter)
+		}()
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
+			assert.NoError(t, err, "mount1 write iteration %d", iter)
+		}()
+
+		wg.Wait()
+
+		// Verify file is readable and contains one of the expected payloads
+		// (read from mount0 — its own view is authoritative for write success).
+		content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+		require.NoError(t, err, "read from mount0 iteration %d", iter)
+		validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
+		assert.True(t, validPayload,
+			"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
+	}
+}
+
+// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
+// from both mounts all succeed without errors.
+func TestDLMRepeatedOpenWriteClose(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const cycles = 20
+	fileName := "repeated_write.txt"
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
+			assert.NoError(t, err, "mount0 cycle %d", i)
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
+			assert.NoError(t, err, "mount1 cycle %d", i)
+		}
+	}()
+
+	wg.Wait()
+
+	// File must be readable from at least one mount
+	content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+	require.NoError(t, err)
+	assert.NotEmpty(t, content, "file must not be empty")
+}
+
+// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
+// mount has a file open for writing, another mount's write-open blocks until
+// the first mount closes the file.
+func TestDLMWriteBlocksSecondWriter(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	fileName := "blocking_test.txt"
+	path0 := filepath.Join(cluster.mountPoints[0], fileName)
+	path1 := filepath.Join(cluster.mountPoints[1], fileName)
+
+	// Mount 0 opens the file for writing and holds it open
+	f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 open")
+	_, err = f.Write([]byte("mount0-holds-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
+	var mount1Completed atomic.Bool
+	mount1Done := make(chan error, 1)
+	go func() {
+		err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
+		mount1Completed.Store(true)
+		mount1Done <- err
+	}()
+
+	// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, mount1Completed.Load(),
+		"mount1 write must not complete while mount0 holds the file open")
+	t.Log("mount1 write is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes the file — this releases the DLM lock
+	require.NoError(t, f.Close(), "mount0 close")
+
+	// Mount 1 should now complete
+	select {
+	case err := <-mount1Done:
+		assert.NoError(t, err, "mount1 write after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("mount1 write did not complete within 30s after mount0 closed")
+	}
+}
+
+// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
+// while mount0 has a file open for writing (re-opened after creation),
+// mount1 cannot rename it until mount0 closes the file.
+func TestDLMRenameWhileWriteOpen(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	origName := "rename_source.txt"
+	newName := "rename_dest.txt"
+
+	// Create and close the file first so it's flushed to the filer and
+	// visible on both mounts.
+	require.NoError(t, os.WriteFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		[]byte("initial-content"), 0644))
+	time.Sleep(2 * time.Second) // metadata propagation
+
+	// Verify mount1 can see the file
+	_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
+	require.NoError(t, err, "mount1 should see the file")
+
+	// Mount 0 re-opens the file for writing and holds it open
+	f, err := os.OpenFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 reopen")
+	_, err = f.Write([]byte("data-while-holding-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to rename — should block because mount0 holds the
+	// DLM lock on the old path
+	var renameCompleted atomic.Bool
+	renameDone := make(chan error, 1)
+	go func() {
+		err := os.Rename(
+			filepath.Join(cluster.mountPoints[1], origName),
+			filepath.Join(cluster.mountPoints[1], newName))
+		renameCompleted.Store(true)
+		renameDone <- err
+	}()
+
+	// Rename must NOT complete while mount0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, renameCompleted.Load(),
+		"rename must not complete while mount0 holds the file open")
+	t.Log("rename is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes → releases DLM lock → rename should proceed
+	require.NoError(t, f.Close(), "mount0 close")
+
+	select {
+	case err := <-renameDone:
+		assert.NoError(t, err, "rename after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("rename did not complete within 30s after mount0 closed")
+	}
+}
+
+// Note: Same-mount rename while a file is open for writing is not tested here
+// because macOS FUSE serializes operations on the same inode, causing a
+// kernel-level deadlock between the Rename handler's internal flush and the
+// pending Close. Same-mount coordination is already handled by the per-mount
+// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
+
+// TestDLMConcurrentRenames verifies that two concurrent renames of the same
+// file from different mounts don't corrupt metadata. DLM locks on both old
+// and new paths ensure renames are serialized.
+func TestDLMConcurrentRenames(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	// Create a file first
+	origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
+	require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
+	time.Sleep(1 * time.Second) // propagation
+
+	// Both mounts try to rename the same file concurrently
+	var wg sync.WaitGroup
+	var errA, errB error
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		errA = os.Rename(
+			filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
+		)
+	}()
+
+	go func() {
+		defer wg.Done()
+		errB = os.Rename(
+			filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
+		)
+	}()
+
+	wg.Wait()
+
+	// At least one rename should succeed; the other may fail with ENOENT
+	// since the source was already moved.
+	succeeded := 0
+	if errA == nil {
+		succeeded++
+		t.Logf("mount0 rename succeeded")
+	} else {
+		t.Logf("mount0 rename failed: %v", errA)
+	}
+	if errB == nil {
+		succeeded++
+		t.Logf("mount1 rename succeeded")
+	} else {
+		t.Logf("mount1 rename failed: %v", errB)
+	}
+	assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
+}

test/fuse_dlm/framework_test.go

@@ -0,0 +1,516 @@
+package fuse_dlm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const filerGroup = "fuse-dlm-test"
+
+// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
+// mounts for testing DLM-based cross-mount write coordination.
+type dlmTestCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPorts     [2]int
+	filerGrpcPorts [2]int
+	mountPoints    [2]string
+
+	masterCmd  *exec.Cmd
+	volumeCmd  *exec.Cmd
+	filerCmds  [2]*exec.Cmd
+	mountCmds  [2]*exec.Cmd
+	logFiles   []*os.File
+
+	cleanupOnce sync.Once
+}
+
+func startDLMTestCluster(t testing.TB) *dlmTestCluster {
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
+	require.NoError(t, err)
+
+	c := &dlmTestCluster{
+		t:          t,
+		baseDir:    baseDir,
+		weedBinary: binary,
+	}
+	// Register cleanup early so processes are stopped even if a require fails below.
+	t.Cleanup(c.Stop)
+
+	// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
+	ports := allocatePorts(t, 8)
+	c.masterPort = ports[0]
+	c.masterGrpcPort = ports[1]
+	c.volumePort = ports[2]
+	c.volumeGrpcPort = ports[3]
+	c.filerPorts[0] = ports[4]
+	c.filerGrpcPorts[0] = ports[5]
+	c.filerPorts[1] = ports[6]
+	c.filerGrpcPorts[1] = ports[7]
+
+	// Write empty security.toml
+	configDir := filepath.Join(baseDir, "config")
+	require.NoError(t, os.MkdirAll(configDir, 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
+
+	// Start master
+	require.NoError(t, c.startMaster(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
+		"master not ready\n%s", c.tailLog("master"))
+
+	// Start volume
+	require.NoError(t, c.startVolume(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
+		"volume not ready\n%s", c.tailLog("volume"))
+
+	// Start 2 filers
+	for i := 0; i < 2; i++ {
+		require.NoError(t, c.startFiler(i, configDir))
+		require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
+			"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
+	}
+	require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
+	require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
+
+	// Start 2 mounts, both pointing at filer0 for metadata consistency.
+	// (filer1 exists for the DLM lock ring but both mounts share filer0's
+	// metadata store since leveldb is per-filer.)
+	for i := 0; i < 2; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i, configDir))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+
+	return c
+}
+
+func (c *dlmTestCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		// Stop mounts first (triggers flush + DLM unlock).
+		// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.mountCmds[i])
+			// Backup unmount in case FUSE teardown didn't clean up
+			exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		// Stop filers, volume, master
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.filerCmds[i])
+		}
+		stopCmd(c.volumeCmd)
+		stopCmd(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			f.Close()
+		}
+
+		// Copy logs for CI
+		c.copyLogsForCI()
+
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+
+		// Wait for ports to be fully released before the next test
+		// allocates new ports (avoids TIME_WAIT collisions).
+		time.Sleep(2 * time.Second)
+	})
+}
+
+// masterAddress returns the master address in the format that encodes both
+// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
+// SeaweedFS uses to communicate non-default gRPC ports between components.
+func (c *dlmTestCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *dlmTestCluster) startMaster(configDir string) error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *dlmTestCluster) startVolume(configDir string) error {
+	volDir := filepath.Join(c.baseDir, "volume")
+	if err := os.MkdirAll(volDir, 0755); err != nil {
+		return fmt.Errorf("create volume dir: %w", err)
+	}
+	c.volumeCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePort),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
+		"-master="+c.masterAddress(),
+		"-dir="+volDir,
+		"-max=10",
+	)
+	return c.startCmd(c.volumeCmd, "volume")
+}
+
+func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
+	filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPorts[idx]),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
+		"-master="+c.masterAddress(),
+		"-filerGroup="+filerGroup,
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
+}
+
+func (c *dlmTestCluster) filerAddress(idx int) string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
+}
+
+func (c *dlmTestCluster) startMount(idx int, configDir string) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"mount",
+		"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-cacheDir="+cacheDir,
+		"-dlm",
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	logFile, err := os.Create(filepath.Join(logPath, name+".log"))
+	if err != nil {
+		return err
+	}
+	c.logFiles = append(c.logFiles, logFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	return cmd.Start()
+}
+
+func (c *dlmTestCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *dlmTestCluster) copyLogsForCI() {
+	ciLogDir := "/tmp/seaweedfs-fuse-dlm-logs"
+	os.MkdirAll(ciLogDir, 0755)
+	logsDir := filepath.Join(c.baseDir, "logs")
+	entries, err := os.ReadDir(logsDir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout", addr)
+}
+
+// waitForMount waits for a FUSE filesystem to actually be mounted at
+// mountPoint by comparing the device ID of the mount point against its parent.
+// A plain directory (pre-created before mount) has the same device as its
+// parent; a mounted FUSE filesystem has a different device.
+func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		parentSys := parentStat.Sys().(*syscall.Stat_t)
+		mountSys := mountStat.Sys().(*syscall.Stat_t)
+		if parentSys.Dev != mountSys.Dev {
+			// Different device = FUSE mounted
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
+	return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
+}
+
+func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "filer",
+			FilerGroup: filerGroup,
+		})
+		cancel()
+		if err == nil && len(resp.ClusterNodes) >= expected {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
+}
+
+// waitForLockRingConverged verifies that both filers have a consistent view of
+// the lock ring by acquiring the same lock through each filer and checking
+// mutual exclusion. Adapted from test/s3/distributed_lock/.
+func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := []pb.ServerAddress{
+		pb.ServerAddress(c.filerGRPCAddress(0)),
+		pb.ServerAddress(c.filerGRPCAddress(1)),
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge")
+}
+
+// convergenceKeysPerPrimary generates one test key per primary filer.
+func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	found := make(map[pb.ServerAddress]bool)
+	var keys []string
+	for i := 0; len(found) < len(owners) && i < 10000; i++ {
+		key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
+		primary, _ := ring.GetPrimaryAndBackup(key)
+		if !found[primary] {
+			found[primary] = true
+			keys = append(keys, key)
+		}
+	}
+	return keys
+}
+
+func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
+	// Try to lock via filer0
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-0",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp0.Error != "" {
+		return false, fmt.Errorf("lock0: %s", resp0.Error)
+	}
+
+	// Try to lock via filer1 — should fail (already locked)
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+	resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-1",
+	})
+
+	// Unlock via filer0
+	client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+		Name:       key,
+		RenewToken: resp0.RenewToken,
+	})
+
+	if err != nil {
+		return false, err
+	}
+	// If filer1 also got the lock, the ring hasn't converged
+	if resp1.Error == "" {
+		// Unlock the second one too
+		client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+			Name:       key,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func stopCmd(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Signal(syscall.SIGTERM)
+	done := make(chan struct{})
+	go func() {
+		cmd.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		cmd.Process.Kill()
+		<-done
+	}
+}
+
+func allocatePorts(t testing.TB, n int) []int {
+	t.Helper()
+	// Hold all listeners open until all ports are collected, then close
+	// them together. This prevents the OS from reassigning a just-freed
+	// port to the next Listen call within the same allocation batch.
+	listeners := make([]net.Listener, 0, n)
+	ports := make([]int, 0, n)
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+	for i := 0; i < n; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	return ports
+}
+
+func findWeedBinary() string {
+	if p := os.Getenv("WEED_BINARY"); p != "" {
+		return p
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	candidates := []string{
+		"../../weed/weed",
+		"./weed",
+	}
+	for _, c := range candidates {
+		if info, err := os.Stat(c); err == nil && !info.IsDir() {
+			abs, _ := filepath.Abs(c)
+			return abs
+		}
+	}
+	return ""
+}

test/fuse_integration/framework_test.go

@@ -82,12 +82,13 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	}
 }
 
-// freePort asks the OS for a free TCP port.
+// freePort asks the OS for a free TCP port in a range where the gRPC
+// offset (port + 10000) won't collide with well-known ports.
 func freePort(t *testing.T) int {
 	t.Helper()
 	const (
 		minServicePort = 20000
-		maxServicePort = 55535 // SeaweedFS gRPC service uses httpPort + 10000.
+		maxServicePort = 55535
 	)
 
 	portCount := maxServicePort - minServicePort + 1

test/kafka/go.mod

@@ -14,7 +14,7 @@ require (
 replace github.com/seaweedfs/seaweedfs => ../../
 
 require (
-	cloud.google.com/go/auth v0.18.1 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -44,23 +44,23 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
@@ -124,8 +124,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
-	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
+	github.com/googleapis/gax-go/v2 v2.19.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -197,7 +197,6 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	github.com/rdleal/intervalst v1.5.0 // indirect
 	github.com/relvacode/iso8601 v1.7.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
@@ -242,30 +241,29 @@ require (
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
 	golang.org/x/image v0.38.0 // indirect
-	golang.org/x/net v0.51.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/term v0.41.0 // indirect
 	golang.org/x/text v0.35.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.267.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/api v0.274.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/mathutil v1.7.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 	storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect

test/kafka/go.sum

@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
-cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -114,12 +114,12 @@ github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
 github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
 github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
 github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
@@ -130,24 +130,24 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgq
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
-github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
+github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -600,8 +600,6 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
 github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
 github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
 github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -754,16 +752,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
@@ -869,8 +867,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -981,8 +979,8 @@ golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -1054,8 +1052,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
-google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
+google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
+google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1089,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1154,8 +1152,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
-modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
 moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
 moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

test/multi_master/cluster.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -17,6 +16,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 const (
@@ -73,23 +74,23 @@ func StartMasterCluster(t testing.TB) *MasterCluster {
 	logsDir := filepath.Join(baseDir, "logs")
 	os.MkdirAll(logsDir, 0o755)
 
-	// Allocate 3 port pairs (http, grpc) atomically to prevent reuse.
-	portPairs, err := allocateMultipleMasterPortPairs(3)
+	// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
+	httpPorts, err := testutil.AllocateMiniPorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}
 	var nodes [3]*masterNode
 	var peerParts []string
-	for i, pp := range portPairs {
+	for i, hp := range httpPorts {
 		dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
 		os.MkdirAll(dataDir, 0o755)
 		nodes[i] = &masterNode{
-			port:     pp[0],
-			grpcPort: pp[1],
+			port:     hp,
+			grpcPort: hp + testutil.GrpcPortOffset,
 			dataDir:  dataDir,
 			logFile:  filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
 		}
-		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", pp[0]))
+		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
 	}
 
 	mc := &MasterCluster{
@@ -357,41 +358,6 @@ func (mc *MasterCluster) tailLog(i int) string {
 	return strings.Join(lines, "\n")
 }
 
-// --- port and binary helpers (adapted from test/volume_server/framework) ---
-
-// allocateMultipleMasterPortPairs finds n non-overlapping (http, grpc) port
-// pairs, holding all listeners until all are found, then releasing them
-// together to avoid races between consecutive allocations.
-func allocateMultipleMasterPortPairs(n int) ([][2]int, error) {
-	var listeners []net.Listener
-	var pairs [][2]int
-
-	defer func() {
-		for _, l := range listeners {
-			l.Close()
-		}
-	}()
-
-	for masterPort := 10000; masterPort <= 55535 && len(pairs) < n; masterPort++ {
-		grpcPort := masterPort + 10000
-		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
-		if err != nil {
-			continue
-		}
-		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)))
-		if err != nil {
-			l1.Close()
-			continue
-		}
-		listeners = append(listeners, l1, l2)
-		pairs = append(pairs, [2]int{masterPort, grpcPort})
-	}
-
-	if len(pairs) < n {
-		return nil, fmt.Errorf("could only allocate %d of %d master port pairs", len(pairs), n)
-	}
-	return pairs, nil
-}
 
 func findOrBuildWeedBinary() (string, error) {
 	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {

test/pjdfstest/Dockerfile

@@ -0,0 +1,28 @@
+FROM chrislusf/seaweedfs:e2e
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    --no-install-recommends \
+    --no-install-suggests \
+    autoconf \
+    automake \
+    build-essential \
+    git \
+    libtap-harness-archive-perl \
+    perl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG PJDFSTEST_REPO=https://github.com/pjd/pjdfstest.git
+ARG PJDFSTEST_REF=03eb25706d8dbf3611c3f820b45b7a5e09a36c06
+
+RUN git clone "${PJDFSTEST_REPO}" /opt/pjdfstest && \
+    cd /opt/pjdfstest && \
+    git checkout "${PJDFSTEST_REF}" && \
+    autoreconf -ifs && \
+    ./configure && \
+    make pjdfstest
+
+COPY run_inside_container.sh /run.sh
+COPY known_failures.txt /known_failures.txt
+RUN chmod +x /run.sh

test/pjdfstest/docker-compose.yml

@@ -0,0 +1,57 @@
+services:
+  master:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  volume:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      master:
+        condition: service_healthy
+
+  filer:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8888/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      volume:
+        condition: service_healthy
+
+  mount:
+    image: chrislusf/seaweedfs:pjdfstest
+    build:
+      context: .
+    command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs -allowOthers"
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - /dev/fuse
+    security_opt:
+      - apparmor:unconfined
+    healthcheck:
+      test: ["CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 10s
+    depends_on:
+      filer:
+        condition: service_healthy

test/pjdfstest/known_failures.txt

@@ -0,0 +1,60 @@
+# Known pjdfstest failures for SeaweedFS FUSE mount.
+#
+# Tests listed here are skipped during CI runs. Each entry must be a path
+# relative to the pjdfstest root (e.g. tests/chmod/02.t).
+#
+# A failure in any test NOT listed here will cause the CI job to fail,
+# catching regressions immediately.
+
+# ── Linux FUSE NAME_MAX=255 limitation ──────────────────────────────────
+# The Linux FUSE kernel module enforces NAME_MAX=255 at the VFS layer.
+# These tests create filenames >255 bytes which cannot be looked up via
+# normal syscalls (stat, chmod, etc.) after creation.
+tests/chmod/02.t
+tests/chmod/03.t
+tests/chown/02.t
+tests/chown/03.t
+tests/ftruncate/02.t
+tests/ftruncate/03.t
+tests/link/02.t
+tests/link/03.t
+tests/mkdir/02.t
+tests/mkdir/03.t
+tests/mkfifo/02.t
+tests/mkfifo/03.t
+tests/mknod/02.t
+tests/mknod/03.t
+tests/open/02.t
+tests/open/03.t
+tests/rename/01.t
+tests/rename/02.t
+tests/rmdir/02.t
+tests/rmdir/03.t
+tests/symlink/02.t
+tests/symlink/03.t
+tests/truncate/02.t
+tests/truncate/03.t
+tests/unlink/02.t
+tests/unlink/03.t
+
+# ── Hard link nlink/ctime tracking (requires filer changes) ────────────
+# nlink counts are not correctly maintained across rename operations.
+tests/rename/23.t
+tests/rename/24.t
+
+# ── Parent directory mtime/ctime on deferred file create ───────────────
+# When file creation is deferred (not flushed to filer immediately),
+# the parent directory mtime/ctime cannot be updated without invalidating
+# the just-cached child entry in the metadata cache.
+tests/open/00.t
+
+# ── Directory rename permission edge case ──────────────────────────────
+# Cross-directory rename of a subdirectory with restricted permissions
+# causes cascading test failures within the test file.
+tests/rename/21.t
+
+# ── rmdir after hard link unlink ───────────────────────────────────────
+# The filer may still report a directory as non-empty after all hard-linked
+# entries have been unlinked.
+tests/unlink/14.t
+

test/pjdfstest/run.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+#
+# Run the pjdfstest POSIX compliance suite against a SeaweedFS FUSE mount.
+#
+# This script:
+#   1. Starts a self-contained "weed mini" server (master+volume+filer in one)
+#   2. Mounts the filesystem with "weed mount"
+#   3. Builds pjdfstest from upstream and runs it under prove(1)
+#
+# Requirements: weed in $PATH, fusermount3, perl with TAP::Harness::Archive,
+# autoconf, make, sudo (pjdfstest exercises chown/chmod which need root).
+#
+# Usage:
+#   test/pjdfstest/run.sh                # runs full suite
+#   PJDFSTEST_TESTS=tests/chmod ./run.sh # runs a subset
+
+set -euo pipefail
+
+WEED_BIN="${WEED_BIN:-weed}"
+WORK_DIR="${WORK_DIR:-$(mktemp -d /tmp/seaweedfs-pjdfstest.XXXXXX)}"
+MOUNT_DIR="${MOUNT_DIR:-${WORK_DIR}/mnt}"
+DATA_DIR="${DATA_DIR:-${WORK_DIR}/data}"
+LOG_DIR="${LOG_DIR:-${WORK_DIR}/logs}"
+FILER_PORT="${FILER_PORT:-28888}"
+FILER_ADDR="127.0.0.1:${FILER_PORT}"
+
+# Pin to an immutable upstream commit so CI is reproducible. Override via env
+# if you want to test against a different ref or fork.
+PJDFSTEST_REPO="${PJDFSTEST_REPO:-https://github.com/pjd/pjdfstest.git}"
+PJDFSTEST_REF="${PJDFSTEST_REF:-03eb25706d8dbf3611c3f820b45b7a5e09a36c06}"
+PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
+
+mini_pid=""
+mount_pid=""
+
+CI_LOG_DIR="/tmp/seaweedfs-pjdfstest-logs"
+
+cleanup() {
+  set +e
+  if [[ -n "${mount_pid}" ]] && kill -0 "${mount_pid}" 2>/dev/null; then
+    kill -TERM "${mount_pid}" 2>/dev/null || true
+    wait "${mount_pid}" 2>/dev/null || true
+  fi
+  if mountpoint -q "${MOUNT_DIR}" 2>/dev/null; then
+    fusermount3 -u "${MOUNT_DIR}" 2>/dev/null || \
+      fusermount -u "${MOUNT_DIR}" 2>/dev/null || \
+      sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+  fi
+  if [[ -n "${mini_pid}" ]] && kill -0 "${mini_pid}" 2>/dev/null; then
+    kill -TERM "${mini_pid}" 2>/dev/null || true
+    wait "${mini_pid}" 2>/dev/null || true
+  fi
+  # Copy logs to a fixed path for CI artifact upload.
+  mkdir -p "${CI_LOG_DIR}"
+  cp "${LOG_DIR}"/*.log "${CI_LOG_DIR}/" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p "${MOUNT_DIR}" "${DATA_DIR}" "${LOG_DIR}"
+
+echo "==> Starting weed mini on ${FILER_ADDR}"
+"${WEED_BIN}" mini \
+  -dir="${DATA_DIR}" \
+  -ip=127.0.0.1 \
+  -filer.port="${FILER_PORT}" \
+  -s3=false \
+  -webdav=false \
+  -admin.ui=false \
+  >"${LOG_DIR}/mini.log" 2>&1 &
+mini_pid=$!
+
+# Wait for filer to accept connections.
+for i in $(seq 1 60); do
+  if (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+    break
+  fi
+  if ! kill -0 "${mini_pid}" 2>/dev/null; then
+    echo "weed mini exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+
+if ! (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+  echo "weed mini filer did not become reachable within 30s; log tail:" >&2
+  tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+  exit 1
+fi
+
+echo "==> Mounting SeaweedFS at ${MOUNT_DIR}"
+# allowOthers is required so that pjdfstest's setuid/setgid sub-tests (run via
+# sudo) can access files created as the invoking user.
+sudo "${WEED_BIN}" mount \
+  -filer="${FILER_ADDR}" \
+  -dir="${MOUNT_DIR}" \
+  -filer.path=/ \
+  -dirAutoCreate \
+  -allowOthers=true \
+  >"${LOG_DIR}/mount.log" 2>&1 &
+mount_pid=$!
+
+for i in $(seq 1 60); do
+  if mountpoint -q "${MOUNT_DIR}"; then
+    break
+  fi
+  if ! kill -0 "${mount_pid}" 2>/dev/null; then
+    echo "weed mount exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+
+if ! mountpoint -q "${MOUNT_DIR}"; then
+  echo "FUSE mount did not come up within 30s" >&2
+  tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
+  exit 1
+fi
+
+echo "==> Cloning and building pjdfstest"
+PJDFSTEST_DIR="${WORK_DIR}/pjdfstest"
+if [[ ! -d "${PJDFSTEST_DIR}/.git" ]]; then
+  git clone "${PJDFSTEST_REPO}" "${PJDFSTEST_DIR}"
+fi
+git -C "${PJDFSTEST_DIR}" remote set-url origin "${PJDFSTEST_REPO}"
+git -C "${PJDFSTEST_DIR}" fetch --depth 1 origin "${PJDFSTEST_REF}"
+git -C "${PJDFSTEST_DIR}" checkout --detach FETCH_HEAD
+(
+  cd "${PJDFSTEST_DIR}"
+  autoreconf -ifs
+  ./configure
+  make pjdfstest
+)
+
+# pjdfstest must run inside the filesystem under test.
+TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
+sudo mkdir -p "${TEST_ROOT}"
+sudo cp -r "${PJDFSTEST_DIR}/." "${TEST_ROOT}/"
+
+cd "${TEST_ROOT}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KNOWN_FAILURES="${SCRIPT_DIR}/known_failures.txt"
+
+# Build the list of tests to run, excluding known failures.
+if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
+  mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
+  all_tests=()
+  while IFS= read -r -d '' t; do
+    all_tests+=("$t")
+  done < <(find tests/ -name '*.t' -print0 | sort -z)
+
+  run_tests=()
+  for t in "${all_tests[@]}"; do
+    is_skipped=false
+    for s in "${skip[@]}"; do
+      if [[ "$t" == "$s" ]]; then
+        is_skipped=true
+        break
+      fi
+    done
+    if ! $is_skipped; then
+      run_tests+=("$t")
+    fi
+  done
+
+  echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
+  sudo prove -rv "${run_tests[@]}"
+else
+  echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
+  sudo prove -rv "${PJDFSTEST_TESTS}"
+fi

test/pjdfstest/run_inside_container.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# Runs pjdfstest inside the mount container against /mnt/seaweedfs.
+# Invoked via: docker compose exec mount /run.sh
+#
+# Uses known_failures.txt to skip tests with known issues. Any failure
+# in a test NOT in the skip list causes a non-zero exit (regression).
+set -euo pipefail
+
+MOUNT_DIR="/mnt/seaweedfs"
+PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
+KNOWN_FAILURES="/known_failures.txt"
+
+# Copy pjdfstest into the mounted filesystem so tests exercise the FS under test.
+TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
+mkdir -p "${TEST_ROOT}"
+cp -r /opt/pjdfstest/. "${TEST_ROOT}/"
+
+cd "${TEST_ROOT}"
+
+# Build the list of tests to run, excluding known failures.
+if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
+  mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
+  all_tests=()
+  while IFS= read -r -d '' t; do
+    all_tests+=("$t")
+  done < <(find tests/ -name '*.t' -print0 | sort -z)
+
+  run_tests=()
+  for t in "${all_tests[@]}"; do
+    is_skipped=false
+    for s in "${skip[@]}"; do
+      if [[ "$t" == "$s" ]]; then
+        is_skipped=true
+        break
+      fi
+    done
+    if ! $is_skipped; then
+      run_tests+=("$t")
+    fi
+  done
+
+  echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
+  prove -rv "${run_tests[@]}"
+else
+  echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
+  prove -rv "${PJDFSTEST_TESTS}"
+fi

test/s3/catalog_trino/trino_catalog_test.go

@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -13,6 +12,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 type TestEnvironment struct {
@@ -96,29 +97,26 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 		t.Fatalf("Failed to create temp dir: %v", err)
 	}
 
-	bindIP := findBindIP()
+	bindIP := testutil.FindBindIP()
 
-	masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
-	icebergPort := mustFreePort(t, "Iceberg")
+	// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
+	ports := testutil.MustAllocatePorts(t, 9)
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
 		weedBinary:      weedBinary,
 		dataDir:         dataDir,
 		bindIP:          bindIP,
-		s3Port:          s3Port,
-		s3GrpcPort:      s3GrpcPort,
-		icebergPort:     icebergPort,
-		masterPort:      masterPort,
-		masterGrpcPort:  masterGrpcPort,
-		filerPort:       filerPort,
-		filerGrpcPort:   filerGrpcPort,
-		volumePort:      volumePort,
-		volumeGrpcPort:  volumeGrpcPort,
-		dockerAvailable: hasDocker(),
+		masterPort:      ports[0],
+		masterGrpcPort:  ports[1],
+		volumePort:      ports[2],
+		volumeGrpcPort:  ports[3],
+		filerPort:       ports[4],
+		filerGrpcPort:   ports[5],
+		s3Port:          ports[6],
+		s3GrpcPort:      ports[7],
+		icebergPort:     ports[8],
+		dockerAvailable: testutil.HasDocker(),
 	}
 }
 
@@ -333,73 +331,6 @@ func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	}
 }
 
-func hasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	port, err := getFreePort()
-	if err != nil {
-		t.Fatalf("Failed to get free port for %s: %v", name, err)
-	}
-	return port
-}
-
-func mustFreePortPair(t *testing.T, name string) (int, int) {
-	t.Helper()
-
-	httpPort, grpcPort, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	return httpPort, grpcPort
-}
-
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := getFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	grpcPort, err := getFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	return httpPort, grpcPort, nil
-}
-
-func getFreePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}
-
-func findBindIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return "127.0.0.1"
-	}
-	for _, addr := range addrs {
-		ipNet, ok := addr.(*net.IPNet)
-		if !ok || ipNet.IP == nil {
-			continue
-		}
-		ip := ipNet.IP.To4()
-		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
-			continue
-		}
-		return ip.String()
-	}
-	return "127.0.0.1"
-}
-
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
 	b := make([]byte, length)

test/s3/distributed_lock/distributed_lock_cluster_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -115,7 +116,7 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
 		require.NoError(t, os.MkdirAll(dir, 0o755), "create %s", dir)
 	}
 
-	ports, err := allocatePorts(12)
+	ports, err := testutil.AllocatePorts(12)
 	require.NoError(t, err, "allocate ports")
 	cluster.masterPort = ports[0]
 	cluster.masterGrpcPort = ports[1]
@@ -591,25 +592,6 @@ func (c *distributedLockCluster) tailLog(name string) string {
 	return strings.Join(lines, "\n")
 }
 
-func allocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, openListener := range listeners {
-				_ = openListener.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
 
 func stopProcess(cmd *exec.Cmd) {
 	if cmd == nil || cmd.Process == nil {

test/s3/iam/s3_iam_admin_test.go

@@ -321,6 +321,13 @@ func TestIAMPolicyManagement(t *testing.T) {
 		require.NoError(t, err)
 		defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
 
+		// List user policies before any inline policy is attached
+		listResp, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "New user should have no inline policies")
+
 		policyName := "test-inline-policy"
 		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
 
@@ -332,6 +339,14 @@ func TestIAMPolicyManagement(t *testing.T) {
 		})
 		require.NoError(t, err)
 
+		// List user policies after attaching inline policy
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, listResp.PolicyNames, "User should have inline policies after PutUserPolicy")
+		assert.False(t, *listResp.IsTruncated)
+
 		// Get user policy
 		getResp, err := iamClient.GetUserPolicy(&iam.GetUserPolicyInput{
 			UserName:   aws.String(userName),
@@ -348,5 +363,22 @@ func TestIAMPolicyManagement(t *testing.T) {
 			PolicyName: aws.String(policyName),
 		})
 		require.NoError(t, err)
+
+		// List user policies after deletion
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "User should have no inline policies after DeleteUserPolicy")
+	})
+
+	t.Run("list_user_policies_nonexistent_user", func(t *testing.T) {
+		_, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String("nonexistent-user-for-list-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
 	})
 }

test/s3/iam/s3_iam_group_test.go

@@ -778,6 +778,105 @@ func TestIAMGroupRawAPI(t *testing.T) {
 	})
 }
 
+// TestIAMGroupInlinePolicy tests group inline policy operations:
+// PutGroupPolicy, GetGroupPolicy, ListGroupPolicies, DeleteGroupPolicy
+func TestIAMGroupInlinePolicy(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	// Skip if running against embedded IAM (which returns NotImplemented for group inline policies)
+	_, probeErr := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{GroupName: aws.String("probe-group-inline-support")})
+	if probeErr != nil {
+		if awsErr, ok := probeErr.(awserr.Error); ok && awsErr.Code() == "NotImplemented" {
+			t.Skip("Skipping: group inline policies not supported in embedded IAM mode")
+		}
+	}
+	require.NoError(t, err)
+
+	groupName := "test-group-inline-policy"
+	policyName := "TestInlinePolicy"
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+	defer func() {
+		// Clean up inline policies before deleting the group
+		iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
+	}()
+
+	t.Run("list_empty", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+		assert.False(t, *resp.IsTruncated)
+	})
+
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`
+
+	t.Run("put_group_policy", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_put", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, resp.PolicyNames)
+	})
+
+	t.Run("get_group_policy", func(t *testing.T) {
+		resp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, groupName, *resp.GroupName)
+		assert.Equal(t, policyName, *resp.PolicyName)
+		assert.Contains(t, *resp.PolicyDocument, "s3:GetObject")
+	})
+
+	t.Run("delete_group_policy", func(t *testing.T) {
+		_, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_delete", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+	})
+
+	t.Run("nonexistent_group", func(t *testing.T) {
+		_, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String("nonexistent-group-for-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}
+
 // createS3Client creates an S3 client with static credentials
 func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
 	sess, err := session.NewSession(&aws.Config{

test/s3/normal/s3_integration_test.go

@@ -7,7 +7,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -25,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -112,37 +112,16 @@ func TestS3Integration(t *testing.T) {
 }
 
 // findAvailablePort finds an available port by binding to port 0
-func findAvailablePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}
 
 // startMiniCluster starts a weed mini instance directly without exec.
 // Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
 func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
-	// Find available ports
-	masterPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find master port: %v", err)
-	}
-	volumePort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find volume port: %v", err)
-	}
-	filerPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find filer port: %v", err)
-	}
-	s3Port, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find s3 port: %v", err)
-	}
+	// Allocate non-colliding ports (including gRPC offsets) for weed mini.
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
+	masterPort := ports[0]
+	volumePort := ports[1]
+	filerPort := ports[2]
+	s3Port := ports[3]
 	// Create temporary directory for test data
 	testDir := t.TempDir()
 
@@ -167,7 +146,7 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 
 	// Create empty security.toml to disable JWT authentication in tests
 	securityToml := filepath.Join(testDir, "security.toml")
-	err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
+	err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
 	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
@@ -233,10 +212,9 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 	}()
 
 	// Wait for S3 service to be ready
-	err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
-	if err != nil {
+	if !testutil.WaitForService(cluster.s3Endpoint, 30*time.Second) {
 		cancel()
-		return nil, fmt.Errorf("S3 service failed to start: %v", err)
+		return nil, fmt.Errorf("S3 service failed to start at %s", cluster.s3Endpoint)
 	}
 
 	// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
@@ -277,14 +255,12 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
 		return fmt.Errorf("resolve rust volume binary: %v", err)
 	}
 
-	rustVolumePort, err := findAvailablePort()
+	rustPorts, err := testutil.AllocatePorts(2)
 	if err != nil {
-		return fmt.Errorf("find rust volume port: %v", err)
-	}
-	rustVolumeGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return fmt.Errorf("find rust volume grpc port: %v", err)
+		return fmt.Errorf("find rust volume ports: %v", err)
 	}
+	rustVolumePort := rustPorts[0]
+	rustVolumeGrpcPort := rustPorts[1]
 
 	rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
 	if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {
@@ -377,25 +353,6 @@ func (c *TestCluster) Stop() {
 	}
 }
 
-// waitForS3Ready waits for the S3 service to be ready
-func waitForS3Ready(endpoint string, timeout time.Duration) error {
-	client := &http.Client{Timeout: 1 * time.Second}
-	deadline := time.Now().Add(timeout)
-
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(endpoint)
-		if err == nil {
-			resp.Body.Close()
-			// Wait a bit more to ensure service is fully ready
-			time.Sleep(500 * time.Millisecond)
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-
-	return fmt.Errorf("timeout waiting for S3 service at %s", endpoint)
-}
-
 // Test functions
 
 func testCreateBucket(t *testing.T, cluster *TestCluster) {

test/s3/policy/policy_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -21,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -703,43 +703,13 @@ func uniqueName(prefix string) string {
 
 // --- Test setup helpers ---
 
-func findAvailablePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}
-
-// findAvailablePortPair finds an available http port P such that P and P+10000 (grpc) are both available
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := findAvailablePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	for i := 0; i < 100; i++ {
-		grpcPort, err := findAvailablePort()
-		if err != nil {
-			return 0, 0, err
-		}
-		if grpcPort != httpPort {
-			return httpPort, grpcPort, nil
-		}
-	}
-	return 0, 0, fmt.Errorf("failed to find available port pair")
-}
 
 func startMiniCluster(t *testing.T) (*TestCluster, error) {
-	masterPort, masterGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	volumePort, volumeGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	filerPort, filerGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	s3Port, s3GrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]
 
 	testDir := t.TempDir()
 
@@ -760,7 +730,7 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 
 	// Disable authentication for tests
 	securityToml := filepath.Join(testDir, "security.toml")
-	err = os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
+	err := os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
 	require.NoError(t, err)
 
 	// Configure credential store for IAM tests
@@ -819,10 +789,9 @@ enabled = true
 	}()
 
 	// Wait for S3
-	err = waitForS3Ready(cluster.s3Endpoint, 60*time.Second)
-	if err != nil {
+	if !testutil.WaitForService(cluster.s3Endpoint, 60*time.Second) {
 		cancel()
-		return nil, err
+		return nil, fmt.Errorf("timeout waiting for S3 at %s", cluster.s3Endpoint)
 	}
 
 	// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
@@ -837,19 +806,6 @@ enabled = true
 	return cluster, nil
 }
 
-func waitForS3Ready(endpoint string, timeout time.Duration) error {
-	client := &http.Client{Timeout: 1 * time.Second}
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(endpoint)
-		if err == nil {
-			resp.Body.Close()
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	return fmt.Errorf("timeout waiting for S3")
-}
 
 // startRustVolumeServer starts a Rust volume server that registers with the same master.
 func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
@@ -860,14 +816,12 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
 		return fmt.Errorf("resolve rust volume binary: %v", err)
 	}
 
-	rustVolumePort, err := findAvailablePort()
+	rustPorts, err := testutil.AllocatePorts(2)
 	if err != nil {
-		return fmt.Errorf("find rust volume port: %v", err)
-	}
-	rustVolumeGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return fmt.Errorf("find rust volume grpc port: %v", err)
+		return fmt.Errorf("find rust volume ports: %v", err)
 	}
+	rustVolumePort := rustPorts[0]
+	rustVolumeGrpcPort := rustPorts[1]
 
 	rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
 	if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {

test/s3/spark/setup_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/testcontainers/testcontainers-go"
 )
 
@@ -94,9 +94,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
 
 	bindIP := testutil.FindBindIP()
 	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)

test/s3tables/catalog/duckdb_oauth_test.go

@@ -0,0 +1,465 @@
+package catalog
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+)
+
+// oauthTestEnv holds a weed mini instance started with IAM credentials
+// so that the OAuth token endpoint is functional.
+type oauthTestEnv struct {
+	seaweedDir     string
+	weedBinary     string
+	dataDir        string
+	bindIP         string
+	s3Port         int
+	s3GrpcPort     int
+	icebergPort    int
+	masterPort     int
+	masterGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	volumePort     int
+	volumeGrpcPort int
+	webdavPort     int
+	weedProcess    *exec.Cmd
+	weedCancel     context.CancelFunc
+	accessKey      string
+	secretKey      string
+}
+
+func newOAuthTestEnv(t *testing.T) *oauthTestEnv {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-oauth-test-*")
+	if err != nil {
+		t.Fatalf("create temp dir: %v", err)
+	}
+
+	bindIP := testutil.FindBindIP()
+	ports := testutil.MustAllocatePorts(t, 10)
+
+	return &oauthTestEnv{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		bindIP:         bindIP,
+		masterPort:     ports[0],
+		masterGrpcPort: ports[1],
+		volumePort:     ports[2],
+		volumeGrpcPort: ports[3],
+		filerPort:      ports[4],
+		filerGrpcPort:  ports[5],
+		s3Port:         ports[6],
+		s3GrpcPort:     ports[7],
+		icebergPort:    ports[8],
+		webdavPort:     ports[9],
+		accessKey:      "AKIAIOSFODNN7EXAMPLE",
+		secretKey:      "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	}
+}
+
+func (env *oauthTestEnv) start(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("write IAM config: %v", err)
+	}
+
+	securityToml := filepath.Join(env.dataDir, "security.toml")
+	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+		t.Fatalf("write security.toml: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
+		"-webdav.port", fmt.Sprintf("%d", env.webdavPort),
+		"-s3.config", iamConfigPath,
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+	)
+
+	if err := cmd.Start(); err != nil {
+		cancel()
+		t.Fatalf("start weed mini: %v", err)
+	}
+	env.weedProcess = cmd
+
+	icebergURL := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
+	if !testutil.WaitForService(icebergURL, 30*time.Second) {
+		cancel()
+		cmd.Wait()
+		t.Fatalf("Iceberg REST API did not become ready at %s", icebergURL)
+	}
+}
+
+func (env *oauthTestEnv) cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		env.weedProcess.Wait()
+	}
+	if env.dataDir != "" {
+		os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *oauthTestEnv) icebergURL() string {
+	return fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
+}
+
+// TestOAuthTokenEndpoint tests the /v1/oauth/tokens endpoint directly via HTTP.
+func TestOAuthTokenEndpoint(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := newOAuthTestEnv(t)
+	defer env.cleanup(t)
+	env.start(t)
+
+	t.Run("valid credentials", func(t *testing.T) {
+		token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+		if token == "" {
+			t.Fatal("expected non-empty token")
+		}
+	})
+
+	t.Run("invalid secret", func(t *testing.T) {
+		resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+			"grant_type":    {"client_credentials"},
+			"client_id":     {env.accessKey},
+			"client_secret": {"wrong-secret"},
+		})
+		if err != nil {
+			t.Fatalf("POST /v1/oauth/tokens: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusUnauthorized {
+			body, _ := io.ReadAll(resp.Body)
+			t.Fatalf("expected 401, got %d: %s", resp.StatusCode, body)
+		}
+	})
+
+	t.Run("missing grant_type", func(t *testing.T) {
+		resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+			"client_id":     {env.accessKey},
+			"client_secret": {env.secretKey},
+		})
+		if err != nil {
+			t.Fatalf("POST /v1/oauth/tokens: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusBadRequest {
+			body, _ := io.ReadAll(resp.Body)
+			t.Fatalf("expected 400, got %d: %s", resp.StatusCode, body)
+		}
+	})
+
+	t.Run("bearer token auth on catalog endpoint", func(t *testing.T) {
+		token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+
+		// Use the token to call the catalog
+		req, err := http.NewRequest(http.MethodGet, env.icebergURL()+"/v1/namespaces", nil)
+		if err != nil {
+			t.Fatalf("create request: %v", err)
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("GET /v1/namespaces with Bearer: %v", err)
+		}
+		defer resp.Body.Close()
+
+		// Auth should pass. We accept 200 (success) or 500 (missing warehouse bucket
+		// is an internal error, not an auth error). Reject 401/403/404/405.
+		body, _ := io.ReadAll(resp.Body)
+		switch resp.StatusCode {
+		case http.StatusOK, http.StatusInternalServerError:
+			t.Logf("Bearer token auth succeeded, status=%d", resp.StatusCode)
+		default:
+			t.Fatalf("Bearer auth failed unexpectedly: status=%d body=%s", resp.StatusCode, body)
+		}
+	})
+}
+
+// TestDuckDBOAuthIntegration tests that DuckDB can connect to the Iceberg REST
+// catalog using the OAuth2 client_credentials flow (CREATE SECRET with client_id
+// and client_secret). This is the scenario reported in issue #9015.
+func TestDuckDBOAuthIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available, skipping DuckDB OAuth integration test")
+	}
+
+	env := newOAuthTestEnv(t)
+	defer env.cleanup(t)
+	env.start(t)
+
+	// Create a table bucket and namespace so DuckDB has something to query
+	bucketName := "duckdb-oauth-" + randomSuffix()
+	createTableBucketViaShell(t, env, bucketName)
+
+	// Create a namespace via the Iceberg REST API using OAuth token
+	token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+	createNamespaceWithToken(t, env, token, bucketName, "testns")
+
+	sqlContent := fmt.Sprintf(`
+INSTALL iceberg;
+LOAD iceberg;
+
+CREATE SECRET iceberg_secret (
+    TYPE ICEBERG,
+    ENDPOINT 'http://host.docker.internal:%d',
+    CLIENT_ID '%s',
+    CLIENT_SECRET '%s',
+    SCOPE 's3://%s/'
+);
+
+CREATE SECRET s3_secret (
+    TYPE S3,
+    KEY_ID '%s',
+    SECRET '%s',
+    ENDPOINT 'host.docker.internal:%d',
+    URL_STYLE 'path',
+    USE_SSL false,
+    SCOPE 's3://%s/'
+);
+
+SELECT 'OAuth token obtained successfully' as status;
+
+-- Try listing namespaces via the Iceberg catalog
+SELECT * FROM iceberg_scan('iceberg_secret', ALLOW_MOVED_PATHS => TRUE) LIMIT 0;
+`, env.icebergPort, env.accessKey, env.secretKey, bucketName,
+		env.accessKey, env.secretKey, env.s3Port, bucketName)
+
+	sqlFile := filepath.Join(env.dataDir, "duckdb_oauth_test.sql")
+	if err := os.WriteFile(sqlFile, []byte(sqlContent), 0644); err != nil {
+		t.Fatalf("write SQL file: %v", err)
+	}
+
+	// Run DuckDB in Docker.
+	// We use a simple test: get a token, create a secret, and verify the
+	// Iceberg extension can communicate with the catalog.
+	// The simpler fallback test just verifies CREATE SECRET succeeds (no 404).
+	fallbackSQL := fmt.Sprintf(`
+INSTALL iceberg;
+LOAD iceberg;
+
+CREATE SECRET (
+    TYPE ICEBERG,
+    ENDPOINT 'http://host.docker.internal:%d',
+    CLIENT_ID '%s',
+    CLIENT_SECRET '%s'
+);
+
+SELECT 'DuckDB Iceberg OAuth secret created successfully' as result;
+`, env.icebergPort, env.accessKey, env.secretKey)
+
+	fallbackFile := filepath.Join(env.dataDir, "duckdb_oauth_fallback.sql")
+	if err := os.WriteFile(fallbackFile, []byte(fallbackSQL), 0644); err != nil {
+		t.Fatalf("write fallback SQL file: %v", err)
+	}
+
+	// Try the full SQL script first (CREATE SECRET + catalog access).
+	// Fall back to the simple CREATE SECRET test if iceberg_scan isn't supported.
+	cmd := exec.Command("docker", "run", "--rm",
+		"-v", fmt.Sprintf("%s:/test", env.dataDir),
+		"--add-host", "host.docker.internal:host-gateway",
+		"--entrypoint", "duckdb",
+		"duckdb/duckdb:latest",
+		"-init", "/test/duckdb_oauth_test.sql",
+		"-c", "SELECT 1",
+	)
+
+	output, err := cmd.CombinedOutput()
+	outputStr := string(output)
+	t.Logf("DuckDB output:\n%s", outputStr)
+
+	if err != nil {
+		if strings.Contains(outputStr, "iceberg extension is not available") ||
+			strings.Contains(outputStr, "Failed to load") {
+			t.Skip("Skipping: Iceberg extension not available in DuckDB Docker image")
+		}
+		// The key check: the old error was "HTTP NotFound_404" on /v1/oauth/tokens.
+		// With our fix, this should no longer happen.
+		if strings.Contains(outputStr, "NotFound_404") && strings.Contains(outputStr, "/v1/oauth/tokens") {
+			t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
+		}
+		// If iceberg_scan failed but CREATE SECRET worked, fall back to simpler test
+		if strings.Contains(outputStr, "OAuth token obtained successfully") {
+			t.Logf("Full SQL had partial success (token obtained), iceberg_scan may not be supported. Continuing.")
+		} else {
+			// Try the fallback script that only tests CREATE SECRET
+			t.Logf("Full SQL failed, trying fallback CREATE SECRET test...")
+			fallbackCmd := exec.Command("docker", "run", "--rm",
+				"-v", fmt.Sprintf("%s:/test", env.dataDir),
+				"--add-host", "host.docker.internal:host-gateway",
+				"--entrypoint", "duckdb",
+				"duckdb/duckdb:latest",
+				"-init", "/test/duckdb_oauth_fallback.sql",
+				"-c", "SELECT 1",
+			)
+			fallbackOutput, fallbackErr := fallbackCmd.CombinedOutput()
+			fallbackStr := string(fallbackOutput)
+			t.Logf("DuckDB fallback output:\n%s", fallbackStr)
+
+			if fallbackErr != nil {
+				if strings.Contains(fallbackStr, "NotFound_404") && strings.Contains(fallbackStr, "/v1/oauth/tokens") {
+					t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
+				}
+				t.Fatalf("DuckDB fallback also failed: %v\nOutput: %s", fallbackErr, fallbackStr)
+			}
+			if !strings.Contains(fallbackStr, "DuckDB Iceberg OAuth secret created successfully") {
+				t.Errorf("expected success message in fallback output, got:\n%s", fallbackStr)
+			}
+			return
+		}
+	}
+
+	if strings.Contains(outputStr, "OAuth token obtained successfully") {
+		t.Logf("DuckDB OAuth CREATE SECRET succeeded")
+	} else {
+		t.Errorf("expected OAuth success message in output, got:\n%s", outputStr)
+	}
+}
+
+// requestOAuthToken obtains a bearer token from the OAuth endpoint.
+func requestOAuthToken(t *testing.T, env *oauthTestEnv, accessKey, secretKey string) string {
+	t.Helper()
+
+	resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+		"grant_type":    {"client_credentials"},
+		"client_id":     {accessKey},
+		"client_secret": {secretKey},
+	})
+	if err != nil {
+		t.Fatalf("POST /v1/oauth/tokens: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("OAuth token request failed: status=%d body=%s", resp.StatusCode, body)
+	}
+
+	var tokenResp struct {
+		AccessToken string `json:"access_token"`
+		TokenType   string `json:"token_type"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &tokenResp); err != nil {
+		t.Fatalf("decode token response: %v", err)
+	}
+
+	if tokenResp.AccessToken == "" {
+		t.Fatal("got empty access_token")
+	}
+	if tokenResp.TokenType != "bearer" {
+		t.Errorf("expected token_type=bearer, got %s", tokenResp.TokenType)
+	}
+	return tokenResp.AccessToken
+}
+
+// createTableBucketViaShell creates a table bucket using weed shell,
+// which bypasses S3 auth. This is the same approach used by the Trino tests.
+func createTableBucketViaShell(t *testing.T, env *oauthTestEnv, bucketName string) {
+	t.Helper()
+
+	cmd := exec.Command(env.weedBinary, "shell",
+		fmt.Sprintf("-master=%s:%d.%d", env.bindIP, env.masterPort, env.masterGrpcPort),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, output)
+	}
+	t.Logf("Created table bucket %s", bucketName)
+}
+
+// createNamespaceWithToken creates a namespace using a Bearer token.
+func createNamespaceWithToken(t *testing.T, env *oauthTestEnv, token, bucketName, namespace string) {
+	t.Helper()
+
+	path := fmt.Sprintf("/v1/%s/namespaces", bucketName)
+	body := fmt.Sprintf(`{"namespace":["%s"]}`, namespace)
+	req, err := http.NewRequest(http.MethodPost, env.icebergURL()+path, strings.NewReader(body))
+	if err != nil {
+		t.Fatalf("create request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+token)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("create namespace: %v", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		t.Fatalf("create namespace failed: status=%d body=%s", resp.StatusCode, respBody)
+	}
+	t.Logf("Created namespace %s in bucket %s", namespace, bucketName)
+}

test/s3tables/catalog/iceberg_catalog_test.go

@@ -9,7 +9,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -17,6 +16,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 // sharedEnv is the single TestEnvironment shared across all tests in this package.
@@ -69,23 +70,6 @@ type TestEnvironment struct {
 	dockerAvailable bool
 }
 
-// hasDocker checks if Docker is available
-func hasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-// getFreePort returns an available ephemeral port and its listener
-func getFreePort() (int, net.Listener, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, nil, err
-	}
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, listener, nil
-}
-
 // newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
 // can be used from TestMain (which has no *testing.T).
 func newTestEnvironmentForMain() (*TestEnvironment, error) {
@@ -118,95 +102,27 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
 		return nil, fmt.Errorf("create temp dir: %w", err)
 	}
 
-	// Allocate free ephemeral ports for each service
-	var listeners []net.Listener
-	closeListeners := func() {
-		for _, l := range listeners {
-			l.Close()
-		}
-	}
-
-	var l net.Listener
-	s3Port, l, err := getFreePort()
+	// Allocate 9 unique ports atomically: s3, iceberg, s3Grpc, master, masterGrpc,
+	// filer, filerGrpc, volume, volumeGrpc
+	ports, err := testutil.AllocatePorts(9)
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3: %w", err)
+		return nil, fmt.Errorf("allocate ports: %w", err)
 	}
-	listeners = append(listeners, l)
-
-	icebergPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Iceberg: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	s3GrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	masterPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	masterGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	filerPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	filerGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	volumePort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	volumeGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	// Release the port reservations so weed mini can bind to them
-	closeListeners()
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
 		weedBinary:      weedBinary,
 		dataDir:         dataDir,
-		s3Port:          s3Port,
-		s3GrpcPort:      s3GrpcPort,
-		icebergPort:     icebergPort,
-		masterPort:      masterPort,
-		masterGrpcPort:  masterGrpcPort,
-		filerPort:       filerPort,
-		filerGrpcPort:   filerGrpcPort,
-		volumePort:      volumePort,
-		volumeGrpcPort:  volumeGrpcPort,
-		dockerAvailable: hasDocker(),
+		s3Port:          ports[0],
+		s3GrpcPort:      ports[1],
+		icebergPort:     ports[2],
+		masterPort:      ports[3],
+		masterGrpcPort:  ports[4],
+		filerPort:       ports[5],
+		filerGrpcPort:   ports[6],
+		volumePort:      ports[7],
+		volumeGrpcPort:  ports[8],
+		dockerAvailable: testutil.HasDocker(),
 	}, nil
 }
 

test/s3tables/catalog_risingwave/setup_test.go

@@ -18,7 +18,7 @@ import (
 
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 var (
@@ -114,11 +114,12 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
-	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
-	env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "RisingWave"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+	env.icebergRestPort = ports[3]
+	env.risingwavePort = ports[4]
 
 	env.bindIP = testutil.FindBindIP()
 

test/s3tables/catalog_spark/setup_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/testcontainers/testcontainers-go"
 )
 
@@ -90,10 +90,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
-	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+	env.icebergRestPort = ports[3]
 
 	bindIP := testutil.FindBindIP()
 

test/s3tables/catalog_trino/trino_catalog_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 type TestEnvironment struct {
@@ -40,7 +40,6 @@ type TestEnvironment struct {
 	dockerAvailable bool
 	accessKey       string
 	secretKey       string
-	closers         []io.Closer
 }
 
 func TestTrinoIcebergCatalog(t *testing.T) {
@@ -214,19 +213,24 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	env := &TestEnvironment{
-		seaweedDir: seaweedDir,
-		weedBinary: weedBinary,
-		dataDir:    dataDir,
-		bindIP:     bindIP,
-		closers:    []io.Closer{},
-	}
+	// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
+	ports := testutil.MustAllocatePorts(t, 9)
 
-	env.masterPort, env.masterGrpcPort = env.mustFreePortPair("Master")
-	env.volumePort, env.volumeGrpcPort = env.mustFreePortPair("Volume")
-	env.filerPort, env.filerGrpcPort = env.mustFreePortPair("Filer")
-	env.s3Port, env.s3GrpcPort = env.mustFreePortPair("S3")
-	env.icebergPort = env.mustFreePort("Iceberg")
+	env := &TestEnvironment{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		bindIP:         bindIP,
+		masterPort:     ports[0],
+		masterGrpcPort: ports[1],
+		volumePort:     ports[2],
+		volumeGrpcPort: ports[3],
+		filerPort:      ports[4],
+		filerGrpcPort:  ports[5],
+		s3Port:         ports[6],
+		s3GrpcPort:     ports[7],
+		icebergPort:    ports[8],
+	}
 
 	env.dockerAvailable = hasDocker()
 	env.accessKey = "AKIAIOSFODNN7EXAMPLE"
@@ -235,47 +239,6 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	return env
 }
 
-func (env *TestEnvironment) mustFreePort(name string) int {
-	port, closer, err := getFreePort()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to get free port for %s: %v", name, err))
-	}
-	env.closers = append(env.closers, closer)
-	return port
-}
-
-func (env *TestEnvironment) mustFreePortPair(name string) (int, int) {
-	httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to get free port pair for %s: %v", name, err))
-	}
-	env.closers = append(env.closers, httpCloser, grpcCloser)
-	return httpPort, grpcPort
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	port, closer, err := getFreePort()
-	if err != nil {
-		t.Fatalf("Failed to get free port for %s: %v", name, err)
-	}
-	closer.Close()
-	return port
-}
-
-func mustFreePortPair(t *testing.T, name string) (int, int) {
-	t.Helper()
-
-	httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	httpCloser.Close()
-	grpcCloser.Close()
-	return httpPort, grpcPort
-}
-
 func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	t.Helper()
 
@@ -293,12 +256,6 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	env.weedCancel = cancel
 
-	// Close all port listeners right before starting the weed process
-	for _, closer := range env.closers {
-		closer.Close()
-	}
-	env.closers = nil
-
 	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
 		"-master.port", fmt.Sprintf("%d", env.masterPort),
 		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
@@ -631,29 +588,6 @@ func hasDocker() bool {
 	return cmd.Run() == nil
 }
 
-func findAvailablePortPair() (int, io.Closer, int, io.Closer, error) {
-	httpPort, httpCloser, err := getFreePort()
-	if err != nil {
-		return 0, nil, 0, nil, err
-	}
-	grpcPort, grpcCloser, err := getFreePort()
-	if err != nil {
-		httpCloser.Close()
-		return 0, nil, 0, nil, err
-	}
-	return httpPort, httpCloser, grpcPort, grpcCloser, nil
-}
-
-func getFreePort() (int, io.Closer, error) {
-	listener, err := net.Listen("tcp", "0.0.0.0:0")
-	if err != nil {
-		return 0, nil, err
-	}
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, listener, nil
-}
-
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
 	b := make([]byte, length)

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -25,7 +25,7 @@ import (
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 )
 

test/s3tables/maintenance/maintenance_integration_test.go

@@ -16,7 +16,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"path"
@@ -40,6 +39,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -92,7 +92,7 @@ func TestMain(m *testing.M) {
 }
 
 func startCluster(testDir string, extraArgs []string) (*testCluster, error) {
-	ports, err := findPorts(10)
+	ports, err := testutil.AllocatePorts(10)
 	if err != nil {
 		return nil, err
 	}
@@ -201,26 +201,6 @@ func (c *testCluster) filerConn(t *testing.T) (*grpc.ClientConn, filer_pb.Seawee
 	return conn, filer_pb.NewSeaweedFilerClient(conn)
 }
 
-func findPorts(n int) ([]int, error) {
-	ls := make([]*net.TCPListener, n)
-	ps := make([]int, n)
-	for i := 0; i < n; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for j := 0; j < i; j++ {
-				ls[j].Close()
-			}
-			return nil, err
-		}
-		ls[i] = l.(*net.TCPListener)
-		ps[i] = ls[i].Addr().(*net.TCPAddr).Port
-	}
-	for _, l := range ls {
-		l.Close()
-	}
-	return ps, nil
-}
-
 func waitReady(endpoint string, timeout time.Duration) error {
 	client := &http.Client{Timeout: 1 * time.Second}
 	deadline := time.Now().Add(timeout)

test/s3tables/polaris/polaris_env_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 const (

test/s3tables/sts_integration/sts_integration_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
 // TestEnvironment mirrors the one in trino_catalog_test.go but simplified

test/s3tables/table-buckets/s3tables_integration_test.go

@@ -3,7 +3,6 @@ package s3tables
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -18,6 +17,7 @@ import (
 
 	"flag"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -590,40 +590,12 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) {
 
 // Helper functions
 
-// findAvailablePorts finds n available ports by binding to port 0 multiple times
-// It keeps the listeners open until all ports are found to ensure uniqueness
-func findAvailablePorts(n int) ([]int, error) {
-	listeners := make([]*net.TCPListener, n)
-	ports := make([]int, n)
-
-	// Open all listeners to ensure we get unique ports
-	for i := 0; i < n; i++ {
-		listener, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			// Close valid listeners before returning error
-			for j := 0; j < i; j++ {
-				listeners[j].Close()
-			}
-			return nil, err
-		}
-		listeners[i] = listener.(*net.TCPListener)
-		ports[i] = listeners[i].Addr().(*net.TCPAddr).Port
-	}
-
-	// Close all listeners
-	for _, l := range listeners {
-		l.Close()
-	}
-
-	return ports, nil
-}
-
 // startMiniClusterInDir starts a weed mini instance using testDir as the data
 // directory. It does not require a *testing.T so it can be called from TestMain.
 // extraArgs are appended to the default mini command flags.
 func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
 	// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
-	ports, err := findAvailablePorts(10)
+	ports, err := testutil.AllocatePorts(10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find available ports: %v", err)
 	}

test/s3tables/testutil/docker.go

@@ -1,92 +0,0 @@
-package testutil
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"os/exec"
-	"testing"
-	"time"
-)
-
-func HasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
-// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
-func MustFreePortPair(t *testing.T, name string) (int, int) {
-	ports := MustAllocatePorts(t, 2)
-	return ports[0], ports[1]
-}
-
-// MustAllocatePorts allocates count unique free ports atomically.
-// All listeners are held open until every port is obtained, preventing
-// the OS from recycling a port between successive allocations.
-func MustAllocatePorts(t *testing.T, count int) []int {
-	t.Helper()
-	ports, err := AllocatePorts(count)
-	if err != nil {
-		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
-	}
-	return ports
-}
-
-// AllocatePorts allocates count unique free ports atomically.
-func AllocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "0.0.0.0:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
-func WaitForService(url string, timeout time.Duration) bool {
-	client := &http.Client{Timeout: 2 * time.Second}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return false
-		case <-ticker.C:
-			resp, err := client.Get(url)
-			if err == nil {
-				resp.Body.Close()
-				return true
-			}
-		}
-	}
-}
-
-func WaitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	address := fmt.Sprintf("127.0.0.1:%d", port)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}

test/testutil/helpers.go

@@ -1,17 +1,25 @@
 package testutil
 
 import (
+	"context"
 	"fmt"
-	"math/rand"
 	"net"
+	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"testing"
 	"time"
 )
 
 const SeaweedMiniStartupTimeout = 45 * time.Second
 
+func HasDocker() bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "docker", "version")
+	return cmd.Run() == nil
+}
+
 func FindBindIP() string {
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -60,34 +68,38 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
 	return iamConfigPath, nil
 }
 
-func MustFreeMiniPort(t *testing.T, name string) int {
-	t.Helper()
+func WaitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
 
-	const (
-		minPort = 10000
-		maxPort = 55000
-	)
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
 
-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+			resp, err := client.Get(url)
+			if err == nil {
+				resp.Body.Close()
+				return true
+			}
 		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-
-		return port
 	}
+}
 
-	t.Fatalf("failed to get free weed mini port for %s", name)
-	return 0
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
 }

test/testutil/ports.go

@@ -0,0 +1,120 @@
+// Package testutil provides shared test utilities for SeaweedFS integration tests.
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"testing"
+)
+
+// GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports.
+const GrpcPortOffset = 10000
+
+// AllocatePorts allocates count unique free ports atomically.
+// All listeners are held open until every port is obtained, preventing
+// the OS from recycling a port between successive allocations.
+func AllocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
+}
+
+// MustAllocatePorts is a testing wrapper for AllocatePorts.
+func MustAllocatePorts(t *testing.T, count int) []int {
+	t.Helper()
+	ports, err := AllocatePorts(count)
+	if err != nil {
+		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
+	}
+	return ports
+}
+
+// AllocateMiniPorts allocates n free ports where each port and its gRPC
+// counterpart (port + GrpcPortOffset) are available and don't collide
+// with any other allocated port or its gRPC counterpart. All listeners
+// are held open until the entire batch is allocated, preventing the OS
+// from recycling ports between allocations. Use this when ports will be
+// passed to weed mini without explicit gRPC port flags, so mini will
+// derive gRPC ports as HTTP + 10000.
+func AllocateMiniPorts(count int) ([]int, error) {
+	const (
+		minPort = 10000
+		maxPort = 55000
+	)
+	reserved := make(map[int]bool)
+	ports := make([]int, 0, count)
+	var listeners []net.Listener
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	for idx := 0; idx < count; idx++ {
+		found := false
+		for i := 0; i < 1000; i++ {
+			port := minPort + rand.Intn(maxPort-minPort)
+			grpcPort := port + GrpcPortOffset
+
+			if reserved[port] || reserved[grpcPort] {
+				continue
+			}
+
+			l1, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			if err != nil {
+				continue
+			}
+
+			l2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+			if err != nil {
+				l1.Close()
+				continue
+			}
+
+			listeners = append(listeners, l1, l2)
+			reserved[port] = true
+			reserved[grpcPort] = true
+			ports = append(ports, port)
+			found = true
+			break
+		}
+		if !found {
+			return nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, count)
+		}
+	}
+
+	return ports, nil
+}
+
+// MustFreeMiniPorts allocates n ports suitable for weed mini, ensuring
+// each port's gRPC offset (port + 10000) doesn't collide with any other
+// allocated port. names is used only for error messages.
+func MustFreeMiniPorts(t *testing.T, names []string) []int {
+	t.Helper()
+	ports, err := AllocateMiniPorts(len(names))
+	if err != nil {
+		t.Fatalf("failed to allocate mini ports for %v: %v", names, err)
+	}
+	return ports
+}
+
+// MustFreeMiniPort allocates a single weed mini port.
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+	return MustFreeMiniPorts(t, []string{name})[0]
+}

test/volume_server/framework/cluster.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -84,12 +85,14 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
 		t.Fatalf("write security config: %v", err)
 	}
 
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
 
-	ports, err := allocatePorts(3)
+	ports, err := testutil.AllocatePorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}
@@ -269,44 +272,6 @@ func stopProcess(cmd *exec.Cmd) {
 	}
 }
 
-func allocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
-func allocateMasterPortPair() (int, int, error) {
-	for masterPort := 10000; masterPort <= 55535; masterPort++ {
-		masterGrpcPort := masterPort + 10000
-		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
-		if err != nil {
-			continue
-		}
-		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
-		if err != nil {
-			_ = l1.Close()
-			continue
-		}
-		_ = l2.Close()
-		_ = l1.Close()
-		return masterPort, masterGrpcPort, nil
-	}
-	return 0, 0, errors.New("unable to find available master port pair")
-}
 
 func newWorkDir() (dir string, keepLogs bool, err error) {
 	keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"

test/volume_server/framework/cluster_mixed.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -92,17 +93,19 @@ func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rust
 		t.Fatalf("write security config: %v", err)
 	}
 
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
 
 	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
 	portsPerServer := 2
 	if profile.SplitPublicPort {
 		portsPerServer = 3
 	}
-	ports, err := allocatePorts(total * portsPerServer)
+	ports, err := testutil.AllocatePorts(total * portsPerServer)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}

test/volume_server/framework/cluster_multi.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -74,10 +75,12 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 		t.Fatalf("write security config: %v", err)
 	}
 
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
 
 	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
 	// If SplitPublicPort is true, we need an additional port per server
@@ -86,7 +89,7 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 		portsPerServer = 4
 	}
 	totalPorts := serverCount * portsPerServer
-	ports, err := allocatePorts(totalPorts)
+	ports, err := testutil.AllocatePorts(totalPorts)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}

test/volume_server/framework/cluster_multi_rust.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -85,10 +86,12 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
 		t.Fatalf("write security config: %v", err)
 	}
 
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
 
 	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
 	// If SplitPublicPort is true, we need an additional port per server
@@ -97,7 +100,7 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
 		portsPerServer = 4
 	}
 	totalPorts := serverCount * portsPerServer
-	ports, err := allocatePorts(totalPorts)
+	ports, err := testutil.AllocatePorts(totalPorts)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}

test/volume_server/framework/cluster_rust.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -79,12 +80,14 @@ func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
 		t.Fatalf("write security config: %v", err)
 	}
 
-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
 
-	ports, err := allocatePorts(3)
+	ports, err := testutil.AllocatePorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}

test/volume_server/framework/cluster_with_filer.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )
 
@@ -25,7 +26,7 @@ func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *Cl
 
 	baseCluster := StartSingleVolumeCluster(t, profile)
 
-	ports, err := allocatePorts(2)
+	ports, err := testutil.AllocatePorts(2)
 	if err != nil {
 		t.Fatalf("allocate filer ports: %v", err)
 	}

test/volume_server/grpc/fetch_remote_s3_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -17,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -24,29 +24,6 @@ import (
 )
 
 // findAvailablePort finds a free TCP port on localhost.
-func findAvailablePort() (int, error) {
-	l, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	port := l.Addr().(*net.TCPAddr).Port
-	l.Close()
-	return port, nil
-}
-
-// waitForPort waits until a TCP port is listening, up to timeout.
-func waitForPort(port int, timeout time.Duration) error {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	return fmt.Errorf("port %d not listening after %v", port, timeout)
-}
 
 // startWeedMini starts a weed mini subprocess and returns the S3 endpoint and cleanup func.
 func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
@@ -60,10 +37,11 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
 		}
 	}
 
-	miniMasterPort, _ := findAvailablePort()
-	miniVolumePort, _ := findAvailablePort()
-	miniFilerPort, _ := findAvailablePort()
-	miniS3Port, _ := findAvailablePort()
+	miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
+	miniMasterPort := miniPorts[0]
+	miniVolumePort := miniPorts[1]
+	miniFilerPort := miniPorts[2]
+	miniS3Port := miniPorts[3]
 	miniDir := t.TempDir()
 	os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644)
 
@@ -89,11 +67,11 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
 		t.Fatalf("start weed mini: %v", err)
 	}
 
-	if err := waitForPort(miniS3Port, 30*time.Second); err != nil {
+	if !testutil.WaitForPort(miniS3Port, 30*time.Second) {
 		cancel()
 		miniCmd.Wait()
 		logFile.Close()
-		t.Fatalf("weed mini S3 not ready: %v", err)
+		t.Fatalf("weed mini S3 not ready on port %d", miniS3Port)
 	}
 	t.Logf("weed mini S3 ready on port %d", miniS3Port)
 

weed/cluster/lock_client.go

@@ -62,6 +62,46 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
 	return
 }
 
+// NewBlockingLongLivedLock blocks until the lock is acquired, then starts a
+// background renewal goroutine that keeps the lock alive. This combines the
+// synchronous acquisition of NewShortLivedLock with the auto-renewal of
+// StartLongLivedLock. Release with Stop().
+func (lc *LockClient) NewBlockingLongLivedLock(key, owner string, lockTTL time.Duration) *LiveLock {
+	if lockTTL == 0 {
+		lockTTL = lock_manager.LiveLockTTL
+	}
+	lock := &LiveLock{
+		key:            key,
+		hostFiler:      lc.seedFiler,
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
+		lc:             lc,
+		lockTTL:        lockTTL,
+	}
+	// Block until acquired
+	lock.retryUntilLocked(lockTTL)
+	// Start renewal goroutine using a ticker for interruptible sleep
+	go func() {
+		renewInterval := lockTTL / 2
+		ticker := time.NewTicker(renewInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-lock.cancelCh:
+				return
+			case <-ticker.C:
+				if err := lock.AttemptToLock(lockTTL); err != nil {
+					glog.V(0).Infof("lock renewal failed for %s: %v", key, err)
+					atomic.StoreInt32(&lock.isLocked, 0)
+				}
+			}
+		}
+	}()
+	return lock
+}
+
 // StartLongLivedLock starts a goroutine to lock the key and returns immediately.
 // lockTTL specifies how long the lock should be held. The renewal interval is
 // automatically derived as lockTTL / 2 to ensure timely renewals.

weed/command/master.go

@@ -224,8 +224,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		if raftServer == nil {
 			glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
 		}
-		// For single-master mode, initialize cluster immediately without waiting
-		if isSingleMaster {
+		// For single-master mode with a fresh log, initialize cluster immediately.
+		// When resuming with existing state, the server is already a member and
+		// will self-elect via fastResume — sending another JoinCommand would block
+		// because goraft's setCommitIndex returns early on JoinCommand entries,
+		// preventing the new entry's event from being notified when old uncommitted
+		// JoinCommands exist in the log.
+		if isSingleMaster && !raftServer.HasExistingState() {
 			glog.V(0).Infof("Single-master mode: initializing cluster immediately")
 			raftServer.DoJoinCommand()
 		}

weed/command/mount.go

@@ -52,7 +52,11 @@ type MountOptions struct {
 
 	dirIdleEvictSec *int
 
+	// Distributed lock for cross-mount write coordination
+	distributedLock *bool
+
 	// FUSE performance options
+	posixDirNlink  *bool
 	writebackCache *bool
 	asyncDio       *bool
 	cacheSymlink   *bool
@@ -125,7 +129,11 @@ func init() {
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 	mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
 
+	// Distributed lock for cross-mount write coordination
+	mountOptions.distributedLock = cmdMount.Flag.Bool("dlm", false, "enable distributed lock for cross-mount write coordination (only one mount can write a file at a time)")
+
 	// FUSE performance options
+	mountOptions.posixDirNlink = cmdMount.Flag.Bool("posix.dirNLink", false, "report POSIX-compliant directory nlink (2 + subdirectory count); costs one directory listing per stat")
 	mountOptions.writebackCache = cmdMount.Flag.Bool("writebackCache", false, "enable FUSE writeback cache for improved write performance (at risk of data loss on crash)")
 	mountOptions.asyncDio = cmdMount.Flag.Bool("asyncDio", false, "enable async direct I/O for better concurrency")
 	mountOptions.cacheSymlink = cmdMount.Flag.Bool("cacheSymlink", false, "enable symlink caching to reduce metadata lookups")

weed/command/mount_std.go

@@ -352,7 +352,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		RdmaMaxConcurrent: *option.rdmaMaxConcurrent,
 		RdmaTimeoutMs:     *option.rdmaTimeoutMs,
 		DirIdleEvictSec:   *option.dirIdleEvictSec,
-		WritebackCache:    option.writebackCache != nil && *option.writebackCache,
+		EnableDistributedLock: option.distributedLock != nil && *option.distributedLock,
+		WritebackCache:       option.writebackCache != nil && *option.writebackCache,
+		PosixDirNlink:        option.posixDirNlink != nil && *option.posixDirNlink,
 	})
 
 	// create mount root

weed/command/s3.go

@@ -514,6 +514,7 @@ func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) {
 
 	// Create Iceberg server using the S3ApiServer as filer client
 	icebergServer := iceberg.NewServer(s3ApiServer, s3ApiServer)
+	icebergServer.SetCredentialValidator(s3ApiServer)
 	icebergServer.RegisterRoutes(icebergRouter)
 
 	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portIceberg)

weed/command/shell.go

@@ -15,6 +15,7 @@ var (
 	shellOptions      shell.ShellOptions
 	shellInitialFiler *string
 	shellCluster      *string
+	shellDebug      *bool
 )
 
 func init() {
@@ -23,6 +24,7 @@ func init() {
 	shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers")
 	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888")
 	shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
+	shellDebug = cmdShell.Flag.Bool("debug", false, "print informational logs to stderr")
 }
 
 var cmdShell = &Command{
@@ -61,7 +63,10 @@ func runShell(command *Command, args []string) bool {
 		filerAddress = viper.GetString("cluster." + cluster + ".filer")
 	}
 	shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
-	fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
+	shellOptions.Debug = *shellDebug
+	if shellOptions.Debug {
+		fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
+	}
 
 	shell.RunShell(shellOptions)
 

weed/filer/entry.go

@@ -12,6 +12,7 @@ import (
 type Attr struct {
 	Mtime         time.Time   // time of last modification
 	Crtime        time.Time   // time of creation (OS X only)
+	Ctime         time.Time   // time of last inode change
 	Mode          os.FileMode // file mode
 	Uid           uint32      // owner uid
 	Gid           uint32      // group gid

weed/filer/entry_codec.go

@@ -82,6 +82,9 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 	return &filer_pb.FuseAttributes{
 		Crtime:        entry.Attr.Crtime.Unix(),
 		Mtime:         entry.Attr.Mtime.Unix(),
+		MtimeNs:       int32(entry.Attr.Mtime.Nanosecond()),
+		Ctime:         entry.Attr.Ctime.Unix(),
+		CtimeNs:       int32(entry.Attr.Ctime.Nanosecond()),
 		FileMode:      uint32(entry.Attr.Mode),
 		Uid:           entry.Uid,
 		Gid:           entry.Gid,
@@ -105,6 +108,9 @@ func EntryAttributeToExistingPb(entry *Entry, attr *filer_pb.FuseAttributes) {
 	}
 	attr.Crtime = entry.Attr.Crtime.Unix()
 	attr.Mtime = entry.Attr.Mtime.Unix()
+	attr.MtimeNs = int32(entry.Attr.Mtime.Nanosecond())
+	attr.Ctime = entry.Attr.Ctime.Unix()
+	attr.CtimeNs = int32(entry.Attr.Ctime.Nanosecond())
 	attr.FileMode = uint32(entry.Attr.Mode)
 	attr.Uid = entry.Uid
 	attr.Gid = entry.Gid
@@ -128,7 +134,12 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	}
 
 	t.Crtime = time.Unix(attr.Crtime, 0)
-	t.Mtime = time.Unix(attr.Mtime, 0)
+	t.Mtime = time.Unix(attr.Mtime, int64(attr.MtimeNs))
+	if attr.Ctime != 0 {
+		t.Ctime = time.Unix(attr.Ctime, int64(attr.CtimeNs))
+	} else {
+		t.Ctime = t.Mtime
+	}
 	t.Mode = os.FileMode(attr.FileMode)
 	t.Uid = attr.Uid
 	t.Gid = attr.Gid

weed/filer/filerstore_hardlink.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -113,6 +114,9 @@ func (fsw *FilerStoreWrapper) DeleteHardLink(ctx context.Context, hardLinkId Har
 		return fsw.KvDelete(ctx, key)
 	}
 
+	// POSIX: update ctime when nlink changes (a hard link was removed).
+	entry.Attr.Ctime = time.Now().UTC()
+
 	newBlob, encodeErr := entry.EncodeAttributesAndChunks()
 	if encodeErr != nil {
 		return encodeErr

weed/filer/postgres/pgx_conn.go

@@ -0,0 +1,50 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/stdlib"
+)
+
+// OpenPGXDB parses the given DSN into a pgx ConnConfig, applies PgBouncer
+// compatibility settings when requested, opens a *sql.DB via
+// stdlib.OpenDB, and verifies it with Ping.
+//
+// In pgx/v5 the prefer_simple_protocol DSN parameter was removed, so simple
+// protocol mode must be configured on the ConnConfig via
+// DefaultQueryExecMode. We use stdlib.OpenDB(config) rather than
+// RegisterConnConfig + sql.Open so we don't leak entries in stdlib's global
+// connection config map on either success or failure paths.
+//
+// adaptedSqlUrl is used only for error messages (the caller is expected to
+// have redacted any password).
+func OpenPGXDB(sqlUrl, adaptedSqlUrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (*sql.DB, error) {
+	connConfig, parseErr := pgx.ParseConfig(sqlUrl)
+	if parseErr != nil {
+		return nil, fmt.Errorf("can not parse connection config for %s error:%v", adaptedSqlUrl, parseErr)
+	}
+
+	// PgBouncer compatibility: use the simple query protocol and disable
+	// statement caching. This avoids prepared statement issues with
+	// PgBouncer's transaction pooling mode.
+	if pgbouncerCompatible {
+		connConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol
+		connConfig.StatementCacheCapacity = 0
+		connConfig.DescriptionCacheCapacity = 0
+	}
+
+	db := stdlib.OpenDB(*connConfig)
+	db.SetMaxIdleConns(maxIdle)
+	db.SetMaxOpenConns(maxOpen)
+	db.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
+
+	if err := db.Ping(); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
+	}
+
+	return db, nil
+}

weed/filer/postgres/postgres_store.go

@@ -8,12 +8,8 @@
 package postgres
 
 import (
-	"database/sql"
-	"fmt"
 	"strconv"
-	"time"
 
-	_ "github.com/jackc/pgx/v5/stdlib"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -68,12 +64,6 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
 	// pgx-optimized connection string with better timeouts and connection handling
 	sqlUrl := "connect_timeout=30"
 
-	// PgBouncer compatibility: add prefer_simple_protocol=true when needed
-	// This avoids prepared statement issues with PgBouncer's transaction pooling mode
-	if pgbouncerCompatible {
-		sqlUrl += " prefer_simple_protocol=true"
-	}
-
 	if hostname != "" {
 		sqlUrl += " host=" + hostname
 	}
@@ -113,23 +103,11 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
 		sqlUrl += " search_path=" + schema
 		adaptedSqlUrl += " search_path=" + schema
 	}
-	var dbErr error
-	store.DB, dbErr = sql.Open("pgx", sqlUrl)
-	if dbErr != nil {
-		if store.DB != nil {
-			store.DB.Close()
-		}
-		store.DB = nil
-		return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, dbErr)
-	}
-
-	store.DB.SetMaxIdleConns(maxIdle)
-	store.DB.SetMaxOpenConns(maxOpen)
-	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
-
-	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
+	db, openErr := OpenPGXDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
+	if openErr != nil {
+		return openErr
 	}
+	store.DB = db
 
 	return nil
 }

weed/filer/postgres2/postgres2_store.go

@@ -9,12 +9,9 @@ package postgres2
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strconv"
-	"time"
 
-	_ "github.com/jackc/pgx/v5/stdlib"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
 	"github.com/seaweedfs/seaweedfs/weed/filer/postgres"
@@ -73,12 +70,6 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
 	// pgx-optimized connection string with better timeouts and connection handling
 	sqlUrl := "connect_timeout=30"
 
-	// PgBouncer compatibility: add prefer_simple_protocol=true when needed
-	// This avoids prepared statement issues with PgBouncer's transaction pooling mode
-	if pgbouncerCompatible {
-		sqlUrl += " prefer_simple_protocol=true"
-	}
-
 	if hostname != "" {
 		sqlUrl += " host=" + hostname
 	}
@@ -118,23 +109,11 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
 		sqlUrl += " search_path=" + schema
 		adaptedSqlUrl += " search_path=" + schema
 	}
-	var dbErr error
-	store.DB, dbErr = sql.Open("pgx", sqlUrl)
-	if dbErr != nil {
-		if store.DB != nil {
-			store.DB.Close()
-		}
-		store.DB = nil
-		return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, dbErr)
-	}
-
-	store.DB.SetMaxIdleConns(maxIdle)
-	store.DB.SetMaxOpenConns(maxOpen)
-	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
-
-	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
+	db, openErr := postgres.OpenPGXDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
+	if openErr != nil {
+		return openErr
 	}
+	store.DB = db
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
 		return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)

weed/iam/responses.go

@@ -178,6 +178,17 @@ type ListAttachedUserPoliciesResponse struct {
 	CommonResponse
 }
 
+// ListUserPoliciesResponse is the response for ListUserPolicies action.
+type ListUserPoliciesResponse struct {
+	XMLName                xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUserPoliciesResponse"`
+	ListUserPoliciesResult struct {
+		PolicyNames []string `xml:"PolicyNames>member"`
+		IsTruncated bool     `xml:"IsTruncated"`
+		Marker      string   `xml:"Marker,omitempty"`
+	} `xml:"ListUserPoliciesResult"`
+	CommonResponse
+}
+
 // GetUserPolicyResponse is the response for GetUserPolicy action.
 type GetUserPolicyResponse struct {
 	XMLName             xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserPolicyResponse"`
@@ -360,6 +371,40 @@ type ListAttachedGroupPoliciesResponse struct {
 	CommonResponse
 }
 
+// PutGroupPolicyResponse is the response for PutGroupPolicy action.
+type PutGroupPolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutGroupPolicyResponse"`
+	CommonResponse
+}
+
+// GetGroupPolicyResponse is the response for GetGroupPolicy action.
+type GetGroupPolicyResponse struct {
+	XMLName              xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetGroupPolicyResponse"`
+	GetGroupPolicyResult struct {
+		GroupName      string `xml:"GroupName"`
+		PolicyName     string `xml:"PolicyName"`
+		PolicyDocument string `xml:"PolicyDocument"`
+	} `xml:"GetGroupPolicyResult"`
+	CommonResponse
+}
+
+// DeleteGroupPolicyResponse is the response for DeleteGroupPolicy action.
+type DeleteGroupPolicyResponse struct {
+	XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteGroupPolicyResponse"`
+	CommonResponse
+}
+
+// ListGroupPoliciesResponse is the response for ListGroupPolicies action.
+type ListGroupPoliciesResponse struct {
+	XMLName                 xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListGroupPoliciesResponse"`
+	ListGroupPoliciesResult struct {
+		PolicyNames []string `xml:"PolicyNames>member"`
+		IsTruncated bool     `xml:"IsTruncated"`
+		Marker      string   `xml:"Marker,omitempty"`
+	} `xml:"ListGroupPoliciesResult"`
+	CommonResponse
+}
+
 // ListGroupsForUserResponse is the response for ListGroupsForUser action.
 type ListGroupsForUserResponse struct {
 	XMLName                 xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListGroupsForUserResponse"`

weed/iamapi/iamapi_group_handlers.go

@@ -1,13 +1,17 @@
 package iamapi
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
+	"sort"
 
 	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 )
 
 func (iama *IamApiServer) CreateGroup(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*CreateGroupResponse, *IamError) {
@@ -40,7 +44,19 @@ func (iama *IamApiServer) DeleteGroup(s3cfg *iam_pb.S3ApiConfiguration, values u
 			if len(g.PolicyNames) > 0 {
 				return resp, &IamError{Code: iam.ErrCodeDeleteConflictException, Error: fmt.Errorf("cannot delete group %s: group has %d attached policy(ies)", groupName, len(g.PolicyNames))}
 			}
+			// Check for inline policies
+			policies := Policies{}
+			if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+				return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+			}
+			if gp := policies.GroupInlinePolicies[groupName]; len(gp) > 0 {
+				return resp, &IamError{Code: iam.ErrCodeDeleteConflictException, Error: fmt.Errorf("cannot delete group %s: group has %d inline policy(ies)", groupName, len(gp))}
+			}
 			s3cfg.Groups = append(s3cfg.Groups[:i], s3cfg.Groups[i+1:]...)
+			// Clean up any empty inline policy entries, reuse already-fetched policies
+			if err := cleanupGroupInlinePolicies(iama, groupName, &policies); err != nil {
+				glog.Warningf("Failed to cleanup inline policies for group %s: %v", groupName, err)
+			}
 			return resp, nil
 		}
 	}
@@ -67,7 +83,11 @@ func (iama *IamApiServer) UpdateGroup(s3cfg *iam_pb.S3ApiConfiguration, values u
 						return resp, &IamError{Code: iam.ErrCodeEntityAlreadyExistsException, Error: fmt.Errorf("group %s already exists", newName)}
 					}
 				}
+				oldName := g.Name
 				g.Name = newName
+				if err := migrateGroupInlinePolicies(iama, oldName, newName); err != nil {
+					glog.Warningf("Failed to migrate inline policies for group rename %s -> %s: %v", oldName, newName, err)
+				}
 			}
 			return resp, nil
 		}
@@ -326,3 +346,225 @@ func buildUserGroupsIndex(s3cfg *iam_pb.S3ApiConfiguration) map[string][]string
 	}
 	return index
 }
+
+// PutGroupPolicy attaches an inline policy to a group.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_PutGroupPolicy.html
+func (iama *IamApiServer) PutGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*PutGroupPolicyResponse, *IamError) {
+	resp := &PutGroupPolicyResponse{}
+	groupName := values.Get("GroupName")
+	if groupName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("GroupName is required")}
+	}
+	policyName := values.Get("PolicyName")
+	if policyName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("PolicyName is required")}
+	}
+	policyDocumentString := values.Get("PolicyDocument")
+	if policyDocumentString == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("PolicyDocument is required")}
+	}
+	policyDocument, err := GetPolicyDocument(&policyDocumentString)
+	if err != nil {
+		return resp, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+	}
+	if _, err := GetActions(&policyDocument); err != nil {
+		return resp, &IamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err}
+	}
+
+	// Find group and get its members for action recomputation
+	var targetGroup *iam_pb.Group
+	for _, g := range s3cfg.Groups {
+		if g.Name == groupName {
+			targetGroup = g
+			break
+		}
+	}
+	if targetGroup == nil {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("group %s does not exist", groupName)}
+	}
+
+	// Persist inline policy
+	policies := Policies{}
+	if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+	}
+	groupPolicies := policies.getOrCreateGroupPolicies(groupName)
+	groupPolicies[policyName] = policyDocument
+	if pErr := iama.s3ApiConfig.PutPolicies(&policies); pErr != nil {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+	}
+
+	// Recompute actions for all group members
+	recomputeActionsForGroupMembers(iama, s3cfg, targetGroup, &policies)
+
+	return resp, nil
+}
+
+// GetGroupPolicy gets an inline policy attached to a group.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetGroupPolicy.html
+func (iama *IamApiServer) GetGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*GetGroupPolicyResponse, *IamError) {
+	resp := &GetGroupPolicyResponse{}
+	groupName := values.Get("GroupName")
+	policyName := values.Get("PolicyName")
+
+	// Verify group exists
+	found := false
+	for _, g := range s3cfg.Groups {
+		if g.Name == groupName {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("group %s does not exist", groupName)}
+	}
+
+	policies := Policies{}
+	if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+	}
+
+	if groupPolicies := policies.GroupInlinePolicies[groupName]; groupPolicies != nil {
+		if policyDocument, exists := groupPolicies[policyName]; exists {
+			policyDocumentJSON, err := json.Marshal(policyDocument)
+			if err != nil {
+				return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+			}
+			resp.GetGroupPolicyResult.GroupName = groupName
+			resp.GetGroupPolicyResult.PolicyName = policyName
+			resp.GetGroupPolicyResult.PolicyDocument = string(policyDocumentJSON)
+			return resp, nil
+		}
+	}
+	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("policy %s not found on group %s", policyName, groupName)}
+}
+
+// DeleteGroupPolicy removes an inline policy from a group.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteGroupPolicy.html
+func (iama *IamApiServer) DeleteGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*DeleteGroupPolicyResponse, *IamError) {
+	resp := &DeleteGroupPolicyResponse{}
+	groupName := values.Get("GroupName")
+	if groupName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("GroupName is required")}
+	}
+	policyName := values.Get("PolicyName")
+	if policyName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("PolicyName is required")}
+	}
+
+	// Find group for member action recomputation
+	var targetGroup *iam_pb.Group
+	for _, g := range s3cfg.Groups {
+		if g.Name == groupName {
+			targetGroup = g
+			break
+		}
+	}
+	if targetGroup == nil {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("group %s does not exist", groupName)}
+	}
+
+	policies := Policies{}
+	if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+	}
+	if groupPolicies := policies.GroupInlinePolicies[groupName]; groupPolicies != nil {
+		delete(groupPolicies, policyName)
+		if pErr := iama.s3ApiConfig.PutPolicies(&policies); pErr != nil {
+			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+		}
+	}
+
+	// Recompute actions for all group members
+	recomputeActionsForGroupMembers(iama, s3cfg, targetGroup, &policies)
+
+	return resp, nil
+}
+
+// ListGroupPolicies lists the names of inline policies attached to a group.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_ListGroupPolicies.html
+func (iama *IamApiServer) ListGroupPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*ListGroupPoliciesResponse, *IamError) {
+	resp := &ListGroupPoliciesResponse{}
+	groupName := values.Get("GroupName")
+	if groupName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("GroupName is required")}
+	}
+
+	// Verify group exists
+	found := false
+	for _, g := range s3cfg.Groups {
+		if g.Name == groupName {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("group %s does not exist", groupName)}
+	}
+
+	policies := Policies{}
+	if pErr := iama.s3ApiConfig.GetPolicies(&policies); pErr != nil && !errors.Is(pErr, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: pErr}
+	}
+	if groupPolicies := policies.GroupInlinePolicies[groupName]; groupPolicies != nil {
+		for policyName := range groupPolicies {
+			resp.ListGroupPoliciesResult.PolicyNames = append(resp.ListGroupPoliciesResult.PolicyNames, policyName)
+		}
+		sort.Strings(resp.ListGroupPoliciesResult.PolicyNames)
+	}
+	resp.ListGroupPoliciesResult.IsTruncated = false
+	return resp, nil
+}
+
+// cleanupGroupInlinePolicies removes all inline policies for a group from persistent storage.
+// If policies is provided, it uses that to avoid redundant I/O; otherwise fetches from storage.
+func cleanupGroupInlinePolicies(iama *IamApiServer, groupName string, policies *Policies) error {
+	if policies == nil {
+		policies = &Policies{}
+		if err := iama.s3ApiConfig.GetPolicies(policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+			return err
+		}
+	}
+	if _, exists := policies.GroupInlinePolicies[groupName]; exists {
+		delete(policies.GroupInlinePolicies, groupName)
+		return iama.s3ApiConfig.PutPolicies(policies)
+	}
+	return nil
+}
+
+// migrateGroupInlinePolicies renames the inline policies key when a group is renamed.
+func migrateGroupInlinePolicies(iama *IamApiServer, oldName, newName string) error {
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return err
+	}
+	if oldPolicies, exists := policies.GroupInlinePolicies[oldName]; exists {
+		if policies.GroupInlinePolicies == nil {
+			policies.GroupInlinePolicies = make(map[string]map[string]policy_engine.PolicyDocument)
+		}
+		policies.GroupInlinePolicies[newName] = oldPolicies
+		delete(policies.GroupInlinePolicies, oldName)
+		return iama.s3ApiConfig.PutPolicies(&policies)
+	}
+	return nil
+}
+
+// recomputeActionsForGroupMembers recomputes the aggregated actions for all members of a group.
+// Uses an identity index for O(N+M) complexity instead of O(N*M).
+func recomputeActionsForGroupMembers(iama *IamApiServer, s3cfg *iam_pb.S3ApiConfiguration, group *iam_pb.Group, policies *Policies) {
+	// Build name -> identity index for O(1) lookup
+	identIndex := make(map[string]*iam_pb.Identity, len(s3cfg.Identities))
+	for _, ident := range s3cfg.Identities {
+		identIndex[ident.Name] = ident
+	}
+	for _, memberName := range group.Members {
+		if ident, ok := identIndex[memberName]; ok {
+			aggregatedActions, err := computeAllActionsForUser(iama, memberName, policies, ident, s3cfg)
+			if err != nil {
+				glog.Warningf("Failed to recompute actions for user %s after group policy change: %v", memberName, err)
+			} else {
+				ident.Actions = aggregatedActions
+			}
+		}
+	}
+}

weed/iamapi/iamapi_management_handlers.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strings"
 	"sync"
 
@@ -69,6 +70,17 @@ func (p *Policies) getOrCreateUserPolicies(userName string) map[string]policy_en
 	return p.InlinePolicies[userName]
 }
 
+// getOrCreateGroupPolicies returns the policy map for a group, creating it if needed.
+func (p *Policies) getOrCreateGroupPolicies(groupName string) map[string]policy_engine.PolicyDocument {
+	if p.GroupInlinePolicies == nil {
+		p.GroupInlinePolicies = make(map[string]map[string]policy_engine.PolicyDocument)
+	}
+	if p.GroupInlinePolicies[groupName] == nil {
+		p.GroupInlinePolicies[groupName] = make(map[string]policy_engine.PolicyDocument)
+	}
+	return p.GroupInlinePolicies[groupName]
+}
+
 // computeAggregatedActionsForUser computes the union of actions across all inline policies for a user.
 // Directly accesses user's policies from Policies.InlinePolicies[userName] for O(1) lookup.
 // If policies is non-nil, it uses that instead of fetching from storage (for I/O optimization).
@@ -138,6 +150,10 @@ type Policies struct {
 	// Structure: [userName][policyName] -> PolicyDocument
 	// Enables fast access without iterating all policies
 	InlinePolicies map[string]map[string]policy_engine.PolicyDocument `json:"inlinePolicies"`
+
+	// GroupInlinePolicies: group-indexed inline policies for O(1) lookup
+	// Structure: [groupName][policyName] -> PolicyDocument
+	GroupInlinePolicies map[string]map[string]policy_engine.PolicyDocument `json:"groupInlinePolicies,omitempty"`
 }
 
 func Hash(s *string) string {
@@ -393,7 +409,7 @@ func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values
 	}
 
 	// Recompute aggregated actions (inline + managed)
-	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent, s3cfg)
 	if computeErr != nil {
 		glog.Warningf("Failed to compute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
 	} else {
@@ -442,17 +458,33 @@ func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values
 
 		policyDocument := policy_engine.PolicyDocument{Version: policyDocumentVersion}
 		statements := make(map[string][]string)
+		seenAction := make(map[string]map[string]bool)
 		for _, action := range ident.Actions {
-			// parse "Read:EXAMPLE-BUCKET"
-			act := strings.Split(action, ":")
+			// parse "Read:EXAMPLE-BUCKET" or "Read:EXAMPLE-BUCKET/prefix/*"
+			// Use SplitN so the path component (which may contain ':') is preserved intact.
+			act := strings.SplitN(action, ":", 2)
 
 			resource := "*"
 			if len(act) == 2 {
-				resource = fmt.Sprintf("arn:aws:s3:::%s/*", act[1])
+				// Preserve the stored path verbatim so bucket-level and
+				// object-level resources remain distinguishable. GetActions
+				// stores the path exactly as parsed from the original ARN
+				// (e.g. "b-le*" for the bucket, "b-le*/*" for objects), and
+				// reconstruction should not rewrite one into the other.
+				resource = fmt.Sprintf("arn:aws:s3:::%s", act[1])
 			}
-			statements[resource] = append(statements[resource],
-				fmt.Sprintf("s3:%s", MapToIdentitiesAction(act[0])),
-			)
+			s3Action := fmt.Sprintf("s3:%s", MapToIdentitiesAction(act[0]))
+			// Dedupe actions per resource: the Read/Write/List internal verbs map to
+			// coarse wildcards (s3:Get*, s3:Put*, s3:List*), so multiple distinct
+			// original actions can collapse to the same reconstructed verb.
+			if seenAction[resource] == nil {
+				seenAction[resource] = make(map[string]bool)
+			}
+			if seenAction[resource][s3Action] {
+				continue
+			}
+			seenAction[resource][s3Action] = true
+			statements[resource] = append(statements[resource], s3Action)
 		}
 		for resource, actions := range statements {
 			isEqAction := false
@@ -520,7 +552,7 @@ func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 	}
 
 	// Recompute aggregated actions from remaining inline + managed policies
-	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent)
+	aggregatedActions, computeErr := computeAllActionsForUser(iama, userName, &policies, targetIdent, s3cfg)
 	if computeErr != nil {
 		glog.Warningf("Failed to recompute aggregated actions for user %s: %v; keeping existing actions", userName, computeErr)
 	} else {
@@ -529,6 +561,43 @@ func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 	return resp, nil
 }
 
+// ListUserPolicies lists the names of inline policies attached to a user.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_ListUserPolicies.html
+func (iama *IamApiServer) ListUserPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *ListUserPoliciesResponse, iamError *IamError) {
+	resp = &ListUserPoliciesResponse{}
+	userName := values.Get("UserName")
+	if userName == "" {
+		return resp, &IamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("UserName is required")}
+	}
+
+	// Verify the user exists
+	found := false
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == userName {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
+	}
+
+	// List inline policy names from persistent storage
+	policies := Policies{}
+	if err := iama.s3ApiConfig.GetPolicies(&policies); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+	}
+
+	if userPolicies := policies.InlinePolicies[userName]; userPolicies != nil {
+		for policyName := range userPolicies {
+			resp.ListUserPoliciesResult.PolicyNames = append(resp.ListUserPoliciesResult.PolicyNames, policyName)
+		}
+		sort.Strings(resp.ListUserPoliciesResult.PolicyNames)
+	}
+	resp.ListUserPoliciesResult.IsTruncated = false
+	return resp, nil
+}
+
 // GetPolicy retrieves a managed policy by ARN.
 func (iama *IamApiServer) GetPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp *GetPolicyResponse, iamError *IamError) {
 	resp = &GetPolicyResponse{}
@@ -655,8 +724,8 @@ func (iama *IamApiServer) AttachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 		prevPolicyNames := ident.PolicyNames
 		ident.PolicyNames = append(ident.PolicyNames, policyName)
 
-		// Recompute aggregated actions (inline + managed)
-		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		// Recompute aggregated actions (inline + managed + group)
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident, s3cfg)
 		if err != nil {
 			// Roll back PolicyNames to keep identity consistent
 			ident.PolicyNames = prevPolicyNames
@@ -705,7 +774,7 @@ func (iama *IamApiServer) DetachUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, val
 			ident.PolicyNames = prevPolicyNames
 			return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err}
 		}
-		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident)
+		aggregatedActions, err := computeAllActionsForUser(iama, userName, &policies, ident, s3cfg)
 		if err != nil {
 			// Roll back PolicyNames to keep identity consistent
 			ident.PolicyNames = prevPolicyNames
@@ -738,8 +807,10 @@ func (iama *IamApiServer) ListAttachedUserPolicies(s3cfg *iam_pb.S3ApiConfigurat
 	return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)}
 }
 
-// computeAllActionsForUser computes the union of actions from both inline and managed policies.
-func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Policies, ident *iam_pb.Identity) ([]string, error) {
+// computeAllActionsForUser computes the union of actions from user inline policies,
+// user managed policies, group inline policies, and group managed policies.
+// If s3cfg is provided, group memberships are resolved to include group policies.
+func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Policies, ident *iam_pb.Identity, s3cfgs ...*iam_pb.S3ApiConfiguration) ([]string, error) {
 	actionSet := make(map[string]bool)
 	var aggregatedActions []string
 
@@ -752,14 +823,14 @@ func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Pol
 		}
 	}
 
-	// Include inline policy actions
+	// Include user inline policy actions
 	inlineActions, err := computeAggregatedActionsForUser(iama, userName, policies)
 	if err != nil {
 		return nil, err
 	}
 	addUniqueActions(inlineActions)
 
-	// Include managed policy actions
+	// Include user managed policy actions
 	for _, policyName := range ident.PolicyNames {
 		if policyDoc, exists := policies.Policies[policyName]; exists {
 			actions, err := GetActions(&policyDoc)
@@ -771,6 +842,48 @@ func computeAllActionsForUser(iama *IamApiServer, userName string, policies *Pol
 		}
 	}
 
+	// Include group policies (both inline and managed) if s3cfg is available
+	if len(s3cfgs) > 0 && s3cfgs[0] != nil {
+		s3cfg := s3cfgs[0]
+		for _, g := range s3cfg.Groups {
+			if g.Disabled {
+				continue
+			}
+			isMember := false
+			for _, m := range g.Members {
+				if m == userName {
+					isMember = true
+					break
+				}
+			}
+			if !isMember {
+				continue
+			}
+			// Group managed policies
+			for _, policyName := range g.PolicyNames {
+				if policyDoc, exists := policies.Policies[policyName]; exists {
+					actions, err := GetActions(&policyDoc)
+					if err != nil {
+						glog.Warningf("Failed to get actions from group managed policy '%s' (group %s) for user %s: %v", policyName, g.Name, userName, err)
+						continue
+					}
+					addUniqueActions(actions)
+				}
+			}
+			// Group inline policies
+			if groupPolicies := policies.GroupInlinePolicies[g.Name]; groupPolicies != nil {
+				for policyName, policyDoc := range groupPolicies {
+					actions, err := GetActions(&policyDoc)
+					if err != nil {
+						glog.Warningf("Failed to get actions from group inline policy '%s' (group %s) for user %s: %v", policyName, g.Name, userName, err)
+						continue
+					}
+					addUniqueActions(actions)
+				}
+			}
+		}
+	}
+
 	return aggregatedActions, nil
 }
 
@@ -1062,6 +1175,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+	case "ListUserPolicies":
+		iama.handleImplicitUsername(r, values)
+		var err *IamError
+		response, err = iama.ListUserPolicies(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
 	case "GetPolicy":
 		var err *IamError
 		response, err = iama.GetPolicy(s3cfg, values)
@@ -1177,6 +1299,38 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		changed = false
+	case "PutGroupPolicy":
+		var err *IamError
+		response, err = iama.PutGroupPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		// changed = true: PutGroupPolicy recomputes member Identity.Actions
+	case "GetGroupPolicy":
+		var err *IamError
+		response, err = iama.GetGroupPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
+	case "DeleteGroupPolicy":
+		var err *IamError
+		response, err = iama.DeleteGroupPolicy(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		// changed = true: DeleteGroupPolicy recomputes member Identity.Actions
+	case "ListGroupPolicies":
+		var err *IamError
+		response, err = iama.ListGroupPolicies(s3cfg, values)
+		if err != nil {
+			writeIamErrorResponse(w, r, reqID, err)
+			return
+		}
+		changed = false
 	case "ListGroupsForUser":
 		var err *IamError
 		response, err = iama.ListGroupsForUser(s3cfg, values)

weed/iamapi/iamapi_management_handlers_test.go

@@ -186,6 +186,77 @@ func TestPutGetUserPolicyPreservesStatements(t *testing.T) {
 	assert.True(t, deleteObjectFound, "s3:DeleteObject action was lost")
 }
 
+// TestPutGetUserPolicyIssue9008 is a regression test for
+// https://github.com/seaweedfs/seaweedfs/issues/9008: put-user-policy followed
+// by get-user-policy must return the same policy document that was submitted,
+// with Action and Resource lists intact (no duplication, no collapsing).
+func TestPutGetUserPolicyIssue9008(t *testing.T) {
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "steward"}},
+	}
+	policyJSON := `{
+    "Version": "2012-10-17",
+    "Statement": [{
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:PutObject", "s3:ListBucket"],
+      "Resource": ["arn:aws:s3:::b-le*", "arn:aws:s3:::b-le*/*"]
+    }]
+  }`
+
+	mockCfg := &mockIamS3ApiConfig{}
+	iama := &IamApiServer{s3ApiConfig: mockCfg}
+	_, iamErr := iama.PutUserPolicy(s3cfg, url.Values{
+		"UserName":       []string{"steward"},
+		"PolicyName":     []string{"steward_policy"},
+		"PolicyDocument": []string{policyJSON},
+	})
+	assert.Nil(t, iamErr)
+
+	// Part 1: verbatim round-trip. GetUserPolicy returns the exact document
+	// that was persisted, with Action and Resource lists intact.
+	resp, iamErr := iama.GetUserPolicy(s3cfg, url.Values{
+		"UserName":   []string{"steward"},
+		"PolicyName": []string{"steward_policy"},
+	})
+	assert.Nil(t, iamErr)
+
+	var got policy_engine.PolicyDocument
+	assert.NoError(t, json.Unmarshal([]byte(resp.GetUserPolicyResult.PolicyDocument), &got))
+
+	assert.Equal(t, "2012-10-17", got.Version)
+	assert.Equal(t, 1, len(got.Statement))
+	stmt := got.Statement[0]
+	assert.Equal(t, policy_engine.PolicyEffectAllow, stmt.Effect)
+	assert.ElementsMatch(t, []string{"s3:GetObject", "s3:PutObject", "s3:ListBucket"}, stmt.Action.Strings())
+	assert.ElementsMatch(t, []string{"arn:aws:s3:::b-le*", "arn:aws:s3:::b-le*/*"}, stmt.Resource.Strings())
+
+	// Part 2: fallback reconstruction. Clear the persisted inline policy so
+	// GetUserPolicy must rebuild the document from ident.Actions. The fallback
+	// is lossy (distinct S3 verbs collapse to wildcards like s3:Get*), but it
+	// must not duplicate actions nor conflate bucket-level and object-level
+	// resources.
+	mockCfg.policies = Policies{}
+
+	resp, iamErr = iama.GetUserPolicy(s3cfg, url.Values{
+		"UserName":   []string{"steward"},
+		"PolicyName": []string{"steward_policy"},
+	})
+	assert.Nil(t, iamErr)
+
+	var fallback policy_engine.PolicyDocument
+	assert.NoError(t, json.Unmarshal([]byte(resp.GetUserPolicyResult.PolicyDocument), &fallback))
+	assert.Equal(t, 1, len(fallback.Statement), "fallback should merge equal-action statements")
+	fstmt := fallback.Statement[0]
+
+	// Each coarse verb appears exactly once (no duplication from the
+	// Read/Write/List -> s3:Get*/s3:Put*/s3:List* expansion).
+	assert.ElementsMatch(t, []string{"s3:Get*", "s3:Put*", "s3:List*"}, fstmt.Action.Strings())
+
+	// Bucket-level and object-level resources stay distinct — the bare bucket
+	// pattern must not be rewritten to an object ARN.
+	assert.ElementsMatch(t, []string{"arn:aws:s3:::b-le*", "arn:aws:s3:::b-le*/*"}, fstmt.Resource.Strings())
+}
+
 func TestMultipleInlinePoliciesAggregateActions(t *testing.T) {
 	s3cfg := &iam_pb.S3ApiConfiguration{
 		Identities: []*iam_pb.Identity{{Name: "alice"}},

weed/iamapi/iamapi_response.go

@@ -23,6 +23,7 @@ type (
 	PutUserPolicyResponse            = iamlib.PutUserPolicyResponse
 	DeleteUserPolicyResponse         = iamlib.DeleteUserPolicyResponse
 	GetUserPolicyResponse            = iamlib.GetUserPolicyResponse
+	ListUserPoliciesResponse         = iamlib.ListUserPoliciesResponse
 	GetPolicyResponse                = iamlib.GetPolicyResponse
 	DeletePolicyResponse             = iamlib.DeletePolicyResponse
 	ListPoliciesResponse             = iamlib.ListPoliciesResponse
@@ -47,5 +48,9 @@ type (
 	AttachGroupPolicyResponse         = iamlib.AttachGroupPolicyResponse
 	DetachGroupPolicyResponse         = iamlib.DetachGroupPolicyResponse
 	ListAttachedGroupPoliciesResponse = iamlib.ListAttachedGroupPoliciesResponse
+	PutGroupPolicyResponse            = iamlib.PutGroupPolicyResponse
+	GetGroupPolicyResponse            = iamlib.GetGroupPolicyResponse
+	DeleteGroupPolicyResponse         = iamlib.DeleteGroupPolicyResponse
+	ListGroupPoliciesResponse         = iamlib.ListGroupPoliciesResponse
 	ListGroupsForUserResponse         = iamlib.ListGroupsForUserResponse
 )

weed/mount/filehandle.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -38,6 +39,11 @@ type FileHandle struct {
 	isDeleted bool
 	isRenamed bool // set by Rename before waiting for async flush; skips old-path metadata flush
 
+	// dlmLock holds the distributed lock for cross-mount write coordination.
+	// Non-nil only when -dlm is enabled and the file was opened for writing.
+	// Acquired in AcquireHandle, released in ReleaseHandle.
+	dlmLock *cluster.LiveLock
+
 	// RDMA chunk offset cache for performance optimization
 	chunkOffsetCache []int64
 	chunkCacheValid  bool
@@ -137,6 +143,13 @@ func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
 }
 
 func (fh *FileHandle) ReleaseHandle() {
+	// Release distributed lock before cleaning up, so other mounts can
+	// proceed as soon as this handle is done flushing.
+	if fh.dlmLock != nil {
+		fh.dlmLock.Stop()
+		fh.dlmLock = nil
+		glog.V(1).Infof("DLM lock released for inode %d", fh.inode)
+	}
 
 	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
 	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)

weed/mount/weedfs.go

@@ -13,6 +13,7 @@ import (
 	"github.com/seaweedfs/go-fuse/v2/fuse"
 	"google.golang.org/grpc"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
@@ -82,10 +83,22 @@ type Option struct {
 	// Directory cache refresh/eviction controls
 	DirIdleEvictSec int
 
+	// EnableDistributedLock enables DLM-based write coordination across mounts.
+	// When true, opening a file for write acquires a distributed lock that is
+	// held (with auto-renewal) until the file is closed. Only one mount can
+	// have a file open for writing at a time.
+	EnableDistributedLock bool
+
 	// WritebackCache enables async flush on close for improved small file write performance.
 	// When true, Flush() returns immediately and data upload + metadata flush happen in background.
 	WritebackCache bool
 
+	// PosixDirNlink enables POSIX-compliant directory nlink counting
+	// (nlink = 2 + number_of_subdirectories). This requires listing
+	// cached directory entries on every stat, which has a performance cost.
+	// When false (default), directories report nlink=2.
+	PosixDirNlink bool
+
 	uniqueCacheDirForRead  string
 	uniqueCacheDirForWrite string
 }
@@ -116,6 +129,8 @@ type WFS struct {
 	filerClient          *wdclient.FilerClient // Cached volume location client
 	refreshMu            sync.Mutex
 	refreshingDirs       map[util.FullPath]struct{}
+	atimeMu              sync.Mutex
+	atimeMap             map[uint64]time.Time // inode -> atime, in-memory only, bounded
 	dirHotWindow         time.Duration
 	dirHotThreshold      int
 	dirIdleEvict         time.Duration
@@ -139,6 +154,10 @@ type WFS struct {
 	// mutations (create, update, delete, rename). All mutations go through one
 	// ordered stream to prevent cross-operation reordering.
 	streamMutate *streamMutateMux
+
+	// lockClient is the DLM client for cross-mount write coordination.
+	// Non-nil only when EnableDistributedLock is true.
+	lockClient *cluster.LockClient
 }
 
 const (
@@ -191,11 +210,17 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 		fhLockTable:       util.NewLockTable[FileHandleId](),
 		posixLocks:        NewPosixLockTable(),
 		refreshingDirs:    make(map[util.FullPath]struct{}),
+		atimeMap:          make(map[uint64]time.Time, 8192),
 		dirHotWindow:      dirHotWindow,
 		dirHotThreshold:   dirHotThreshold,
 		dirIdleEvict:      dirIdleEvict,
 	}
 
+	if option.EnableDistributedLock && len(option.FilerAddresses) > 0 {
+		wfs.lockClient = cluster.NewLockClient(option.GrpcDialOption, option.FilerAddresses[0])
+		glog.V(0).Infof("distributed lock manager enabled for mount")
+	}
+
 	wfs.option.filerIndex = int32(rand.IntN(len(option.FilerAddresses)))
 	wfs.option.setupUniqueCacheDirectory()
 	if option.CacheSizeMBForRead > 0 {

weed/mount/weedfs_access.go

@@ -69,12 +69,11 @@ func hasAccess(callerUid, callerGid, fileUid, fileGid uint32, perm uint32, mask
 	if !isMember {
 		groupIDs, err := lookupSupplementaryGroupIDs(callerUid)
 		if err != nil {
-			// Cannot determine group membership; require both group and
-			// other permission classes to satisfy the mask so we never
-			// overgrant when the lookup fails.
-			groupMatch := ((perm >> 3) & mask) == mask
-			otherMatch := (perm & mask) == mask
-			return groupMatch && otherMatch
+			// Cannot determine supplementary group membership.
+			// Fall through to "other" permission check since we already
+			// know the caller is not the owner (checked above) and not
+			// in the primary group.
+			return (perm & mask) == mask
 		}
 		fileGidStr := strconv.Itoa(int(fileGid))
 		for _, gidStr := range groupIDs {
@@ -92,6 +91,19 @@ func hasAccess(callerUid, callerGid, fileUid, fileGid uint32, perm uint32, mask
 	return (perm & mask) == mask
 }
 
+// checkStickyBit enforces the POSIX sticky-bit rule: when a directory has the
+// sticky bit set, only the file owner, the directory owner, or root may
+// delete or rename entries within it.
+func checkStickyBit(dirMode, dirUid, targetUid, callerUid uint32) fuse.Status {
+	if dirMode&0o1000 == 0 {
+		return fuse.OK
+	}
+	if callerUid == 0 || callerUid == dirUid || callerUid == targetUid {
+		return fuse.OK
+	}
+	return fuse.EPERM
+}
+
 // openFlagsToAccessMask converts open(2) flags to an access permission mask.
 func openFlagsToAccessMask(flags uint32) uint32 {
 	switch flags & uint32(syscall.O_ACCMODE) {

weed/mount/weedfs_attr.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"context"
 	"os"
 	"syscall"
 	"time"
@@ -16,14 +17,21 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
 	glog.V(4).Infof("GetAttr %v", input.NodeId)
 	if input.NodeId == 1 {
 		wfs.setRootAttr(out)
+		if wfs.option.PosixDirNlink {
+			wfs.applyDirNlink(&out.Attr, util.FullPath(wfs.option.FilerMountRootPath))
+		}
 		return fuse.OK
 	}
 
 	inode := input.NodeId
-	_, _, entry, status := wfs.maybeReadEntry(inode)
+	path, _, entry, status := wfs.maybeReadEntry(inode)
 	if status == fuse.OK {
 		out.AttrValid = 1
 		wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
+		wfs.applyInMemoryAtime(&out.Attr, inode)
+		if entry.IsDirectory && wfs.option.PosixDirNlink {
+			wfs.applyDirNlink(&out.Attr, path)
+		}
 		return status
 	} else {
 		if fh, found := wfs.fhMap.FindFileHandle(inode); found {
@@ -32,6 +40,7 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
 			fhActiveLock := wfs.fhLockTable.AcquireLock("GetAttr", fh.fh, util.SharedLock)
 			wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry(), true)
 			wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+			wfs.applyInMemoryAtime(&out.Attr, inode)
 			out.Nlink = 0
 			return fuse.OK
 		}
@@ -90,7 +99,9 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 				fh.entryChunkGroup.SetChunks(chunks)
 			}
 		}
-		entry.Attributes.Mtime = time.Now().Unix()
+		truncNow := time.Now()
+		entry.Attributes.Mtime = truncNow.Unix()
+		entry.Attributes.MtimeNs = int32(truncNow.Nanosecond())
 		entry.Attributes.FileSize = size
 
 	}
@@ -108,8 +119,10 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 		}
 	}
 
+	ownerChanged := false
 	if uid, ok := input.GetUID(); ok {
 		entry.Attributes.Uid = uid
+		ownerChanged = true
 		if input.NodeId == 1 {
 			wfs.option.MountUid = uid
 		}
@@ -117,25 +130,38 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
 
 	if gid, ok := input.GetGID(); ok {
 		entry.Attributes.Gid = gid
+		ownerChanged = true
 		if input.NodeId == 1 {
 			wfs.option.MountGid = gid
 		}
 	}
 
+	// POSIX: clear SUID/SGID bits when ownership changes (unless caller is root).
+	if ownerChanged && input.Uid != 0 {
+		entry.Attributes.FileMode &^= 0o6000
+	}
+
 	if atime, ok := input.GetATime(); ok {
-		entry.Attributes.Mtime = atime.Unix()
+		wfs.setAtime(input.NodeId, atime)
 	}
 
 	if mtime, ok := input.GetMTime(); ok {
 		entry.Attributes.Mtime = mtime.Unix()
+		entry.Attributes.MtimeNs = int32(mtime.Nanosecond())
 	}
 
+	// POSIX: update ctime on any metadata change.
+	now := time.Now()
+	entry.Attributes.Ctime = now.Unix()
+	entry.Attributes.CtimeNs = int32(now.Nanosecond())
+
 	out.AttrValid = 1
 	size, includeSize := input.GetSize()
 	if includeSize {
 		out.Attr.Size = size
 	}
 	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry, !includeSize)
+	wfs.applyInMemoryAtime(&out.Attr, input.NodeId)
 
 	if fh != nil {
 		fh.dirtyMetadata = true
@@ -157,7 +183,7 @@ func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
 	out.Ctime = now
 	out.Atime = now
 	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
-	out.Nlink = 1
+	out.Nlink = 2
 }
 
 func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
@@ -177,10 +203,21 @@ func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.E
 	}
 	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	out.Mtime = uint64(entry.Attributes.Mtime)
-	out.Ctime = uint64(entry.Attributes.Mtime)
+	out.Mtimensec = uint32(entry.Attributes.MtimeNs)
+	if entry.Attributes.Ctime != 0 {
+		out.Ctime = uint64(entry.Attributes.Ctime)
+		out.Ctimensec = uint32(entry.Attributes.CtimeNs)
+	} else {
+		out.Ctime = uint64(entry.Attributes.Mtime)
+		out.Ctimensec = uint32(entry.Attributes.MtimeNs)
+	}
 	out.Atime = uint64(entry.Attributes.Mtime)
+	out.Atimensec = uint32(entry.Attributes.MtimeNs)
+	// In-memory atime overlay is applied by the caller via applyInMemoryAtime.
 	out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
-	if entry.HardLinkCounter > 0 {
+	if entry.IsDirectory {
+		out.Nlink = 2
+	} else if entry.HardLinkCounter > 0 {
 		out.Nlink = uint32(entry.HardLinkCounter)
 	} else {
 		out.Nlink = 1
@@ -199,10 +236,20 @@ func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.E
 	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	setBlksize(out, blockSize)
 	out.Atime = uint64(entry.Attr.Mtime.Unix())
+	out.Atimensec = uint32(entry.Attr.Mtime.Nanosecond())
 	out.Mtime = uint64(entry.Attr.Mtime.Unix())
-	out.Ctime = uint64(entry.Attr.Mtime.Unix())
+	out.Mtimensec = uint32(entry.Attr.Mtime.Nanosecond())
+	if !entry.Attr.Ctime.IsZero() {
+		out.Ctime = uint64(entry.Attr.Ctime.Unix())
+		out.Ctimensec = uint32(entry.Attr.Ctime.Nanosecond())
+	} else {
+		out.Ctime = uint64(entry.Attr.Mtime.Unix())
+		out.Ctimensec = uint32(entry.Attr.Mtime.Nanosecond())
+	}
 	out.Mode = toSyscallMode(entry.Attr.Mode)
-	if entry.HardLinkCounter > 0 {
+	if entry.IsDirectory() {
+		out.Nlink = 2
+	} else if entry.HardLinkCounter > 0 {
 		out.Nlink = uint32(entry.HardLinkCounter)
 	} else {
 		out.Nlink = 1
@@ -228,6 +275,64 @@ func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.
 	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
 }
 
+// touchDirMtimeCtime updates a directory's mtime and ctime on the filer.
+// POSIX requires this when entries are created or removed in the directory.
+func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) {
+	dirEntry, code := wfs.maybeLoadEntry(dirPath)
+	if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil {
+		return
+	}
+	now := time.Now()
+	dirEntry.Attributes.Mtime = now.Unix()
+	dirEntry.Attributes.MtimeNs = int32(now.Nanosecond())
+	dirEntry.Attributes.Ctime = now.Unix()
+	dirEntry.Attributes.CtimeNs = int32(now.Nanosecond())
+	wfs.saveEntry(dirPath, dirEntry)
+}
+
+const atimeMapMaxSize = 8192
+
+// setAtime stores an in-memory atime for an inode. The map is bounded;
+// when full, a random entry is evicted.
+func (wfs *WFS) setAtime(inode uint64, t time.Time) {
+	wfs.atimeMu.Lock()
+	defer wfs.atimeMu.Unlock()
+	if len(wfs.atimeMap) >= atimeMapMaxSize {
+		// evict one random entry
+		for k := range wfs.atimeMap {
+			delete(wfs.atimeMap, k)
+			break
+		}
+	}
+	wfs.atimeMap[inode] = t
+}
+
+// applyInMemoryAtime overlays the in-memory atime onto a fuse.Attr if present.
+func (wfs *WFS) applyInMemoryAtime(out *fuse.Attr, inode uint64) {
+	wfs.atimeMu.Lock()
+	if t, ok := wfs.atimeMap[inode]; ok {
+		out.Atime = uint64(t.Unix())
+		out.Atimensec = uint32(t.Nanosecond())
+	}
+	wfs.atimeMu.Unlock()
+}
+
+// applyDirNlink sets nlink = 2 + number_of_subdirectories for a directory.
+// Only counts from the local metacache to avoid expensive filer queries.
+// When the cache has no entries (e.g. before readdir), keeps nlink=2.
+func (wfs *WFS) applyDirNlink(out *fuse.Attr, dirPath util.FullPath) {
+	var subdirCount uint32
+	wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, 100000, func(entry *filer.Entry) (bool, error) {
+		if entry.IsDirectory() {
+			subdirCount++
+		}
+		return true, nil
+	})
+	if subdirCount > 0 {
+		out.Nlink = 2 + subdirCount
+	}
+}
+
 func chmod(existing uint32, mode uint32) uint32 {
 	return existing&^07777 | mode&07777
 }

weed/mount/weedfs_dir_lookup.go

@@ -44,6 +44,10 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
 
 	wfs.outputFilerEntry(out, inode, localEntry)
 
+	if localEntry.IsDirectory() && wfs.option.PosixDirNlink {
+		wfs.applyDirNlink(&out.Attr, fullFilePath)
+	}
+
 	return fuse.OK
 
 }

weed/mount/weedfs_dir_mkrm.go

@@ -30,12 +30,14 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
 		return s
 	}
 
+	now := time.Now().Unix()
 	newEntry := &filer_pb.Entry{
 		Name:        name,
 		IsDirectory: true,
 		Attributes: &filer_pb.FuseAttributes{
-			Mtime:    time.Now().Unix(),
-			Crtime:   time.Now().Unix(),
+			Mtime:    now,
+			Crtime:   now,
+			Ctime:    now,
 			FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask),
 			Uid:      in.Uid,
 			Gid:      in.Gid,
@@ -76,6 +78,7 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
 			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
 		}
 		wfs.inodeToPath.TouchDirectory(dirFullPath)
+		wfs.touchDirMtimeCtime(dirFullPath)
 	}
 
 	glog.V(3).Infof("mkdir %s: %v", entryFullPath, err)
@@ -114,6 +117,17 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
 	}
 	entryFullPath := dirFullPath.Child(name)
 
+	// POSIX: enforce sticky bit on the parent directory.
+	if dirEntry, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
+		targetUid := uint32(0)
+		if targetEntry, targetCode := wfs.maybeLoadEntry(entryFullPath); targetCode == fuse.OK && targetEntry != nil && targetEntry.Attributes != nil {
+			targetUid = targetEntry.Attributes.Uid
+		}
+		if code := checkStickyBit(dirEntry.Attributes.FileMode, dirEntry.Attributes.Uid, targetUid, header.Uid); code != fuse.OK {
+			return code
+		}
+	}
+
 	glog.V(3).Infof("remove directory: %v", entryFullPath)
 	deleteReq := &filer_pb.DeleteEntryRequest{
 		Directory:            string(dirFullPath),
@@ -141,6 +155,7 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
 	}
 	wfs.inodeToPath.RemovePath(entryFullPath)
 	wfs.inodeToPath.TouchDirectory(dirFullPath)
+	wfs.touchDirMtimeCtime(dirFullPath)
 
 	return fuse.OK
 

weed/mount/weedfs_file_copy_range.go

@@ -288,8 +288,12 @@ func synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut *FileHandle, sou
 		}
 	}
 
+	copyNow := time.Now()
 	localEntry.Attributes.FileSize = uint64(sourceSize)
-	localEntry.Attributes.Mtime = time.Now().Unix()
+	localEntry.Attributes.Mtime = copyNow.Unix()
+	localEntry.Attributes.MtimeNs = int32(copyNow.Nanosecond())
+	localEntry.Attributes.Ctime = copyNow.Unix()
+	localEntry.Attributes.CtimeNs = int32(copyNow.Nanosecond())
 	return localEntry
 }
 

weed/mount/weedfs_file_mkrm.go

@@ -2,10 +2,12 @@ package mount
 
 import (
 	"context"
+	"fmt"
 	"syscall"
 	"time"
 
 	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -110,6 +112,18 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
 	// Mark dirty so the deferred filer create happens on Flush,
 	// even if the file is closed without any writes.
 	fileHandle.dirtyMetadata = true
+
+	// Acquire DLM lock for new file creation (Create bypasses AcquireHandle
+	// so we must acquire the lock here). Always lock on Create since file
+	// creation is inherently a write operation.
+	if wfs.lockClient != nil && fileHandle.dlmLock == nil {
+		owner := fmt.Sprintf("mount-%d", wfs.signature)
+		fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+			string(entryFullPath), owner, lock_manager.LiveLockTTL,
+		)
+		glog.V(1).Infof("DLM lock acquired for new file %s", entryFullPath)
+	}
+
 	out.Fh = uint64(fileHandle.fh)
 	out.OpenFlags = 0
 
@@ -172,6 +186,17 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
 		return fuse.EPERM
 	}
 
+	// POSIX: enforce sticky bit on the parent directory.
+	if dirEntry, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
+		targetUid := uint32(0)
+		if entry != nil && entry.Attributes != nil {
+			targetUid = entry.Attributes.Uid
+		}
+		if code := checkStickyBit(dirEntry.Attributes.FileMode, dirEntry.Attributes.Uid, targetUid, header.Uid); code != fuse.OK {
+			return code
+		}
+	}
+
 	// Before deleting from the filer, mark any draining async-flush handle
 	// as deleted and wait for it to complete.  Without this, the async flush
 	// can race with the filer delete and recreate the just-unlinked entry
@@ -219,6 +244,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
 		wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
 	}
 	wfs.inodeToPath.TouchDirectory(dirFullPath)
+	wfs.touchDirMtimeCtime(dirFullPath)
 
 	wfs.inodeToPath.RemovePath(entryFullPath)
 
@@ -265,6 +291,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
 		Attributes: &filer_pb.FuseAttributes{
 			Mtime:    now,
 			Crtime:   now,
+			Ctime:    now,
 			FileMode: uint32(fileMode),
 			Uid:      uid,
 			Gid:      gid,
@@ -313,6 +340,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
 			wfs.inodeToPath.InvalidateChildrenCache(dirFullPath)
 		}
 		wfs.inodeToPath.TouchDirectory(dirFullPath)
+		wfs.touchDirMtimeCtime(dirFullPath)
 	}
 
 	glog.V(3).Infof("createFile %s: %v", entryFullPath, err)
@@ -335,7 +363,11 @@ func (wfs *WFS) truncateEntry(entryFullPath util.FullPath, entry *filer_pb.Entry
 	entry.Content = nil
 	entry.Chunks = nil
 	entry.Attributes.FileSize = 0
-	entry.Attributes.Mtime = time.Now().Unix()
+	truncNow := time.Now()
+	entry.Attributes.Mtime = truncNow.Unix()
+	entry.Attributes.MtimeNs = int32(truncNow.Nanosecond())
+	entry.Attributes.Ctime = truncNow.Unix()
+	entry.Attributes.CtimeNs = int32(truncNow.Nanosecond())
 
 	if code := wfs.saveEntry(entryFullPath, entry); code != fuse.OK {
 		return code

weed/mount/weedfs_file_sync.go

@@ -167,6 +167,10 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32, allowAsync bool) fuse.S
 
 // flushMetadataToFiler sends the file's chunk references and attributes to the filer.
 // This is shared between the synchronous doFlush path and the async flush completion.
+//
+// When -dlm is enabled, the distributed lock is already held by the FileHandle
+// from open-for-write through close, so no additional distributed lock is
+// needed here. The local fhLockTable lock below serializes within this mount.
 func (wfs *WFS) flushMetadataToFiler(fh *FileHandle, dir, name string, uid, gid uint32) error {
 	fileFullPath := fh.FullPath()
 	glog.V(4).Infof("flushMetadataToFiler %s/%s inode %d fh %d", dir, name, fh.inode, fh.fh)
@@ -185,7 +189,11 @@ func (wfs *WFS) flushMetadataToFiler(fh *FileHandle, dir, name string, uid, gid
 		if entry.Attributes.Gid == 0 {
 			entry.Attributes.Gid = gid
 		}
-		entry.Attributes.Mtime = time.Now().Unix()
+		flushNow := time.Now()
+		entry.Attributes.Mtime = flushNow.Unix()
+		entry.Attributes.MtimeNs = int32(flushNow.Nanosecond())
+		entry.Attributes.Ctime = flushNow.Unix()
+		entry.Attributes.CtimeNs = int32(flushNow.Nanosecond())
 	}
 
 	glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks()))

weed/mount/weedfs_file_write.go

@@ -88,6 +88,11 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
 
 	fh.dirtyMetadata = true
 
+	// POSIX: clear SUID/SGID bits on write by non-root users.
+	if in.Uid != 0 {
+		entry.Attributes.FileMode &^= 0o6000
+	}
+
 	if IsDebugFileReadWrite {
 		// print("+")
 		fh.mirrorFile.WriteAt(data, offset)

weed/mount/weedfs_filehandle.go

@@ -1,7 +1,10 @@
 package mount
 
 import (
+	"fmt"
+
 	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -23,13 +26,29 @@ func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle
 		}
 		// Check unix permission bits for the requested access mode.
 		if entry != nil && entry.Attributes != nil {
-			if mask := openFlagsToAccessMask(flags); mask != 0 && !hasAccess(uid, gid, entry.Attributes.Uid, entry.Attributes.Gid, entry.Attributes.FileMode, mask) {
+			fileUid, fileGid := entry.Attributes.Uid, entry.Attributes.Gid
+			if wfs.option.UidGidMapper != nil {
+				fileUid, fileGid = wfs.option.UidGidMapper.FilerToLocal(fileUid, fileGid)
+			}
+			if mask := openFlagsToAccessMask(flags); mask != 0 && !hasAccess(uid, gid, fileUid, fileGid, entry.Attributes.FileMode, mask) {
 				return nil, fuse.EACCES
 			}
 		}
 		// need to AcquireFileHandle again to ensure correct handle counter
 		fileHandle = wfs.fhMap.AcquireFileHandle(wfs, inode, entry)
 		fileHandle.RememberPath(path)
+
+		// Acquire distributed lock for write opens. The lock is held with
+		// auto-renewal until the file handle is released (close).
+		// Use the filer path as the lock key since inode numbers are
+		// assigned per-mount and differ across mount instances.
+		if wfs.lockClient != nil && flags&fuse.O_ANYWRITE != 0 && fileHandle.dlmLock == nil {
+			owner := fmt.Sprintf("mount-%d", wfs.signature)
+			fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+				string(path), owner, lock_manager.LiveLockTTL,
+			)
+			glog.V(1).Infof("DLM lock acquired for %s", path)
+		}
 	}
 	return
 }

weed/mount/weedfs_link.go

@@ -72,7 +72,11 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 	}
 
 	// CreateLink 1.2 : update new file to hardlink mode
-	oldEntry.Attributes.Mtime = time.Now().Unix()
+	linkNow := time.Now()
+	oldEntry.Attributes.Mtime = linkNow.Unix()
+	oldEntry.Attributes.MtimeNs = int32(linkNow.Nanosecond())
+	oldEntry.Attributes.Ctime = linkNow.Unix()
+	oldEntry.Attributes.CtimeNs = int32(linkNow.Nanosecond())
 	request := &filer_pb.CreateEntryRequest{
 		Directory: string(newParentPath),
 		Entry: &filer_pb.Entry{
@@ -127,6 +131,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 				glog.Warningf("link %s: best-effort metadata apply failed: %v", newParentPath.Child(name), applyErr)
 				wfs.inodeToPath.InvalidateChildrenCache(newParentPath)
 			}
+			wfs.touchDirMtimeCtime(newParentPath)
 		}
 	}
 

weed/mount/weedfs_metadata_flush.go

@@ -85,6 +85,10 @@ func (wfs *WFS) flushAllDirtyMetadata() {
 // flushFileMetadata flushes the current file metadata to the filer without
 // flushing dirty pages from memory. This updates chunk references in the filer
 // so volume.fsck can see them, while keeping data in the write buffer.
+//
+// When -dlm is enabled, the distributed lock is already held by the FileHandle
+// from open-for-write through close, so no additional distributed lock is
+// needed here. The local fhLockTable lock below serializes within this mount.
 func (wfs *WFS) flushFileMetadata(fh *FileHandle) error {
 	// Acquire exclusive lock on the file handle
 	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("flushMetadata", fh.fh, util.ExclusiveLock)
@@ -107,7 +111,11 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error {
 	entry.Name = name
 
 	if entry.Attributes != nil {
-		entry.Attributes.Mtime = time.Now().Unix()
+		metaNow := time.Now()
+		entry.Attributes.Mtime = metaNow.Unix()
+		entry.Attributes.MtimeNs = int32(metaNow.Nanosecond())
+		entry.Attributes.Ctime = metaNow.Unix()
+		entry.Attributes.CtimeNs = int32(metaNow.Nanosecond())
 	}
 
 	// Get current chunks - these include chunks that have been uploaded

weed/mount/weedfs_rename.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/seaweedfs/go-fuse/v2/fs"
 	"github.com/seaweedfs/go-fuse/v2/fuse"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -198,6 +199,32 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 		return status
 	}
 
+	// POSIX: enforce sticky bit on the source directory.
+	if oldDirEntry, dirCode := wfs.maybeLoadEntry(oldDir); dirCode == fuse.OK && oldDirEntry != nil && oldDirEntry.Attributes != nil {
+		targetUid := uint32(0)
+		if oldEntry != nil && oldEntry.Attributes != nil {
+			targetUid = oldEntry.Attributes.Uid
+		}
+		if code := checkStickyBit(oldDirEntry.Attributes.FileMode, oldDirEntry.Attributes.Uid, targetUid, in.Uid); code != fuse.OK {
+			return code
+		}
+	}
+
+	// POSIX: enforce sticky bit on the destination directory when replacing an existing entry.
+	if in.Flags != RenameNoReplace {
+		if newEntry, newStatus := wfs.maybeLoadEntry(newPath); newStatus == fuse.OK && newEntry != nil {
+			if newDirEntry, dirCode := wfs.maybeLoadEntry(newDir); dirCode == fuse.OK && newDirEntry != nil && newDirEntry.Attributes != nil {
+				targetUid := uint32(0)
+				if newEntry.Attributes != nil {
+					targetUid = newEntry.Attributes.Uid
+				}
+				if code := checkStickyBit(newDirEntry.Attributes.FileMode, newDirEntry.Attributes.Uid, targetUid, in.Uid); code != fuse.OK {
+					return code
+				}
+			}
+		}
+	}
+
 	if wormEnforced, _ := wfs.wormEnforcedForEntry(oldPath, oldEntry); wormEnforced {
 		return fuse.EPERM
 	}
@@ -233,6 +260,41 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 		wfs.waitForPendingAsyncFlush(inode)
 	}
 
+	// Acquire DLM locks on both old and new paths to prevent another mount
+	// from opening either path for writing during the rename. Lock in
+	// sorted order to prevent deadlocks when two mounts rename in opposite
+	// directions (A→B vs B→A).
+	//
+	// Skip the old-path lock if this mount already holds it via an open
+	// file handle (otherwise we'd deadlock trying to re-acquire our own lock).
+	if wfs.lockClient != nil {
+		owner := fmt.Sprintf("mount-%d", wfs.signature)
+
+		// Check if the source file handle already holds a DLM lock on oldPath
+		oldPathAlreadyLocked := false
+		if sourceInode, found := wfs.inodeToPath.GetInode(oldPath); found {
+			if fh, ok := wfs.fhMap.FindFileHandle(sourceInode); ok && fh.dlmLock != nil {
+				oldPathAlreadyLocked = true
+			}
+		}
+
+		// Determine which paths need new DLM locks
+		pathsToLock := []string{string(newPath)}
+		if !oldPathAlreadyLocked {
+			pathsToLock = append(pathsToLock, string(oldPath))
+		}
+		// Sort for consistent lock ordering
+		if len(pathsToLock) == 2 && pathsToLock[0] > pathsToLock[1] {
+			pathsToLock[0], pathsToLock[1] = pathsToLock[1], pathsToLock[0]
+		}
+
+		for _, p := range pathsToLock {
+			dlmLock := wfs.lockClient.NewBlockingLongLivedLock(p, owner, lock_manager.LiveLockTTL)
+			defer dlmLock.Stop()
+		}
+		glog.V(1).Infof("DLM locks acquired for rename %s => %s (oldPathAlreadyLocked=%v)", oldPath, newPath, oldPathAlreadyLocked)
+	}
+
 	// update remote filer
 	request := &filer_pb.StreamRenameEntryRequest{
 		OldDirectory: string(oldDir),
@@ -261,6 +323,10 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
 	}
 	wfs.inodeToPath.TouchDirectory(oldDir)
 	wfs.inodeToPath.TouchDirectory(newDir)
+	wfs.touchDirMtimeCtime(oldDir)
+	if oldDir != newDir {
+		wfs.touchDirMtimeCtime(newDir)
+	}
 
 	return fuse.OK
 
@@ -296,6 +362,23 @@ func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 				// Keep the saved handle path current so any flush fallback
 				// after Forget uses the post-rename location, not the old one.
 				fh.RememberPath(newPath)
+
+				// Migrate the DLM lock from old path to new path so the
+				// lock key matches the current file location. Hold the
+				// fhLockTable to prevent ReleaseHandle from concurrently
+				// stopping the lock during migration.
+				if wfs.lockClient != nil {
+					fhActiveLock := wfs.fhLockTable.AcquireLock("renameDLM", fh.fh, util.ExclusiveLock)
+					if fh.dlmLock != nil {
+						owner := fmt.Sprintf("mount-%d", wfs.signature)
+						fh.dlmLock.Stop()
+						fh.dlmLock = wfs.lockClient.NewBlockingLongLivedLock(
+							string(newPath), owner, lock_manager.LiveLockTTL,
+						)
+						glog.V(1).Infof("DLM lock migrated from %s to %s", oldPath, newPath)
+					}
+					wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+				}
 			}
 			// invalidate attr and data
 			// wfs.fuseServer.InodeNotify(sourceInode, 0, -1)

weed/mount/weedfs_symlink.go

@@ -28,14 +28,16 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
 	}
 	entryFullPath := dirPath.Child(name)
 
+	now := time.Now().Unix()
 	request := &filer_pb.CreateEntryRequest{
 		Directory: string(dirPath),
 		Entry: &filer_pb.Entry{
 			Name:        name,
 			IsDirectory: false,
 			Attributes: &filer_pb.FuseAttributes{
-				Mtime:         time.Now().Unix(),
-				Crtime:        time.Now().Unix(),
+				Mtime:         now,
+				Crtime:        now,
+				Ctime:         now,
 				FileMode:      uint32(os.FileMode(0777) | os.ModeSymlink),
 				Uid:           header.Uid,
 				Gid:           header.Gid,
@@ -58,6 +60,7 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
 			glog.Warningf("symlink %s: best-effort metadata apply failed: %v", entryFullPath, applyErr)
 			wfs.inodeToPath.InvalidateChildrenCache(dirPath)
 		}
+		wfs.touchDirMtimeCtime(dirPath)
 	}
 
 	// Map back to local uid/gid before writing to the kernel.

weed/mount/wfs_save.go

@@ -66,7 +66,10 @@ func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) {
 }
 
 func checkName(name string) fuse.Status {
-	if len(name) >= 4096 {
+	// The Linux FUSE kernel module enforces NAME_MAX=255 at the VFS layer.
+	// Return ENAMETOOLONG early to avoid creating entries that cannot be
+	// looked up via normal syscalls (stat, chmod, etc.).
+	if len(name) > 255 {
 		return fuse.Status(syscall.ENAMETOOLONG)
 	}
 	return fuse.OK

weed/mq/kafka/protocol/fetch_partition_reader.go

@@ -29,10 +29,9 @@ type partitionReader struct {
 
 // bufferedRecords represents a batch of pre-fetched records
 type bufferedRecords struct {
-	recordBatch   []byte
-	startOffset   int64
-	endOffset     int64
-	highWaterMark int64
+	recordBatch []byte
+	startOffset int64
+	endOffset   int64
 }
 
 // partitionFetchRequest represents a request to fetch data from this partition
@@ -43,7 +42,6 @@ type partitionFetchRequest struct {
 	resultChan      chan *partitionFetchResult
 	isSchematized   bool
 	apiVersion      uint16
-	correlationID   int32 // Added for correlation tracking
 }
 
 // newPartitionReader creates and starts a new partition reader with pre-fetch buffering

weed/mq/kafka/protocol/handler.go

@@ -3323,33 +3323,6 @@ func (h *Handler) writeResponseWithCorrelationID(w *bufio.Writer, correlationID
 	return h.writeResponseWithHeader(w, correlationID, 0, 0, responseBody, timeout)
 }
 
-// writeResponseWithTimeout writes a Kafka response with timeout handling
-// DEPRECATED: Use writeResponseWithCorrelationID instead
-func (h *Handler) writeResponseWithTimeout(w *bufio.Writer, response []byte, timeout time.Duration) error {
-	// This old function expects response to include correlation ID at the start
-	// For backward compatibility with any remaining callers
-
-	// Write response size (4 bytes)
-	responseSizeBytes := make([]byte, 4)
-	binary.BigEndian.PutUint32(responseSizeBytes, uint32(len(response)))
-
-	if _, err := w.Write(responseSizeBytes); err != nil {
-		return fmt.Errorf("write response size: %w", err)
-	}
-
-	// Write response data
-	if _, err := w.Write(response); err != nil {
-		return fmt.Errorf("write response data: %w", err)
-	}
-
-	// Flush the buffer
-	if err := w.Flush(); err != nil {
-		return fmt.Errorf("flush response: %w", err)
-	}
-
-	return nil
-}
-
 // EnableSchemaManagement enables schema management with the given configuration
 func (h *Handler) EnableSchemaManagement(config schema.ManagerConfig) error {
 	manager, err := schema.NewManagerWithHealthCheck(config)

weed/mq/kafka/protocol/produce.go

@@ -803,187 +803,6 @@ func (h *Handler) handleProduceV2Plus(ctx context.Context, correlationID uint32,
 	return response, nil
 }
 
-// performSchemaValidation performs comprehensive schema validation for a topic
-func (h *Handler) performSchemaValidation(topicName string, schemaID uint32, messageFormat schema.Format, messageBytes []byte) error {
-	// 1. Check if topic is configured to require schemas
-	if !h.isSchematizedTopic(topicName) {
-		// Topic doesn't require schemas, but message is schematized - this is allowed
-		return nil
-	}
-
-	// 2. Get expected schema metadata for the topic
-	expectedMetadata, err := h.getSchemaMetadataForTopic(topicName)
-	if err != nil {
-		// No expected schema found - in strict mode this would be an error
-		// In permissive mode, allow any valid schema
-		if h.isStrictSchemaValidation() {
-			// Add delay before returning schema validation error to prevent overloading
-			time.Sleep(100 * time.Millisecond)
-			return fmt.Errorf("topic %s requires schema but no expected schema found: %w", topicName, err)
-		}
-		return nil
-	}
-
-	// 3. Validate schema ID matches expected schema
-	expectedSchemaID, err := h.parseSchemaID(expectedMetadata["schema_id"])
-	if err != nil {
-		// Add delay before returning schema validation error to prevent overloading
-		time.Sleep(100 * time.Millisecond)
-		return fmt.Errorf("invalid expected schema ID for topic %s: %w", topicName, err)
-	}
-
-	// 4. Check schema compatibility
-	if schemaID != expectedSchemaID {
-		// Schema ID doesn't match - check if it's a compatible evolution
-		compatible, err := h.checkSchemaEvolution(topicName, expectedSchemaID, schemaID, messageFormat)
-		if err != nil {
-			// Add delay before returning schema validation error to prevent overloading
-			time.Sleep(100 * time.Millisecond)
-			return fmt.Errorf("failed to check schema evolution for topic %s: %w", topicName, err)
-		}
-		if !compatible {
-			// Add delay before returning schema validation error to prevent overloading
-			time.Sleep(100 * time.Millisecond)
-			return fmt.Errorf("schema ID %d is not compatible with expected schema %d for topic %s",
-				schemaID, expectedSchemaID, topicName)
-		}
-	}
-
-	// 5. Validate message format matches expected format
-	expectedFormatStr := expectedMetadata["schema_format"]
-	var expectedFormat schema.Format
-	switch expectedFormatStr {
-	case "AVRO":
-		expectedFormat = schema.FormatAvro
-	case "PROTOBUF":
-		expectedFormat = schema.FormatProtobuf
-	case "JSON_SCHEMA":
-		expectedFormat = schema.FormatJSONSchema
-	default:
-		expectedFormat = schema.FormatUnknown
-	}
-	if messageFormat != expectedFormat {
-		return fmt.Errorf("message format %s does not match expected format %s for topic %s",
-			messageFormat, expectedFormat, topicName)
-	}
-
-	// 6. Perform message-level validation
-	return h.validateMessageContent(schemaID, messageFormat, messageBytes)
-}
-
-// checkSchemaEvolution checks if a schema evolution is compatible
-func (h *Handler) checkSchemaEvolution(topicName string, expectedSchemaID, actualSchemaID uint32, format schema.Format) (bool, error) {
-	// Get both schemas
-	expectedSchema, err := h.schemaManager.GetSchemaByID(expectedSchemaID)
-	if err != nil {
-		return false, fmt.Errorf("failed to get expected schema %d: %w", expectedSchemaID, err)
-	}
-
-	actualSchema, err := h.schemaManager.GetSchemaByID(actualSchemaID)
-	if err != nil {
-		return false, fmt.Errorf("failed to get actual schema %d: %w", actualSchemaID, err)
-	}
-
-	// Since we're accessing schema from registry for this topic, ensure topic config is updated
-	h.ensureTopicSchemaFromRegistryCache(topicName, expectedSchema, actualSchema)
-
-	// Check compatibility based on topic's compatibility level
-	compatibilityLevel := h.getTopicCompatibilityLevel(topicName)
-
-	result, err := h.schemaManager.CheckSchemaCompatibility(
-		expectedSchema.Schema,
-		actualSchema.Schema,
-		format,
-		compatibilityLevel,
-	)
-	if err != nil {
-		return false, fmt.Errorf("failed to check schema compatibility: %w", err)
-	}
-
-	return result.Compatible, nil
-}
-
-// validateMessageContent validates the message content against its schema
-func (h *Handler) validateMessageContent(schemaID uint32, format schema.Format, messageBytes []byte) error {
-	// Decode the message to validate it can be parsed correctly
-	_, err := h.schemaManager.DecodeMessage(messageBytes)
-	if err != nil {
-		return fmt.Errorf("message validation failed for schema %d: %w", schemaID, err)
-	}
-
-	// Additional format-specific validation could be added here
-	switch format {
-	case schema.FormatAvro:
-		return h.validateAvroMessage(schemaID, messageBytes)
-	case schema.FormatProtobuf:
-		return h.validateProtobufMessage(schemaID, messageBytes)
-	case schema.FormatJSONSchema:
-		return h.validateJSONSchemaMessage(schemaID, messageBytes)
-	default:
-		return fmt.Errorf("unsupported schema format for validation: %s", format)
-	}
-}
-
-// validateAvroMessage performs Avro-specific validation
-func (h *Handler) validateAvroMessage(schemaID uint32, messageBytes []byte) error {
-	// Basic validation is already done in DecodeMessage
-	// Additional Avro-specific validation could be added here
-	return nil
-}
-
-// validateProtobufMessage performs Protobuf-specific validation
-func (h *Handler) validateProtobufMessage(schemaID uint32, messageBytes []byte) error {
-	// Get the schema for additional validation
-	cachedSchema, err := h.schemaManager.GetSchemaByID(schemaID)
-	if err != nil {
-		return fmt.Errorf("failed to get Protobuf schema %d: %w", schemaID, err)
-	}
-
-	// Parse the schema to get the descriptor
-	parser := schema.NewProtobufDescriptorParser()
-	protobufSchema, err := parser.ParseBinaryDescriptor([]byte(cachedSchema.Schema), "")
-	if err != nil {
-		return fmt.Errorf("failed to parse Protobuf schema: %w", err)
-	}
-
-	// Validate message against schema
-	envelope, ok := schema.ParseConfluentEnvelope(messageBytes)
-	if !ok {
-		return fmt.Errorf("invalid Confluent envelope")
-	}
-
-	return protobufSchema.ValidateMessage(envelope.Payload)
-}
-
-// validateJSONSchemaMessage performs JSON Schema-specific validation
-func (h *Handler) validateJSONSchemaMessage(schemaID uint32, messageBytes []byte) error {
-	// Get the schema for validation
-	cachedSchema, err := h.schemaManager.GetSchemaByID(schemaID)
-	if err != nil {
-		return fmt.Errorf("failed to get JSON schema %d: %w", schemaID, err)
-	}
-
-	// Create JSON Schema decoder for validation
-	decoder, err := schema.NewJSONSchemaDecoder(cachedSchema.Schema)
-	if err != nil {
-		return fmt.Errorf("failed to create JSON Schema decoder: %w", err)
-	}
-
-	// Parse envelope and validate payload
-	envelope, ok := schema.ParseConfluentEnvelope(messageBytes)
-	if !ok {
-		return fmt.Errorf("invalid Confluent envelope")
-	}
-
-	// Validate JSON payload against schema
-	_, err = decoder.Decode(envelope.Payload)
-	if err != nil {
-		return fmt.Errorf("JSON Schema validation failed: %w", err)
-	}
-
-	return nil
-}
-
 // Helper methods for configuration
 
 // isSchemaValidationError checks if an error is related to schema validation
@@ -1001,38 +820,6 @@ func (h *Handler) isSchemaValidationError(err error) bool {
 		strings.Contains(errStr, "json schema")
 }
 
-// isStrictSchemaValidation returns whether strict schema validation is enabled
-func (h *Handler) isStrictSchemaValidation() bool {
-	// This could be configurable per topic or globally
-	// For now, default to permissive mode
-	return false
-}
-
-// getTopicCompatibilityLevel returns the compatibility level for a topic
-func (h *Handler) getTopicCompatibilityLevel(topicName string) schema.CompatibilityLevel {
-	// This could be configurable per topic
-	// For now, default to backward compatibility
-	return schema.CompatibilityBackward
-}
-
-// parseSchemaID parses a schema ID from string
-func (h *Handler) parseSchemaID(schemaIDStr string) (uint32, error) {
-	if schemaIDStr == "" {
-		return 0, fmt.Errorf("empty schema ID")
-	}
-
-	var schemaID uint64
-	if _, err := fmt.Sscanf(schemaIDStr, "%d", &schemaID); err != nil {
-		return 0, fmt.Errorf("invalid schema ID format: %w", err)
-	}
-
-	if schemaID > 0xFFFFFFFF {
-		return 0, fmt.Errorf("schema ID too large: %d", schemaID)
-	}
-
-	return uint32(schemaID), nil
-}
-
 // isSystemTopic checks if a topic should bypass schema processing
 func (h *Handler) isSystemTopic(topicName string) bool {
 	// System topics that should be stored as-is without schema processing

weed/pb/filer.proto

@@ -197,6 +197,9 @@ message FuseAttributes {
     bytes md5 = 14;
     uint32 rdev = 16;
     uint64 inode = 17;
+    int64 ctime = 18; // unix time in seconds, inode change time
+    int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
+    int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
 }
 
 message CreateEntryRequest {

weed/pb/filer_pb/filer.pb.go

@@ -961,6 +961,9 @@ type FuseAttributes struct {
 	Md5           []byte                 `protobuf:"bytes,14,opt,name=md5,proto3" json:"md5,omitempty"`
 	Rdev          uint32                 `protobuf:"varint,16,opt,name=rdev,proto3" json:"rdev,omitempty"`
 	Inode         uint64                 `protobuf:"varint,17,opt,name=inode,proto3" json:"inode,omitempty"`
+	Ctime         int64                  `protobuf:"varint,18,opt,name=ctime,proto3" json:"ctime,omitempty"`                    // unix time in seconds, inode change time
+	MtimeNs       int32                  `protobuf:"varint,19,opt,name=mtime_ns,json=mtimeNs,proto3" json:"mtime_ns,omitempty"` // nanosecond component of mtime (0-999999999)
+	CtimeNs       int32                  `protobuf:"varint,20,opt,name=ctime_ns,json=ctimeNs,proto3" json:"ctime_ns,omitempty"` // nanosecond component of ctime (0-999999999)
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -1093,6 +1096,27 @@ func (x *FuseAttributes) GetInode() uint64 {
 	return 0
 }
 
+func (x *FuseAttributes) GetCtime() int64 {
+	if x != nil {
+		return x.Ctime
+	}
+	return 0
+}
+
+func (x *FuseAttributes) GetMtimeNs() int32 {
+	if x != nil {
+		return x.MtimeNs
+	}
+	return 0
+}
+
+func (x *FuseAttributes) GetCtimeNs() int32 {
+	if x != nil {
+		return x.CtimeNs
+	}
+	return 0
+}
+
 type CreateEntryRequest struct {
 	state                    protoimpl.MessageState `protogen:"open.v1"`
 	Directory                string                 `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"`
@@ -5136,7 +5160,7 @@ const file_filer_proto_rawDesc = "" +
 	"\x06FileId\x12\x1b\n" +
 	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x19\n" +
 	"\bfile_key\x18\x02 \x01(\x04R\afileKey\x12\x16\n" +
-	"\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xe8\x02\n" +
+	"\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xb4\x03\n" +
 	"\x0eFuseAttributes\x12\x1b\n" +
 	"\tfile_size\x18\x01 \x01(\x04R\bfileSize\x12\x14\n" +
 	"\x05mtime\x18\x02 \x01(\x03R\x05mtime\x12\x1b\n" +
@@ -5153,7 +5177,10 @@ const file_filer_proto_rawDesc = "" +
 	"\x0esymlink_target\x18\r \x01(\tR\rsymlinkTarget\x12\x10\n" +
 	"\x03md5\x18\x0e \x01(\fR\x03md5\x12\x12\n" +
 	"\x04rdev\x18\x10 \x01(\rR\x04rdev\x12\x14\n" +
-	"\x05inode\x18\x11 \x01(\x04R\x05inode\"\x82\x02\n" +
+	"\x05inode\x18\x11 \x01(\x04R\x05inode\x12\x14\n" +
+	"\x05ctime\x18\x12 \x01(\x03R\x05ctime\x12\x19\n" +
+	"\bmtime_ns\x18\x13 \x01(\x05R\amtimeNs\x12\x19\n" +
+	"\bctime_ns\x18\x14 \x01(\x05R\actimeNs\"\x82\x02\n" +
 	"\x12CreateEntryRequest\x12\x1c\n" +
 	"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" +
 	"\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12\x15\n" +

weed/s3api/iceberg/handlers_oauth.go

@@ -0,0 +1,196 @@
+package iceberg
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	jwt "github.com/golang-jwt/jwt/v5"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// OAuthTokenResponse is the response for POST /v1/oauth/tokens.
+type OAuthTokenResponse struct {
+	AccessToken string `json:"access_token"`
+	TokenType   string `json:"token_type"`
+	ExpiresIn   int    `json:"expires_in"`
+	Scope       string `json:"scope,omitempty"`
+}
+
+// OAuthErrorResponse is the error response for the OAuth endpoint.
+type OAuthErrorResponse struct {
+	Error       string `json:"error"`
+	Description string `json:"error_description,omitempty"`
+}
+
+// IcebergClaims are JWT claims for Iceberg catalog OAuth tokens.
+type IcebergClaims struct {
+	IdentityName string `json:"identity_name"`
+	AccessKey    string `json:"access_key"`
+	jwt.RegisteredClaims
+}
+
+const oauthTokenExpiry = 3600 // 1 hour in seconds
+
+// handleOAuthTokens implements the OAuth2 client_credentials flow.
+// POST /v1/oauth/tokens
+func (s *Server) handleOAuthTokens(w http.ResponseWriter, r *http.Request) {
+	if err := r.ParseForm(); err != nil {
+		writeOAuthError(w, http.StatusBadRequest, "invalid_request", "Could not parse form body")
+		return
+	}
+
+	// Reject credentials in query string to prevent leaking secrets into logs and caches.
+	if r.URL.Query().Get("client_secret") != "" {
+		writeOAuthError(w, http.StatusBadRequest, "invalid_request", "client_secret must not be sent in the URL")
+		return
+	}
+
+	grantType := r.PostFormValue("grant_type")
+	if grantType != "client_credentials" {
+		writeOAuthError(w, http.StatusBadRequest, "unsupported_grant_type",
+			fmt.Sprintf("Unsupported grant_type: %s", grantType))
+		return
+	}
+
+	clientID := r.PostFormValue("client_id")
+	clientSecret := r.PostFormValue("client_secret")
+
+	// Also support HTTP Basic auth per OAuth2 spec
+	if clientID == "" && clientSecret == "" {
+		var ok bool
+		clientID, clientSecret, ok = r.BasicAuth()
+		if !ok {
+			writeOAuthError(w, http.StatusUnauthorized, "invalid_client", "Missing client credentials")
+			return
+		}
+	}
+
+	if clientID == "" || clientSecret == "" {
+		writeOAuthError(w, http.StatusUnauthorized, "invalid_client", "Missing client_id or client_secret")
+		return
+	}
+
+	if s.credentialValidator == nil {
+		writeOAuthError(w, http.StatusInternalServerError, "server_error", "Credential validation not configured")
+		return
+	}
+
+	identityName, _, err := s.credentialValidator.ValidateS3Credential(clientID, clientSecret)
+	if err != nil {
+		glog.V(2).Infof("Iceberg OAuth: credential validation failed for client_id=%s: %v", clientID, err)
+		writeOAuthError(w, http.StatusUnauthorized, "invalid_client", "Invalid client credentials")
+		return
+	}
+
+	// Generate a JWT signed with a key derived from the client secret.
+	// Include the access key in claims so we can look up the exact credential for verification.
+	signingKey := deriveSigningKey(clientID, clientSecret)
+	now := time.Now()
+	claims := IcebergClaims{
+		IdentityName: identityName,
+		AccessKey:    clientID,
+		RegisteredClaims: jwt.RegisteredClaims{
+			IssuedAt:  jwt.NewNumericDate(now),
+			ExpiresAt: jwt.NewNumericDate(now.Add(time.Duration(oauthTokenExpiry) * time.Second)),
+			Issuer:    "seaweedfs-iceberg",
+		},
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	tokenString, err := token.SignedString(signingKey)
+	if err != nil {
+		glog.Errorf("Iceberg OAuth: failed to sign token: %v", err)
+		writeOAuthError(w, http.StatusInternalServerError, "server_error", "Failed to generate token")
+		return
+	}
+
+	scope := r.PostFormValue("scope")
+	resp := OAuthTokenResponse{
+		AccessToken: tokenString,
+		TokenType:   "bearer",
+		ExpiresIn:   oauthTokenExpiry,
+		Scope:       scope,
+	}
+	w.Header().Set("Cache-Control", "no-store")
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// authenticateBearer validates a Bearer token from the Authorization header.
+// Returns the identity name, identity object, and whether auth succeeded.
+func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) {
+	auth := r.Header.Get("Authorization")
+	if auth == "" {
+		return "", nil, false
+	}
+	if !strings.HasPrefix(strings.ToLower(auth), "bearer ") {
+		return "", nil, false
+	}
+	tokenString := strings.TrimSpace(auth[7:])
+	if tokenString == "" {
+		return "", nil, false
+	}
+
+	if s.credentialValidator == nil {
+		return "", nil, false
+	}
+
+	// Parse the token without verification first to get the access key,
+	// then look up the exact credential to verify the signature.
+	parser := jwt.NewParser(jwt.WithoutClaimsValidation())
+	unverified := &IcebergClaims{}
+	_, _, err := parser.ParseUnverified(tokenString, unverified)
+	if err != nil {
+		glog.V(2).Infof("Iceberg OAuth: failed to parse token: %v", err)
+		return "", nil, false
+	}
+
+	if unverified.AccessKey == "" {
+		return "", nil, false
+	}
+
+	// Look up the credential by access key to get the signing key for verification
+	identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey)
+	if err != nil {
+		glog.V(2).Infof("Iceberg OAuth: failed to get credential for access key: %v", err)
+		return "", nil, false
+	}
+
+	signingKey := deriveSigningKey(unverified.AccessKey, secretKey)
+	claims := &IcebergClaims{}
+	verified, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
+		}
+		return signingKey, nil
+	})
+	if err != nil || !verified.Valid {
+		glog.V(2).Infof("Iceberg OAuth: token verification failed: %v", err)
+		return "", nil, false
+	}
+
+	return identityName, identity, true
+}
+
+// deriveSigningKey derives a signing key from the access key and secret using HMAC-SHA256.
+// Including the access key prevents cross-credential token forgery when two
+// credentials happen to share the same secret.
+func deriveSigningKey(accessKey, secret string) []byte {
+	h := hmac.New(sha256.New, []byte("seaweedfs-iceberg-oauth"))
+	h.Write([]byte(accessKey))
+	h.Write([]byte{0}) // null separator
+	h.Write([]byte(secret))
+	return h.Sum(nil)
+}
+
+func writeOAuthError(w http.ResponseWriter, status int, errCode, description string) {
+	resp := OAuthErrorResponse{
+		Error:       errCode,
+		Description: description,
+	}
+	writeJSON(w, status, resp)
+}
+

weed/s3api/iceberg/handlers_oauth_test.go

@@ -0,0 +1,155 @@
+package iceberg
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+type mockCredentialValidator struct {
+	credentials map[string]string // accessKey -> secretKey
+	identities  map[string]string // accessKey -> identityName
+}
+
+func (m *mockCredentialValidator) ValidateS3Credential(accessKey, secretKey string) (string, interface{}, error) {
+	expected, ok := m.credentials[accessKey]
+	if !ok {
+		return "", nil, fmt.Errorf("access key not found")
+	}
+	if expected != secretKey {
+		return "", nil, fmt.Errorf("invalid secret key")
+	}
+	return m.identities[accessKey], nil, nil
+}
+
+func (m *mockCredentialValidator) GetCredentialByAccessKey(accessKey string) (string, interface{}, string, error) {
+	secret, ok := m.credentials[accessKey]
+	if !ok {
+		return "", nil, "", fmt.Errorf("access key not found")
+	}
+	return m.identities[accessKey], nil, secret, nil
+}
+
+func newTestServerWithOAuth() *Server {
+	cv := &mockCredentialValidator{
+		credentials: map[string]string{"AKID123": "secret456"},
+		identities:  map[string]string{"AKID123": "testuser"},
+	}
+	s := &Server{
+		credentialValidator: cv,
+	}
+	return s
+}
+
+func TestHandleOAuthTokens_Success(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	body := "grant_type=client_credentials&client_id=AKID123&client_secret=secret456"
+	req := httptest.NewRequest(http.MethodPost, "/v1/oauth/tokens", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	w := httptest.NewRecorder()
+
+	s.handleOAuthTokens(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+
+	var resp OAuthTokenResponse
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp.TokenType != "bearer" {
+		t.Errorf("expected token_type=bearer, got %s", resp.TokenType)
+	}
+	if resp.AccessToken == "" {
+		t.Error("expected non-empty access_token")
+	}
+	if resp.ExpiresIn != oauthTokenExpiry {
+		t.Errorf("expected expires_in=%d, got %d", oauthTokenExpiry, resp.ExpiresIn)
+	}
+}
+
+func TestHandleOAuthTokens_InvalidCredentials(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	body := "grant_type=client_credentials&client_id=AKID123&client_secret=wrongsecret"
+	req := httptest.NewRequest(http.MethodPost, "/v1/oauth/tokens", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	w := httptest.NewRecorder()
+
+	s.handleOAuthTokens(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("expected 401, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+func TestHandleOAuthTokens_UnsupportedGrantType(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	body := "grant_type=authorization_code&client_id=AKID123&client_secret=secret456"
+	req := httptest.NewRequest(http.MethodPost, "/v1/oauth/tokens", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	w := httptest.NewRecorder()
+
+	s.handleOAuthTokens(w, req)
+
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+func TestBearerTokenRoundTrip(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	// Get a token
+	body := "grant_type=client_credentials&client_id=AKID123&client_secret=secret456"
+	req := httptest.NewRequest(http.MethodPost, "/v1/oauth/tokens", strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	w := httptest.NewRecorder()
+	s.handleOAuthTokens(w, req)
+
+	var resp OAuthTokenResponse
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatal(err)
+	}
+
+	// Use the token for Bearer auth
+	authReq := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+	authReq.Header.Set("Authorization", "Bearer "+resp.AccessToken)
+
+	identityName, _, ok := s.authenticateBearer(authReq)
+	if !ok {
+		t.Fatal("expected Bearer auth to succeed")
+	}
+	if identityName != "testuser" {
+		t.Errorf("expected identity 'testuser', got '%s'", identityName)
+	}
+}
+
+func TestBearerTokenInvalid(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+	req.Header.Set("Authorization", "Bearer invalid-token")
+
+	_, _, ok := s.authenticateBearer(req)
+	if ok {
+		t.Error("expected Bearer auth to fail with invalid token")
+	}
+}
+
+func TestBearerTokenNone(t *testing.T) {
+	s := newTestServerWithOAuth()
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+
+	_, _, ok := s.authenticateBearer(req)
+	if ok {
+		t.Error("expected Bearer auth to fail with no token")
+	}
+}

weed/s3api/iceberg/server.go

@@ -21,12 +21,25 @@ type S3Authenticator interface {
 	DefaultAllow() bool
 }
 
+// CredentialValidator validates S3 access key / secret key pairs
+// and provides credential lookup for OAuth token verification.
+type CredentialValidator interface {
+	// ValidateS3Credential checks if the access key and secret key are valid.
+	// Returns the identity name and identity object on success.
+	ValidateS3Credential(accessKey, secretKey string) (identityName string, identity interface{}, err error)
+	// GetCredentialByAccessKey looks up a credential by access key.
+	// Returns the identity name, identity object, and secret key.
+	// Used for verifying Bearer tokens signed with a specific credential.
+	GetCredentialByAccessKey(accessKey string) (identityName string, identity interface{}, secretKey string, err error)
+}
+
 // Server implements the Iceberg REST Catalog API.
 type Server struct {
-	filerClient   FilerClient
-	tablesManager *s3tables.Manager
-	prefix        string // optional prefix for routes
-	authenticator S3Authenticator
+	filerClient         FilerClient
+	tablesManager       *s3tables.Manager
+	prefix              string // optional prefix for routes
+	authenticator       S3Authenticator
+	credentialValidator CredentialValidator
 }
 
 // NewServer creates a new Iceberg REST Catalog server.
@@ -40,6 +53,11 @@ func NewServer(filerClient FilerClient, authenticator S3Authenticator) *Server {
 	}
 }
 
+// SetCredentialValidator sets the credential validator for OAuth token support.
+func (s *Server) SetCredentialValidator(cv CredentialValidator) {
+	s.credentialValidator = cv
+}
+
 // RegisterRoutes registers Iceberg REST API routes on the provided router.
 func (s *Server) RegisterRoutes(router *mux.Router) {
 	// Add middleware to log all requests/responses
@@ -48,6 +66,9 @@ func (s *Server) RegisterRoutes(router *mux.Router) {
 	// Configuration endpoint - no auth needed for config
 	router.HandleFunc("/v1/config", s.handleConfig).Methods(http.MethodGet)
 
+	// OAuth2 token endpoint - no auth needed (this IS the auth endpoint)
+	router.HandleFunc("/v1/oauth/tokens", s.handleOAuthTokens).Methods(http.MethodPost)
+
 	// Namespace endpoints - wrapped with Auth middleware
 	router.HandleFunc("/v1/namespaces", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
 	router.HandleFunc("/v1/namespaces", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
@@ -122,6 +143,18 @@ func (w *responseWriter) WriteHeader(code int) {
 
 func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		// Try Bearer token authentication first (from OAuth2 flow)
+		if identityName, identity, ok := s.authenticateBearer(r); ok {
+			ctx := r.Context()
+			ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
+			if identity != nil {
+				ctx = s3_constants.SetIdentityInContext(ctx, identity)
+			}
+			r = r.WithContext(ctx)
+			handler(w, r)
+			return
+		}
+
 		if s.authenticator == nil {
 			writeError(w, http.StatusUnauthorized, "NotAuthorizedException", "Authentication required")
 			return

weed/s3api/s3api_embedded_iam.go

@@ -99,6 +99,7 @@ type (
 	iamPutUserPolicyResponse            = iamlib.PutUserPolicyResponse
 	iamDeleteUserPolicyResponse         = iamlib.DeleteUserPolicyResponse
 	iamGetUserPolicyResponse            = iamlib.GetUserPolicyResponse
+	iamListUserPoliciesResponse         = iamlib.ListUserPoliciesResponse
 	iamAttachUserPolicyResponse         = iamlib.AttachUserPolicyResponse
 	iamDetachUserPolicyResponse         = iamlib.DetachUserPolicyResponse
 	iamListAttachedUserPoliciesResponse = iamlib.ListAttachedUserPoliciesResponse
@@ -124,6 +125,10 @@ type (
 	iamAttachGroupPolicyResponse         = iamlib.AttachGroupPolicyResponse
 	iamDetachGroupPolicyResponse         = iamlib.DetachGroupPolicyResponse
 	iamListAttachedGroupPoliciesResponse = iamlib.ListAttachedGroupPoliciesResponse
+	iamPutGroupPolicyResponse            = iamlib.PutGroupPolicyResponse
+	iamGetGroupPolicyResponse            = iamlib.GetGroupPolicyResponse
+	iamDeleteGroupPolicyResponse         = iamlib.DeleteGroupPolicyResponse
+	iamListGroupPoliciesResponse         = iamlib.ListGroupPoliciesResponse
 	iamListGroupsForUserResponse         = iamlib.ListGroupsForUserResponse
 )
 
@@ -939,6 +944,23 @@ func (e *EmbeddedIamApi) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, valu
 	return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)}
 }
 
+// ListUserPolicies lists the names of inline policies attached to a user.
+// https://docs.aws.amazon.com/IAM/latest/APIReference/API_ListUserPolicies.html
+func (e *EmbeddedIamApi) ListUserPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*iamListUserPoliciesResponse, *iamError) {
+	resp := &iamListUserPoliciesResponse{}
+	userName := values.Get("UserName")
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == userName {
+			if len(ident.Actions) > 0 {
+				resp.ListUserPoliciesResult.PolicyNames = []string{userName + "_policy"}
+			}
+			resp.ListUserPoliciesResult.IsTruncated = false
+			return resp, nil
+		}
+	}
+	return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)}
+}
+
 // AttachUserPolicy attaches a managed policy to a user.
 func (e *EmbeddedIamApi) AttachUserPolicy(ctx context.Context, values url.Values) (*iamAttachUserPolicyResponse, *iamError) {
 	resp := &iamAttachUserPolicyResponse{}
@@ -1753,6 +1775,31 @@ func (e *EmbeddedIamApi) ListGroupsForUser(s3cfg *iam_pb.S3ApiConfiguration, val
 	return resp, nil
 }
 
+// notImplementedError returns a NotImplemented IAM error for the embedded server.
+func notImplementedGroupInlineError() *iamError {
+	return &iamError{Code: s3err.GetAPIError(s3err.ErrNotImplemented).Code, Error: fmt.Errorf("group inline policies are not supported in embedded IAM mode; use the standalone IAM server or managed policies (AttachGroupPolicy)")}
+}
+
+// PutGroupPolicy is not supported in embedded IAM mode.
+func (e *EmbeddedIamApi) PutGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*iamPutGroupPolicyResponse, *iamError) {
+	return &iamPutGroupPolicyResponse{}, notImplementedGroupInlineError()
+}
+
+// GetGroupPolicy is not supported in embedded IAM mode.
+func (e *EmbeddedIamApi) GetGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*iamGetGroupPolicyResponse, *iamError) {
+	return &iamGetGroupPolicyResponse{}, notImplementedGroupInlineError()
+}
+
+// DeleteGroupPolicy is not supported in embedded IAM mode.
+func (e *EmbeddedIamApi) DeleteGroupPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*iamDeleteGroupPolicyResponse, *iamError) {
+	return &iamDeleteGroupPolicyResponse{}, notImplementedGroupInlineError()
+}
+
+// ListGroupPolicies is not supported in embedded IAM mode.
+func (e *EmbeddedIamApi) ListGroupPolicies(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (*iamListGroupPoliciesResponse, *iamError) {
+	return &iamListGroupPoliciesResponse{}, notImplementedGroupInlineError()
+}
+
 // handleImplicitUsername adds username who signs the request to values if 'username' is not specified.
 // According to AWS documentation: "If you do not specify a user name, IAM determines the user name
 // implicitly based on the Amazon Web Services access key ID signing the request."
@@ -1906,8 +1953,8 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 	action := values.Get("Action")
 	if e.readOnly {
 		switch action {
-		case "ListUsers", "ListAccessKeys", "GetUser", "GetUserPolicy", "ListAttachedUserPolicies", "ListPolicies", "GetPolicy", "ListPolicyVersions", "GetPolicyVersion", "ListServiceAccounts", "GetServiceAccount",
-			"GetGroup", "ListGroups", "ListAttachedGroupPolicies", "ListGroupsForUser":
+		case "ListUsers", "ListAccessKeys", "GetUser", "GetUserPolicy", "ListUserPolicies", "ListAttachedUserPolicies", "ListPolicies", "GetPolicy", "ListPolicyVersions", "GetPolicyVersion", "ListServiceAccounts", "GetServiceAccount",
+			"GetGroup", "ListGroups", "ListAttachedGroupPolicies", "GetGroupPolicy", "ListGroupPolicies", "ListGroupsForUser":
 			// Allowed read-only actions
 		default:
 			return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code, Error: fmt.Errorf("IAM write operations are disabled on this server")}
@@ -2003,6 +2050,13 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		if iamErr != nil {
 			return nil, iamErr
 		}
+	case "ListUserPolicies":
+		var iamErr *iamError
+		response, iamErr = e.ListUserPolicies(s3cfg, values)
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		changed = false
 	case "AttachUserPolicy":
 		var iamErr *iamError
 		response, iamErr = e.AttachUserPolicy(ctx, values)
@@ -2154,6 +2208,34 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 			return nil, iamErr
 		}
 		changed = false
+	case "PutGroupPolicy":
+		var iamErr *iamError
+		response, iamErr = e.PutGroupPolicy(s3cfg, values)
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		changed = false
+	case "GetGroupPolicy":
+		var iamErr *iamError
+		response, iamErr = e.GetGroupPolicy(s3cfg, values)
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		changed = false
+	case "DeleteGroupPolicy":
+		var iamErr *iamError
+		response, iamErr = e.DeleteGroupPolicy(s3cfg, values)
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		changed = false
+	case "ListGroupPolicies":
+		var iamErr *iamError
+		response, iamErr = e.ListGroupPolicies(s3cfg, values)
+		if iamErr != nil {
+			return nil, iamErr
+		}
+		changed = false
 	case "ListGroupsForUser":
 		var iamErr *iamError
 		response, iamErr = e.ListGroupsForUser(s3cfg, values)
@@ -2198,7 +2280,7 @@ func (e *EmbeddedIamApi) DoActions(w http.ResponseWriter, r *http.Request) {
 
 	// Handle implicit username for HTTP requests
 	switch r.Form.Get("Action") {
-	case "ListAccessKeys", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey":
+	case "ListAccessKeys", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey", "ListUserPolicies":
 		e.handleImplicitUsername(r, values)
 	case "CreateServiceAccount":
 		createdBy := s3_constants.GetIdentityNameFromContext(r)

weed/s3api/s3api_embedded_iam_test.go

@@ -461,6 +461,112 @@ func TestEmbeddedIamDeleteUserPolicyUserNotFound(t *testing.T) {
 	assert.Equal(t, http.StatusNotFound, rr.Code)
 }
 
+// TestEmbeddedIamListUserPolicies tests listing inline policies for a user.
+func TestEmbeddedIamListUserPolicies(t *testing.T) {
+	api := NewEmbeddedIamApiForTest()
+	api.mockConfig = &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{
+			{
+				Name:    "UserWithPolicy",
+				Actions: []string{"Read", "Write"},
+				Credentials: []*iam_pb.Credential{
+					{AccessKey: UserAccessKeyPrefix + "TEST12345", SecretKey: "secret"},
+				},
+			},
+			{
+				Name: "UserWithoutPolicy",
+				Credentials: []*iam_pb.Credential{
+					{AccessKey: UserAccessKeyPrefix + "TEST67890", SecretKey: "secret"},
+				},
+			},
+		},
+	}
+
+	// List policies for user with actions
+	form := url.Values{}
+	form.Set("Action", "ListUserPolicies")
+	form.Set("UserName", "UserWithPolicy")
+
+	req, _ := http.NewRequest("POST", "/", nil)
+	req.PostForm = form
+	req.Form = form
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	rr := httptest.NewRecorder()
+	apiRouter := mux.NewRouter().SkipClean(true)
+	apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions)
+	apiRouter.ServeHTTP(rr, req)
+
+	assert.Equal(t, http.StatusOK, rr.Code)
+	assert.Contains(t, rr.Body.String(), "ListUserPoliciesResponse")
+	assert.Contains(t, rr.Body.String(), "PolicyNames")
+	assert.Contains(t, rr.Body.String(), "UserWithPolicy_policy")
+
+	// List policies for user without actions
+	form2 := url.Values{}
+	form2.Set("Action", "ListUserPolicies")
+	form2.Set("UserName", "UserWithoutPolicy")
+
+	req2, _ := http.NewRequest("POST", "/", nil)
+	req2.PostForm = form2
+	req2.Form = form2
+	req2.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	rr2 := httptest.NewRecorder()
+	apiRouter.ServeHTTP(rr2, req2)
+
+	assert.Equal(t, http.StatusOK, rr2.Code)
+	assert.Contains(t, rr2.Body.String(), "ListUserPoliciesResponse")
+
+	// List policies for nonexistent user
+	form3 := url.Values{}
+	form3.Set("Action", "ListUserPolicies")
+	form3.Set("UserName", "NonExistentUser")
+
+	req3, _ := http.NewRequest("POST", "/", nil)
+	req3.PostForm = form3
+	req3.Form = form3
+	req3.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	rr3 := httptest.NewRecorder()
+	apiRouter.ServeHTTP(rr3, req3)
+
+	assert.Equal(t, http.StatusNotFound, rr3.Code)
+}
+
+// TestEmbeddedIamGroupInlinePoliciesNotImplemented tests that group inline policies
+// return NotImplemented in embedded IAM mode.
+func TestEmbeddedIamGroupInlinePoliciesNotImplemented(t *testing.T) {
+	api := NewEmbeddedIamApiForTest()
+	s3cfg := &iam_pb.S3ApiConfiguration{
+		Groups: []*iam_pb.Group{
+			{Name: "developers", Members: []string{"alice"}},
+		},
+	}
+
+	notImpl := s3err.GetAPIError(s3err.ErrNotImplemented).Code
+
+	_, iamErr := api.PutGroupPolicy(s3cfg, url.Values{
+		"GroupName":      {"developers"},
+		"PolicyName":     {"DevPolicy"},
+		"PolicyDocument": {`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`},
+	})
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, notImpl, iamErr.Code)
+
+	_, iamErr = api.GetGroupPolicy(s3cfg, url.Values{"GroupName": {"developers"}, "PolicyName": {"DevPolicy"}})
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, notImpl, iamErr.Code)
+
+	_, iamErr = api.DeleteGroupPolicy(s3cfg, url.Values{"GroupName": {"developers"}, "PolicyName": {"DevPolicy"}})
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, notImpl, iamErr.Code)
+
+	_, iamErr = api.ListGroupPolicies(s3cfg, url.Values{"GroupName": {"developers"}})
+	assert.NotNil(t, iamErr)
+	assert.Equal(t, notImpl, iamErr.Code)
+}
+
 // TestEmbeddedIamAttachUserPolicy tests attaching a managed policy to a user.
 func TestEmbeddedIamAttachUserPolicy(t *testing.T) {
 	api := NewEmbeddedIamApiForTest()

weed/s3api/s3api_handlers.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"net/http"
 
@@ -46,8 +47,13 @@ func (s3a *S3ApiServer) withFilerClientFailover(streamingMode bool, fn func(file
 		return nil
 	}
 
-	// Record failure for current filer
-	s3a.filerClient.RecordFilerFailure(currentFiler)
+	// ErrNotFound is a valid application-level response (entry doesn't exist on this filer),
+	// not a filer health issue. Only record true failures (transport errors, timeouts, etc.)
+	// in the health tracker to avoid poisoning the circuit breaker with normal "not found"
+	// responses in multi-filer setups.
+	if !errors.Is(err, filer_pb.ErrNotFound) {
+		s3a.filerClient.RecordFilerFailure(currentFiler)
+	}
 
 	// Current filer failed - try all other filers with health-aware selection
 	filers := s3a.filerClient.GetAllFilers()
@@ -77,8 +83,10 @@ func (s3a *S3ApiServer) withFilerClientFailover(streamingMode bool, fn func(file
 			return nil
 		}
 
-		// Record failure for health tracking
-		s3a.filerClient.RecordFilerFailure(filer)
+		// Only record real failures, not ErrNotFound
+		if !errors.Is(err, filer_pb.ErrNotFound) {
+			s3a.filerClient.RecordFilerFailure(filer)
+		}
 		glog.V(2).Infof("WithFilerClient: failover to %s failed: %v", filer, err)
 		lastErr = err
 	}

weed/s3api/s3api_server.go

@@ -1055,3 +1055,52 @@ func (s3a *S3ApiServer) DefaultAllow() bool {
 	}
 	return s3a.iam.iamIntegration.DefaultAllow()
 }
+
+// ValidateS3Credential validates an S3 access key / secret key pair.
+// Returns the identity name and identity object on success.
+func (s3a *S3ApiServer) ValidateS3Credential(accessKey, secretKey string) (string, interface{}, error) {
+	if s3a.iam == nil {
+		return "", nil, fmt.Errorf("IAM not configured")
+	}
+	identity, cred, found := s3a.iam.LookupByAccessKey(accessKey)
+	if !found {
+		return "", nil, fmt.Errorf("access key not found")
+	}
+	if cred.SecretKey != secretKey {
+		return "", nil, fmt.Errorf("invalid secret key")
+	}
+	if identity.Disabled {
+		return "", nil, fmt.Errorf("identity is disabled")
+	}
+	if cred.isCredentialExpired() {
+		return "", nil, fmt.Errorf("credential expired")
+	}
+	if cred.Status == "Inactive" {
+		return "", nil, fmt.Errorf("credential is inactive")
+	}
+	return identity.Name, identity, nil
+}
+
+// GetCredentialByAccessKey looks up a credential by access key.
+// Returns the identity name, identity object, and secret key.
+// Used for verifying Iceberg OAuth Bearer tokens with the exact credential
+// that was used to sign the token.
+func (s3a *S3ApiServer) GetCredentialByAccessKey(accessKey string) (string, interface{}, string, error) {
+	if s3a.iam == nil {
+		return "", nil, "", fmt.Errorf("IAM not configured")
+	}
+	identity, cred, found := s3a.iam.LookupByAccessKey(accessKey)
+	if !found {
+		return "", nil, "", fmt.Errorf("access key not found")
+	}
+	if identity.Disabled {
+		return "", nil, "", fmt.Errorf("identity is disabled")
+	}
+	if cred.isCredentialExpired() {
+		return "", nil, "", fmt.Errorf("credential expired")
+	}
+	if cred.Status == "Inactive" {
+		return "", nil, "", fmt.Errorf("credential is inactive")
+	}
+	return identity.Name, identity, cred.SecretKey, nil
+}

weed/server/filer_server.go

@@ -99,7 +99,6 @@ type FilerServer struct {
 
 	filer_pb.UnimplementedSeaweedFilerServer
 	option         *FilerOption
-	secret         security.SigningKey
 	filer          *filer.Filer
 	filerGuard     *security.Guard
 	volumeGuard    *security.Guard

weed/server/raft_server.go

@@ -290,6 +290,19 @@ func recoverTopologyIdFromSnapshot(dataDir string, topo *topology.Topology) {
 	recoverTopologyIdFromState(snap.State, topo)
 }
 
+// HasExistingState returns true when the raft log already contains entries,
+// indicating this server was previously joined and does not need a new
+// JoinCommand on startup.
+func (s *RaftServer) HasExistingState() bool {
+	if s.raftServer != nil {
+		return !s.raftServer.IsLogEmpty()
+	}
+	if s.RaftHashicorp != nil {
+		return s.RaftHashicorp.LastIndex() > 0
+	}
+	return false
+}
+
 func (s *RaftServer) DoJoinCommand() {
 
 	glog.V(0).Infoln("Initializing new cluster")