s3tables: let resource-policy denies veto the learn-absence answer

Review feedback: a caller whose only grant was an identity-level action could still learn absence under a resource-policy Deny, and authorizeAbsentResource evaluated the policy against the caller-supplied ARN, which accepts any account. Denies are now checked before the learn-absence fallback and the policy is evaluated against the canonical ARN.

README.md

@@ -225,7 +225,7 @@ The full operation list is in [Amazon S3 API][AmazonS3API], and [Supported APIs
 
 SeaweedFS is a lakehouse in one system. [S3 Table Buckets][S3TableBucket] hold Apache Iceberg tables by default, or [Lance][LanceCatalog] tables for vectors and multimodal data, and the built-in [Iceberg REST Catalog][IcebergCatalog] and Lance namespace serve them directly. There is no Hive Metastore, Glue, or separate catalog service to deploy, secure, and back up.
 
-* Query engines operate on the same tables at the same time: [Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], [Apache Doris][DorisIceberg], [RisingWave][RisingWaveIceberg], ClickHouse, and [LanceDB][LanceDB]. Catalog commits are atomic compare-and-swap, so concurrent writers are safe. [Lakekeeper][Lakekeeper] can front the same storage with STS-vended credentials.
+* Query engines operate on the same tables at the same time: [Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], [Apache Doris][DorisIceberg], [RisingWave][RisingWaveIceberg], ClickHouse, and [LanceDB][LanceDB]. Catalog commits are atomic compare-and-swap, so concurrent writers are safe. [Lakekeeper][Lakekeeper] can front the same storage with STS-vended credentials. [OLake Go][OLake] is a high-performance, open-source EL (Extract–Load) platform that bridges operational databases and open lakehouse storage - writing Apache Iceberg tables or Parquet files on Amazon S3.
 * [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan file removal, and manifest rewriting, configured per bucket or table through the S3 Tables maintenance APIs, and the same for [Lance][LanceMaintenance].
 * IAM at the bucket, namespace, and table level with standard bucket policies, see [S3 Tables Security][S3TablesSecurity].
 * A [Hadoop compatible file system][Hadoop] for Spark, Flink, and HBase.
@@ -487,6 +487,7 @@ Your support will be really appreciated by me and other supporters!
 [RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
 [LanceDB]: https://github.com/seaweedfs/seaweedfs/wiki/LanceDB-Integration
 [Lakekeeper]: https://github.com/seaweedfs/seaweedfs/wiki/Lakekeeper-Iceberg-Integration
+[OLake]: https://github.com/seaweedfs/seaweedfs/wiki/OLake-Go-Iceberg-Integration
 [IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
 [LanceMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Lance-Maintenance-Worker
 [S3TablesSecurity]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Tables-Security

seaweed-volume/README.md

@@ -68,7 +68,10 @@ seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
 | `--pprof` | `false` | Enable pprof HTTP handlers |
 
 Set `RUST_LOG=debug` (or `trace`, `info`, `warn`) for log level control.
-Set `SEAWEED_WRITE_QUEUE=1` to enable batched async write processing.
+
+Durable (`fsync=true`) uploads always go through per-volume batching workers
+that share one `.dat`/`.idx` sync per batch, matching the Go volume server;
+non-fsync writes take the synchronous path.
 
 ## Features
 

seaweed-volume/proto/master.proto

@@ -59,6 +59,8 @@ service Seaweed {
   }
   rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
   }
+  rpc ResumeVolumeGrowth (ResumeVolumeGrowthRequest) returns (ResumeVolumeGrowthResponse) {
+  }
   rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
   }
   rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
@@ -324,6 +326,13 @@ message VolumeGrowRequest {
   float grow_avg_fill_threshold = 10;
 }
 
+message ResumeVolumeGrowthRequest {
+}
+
+message ResumeVolumeGrowthResponse {
+  bool was_frozen = 1;
+}
+
 message AssignResponse {
   string fid = 1;
   uint64 count = 4;

seaweed-volume/src/config.rs

@@ -303,8 +303,6 @@ pub struct VolumeServerConfig {
     pub grpc_allowed_wildcard_domain: String,
     pub grpc_volume_allowed_common_names: Vec<String>,
     pub tls_policy: TlsPolicy,
-    /// Enable batched write queue for improved throughput under load.
-    pub enable_write_queue: bool,
     /// Path to security.toml — stored for SIGHUP reload.
     pub security_file: String,
     /// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
@@ -968,8 +966,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         grpc_allowed_wildcard_domain: sec.grpc_allowed_wildcard_domain,
         grpc_volume_allowed_common_names: sec.grpc_volume_allowed_common_names,
         tls_policy: sec.tls_policy,
-        enable_write_queue: env_string(env, "SEAWEED_WRITE_QUEUE")
-            .is_some_and(|v| v == "1" || v == "true"),
         security_file: cli.security_file,
         allow_untrusted_remote_endpoints: cli.allow_untrusted_remote_endpoints,
     }

seaweed-volume/src/main.rs

@@ -433,12 +433,11 @@ async fn run(
         }
     }
 
-    // Initialize the batched write queue if enabled
-    if config.enable_write_queue {
-        info!("Batched write queue enabled");
-        let wq = WriteQueue::new(state.clone(), 128);
-        let _ = state.write_queue.set(wq);
-    }
+    // The write queue is always on: durable (fsync) uploads go through it so
+    // a batch shares one .dat/.idx sync, the way Go's per-volume async writer
+    // group-commits. Non-fsync writes stay on the synchronous path.
+    let wq = WriteQueue::new(state.clone(), 128);
+    let _ = state.write_queue.set(wq);
 
     // Set initial metric gauges for concurrent limits and max volumes
     metrics::CONCURRENT_UPLOAD_LIMIT.set(state.concurrent_upload_limit);

seaweed-volume/src/server/grpc_client.rs

@@ -431,7 +431,6 @@ mod tests {
             grpc_allowed_wildcard_domain: String::new(),
             grpc_volume_allowed_common_names: vec![],
             tls_policy: TlsPolicy::default(),
-            enable_write_queue: false,
             security_file: String::new(),
         }
     }

seaweed-volume/src/server/grpc_server.rs

@@ -16,7 +16,7 @@ use crate::pb::filer_pb;
 use crate::pb::master_pb;
 use crate::pb::volume_server_pb;
 use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
-use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from};
+use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from};
 use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder;
 use crate::storage::needle::needle::{self, Needle};
 use crate::storage::types::*;
@@ -1353,6 +1353,7 @@ impl VolumeServer for VolumeGrpcService {
                     // Go's CommitCompact waits out leased readers under
                     // dataFileAccessLock — do it without holding the store
                     // guard, so a slow download stalls only this vacuum.
+                    let _sync_guard = vol.data_file_sync_lock().lock().unwrap();
                     let _lease = vol.instance().write_lock();
                     let is_read_only = vol.is_read_only();
                     vol.commit_compact_held()
@@ -2317,9 +2318,9 @@ impl VolumeServer for VolumeGrpcService {
                             // owns the .ecx on disk (volume not yet mounted — relevant
                             // when shards stream in mid-rebuild before any
                             // VolumeEcShardsMount has happened; see #9212), then any
-                            // HDD, then any disk. Pass the build's default data-shard
-                            // count for free-slot maths; the helper takes it as a
-                            // parameter so custom-ratio builds can swap it.
+                            // HDD, then any disk. The free-slot maths uses the
+                            // volume's own data-shard count from its .vif so a
+                            // custom-ratio volume is not scored as 10+4.
                             let vid = VolumeId(info.volume_id);
                             let dir = if info.disk_id > 0 {
                                 let count = store.locations.len();
@@ -2334,11 +2335,13 @@ impl VolumeServer for VolumeGrpcService {
                             } else {
                                 // The mounted-volume refusal above means no disk
                                 // holds an in-memory claim here.
+                                let data_shards =
+                                    store.lookup_ec_data_shards(&info.collection, vid);
                                 store
                                     .find_ec_shard_target_location(
                                         &info.collection,
                                         vid,
-                                        DATA_SHARDS_COUNT as u32,
+                                        data_shards,
                                         &[],
                                     )
                                     .map(|i| store.locations[i].directory.clone())
@@ -3288,9 +3291,9 @@ impl VolumeServer for VolumeGrpcService {
         //   volume mounted, then a disk that owns the .ecx on disk (volume
         //   not yet mounted — relevant for ec.rebuild, where only the
         //   first shard carries .ecx and subsequent shards must land on
-        //   the same disk; see #9212), then any HDD, then any disk. Pass
-        //   the build's default data-shard count; the helper takes it as a
-        //   parameter so custom-ratio builds can swap it.
+        //   the same disk; see #9212), then any HDD, then any disk. The
+        //   free-slot maths uses the volume's own data-shard count from its
+        //   .vif so a custom-ratio volume is not scored as 10+4.
         let (dest_dir, dest_idx_dir) = {
             let store = self.state.store.read().unwrap();
             let count = store.locations.len();
@@ -3322,10 +3325,11 @@ impl VolumeServer for VolumeGrpcService {
                         req.volume_id, shard_ids, dirs
                     )));
                 }
+                let data_shards = store.lookup_ec_data_shards(&req.collection, vid);
                 match store.find_ec_shard_target_location(
                     &req.collection,
                     vid,
-                    DATA_SHARDS_COUNT as u32,
+                    data_shards,
                     &shard_ids,
                 ) {
                     Some(i) => {
@@ -4723,6 +4727,7 @@ impl VolumeServer for VolumeGrpcService {
                     // backend swap: between clearing the remote-only write
                     // restriction and attaching the local .dat a write would
                     // pass check_writable and then fail on a missing dat_file.
+                    let _sync_guard = vol.data_file_sync_lock().lock().unwrap();
                     let _lease = vol.instance().write_lock();
 
                     // The store guard dropped before the lease was taken, so an

seaweed-volume/src/server/handlers.rs

@@ -2433,6 +2433,9 @@ fn is_zero_u32(v: &u32) -> bool {
     *v == 0
 }
 
+/// Store an uploaded needle on the local volume — durable (fsync) uploads
+/// through the per-volume write queue, the rest synchronously — and then
+/// replicate it if the volume's placement calls for more copies.
 pub async fn post_handler(
     State(state): State<Arc<VolumeServerState>>,
     request: Request<Body>,
@@ -2953,15 +2956,7 @@ pub async fn post_handler(
     let needle_etag = n.etag();
     let needle_has_name = n.has_name();
 
-    let write_result = if let Some(wq) = state.write_queue.get() {
-        wq.submit(vid, n, fsync).await
-    } else {
-        let vol = {
-            let store = state.store.read().unwrap();
-            store.writable_volume(vid)
-        };
-        vol.and_then(|vol| vol.write_needle(&mut n, true, fsync))
-    };
+    let write_result = write_upload_needle(&state, vid, n, fsync).await;
 
     // Replicate to remote volume servers if this volume has replicas.
     // Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
@@ -3055,6 +3050,28 @@ pub async fn post_handler(
     resp
 }
 
+/// Append an uploaded needle the way Go's `writeNeedle2` does: a durable
+/// write goes to the volume's batching worker, which group-commits the
+/// batch's .dat/.idx syncs, while anything else — a non-durable write, or a
+/// durable one landing while the server is stopping and the workers are
+/// winding down — takes the synchronous path and fsyncs inline.
+async fn write_upload_needle(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    mut n: Needle,
+    fsync: bool,
+) -> Result<(u64, Size, bool), crate::storage::volume::VolumeError> {
+    let queued = fsync && !*state.is_stopping.read().unwrap();
+    if queued && let Some(wq) = state.write_queue.get() {
+        return wq.submit(vid, n, fsync).await;
+    }
+    let vol = {
+        let store = state.store.read().unwrap();
+        store.writable_volume(vid)
+    };
+    vol.and_then(|vol| vol.write_needle(&mut n, true, fsync))
+}
+
 // ============================================================================
 // Delete Handler
 // ============================================================================
@@ -7188,4 +7205,52 @@ mod tests {
             "the vacuum moves the needle once the stream drains"
         );
     }
+
+    /// Issue #801: an fsync upload takes the per-volume worker so its sync is
+    /// shared across the batch (Go's `writeNeedle2` split); a non-fsync write,
+    /// or an fsync write landing while the server is stopping, takes the
+    /// synchronous path instead.
+    #[tokio::test]
+    async fn test_write_upload_needle_routes_fsync_to_queue() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        assert!(
+            state
+                .write_queue
+                .set(crate::server::write_queue::WriteQueue::new(
+                    state.clone(),
+                    128,
+                ))
+                .is_ok()
+        );
+        let queue = state.write_queue.get().unwrap();
+
+        let needle = |id: u64| {
+            let mut n = Needle {
+                id: NeedleId(id),
+                cookie: Cookie(TEST_COOKIE),
+                data: vec![id as u8; 8],
+                data_size: 8,
+                ..Needle::default()
+            };
+            n.checksum = CRC::new(&n.data);
+            n
+        };
+
+        write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b01), true)
+            .await
+            .unwrap();
+        assert_eq!(queue.submits_for_test(), 1);
+
+        write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b02), false)
+            .await
+            .unwrap();
+        assert_eq!(queue.submits_for_test(), 1);
+
+        *state.is_stopping.write().unwrap() = true;
+        write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b03), true)
+            .await
+            .unwrap();
+        assert_eq!(queue.submits_for_test(), 1);
+    }
 }

seaweed-volume/src/server/profiling.rs

@@ -170,7 +170,6 @@ mod tests {
             grpc_allowed_wildcard_domain: String::new(),
             grpc_volume_allowed_common_names: vec![],
             tls_policy: TlsPolicy::default(),
-            enable_write_queue: false,
             security_file: String::new(),
         }
     }

seaweed-volume/src/server/volume_server.rs

@@ -77,7 +77,9 @@ pub struct VolumeServerState {
     pub pre_stop_seconds: u32,
     /// Notify heartbeat to send an immediate update when volume state changes.
     pub volume_state_notify: tokio::sync::Notify,
-    /// Optional batched write queue for improved throughput under load.
+    /// Per-volume write workers that group-commit durable (fsync) uploads.
+    /// Set once at startup in production; a test state may leave it unset,
+    /// which sends every write down the synchronous path.
     pub write_queue: std::sync::OnceLock<WriteQueue>,
     /// Read mode: local, proxy, or redirect for non-local volumes.
     pub read_mode: ReadMode,

seaweed-volume/src/server/write_queue.rs

@@ -1,7 +1,8 @@
 //! Async batched write processing for the volume server.
 //!
-//! Instead of each upload handler directly calling `write_needle`, writes are
-//! submitted to a queue. Each volume gets its own FIFO worker (Go's
+//! Durable (fsync) uploads are submitted here instead of each handler calling
+//! `write_needle` directly — Go's `writeNeedle2` makes the same split, so the
+//! queue is always constructed. Each volume gets its own FIFO worker (Go's
 //! volume_write.go keeps one batching worker per volume), which drains its
 //! channel in batches of up to 128 entries or 4 MiB and shares the .dat/.idx
 //! flushes across durable writes (see `Volume::write_needles_grouped`).
@@ -66,6 +67,9 @@ pub struct WriteQueue {
     /// alone would multiply the old single queue's capacity by volume count.
     queue_permits: Arc<Semaphore>,
     workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
+    /// Submissions accepted, for tests asserting which writes route here.
+    #[cfg(test)]
+    submits: Arc<std::sync::atomic::AtomicU64>,
 }
 
 impl WriteQueue {
@@ -78,9 +82,17 @@ impl WriteQueue {
             io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
             queue_permits: Arc::new(Semaphore::new(capacity)),
             workers: Arc::new(Mutex::new(HashMap::new())),
+            #[cfg(test)]
+            submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
         }
     }
 
+    /// How many writes were submitted, for tests asserting upload routing.
+    #[cfg(test)]
+    pub fn submits_for_test(&self) -> u64 {
+        self.submits.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
     /// Sender for the volume's worker, spawning one on first use.
     fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
         let mut workers = self.workers.lock().unwrap();
@@ -110,6 +122,9 @@ impl WriteQueue {
     pub async fn submit(&self, volume_id: VolumeId, needle: Needle, fsync: bool) -> WriteResult {
         // Workers spawn only for mounted volumes — a bogus id never creates
         // a channel, so volume-id sprays can't grow the worker map.
+        #[cfg(test)]
+        self.submits
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
         {
             let store = self.state.store.read().unwrap();
             if store.find_volume(volume_id).is_none() {

seaweed-volume/src/storage/mod.rs

@@ -18,3 +18,4 @@ pub mod volume_idx_repair;
 pub mod volume_open;
 pub mod volume_report;
 pub mod volume_report_hash;
+pub mod volume_tail_repair;

seaweed-volume/src/storage/store.rs

@@ -14,7 +14,7 @@ use crate::pb::master_pb;
 use crate::storage::disk_location::DiskLocation;
 use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
 use crate::storage::erasure_coding::ec_volume::{
-    ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
+    ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs,
 };
 use crate::storage::needle::needle::{Needle, get_actual_size};
 use crate::storage::needle_map::NeedleMapKind;
@@ -416,6 +416,39 @@ impl Store {
             .collect()
     }
 
+    /// Look up the per-volume EC data-shard count, preferring a mounted EC
+    /// volume's own config and then the `.vif` on whichever disk holds it.
+    /// Falls back to the default when neither exists yet (e.g. before the
+    /// first shard streams in during ec.rebuild), so
+    /// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio
+    /// volumes against their real width instead of the build default.
+    pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 {
+        use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
+
+        for loc in &self.locations {
+            if let Some(ec_vol) = loc.find_ec_volume(vid) {
+                return ec_vol.data_shards;
+            }
+        }
+        let Some(first) = self.locations.first() else {
+            return DATA_SHARDS_COUNT as u32;
+        };
+        let other_dirs: Vec<String> = self.locations[1..]
+            .iter()
+            .flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()])
+            .filter(|d| !d.is_empty())
+            .collect();
+        read_ec_shard_config_across_dirs(
+            &first.directory,
+            &first.idx_directory,
+            &other_dirs,
+            collection,
+            vid,
+        )
+        .map(|(data_shards, _, _)| data_shards)
+        .unwrap_or(DATA_SHARDS_COUNT as u32)
+    }
+
     /// Create a new volume, placing it on the location with the most free space.
     pub fn add_volume(
         &mut self,
@@ -2980,6 +3013,35 @@ mod tests {
         );
     }
 
+    /// A volume whose `.vif` declares a custom EC ratio must feed its own
+    /// data-shard count into the free-slot maths — not the build default —
+    /// wherever the `.vif` happens to sit. With no `.vif` anywhere (a
+    /// brand-new volume mid-ec.rebuild) the answer is the default.
+    #[test]
+    fn test_lookup_ec_data_shards_reads_vif_on_any_disk() {
+        use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
+
+        let (store, _tmp) = make_ec_target_test_store(3);
+        let collection = "custom-ratio";
+        let vid = VolumeId(4242);
+
+        assert_eq!(
+            store.lookup_ec_data_shards(collection, vid),
+            DATA_SHARDS_COUNT as u32,
+            "no .vif anywhere must answer the default"
+        );
+
+        // A 6+2 .vif lands on disk 2 only — the rebuild staging disk.
+        let base = volume_file_name(&store.locations[2].directory, collection, vid);
+        std::fs::write(
+            format!("{}.vif", base),
+            br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#,
+        )
+        .unwrap();
+
+        assert_eq!(store.lookup_ec_data_shards(collection, vid), 6);
+    }
+
     /// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`;
     /// a disk this delete never changed keeps its sidecar. Disk 0 models
     /// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an

seaweed-volume/src/storage/volume.rs

@@ -29,6 +29,7 @@ use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, Red
 use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
 use crate::storage::types::*;
 use crate::storage::volume_open::open_volume_file;
+use crate::storage::volume_tail_repair::TailRepairError;
 
 // ============================================================================
 // Errors
@@ -1247,6 +1248,12 @@ pub struct Volume {
     idx_syncs_for_test: std::sync::atomic::AtomicUsize,
     needle_map_kind: NeedleMapKind,
     data_file_access_control: Arc<DataFileAccessControl>,
+    /// Serializes a data-file mutation with the sync that makes it durable:
+    /// write, delete and lifecycle paths hold it for their whole critical
+    /// section, while the fsync itself runs with only the `io` read guard so
+    /// readers of this volume do not wait on it. Lock order is
+    /// `data_file_sync_lock` -> `data_file_access_control` -> `io`.
+    data_file_sync_lock: std::sync::Mutex<()>,
 
     /// Go's superBlockAccessLock.
     super_block: RwLock<SuperBlock>,
@@ -1370,6 +1377,7 @@ impl Volume {
             idx_syncs_for_test: Default::default(),
             needle_map_kind,
             data_file_access_control: Arc::new(DataFileAccessControl::default()),
+            data_file_sync_lock: std::sync::Mutex::new(()),
             super_block: RwLock::new(SuperBlock {
                 replica_placement: replica_placement.unwrap_or_default(),
                 ttl: ttl.unwrap_or(crate::storage::needle::ttl::TTL::EMPTY),
@@ -1426,6 +1434,7 @@ impl Volume {
             idx_syncs_for_test: Default::default(),
             needle_map_kind: NeedleMapKind::InMemory,
             data_file_access_control: Arc::new(DataFileAccessControl::default()),
+            data_file_sync_lock: std::sync::Mutex::new(()),
             super_block: RwLock::new(SuperBlock::default()),
             no_write: AtomicU8::new(0),
             io_unavailable: Arc::default(),
@@ -1446,6 +1455,13 @@ impl Volume {
         self.data_file_access_control.clone()
     }
 
+    /// The mutation/sync serializer, for callers that manage the access
+    /// lease themselves (the `_held` lifecycle paths). Same lock order as
+    /// everywhere else: sync lock first, then the write lease, then `io`.
+    pub(crate) fn data_file_sync_lock(&self) -> &std::sync::Mutex<()> {
+        &self.data_file_sync_lock
+    }
+
     pub(crate) fn is_instance(&self, instance: &Arc<DataFileAccessControl>) -> bool {
         Arc::ptr_eq(instance, &self.data_file_access_control)
     }
@@ -1709,6 +1725,48 @@ impl Volume {
                 ),
             }
 
+            // A crash between the .dat append and the .idx append leaves a
+            // .dat tail no index entry covers. Re-index complete needles and
+            // cut an incomplete remainder before the index loads, so the
+            // recovered rows land in the needle map and inside its append
+            // offset. Mirrors Go running repairUnindexedDatTail inside
+            // CheckVolumeDataIntegrity ahead of the needle map load. Only a
+            // volume about to load writable is repaired.
+            // A no_write_can_delete volume still appends tombstone records on
+            // delete, so it tears tails the same way and gets the same repair.
+            if !self.has_remote_file() && !self.no_write_flags().0 {
+                // An empty .idx gets the same repair: every check below
+                // skips an empty index on its own, so a tail the repair
+                // cannot explain there fails the load like in Go rather
+                // than warn and load writable.
+                let idx_empty = fs::metadata(self.file_name(".idx"))
+                    .map(|m| m.len() == 0)
+                    .unwrap_or(false);
+                match self.repair_unindexed_dat_tail(io) {
+                    Ok(()) => {}
+                    Err(e) if e.is_not_torn() && !idx_empty => warn!(
+                        volume_id = self.id.0,
+                        error = %e,
+                        "unindexed .dat tail not repaired"
+                    ),
+                    // The index refusing a read-write open is the same failure
+                    // Go reports opening it ahead of the check: it fails the
+                    // load rather than only the repair.
+                    Err(TailRepairError::OpenIndex(e)) => return Err(e),
+                    Err(e) => {
+                        // A repair that could not complete must not load the
+                        // volume writable on files it half-fixed: the check
+                        // fails as in Go, so the volume loads read-only.
+                        self.set_no_write_flag(NO_WRITE_OR_DELETE, true);
+                        warn!(
+                            volume_id = self.id.0,
+                            error = %e,
+                            "repair unindexed .dat tail"
+                        );
+                    }
+                }
+            }
+
             self.load_index(io)?;
 
             // Match Go: CheckVolumeDataIntegrity after loading index (volume_loading.go L154-159)
@@ -1980,6 +2038,7 @@ impl Volume {
     }
 
     pub(crate) fn load_remote_dat_file(&self) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         let mut io = self.io.write().unwrap();
         self.load_remote_dat_file_io(&mut io)
@@ -2066,7 +2125,7 @@ impl Volume {
         self.current_dat_file_size_io(&io)
     }
 
-    fn current_dat_file_size_io(&self, io: &VolumeIo) -> io::Result<u64> {
+    pub(crate) fn current_dat_file_size_io(&self, io: &VolumeIo) -> io::Result<u64> {
         if let Some(ref f) = io.dat_file {
             Ok(f.metadata()?.len())
         } else if let Some(ref remote_dat_file) = io.remote_dat_file {
@@ -2290,7 +2349,7 @@ impl Volume {
         self.read_needle_blob_unlocked(&io, offset, size)
     }
 
-    fn read_needle_blob_unlocked(
+    pub(crate) fn read_needle_blob_unlocked(
         &self,
         io: &VolumeIo,
         offset: i64,
@@ -2509,10 +2568,14 @@ impl Volume {
         check_cookie: bool,
         fsync: bool,
     ) -> Result<(u64, Size, bool), VolumeError> {
-        let _guard = self.data_file_access_control.write_lock();
-        let mut io = self.io.write().unwrap();
-        self.check_writable()?;
-        self.do_write_request(&mut io, n, check_cookie, fsync)
+        // A single write is a one-entry run of the grouped path: identical
+        // outcome, and the fsync handling lives in one place.
+        debug_assert!(check_cookie, "every write_needle caller checks the cookie");
+        let _ = check_cookie;
+        let mut writes = [(std::mem::take(n), fsync)];
+        let mut results = self.write_needles_grouped(&mut writes);
+        *n = std::mem::replace(&mut writes[0].0, Needle::default());
+        results.pop().unwrap_or(Err(VolumeError::NotFound))
     }
 
     /// Write a batch of needles the way Go's processBatch does: one .dat
@@ -2527,8 +2590,13 @@ impl Volume {
         &self,
         writes: &mut [(Needle, bool)],
     ) -> Vec<Result<(u64, Size, bool), VolumeError>> {
-        let _guard = self.data_file_access_control.write_lock();
-        let mut io = self.io.write().unwrap();
+        // The sync lock serializes a .dat mutation with the flush that makes
+        // it durable: a failed sync's rollback can never clobber an append
+        // that slipped in behind it, and a compaction swap or unlink cannot
+        // land between the append and its sync. Readers never take it, and
+        // the write-side guards are dropped for the sync itself, so reads of
+        // this volume do not wait on fsync.
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let mut results = Vec::with_capacity(writes.len());
         let mut rest = writes;
         while !rest.is_empty() {
@@ -2536,8 +2604,10 @@ impl Volume {
             let len = rest.iter().take_while(|(n, _)| ids.insert(n.id)).count();
             let (run, tail) = std::mem::take(&mut rest).split_at_mut(len);
             if run.iter().any(|(_, fsync)| *fsync) {
-                results.extend(self.write_synced_run(&mut io, run));
+                results.extend(self.write_synced_run(run));
             } else {
+                let _guard = self.data_file_access_control.write_lock();
+                let mut io = self.io.write().unwrap();
                 for (n, _) in run.iter_mut() {
                     results.push(
                         self.check_writable()
@@ -2552,7 +2622,6 @@ impl Volume {
 
     fn write_synced_run(
         &self,
-        io: &mut VolumeIo,
         run: &mut [(Needle, bool)],
     ) -> Vec<Result<(u64, Size, bool), VolumeError>> {
         // Per entry: Some(offset) once appended, None when it dedups.
@@ -2560,63 +2629,82 @@ impl Volume {
         // Per entry: the I/O error streak once it is staged, where a write
         // sent on its own would have recorded its success.
         let mut marks = Vec::with_capacity(run.len());
-        let mut last_append_at_ns = io.last_append_at_ns;
+        let mut last_append_at_ns;
         let mut run_start = None;
         let mut sync = false;
-        for (n, fsync) in run.iter_mut() {
-            let r = self.append_unpublished(io, n, &mut last_append_at_ns);
-            marks.push(self.io_errors.mark());
-            if let Ok(Some(offset)) = r {
-                run_start.get_or_insert(offset);
-            }
-            // Only a durable entry that got this far needs the sync.
-            sync |= *fsync && r.is_ok();
-            staged.push(r);
-        }
-
-        if sync && let Err(e) = self.flush_dat(io) {
-            self.check_read_write_error(Some(&e));
-            if let Some(start) = run_start {
-                self.undo_unsynced_append(io, start);
-            }
-            let e = VolumeError::Io(e);
-            return staged
-                .into_iter()
-                .map(|r| r.and_then(|_| Err(run_error(&e))))
-                .collect();
-        }
-        io.last_append_at_ns = last_append_at_ns;
-        for ((n, _), r) in run.iter().zip(&staged) {
-            if matches!(r, Ok(Some(_))) && n.append_at_ns > io.last_write_append_at_ns {
-                io.last_write_append_at_ns = n.append_at_ns;
-                io.last_write_needle_key = n.id;
-                io.last_write_deleted = false;
+        {
+            // Append under the write guards, then drop them for the flush:
+            // readers need only `io`'s read side, while mutators and a
+            // compaction swap stay excluded by the caller's sync lock.
+            let _guard = self.data_file_access_control.write_lock();
+            let mut io = self.io.write().unwrap();
+            last_append_at_ns = io.last_append_at_ns;
+            for (n, fsync) in run.iter_mut() {
+                let r = self.append_unpublished(&mut io, n, &mut last_append_at_ns);
+                marks.push(self.io_errors.mark());
+                if let Ok(Some(offset)) = r {
+                    run_start.get_or_insert(offset);
+                }
+                // Only a durable entry that got this far needs the sync.
+                sync |= *fsync && r.is_ok();
+                staged.push(r);
             }
         }
 
-        // A durable entry that fails to publish stops the volume taking
-        // writes, so the entries after it are refused the way a lone
-        // write would be.
-        let mut refused = false;
-        for ((n, fsync), r) in run.iter().zip(staged.iter_mut()) {
-            if refused {
-                *r = Err(self
-                    .check_writable()
-                    .err()
-                    .unwrap_or(VolumeError::ReadOnly(self.id)));
-            } else if let Ok(Some(offset)) = *r
-                && let Err(e) = self.publish_write(io, n, offset, *fsync)
-            {
-                *r = Err(e);
-                refused = *fsync;
+        if sync {
+            let io = self.io.read().unwrap();
+            if let Err(e) = self.flush_dat(&io) {
+                self.check_read_write_error(Some(&e));
+                if let Some(start) = run_start {
+                    self.undo_unsynced_append(&io, start);
+                }
+                let e = VolumeError::Io(e);
+                return staged
+                    .into_iter()
+                    .map(|r| r.and_then(|_| Err(run_error(&e))))
+                    .collect();
             }
         }
 
-        if sync && let Err(e) = self.flush_idx(io) {
-            return staged
-                .into_iter()
-                .map(|r| r.and_then(|_| Err(run_error(&e))))
-                .collect();
+        {
+            let _guard = self.data_file_access_control.write_lock();
+            let mut io = self.io.write().unwrap();
+            io.last_append_at_ns = last_append_at_ns;
+            for ((n, _), r) in run.iter().zip(&staged) {
+                if matches!(r, Ok(Some(_))) && n.append_at_ns > io.last_write_append_at_ns {
+                    io.last_write_append_at_ns = n.append_at_ns;
+                    io.last_write_needle_key = n.id;
+                    io.last_write_deleted = false;
+                }
+            }
+
+            // A durable entry that fails to publish stops the volume taking
+            // writes, so the entries after it are refused the way a lone
+            // write would be.
+            let mut refused = false;
+            for ((n, fsync), r) in run.iter().zip(staged.iter_mut()) {
+                if refused {
+                    *r = Err(self
+                        .check_writable()
+                        .err()
+                        .unwrap_or(VolumeError::ReadOnly(self.id)));
+                } else if let Ok(Some(offset)) = *r
+                    && let Err(e) = self.publish_write(&mut io, n, offset, *fsync)
+                {
+                    *r = Err(e);
+                    refused = *fsync;
+                }
+            }
+        }
+
+        if sync {
+            let io = self.io.read().unwrap();
+            if let Err(e) = self.flush_idx(&io) {
+                return staged
+                    .into_iter()
+                    .map(|r| r.and_then(|_| Err(run_error(&e))))
+                    .collect();
+            }
         }
 
         let written = run
@@ -2630,7 +2718,9 @@ impl Volume {
             // Sent one at a time, the last write to land would have cleared
             // the streak of the appends before it, and the ones after it
             // would have failed on top of that.
-            self.finish_write(io, last_modified, sync, marks[last]);
+            let _guard = self.data_file_access_control.write_lock();
+            let mut io = self.io.write().unwrap();
+            self.finish_write(&mut io, last_modified, sync, marks[last]);
         }
 
         run.iter()
@@ -2679,7 +2769,7 @@ impl Volume {
     /// Flush the .dat, the first half of a durable write. The .idx is flushed
     /// separately by flush_idx once the row is published; the two are split so
     /// nothing is indexed before the bytes it points at are down.
-    fn flush_dat(&self, io: &VolumeIo) -> io::Result<()> {
+    pub(crate) fn flush_dat(&self, io: &VolumeIo) -> io::Result<()> {
         #[cfg(test)]
         self.dat_syncs_for_test.fetch_add(1, Ordering::Relaxed);
         #[cfg(test)]
@@ -2731,7 +2821,7 @@ impl Volume {
     }
 
     /// Take an unflushed append back off the .dat after its sync failed.
-    fn truncate_dat(&self, io: &VolumeIo, len: u64) -> io::Result<()> {
+    pub(crate) fn truncate_dat(&self, io: &VolumeIo, len: u64) -> io::Result<()> {
         #[cfg(test)]
         if io.fail_truncate_for_test {
             return Err(io::Error::other("injected truncate failure"));
@@ -3082,6 +3172,7 @@ impl Volume {
 
     /// Delete a needle from the volume.
     pub fn delete_needle(&self, n: &mut Needle) -> Result<Size, VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         let mut io = self.io.write().unwrap();
         if self.is_closed() {
@@ -3464,7 +3555,7 @@ impl Volume {
     /// key-sorted .idx (e.g. legacy weed fix output) falls back to a single
     /// linear scan for the maximum offset. Returns -1 when no entry has a
     /// non-zero offset (a pre-allocated / all-zero index).
-    fn find_dat_tail_entry_offset(
+    pub(crate) fn find_dat_tail_entry_offset(
         &self,
         io: &VolumeIo,
         idx_file: &mut File,
@@ -3981,6 +4072,7 @@ impl Volume {
         offset: Offset,
         size: Size,
     ) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         let mut io = self.io.write().unwrap();
         if let Some(ref mut nm) = io.nm {
@@ -4005,6 +4097,7 @@ impl Volume {
         // The flag flip and the .vif save are one critical section with the idx
         // attach: Go's markVolumeReadonly runs under dataFileAccessLock, so a
         // racing mark-writable cannot interleave a save of half-applied state.
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         self.set_read_only_persist_held(can_delete, persist)
     }
@@ -4089,6 +4182,7 @@ impl Volume {
         if let Some(e) = self.unavailable_error() {
             return Err(e);
         }
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         let (was_no_write_or_delete, was_no_write_can_delete) = self.no_write_flags();
         // Remote-tiered volumes must stay no_write_can_delete regardless of marks.
@@ -4187,6 +4281,7 @@ impl Volume {
         &self,
         f: impl FnOnce(&mut Vec<PbRemoteFile>),
     ) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         self.update_remote_files_held(f)
     }
@@ -4451,6 +4546,7 @@ impl Volume {
 
     /// Change the replication placement and rewrite the super block.
     pub fn set_replica_placement(&self, rp: ReplicaPlacement) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         let bytes = {
             let mut sb = self.super_block.write().unwrap();
@@ -4621,6 +4717,7 @@ impl Volume {
 
     /// Write a raw needle blob at a specific offset in the .dat file.
     pub fn write_needle_blob(&self, offset: i64, needle_blob: &[u8]) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         if self.is_read_only() {
             return Err(VolumeError::ReadOnly(self.id));
@@ -4652,6 +4749,7 @@ impl Volume {
         needle_blob: &[u8],
         size: Size,
     ) -> Result<(), VolumeError> {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         // nm.put on a read-only volume fails only after the blob is appended to .dat.
         if self.is_read_only() {
@@ -4956,6 +5054,16 @@ impl Volume {
         // would stall every volume behind one slow client. The vacuum-commit
         // RPC uses the lease-holding path instead — it waits with no store
         // guard held, matching Go's blocking commit.
+        // The sync lock must come before the lease and be non-blocking like
+        // it: a durable write holds it across its fsync, and the commit must
+        // not swap files over an unpublished write — nor stall the store
+        // behind one.
+        let Ok(_sync_guard) = self.data_file_sync_lock.try_lock() else {
+            return Err(VolumeError::Io(io::Error::new(
+                io::ErrorKind::WouldBlock,
+                COMMIT_WAITS_FOR_READS,
+            )));
+        };
         let Some(_lease) = self.data_file_access_control.try_write_lock() else {
             return Err(VolumeError::Io(io::Error::new(
                 io::ErrorKind::WouldBlock,
@@ -4965,7 +5073,9 @@ impl Volume {
         self.commit_compact_held()
     }
 
-    /// `commit_compact` for callers already holding the write lease.
+    /// `commit_compact` for callers already holding `data_file_sync_lock`
+    /// and the write lease — the swap must not land between a durable
+    /// write's append and its publish.
     pub(crate) fn commit_compact_held(&self) -> Result<(), VolumeError> {
         // A handle detached between lookup and lease must not swap compacted
         // files over the ones its remounted replacement now owns.
@@ -5356,6 +5466,7 @@ impl Volume {
     /// Go's `Volume.Close`: the write lease excludes in-flight I/O — with a
     /// shared handle, `&mut self` no longer provides that exclusion.
     pub fn close(&self) {
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         self.closed.store(true, Ordering::Release);
         self.close_locked();
@@ -5401,6 +5512,7 @@ impl Volume {
         if self.is_compacting() {
             return Err(self.compacting_error());
         }
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
 
         if self.dir_idx() == new_idx_dir {
@@ -5463,6 +5575,7 @@ impl Volume {
     ) -> Result<(), VolumeError> {
         // Go's Destroy holds dataFileAccessLock: file unlinking must exclude
         // in-flight I/O from a shared handle, not just borrow exclusivity.
+        let _sync_guard = self.data_file_sync_lock.lock().unwrap();
         let _guard = self.data_file_access_control.write_lock();
         // Either enabled check may pass: a volume with no live data qualifies
         // whether it reads empty or as all garbage. Byte counters, not counts:

test/kafka/kafka-client-loadtest/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect

test/kafka/kafka-client-loadtest/go.sum

@@ -38,8 +38,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=

test/kafka/kafka-client-loadtest/tools/go.mod

@@ -5,6 +5,6 @@ go 1.24.7
 require github.com/segmentio/kafka-go v0.4.49
 
 require (
-	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 )

test/kafka/kafka-client-loadtest/tools/go.sum

@@ -1,7 +1,7 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

test/kafka/simple-consumer/go.mod

@@ -1,10 +1,10 @@
 module simple-consumer
 
-go 1.21
+go 1.24
 
 require github.com/segmentio/kafka-go v0.4.47
 
 require (
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 )

test/kafka/simple-consumer/go.sum

@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

test/master_cold_start/assign_test.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
-	"strconv"
 	"testing"
 	"time"
 
@@ -45,11 +44,7 @@ func TestColdStartFirstWrite(t *testing.T) {
 			body, _ := io.ReadAll(resp.Body)
 			resp.Body.Close()
 			if resp.StatusCode == http.StatusServiceUnavailable && time.Now().Before(deadline) {
-				delay, _ := strconv.Atoi(resp.Header.Get("Retry-After"))
-				if delay <= 0 {
-					t.Fatalf("dir/assign shed without a usable Retry-After: %d %s", resp.StatusCode, body)
-				}
-				time.Sleep(time.Duration(delay) * time.Second)
+				time.Sleep(time.Second)
 				continue
 			}
 			if resp.StatusCode != http.StatusOK {

test/s3tables/catalog_olake/README.md

@@ -3,7 +3,7 @@
 An integration test for [OLake](https://github.com/datazip-inc/olake) against
 SeaweedFS's Iceberg REST Catalog, in the same shape as `catalog_clickhouse`.
 
-## Why OLake, given we already test five engines
+## Why OLake, given we already test other engines
 
 Two things here are covered by nothing else in this directory.
 
@@ -73,8 +73,9 @@ Nothing in the destination config is SeaweedFS-specific:
 ```
 
 `catalog_type` is the generic `rest`, auth is the standard OAuth2
-client-credentials flow, and `s3_path_style` does not even need setting —
-OLake turns it on by itself whenever `s3_endpoint` is non-empty. As of
+client-credentials flow, and the test config sets `s3_path_style` explicitly;
+OLake also defaults to path style whenever `s3_endpoint` is non-empty, so
+either form works. As of
 OLake v0.10.1 this works with no change on either side.
 
 ## Running it

test/s3tables/catalog_olake/olake_catalog_test.go

@@ -376,8 +376,8 @@ func (env *TestEnvironment) execSQL(t *testing.T, sqlText string) {
 // writeOLakeConfigs writes the source and destination configs. The destination
 // is the point of this test: nothing in it is SeaweedFS-specific. catalog_type
 // is the generic "rest", auth is the standard OAuth2 client-credentials flow,
-// and path-style access is not even set here because OLake turns it on by
-// itself whenever s3_endpoint is non-empty.
+// and path-style access is set explicitly here; OLake also turns it on by
+// itself whenever s3_endpoint is non-empty, so either form works.
 func (env *TestEnvironment) writeOLakeConfigs(t *testing.T) {
 	t.Helper()
 

weed/admin/view/app/s3tables_buckets.templ

@@ -342,7 +342,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`
 									}
 									<label class="btn btn-outline-primary w-100 text-start p-3" for="s3tablesBucketFormatIceberg">
 										<span class="d-block fw-semibold"><i class="bi bi-snow me-1"></i>Iceberg</span>
-										<span class="d-block small text-muted mt-1">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB.</span>
+										<span class="d-block small text-muted mt-1">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span>
 									</label>
 								</div>
 								<div class="col-6">

weed/admin/view/app/s3tables_buckets_templ.go

@@ -636,7 +636,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB.</span></label></div><div class=\"col-6\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}

weed/command/filer_remote_sync_dir.go

@@ -384,6 +384,16 @@ func processUpdateEvent(
 		}
 		glog.V(0).Infof("never replicated, uploading %s", remote_storage.FormatLocation(dest))
 	}
+	if proto.Equal(oldDest, dest) && isDroppedLocalCopy(message.NewEntry) {
+		// The update dropped the local copy: remote.uncache, eviction under
+		// storage pressure, or remote.meta.sync pulling a newer remote
+		// version. Each clears the sync stamp, so shouldSendToRemote reads
+		// the entry as dirty when the remote mtime shares its second. The
+		// content is the object at this very key, and uploading the chunkless
+		// entry would replace it with zero bytes.
+		glog.V(2).Infof("skipping update of remote-only entry: %+v", resp)
+		return nil
+	}
 	glog.V(2).Infof("update: %+v", resp)
 	if !proto.Equal(oldDest, dest) {
 		// A renamed entry that holds no local data now (the snapshot was
@@ -488,6 +498,15 @@ func isRemoteOnly(entry *filer_pb.Entry) bool {
 	return entry != nil && !filer.HasData(entry) && entry.IsInRemoteOnly()
 }
 
+// isDroppedLocalCopy reports a remote-only entry whose size is still the
+// remote object's: its local copy was dropped and the object describes it.
+// Uncache and eviction keep the size, and remote.meta.sync sets it from the
+// object. A truncate drops the chunks too but changes the size, and the
+// truncated content has to reach the remote.
+func isDroppedLocalCopy(entry *filer_pb.Entry) bool {
+	return isRemoteOnly(entry) && entry.GetAttributes().GetFileSize() == uint64(entry.RemoteEntry.RemoteSize)
+}
+
 // completeRemoteOnlyRename finishes a rename whose content exists only on the
 // remote. When the destination object already holds what the entry's stamp
 // describes (the rename ran before, its offset was not persisted, and

weed/command/filer_remote_sync_dir_test.go

@@ -1722,6 +1722,206 @@ func TestMetadataOnlyUpdateRestampsSyncTs(t *testing.T) {
 	}
 }
 
+// The droppedCopy fixtures are a small synced file under a mount: a Delta log
+// entry whose remote object was written in the second of its mtime, as a quick
+// sync of a small file is.
+const droppedCopyMtime = 1786096669
+
+var (
+	droppedCopyDir    = "/buckets/gold/sales/_delta_log"
+	droppedCopyName   = "00000000000000000007.json"
+	droppedCopyMount  = &remote_pb.RemoteStorageLocation{Name: "s3", Bucket: "gold", Path: "/"}
+	droppedCopyDest   = &remote_pb.RemoteStorageLocation{Name: "s3", Bucket: "gold", Path: "/sales/_delta_log/" + droppedCopyName}
+	droppedCopySynced = &filer_pb.RemoteEntry{StorageName: "s3", RemoteETag: `"abc"`, RemoteSize: 1024, RemoteMtime: droppedCopyMtime, LastLocalSyncTsNs: droppedCopyMtime*1e9 + 500}
+)
+
+// droppedCopyStamp is the synced stamp with LastLocalSyncTsNs cleared, as
+// remote.uncache and the eviction leave it.
+func droppedCopyStamp() *filer_pb.RemoteEntry {
+	stamp := proto.Clone(droppedCopySynced).(*filer_pb.RemoteEntry)
+	stamp.LastLocalSyncTsNs = 0
+	return stamp
+}
+
+// droppedCopyEntry is the fixture file with the given stamp and chunks, sized
+// as its remote object.
+func droppedCopyEntry(stamp *filer_pb.RemoteEntry, chunks ...*filer_pb.FileChunk) *filer_pb.Entry {
+	entry := entryWith(droppedCopyName, stamp, chunks...)
+	entry.Attributes.FileSize = uint64(droppedCopySynced.RemoteSize)
+	return entry
+}
+
+func droppedCopyUpdate(oldEntry, newEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory:         droppedCopyDir,
+		EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewParentPath: droppedCopyDir, NewEntry: newEntry},
+	}
+}
+
+// TestDroppedLocalCopyIsNotUploaded: remote.uncache and the filer's eviction
+// under storage pressure drop a synced entry's chunks and clear its sync
+// stamp; remote.meta.sync does the same when it pulls a newer remote version.
+// With the remote mtime in the entry's second, shouldSendToRemote falls back
+// to the cleared stamp and reads the entry as dirty. The object at the same
+// key is the entry's only content, so uploading the chunkless entry would
+// replace it with a zero-byte object. The update is skipped, and the stamp
+// stays cleared so remote.cache and eviction still see the entry as uncached.
+func TestDroppedLocalCopyIsNotUploaded(t *testing.T) {
+	// the eviction's entry, built the way uncacheRemoteEntry builds it, and
+	// the event the way the filer's CreateEntry emits it
+	cached := &filer.Entry{
+		FullPath: util.NewFullPath(droppedCopyDir, droppedCopyName),
+		Attr:     filer.Attr{Mtime: time.Unix(droppedCopyMtime, 0), Mode: 0660, FileSize: 1024},
+		Chunks:   []*filer_pb.FileChunk{chunk("3,01", "e1")},
+		Remote:   droppedCopySynced,
+	}
+	evicted := cached.ShallowClone()
+	evicted.Chunks = nil
+	evicted.Remote = proto.Clone(cached.Remote).(*filer_pb.RemoteEntry)
+	evicted.Remote.LastLocalSyncTsNs = 0
+
+	// remote.meta.sync found a newer object: its mtime becomes the entry's
+	newer := &filer_pb.RemoteEntry{StorageName: "s3", RemoteETag: `"def"`, RemoteSize: 4096, RemoteMtime: droppedCopyMtime + 60}
+	pulled := &filer_pb.Entry{Name: droppedCopyName, Attributes: &filer_pb.FuseAttributes{Mtime: droppedCopyMtime + 60, FileSize: 4096}, RemoteEntry: newer}
+
+	tests := []struct {
+		name     string
+		oldEntry *filer_pb.Entry
+		newEntry *filer_pb.Entry
+	}{
+		{name: "remote.uncache", oldEntry: droppedCopyEntry(droppedCopySynced, chunk("3,01", "e1")), newEntry: droppedCopyEntry(droppedCopyStamp())},
+		{name: "eviction under storage pressure", oldEntry: cached.ToProtoEntry(), newEntry: evicted.ToProtoEntry()},
+		{name: "remote.meta.sync pulls a newer version", oldEntry: entryWith(droppedCopyName, droppedCopySynced, chunk("3,01", "e1")), newEntry: pulled},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resp := droppedCopyUpdate(tt.oldEntry, tt.newEntry)
+			if !shouldSendToRemote(tt.newEntry) {
+				t.Fatal("precondition: the cleared stamp in the remote mtime's second reads as dirty, the bug's trigger")
+			}
+			if isMetadataOnlyUpdate(resp.Directory, resp.EventNotification) {
+				t.Fatal("precondition: the dropped chunks make the update a content change")
+			}
+
+			remote := &recordingRemote{}
+			filerClient := &stubFilerClient{entry: tt.newEntry}
+			if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
+				t.Fatal(err)
+			}
+			if len(remote.writes) != 0 {
+				t.Errorf("writes = %+v, want none: the remote object is the entry's content and an upload would zero it", remote.writes)
+			}
+			if len(remote.deletes) != 0 || len(remote.metadataUpdates) != 0 {
+				t.Errorf("deletes = %+v, metadata updates = %+v, want the remote object left alone", remote.deletes, remote.metadataUpdates)
+			}
+			if filerClient.updates != 0 {
+				t.Errorf("stamped the entry %d times, want 0: a stamp would mark the uncached entry as cached", filerClient.updates)
+			}
+		})
+	}
+
+	t.Run("a metadata edit of a remote-only entry still reaches the remote", func(t *testing.T) {
+		tagged := droppedCopyEntry(droppedCopyStamp())
+		tagged.Extended = map[string][]byte{"X-Amz-Tagging-tier": []byte("gold")}
+		resp := droppedCopyUpdate(droppedCopyEntry(droppedCopyStamp()), tagged)
+		remote := &recordingRemote{}
+		filerClient := &stubFilerClient{entry: tagged}
+		if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
+			t.Fatal(err)
+		}
+		if len(remote.metadataUpdates) != 1 || len(remote.writes) != 0 {
+			t.Errorf("metadata updates = %+v, writes = %+v, want one metadata update and no upload", remote.metadataUpdates, remote.writes)
+		}
+	})
+}
+
+// TestSameSecondRewriteIsStillUploaded: a rewrite that lands in the remote
+// mtime's second holds local data, so the skip for dropped local copies does
+// not apply and it is uploaded and restamped (#11715).
+func TestSameSecondRewriteIsStillUploaded(t *testing.T) {
+	stale := proto.Clone(droppedCopySynced).(*filer_pb.RemoteEntry)
+	stale.LastLocalSyncTsNs = droppedCopyMtime*1e9 + 100
+	rewrite := func(stamp *filer_pb.RemoteEntry) *filer_pb.Entry {
+		entry := entryWith(droppedCopyName, stamp, chunk("4,02", "e2"))
+		entry.Attributes.MtimeNs = 337
+		return entry
+	}
+
+	tests := []struct {
+		name     string
+		oldEntry *filer_pb.Entry
+		newEntry *filer_pb.Entry
+	}{
+		// Filer.UpdateEntry clears the stamp of a content change
+		{name: "synced entry, stamp cleared by the filer", oldEntry: entryWith(droppedCopyName, droppedCopySynced, chunk("3,01", "e1")), newEntry: rewrite(droppedCopyStamp())},
+		{name: "synced entry, stamp older than the rewrite", oldEntry: entryWith(droppedCopyName, stale, chunk("3,01", "e1")), newEntry: rewrite(stale)},
+		{name: "uncached entry", oldEntry: entryWith(droppedCopyName, droppedCopyStamp()), newEntry: rewrite(droppedCopyStamp())},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.newEntry.Attributes.Mtime != tt.newEntry.RemoteEntry.RemoteMtime {
+				t.Fatal("precondition: the rewrite lands in the remote mtime's second")
+			}
+			remote := &recordingRemote{}
+			filerClient := &stubFilerClient{entry: tt.newEntry}
+			if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, droppedCopyUpdate(tt.oldEntry, tt.newEntry)); err != nil {
+				t.Fatal(err)
+			}
+			if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], droppedCopyDest) {
+				t.Fatalf("writes = %+v, want the rewrite uploaded to %s", remote.writes, remote_storage.FormatLocation(droppedCopyDest))
+			}
+			if !slices.Equal(remote.writtenChunks[0], []string{"4,02"}) {
+				t.Errorf("uploaded chunks %v, want the rewrite's [4,02]", remote.writtenChunks[0])
+			}
+			if filerClient.updates != 1 {
+				t.Errorf("stamped the entry %d times, want 1 after the upload", filerClient.updates)
+			}
+		})
+	}
+}
+
+// TestTruncateToZeroIsUploaded: a FUSE truncate of a cached file to zero
+// (truncateEntry, or SetAttr with size 0) drops the chunks and keeps the
+// sized RemoteEntry, so the entry looks remote-only. Its size is no longer
+// the remote object's, and the empty file has to replace the object.
+func TestTruncateToZeroIsUploaded(t *testing.T) {
+	truncated := func(mtime int64, mtimeNs int32) *filer_pb.Entry {
+		// the filer clears the stamp of a content change
+		entry := entryWith(droppedCopyName, droppedCopyStamp())
+		entry.Attributes.Mtime, entry.Attributes.MtimeNs = mtime, mtimeNs
+		return entry
+	}
+	tests := []struct {
+		name     string
+		newEntry *filer_pb.Entry
+	}{
+		{name: "in a later second", newEntry: truncated(droppedCopyMtime+5, 0)},
+		{name: "in the remote mtime's second", newEntry: truncated(droppedCopyMtime, 337)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if !isRemoteOnly(tt.newEntry) {
+				t.Fatal("precondition: the truncated entry looks remote-only")
+			}
+			remote := &recordingRemote{}
+			filerClient := &stubFilerClient{entry: tt.newEntry}
+			resp := droppedCopyUpdate(droppedCopyEntry(droppedCopySynced, chunk("3,01", "e1")), tt.newEntry)
+			if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
+				t.Fatal(err)
+			}
+			if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], droppedCopyDest) {
+				t.Fatalf("writes = %+v, want the empty file uploaded to %s", remote.writes, remote_storage.FormatLocation(droppedCopyDest))
+			}
+			if len(remote.writtenChunks[0]) != 0 {
+				t.Errorf("uploaded chunks %v, want none: a zero-byte object", remote.writtenChunks[0])
+			}
+			if filerClient.updates != 1 {
+				t.Errorf("stamped the entry %d times, want 1 after the upload", filerClient.updates)
+			}
+		})
+	}
+}
+
 func TestCompleteRemoteOnlyRenameKeepsAReusedOldKey(t *testing.T) {
 	// A was renamed to B, then D to A before the sync got to the first
 	// rename: the object at A is now D's, and the only copy of it

weed/command/s3.go

@@ -762,8 +762,8 @@ type icebergCredentialVendor struct {
 	server *s3api.S3ApiServer
 }
 
-func (v icebergCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*iceberg.VendedCredentials, error) {
-	credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix)
+func (v icebergCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*iceberg.VendedCredentials, error) {
+	credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
 	if err != nil || credentials == nil {
 		return nil, err
 	}
@@ -781,8 +781,8 @@ type lanceCredentialVendor struct {
 	server *s3api.S3ApiServer
 }
 
-func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*lance.VendedCredentials, error) {
-	credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix)
+func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*lance.VendedCredentials, error) {
+	credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
 	if err != nil || credentials == nil {
 		return nil, err
 	}

weed/credential/grpc/grpc_identity.go

@@ -120,13 +120,20 @@ func (store *IamGrpcStore) CreateAccessKey(ctx context.Context, username string,
 }
 
 func (store *IamGrpcStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
-	return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+	err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
 		_, err := client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
 			Username:  username,
 			AccessKey: accessKey,
 		})
 		return err
 	})
+	// The filer-side handler returns gRPC NotFound when the user or the key is
+	// absent; translate back to the sentinel so callers can use
+	// errors.Is(err, credential.ErrAccessKeyNotFound) uniformly across stores.
+	if err != nil && status.Code(err) == codes.NotFound {
+		return credential.ErrAccessKeyNotFound
+	}
+	return err
 }
 
 // AttachUserPolicy attaches a managed policy to a user by policy name

weed/credential/grpc/grpc_store_test.go

@@ -2,6 +2,7 @@ package grpc
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"strings"
@@ -106,3 +107,50 @@ func TestIamGrpcStore_AdminBearerToken(t *testing.T) {
 		}
 	})
 }
+
+// TestIamGrpcStore_DeleteAccessKeyNotFound pins the not-found translation:
+// the filer reports a missing user or key as gRPC NotFound, and the store must
+// surface it as credential.ErrAccessKeyNotFound so repeat deletes stay a no-op
+// for IAM dispatchers that tolerate the sentinel.
+func TestIamGrpcStore_DeleteAccessKeyNotFound(t *testing.T) {
+	const goodKey = "iam-admin-itest-key"
+
+	cm, err := credential.NewCredentialManager(credential.StoreTypeMemory, nil, "")
+	if err != nil {
+		t.Fatalf("NewCredentialManager: %v", err)
+	}
+	defer cm.Shutdown()
+
+	iamSrv := weed_server.NewIamGrpcServer(cm, security.SigningKey(goodKey))
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcSrv := grpc.NewServer()
+	iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcSrv, iamSrv)
+	go func() { _ = grpcSrv.Serve(lis) }()
+	t.Cleanup(grpcSrv.Stop)
+
+	_, portStr, err := net.SplitHostPort(lis.Addr().String())
+	if err != nil {
+		t.Fatalf("split listener addr: %v", err)
+	}
+	serverAddr := pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", portStr))
+
+	store := &IamGrpcStore{}
+	store.SetFilerAddressFunc(func() pb.ServerAddress { return serverAddr }, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	store.SetAdminSigning(security.SigningKey(goodKey), 0)
+
+	ctx := context.Background()
+	if err := store.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil {
+		t.Fatalf("CreateUser: %v", err)
+	}
+
+	if err := store.DeleteAccessKey(ctx, "alice", "AKIAMISSING0000000001"); !errors.Is(err, credential.ErrAccessKeyNotFound) {
+		t.Fatalf("DeleteAccessKey with missing key: got %v, want ErrAccessKeyNotFound", err)
+	}
+	if err := store.DeleteAccessKey(ctx, "ghost", "AKIAMISSING0000000001"); !errors.Is(err, credential.ErrAccessKeyNotFound) {
+		t.Fatalf("DeleteAccessKey with missing user: got %v, want ErrAccessKeyNotFound", err)
+	}
+}

weed/credential/memory/memory_identity.go

@@ -142,6 +142,14 @@ func (store *MemoryStore) UpdateUser(ctx context.Context, username string, ident
 		return credential.ErrUserNotFound
 	}
 
+	if identity.Name != username {
+		// Rename: the destination name must not already be taken, matching
+		// RenameUser's collision check.
+		if _, clash := store.users[identity.Name]; clash {
+			return credential.ErrUserAlreadyExists
+		}
+	}
+
 	// Remove old access keys from index
 	for _, cred := range existingUser.Credentials {
 		delete(store.accessKeys, cred.AccessKey)
@@ -156,11 +164,15 @@ func (store *MemoryStore) UpdateUser(ctx context.Context, username string, ident
 
 	// Deep copy to avoid mutation issues
 	identityCopy := store.deepCopyIdentity(identity)
-	store.users[username] = identityCopy
+	if identityCopy.Name != username {
+		// Rename: re-key the record, the way filer_etc does (delete old, save new).
+		delete(store.users, username)
+	}
+	store.users[identityCopy.Name] = identityCopy
 
 	// Re-index access keys
 	for _, cred := range identity.Credentials {
-		store.accessKeys[cred.AccessKey] = username
+		store.accessKeys[cred.AccessKey] = identityCopy.Name
 	}
 
 	return nil

weed/credential/memory/memory_store_test.go

@@ -378,3 +378,82 @@ func TestMemoryStoreServiceAccountByAccessKey(t *testing.T) {
 		t.Errorf("Expected ErrAccessKeyNotFound after delete, got %v", err)
 	}
 }
+
+// TestMemoryStoreUpdateUserRename: UpdateUser with an identity whose Name
+// differs from the key must re-key the record, the way filer_etc does
+// (delete old, save new). The IAM rename path relies on this.
+func TestMemoryStoreUpdateUserRename(t *testing.T) {
+	store := &MemoryStore{}
+	if err := store.Initialize(util.GetViper(), "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+	ctx := context.Background()
+
+	err := store.CreateUser(ctx, &iam_pb.Identity{
+		Name:        "oldname",
+		Credentials: []*iam_pb.Credential{{AccessKey: "access123", SecretKey: "secret123"}},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create user: %v", err)
+	}
+
+	err = store.UpdateUser(ctx, "oldname", &iam_pb.Identity{
+		Name:        "newname",
+		Credentials: []*iam_pb.Credential{{AccessKey: "access123", SecretKey: "secret123"}},
+	})
+	if err != nil {
+		t.Fatalf("Failed to rename user via UpdateUser: %v", err)
+	}
+
+	if _, err := store.GetUser(ctx, "oldname"); err != credential.ErrUserNotFound {
+		t.Fatalf("Expected ErrUserNotFound for old name, got %v", err)
+	}
+	renamed, err := store.GetUser(ctx, "newname")
+	if err != nil {
+		t.Fatalf("Failed to get renamed user: %v", err)
+	}
+	if renamed.Name != "newname" {
+		t.Errorf("Expected 'newname', got '%s'", renamed.Name)
+	}
+	byKey, err := store.GetUserByAccessKey(ctx, "access123")
+	if err != nil {
+		t.Fatalf("Access key must resolve to the renamed user: %v", err)
+	}
+	if byKey.Name != "newname" {
+		t.Errorf("Expected access key owner 'newname', got '%s'", byKey.Name)
+	}
+}
+
+// TestMemoryStoreUpdateUserRenameCollision: a rename whose destination name is
+// taken must fail rather than overwrite the existing user.
+func TestMemoryStoreUpdateUserRenameCollision(t *testing.T) {
+	store := &MemoryStore{}
+	if err := store.Initialize(util.GetViper(), "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+	ctx := context.Background()
+
+	if err := store.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil {
+		t.Fatalf("Failed to create alice: %v", err)
+	}
+	if err := store.CreateUser(ctx, &iam_pb.Identity{
+		Name:        "bob",
+		Credentials: []*iam_pb.Credential{{AccessKey: "bobkey123", SecretKey: "secret123"}},
+	}); err != nil {
+		t.Fatalf("Failed to create bob: %v", err)
+	}
+
+	err := store.UpdateUser(ctx, "alice", &iam_pb.Identity{Name: "bob"})
+	if err != credential.ErrUserAlreadyExists {
+		t.Fatalf("Expected ErrUserAlreadyExists, got %v", err)
+	}
+
+	// Both users must be intact.
+	if _, err := store.GetUser(ctx, "alice"); err != nil {
+		t.Fatalf("alice must still exist: %v", err)
+	}
+	bob, err := store.GetUserByAccessKey(ctx, "bobkey123")
+	if err != nil || bob.Name != "bob" {
+		t.Fatalf("bob's access key must still resolve to bob: %v", err)
+	}
+}

weed/iam/helpers.go

@@ -104,13 +104,15 @@ var baseS3ActionMap = map[string]string{
 	"DeleteObjectVersion": s3_constants.ACTION_WRITE,
 	"GetObjectVersion":    s3_constants.ACTION_READ,
 	// Tagging operations
-	"GetObjectTagging":        s3_constants.ACTION_TAGGING,
-	"GetObjectVersionTagging": s3_constants.ACTION_TAGGING,
-	"PutObjectTagging":        s3_constants.ACTION_TAGGING,
-	"DeleteObjectTagging":     s3_constants.ACTION_TAGGING,
-	"GetBucketTagging":        s3_constants.ACTION_TAGGING,
-	"PutBucketTagging":        s3_constants.ACTION_TAGGING,
-	"DeleteBucketTagging":     s3_constants.ACTION_TAGGING,
+	"GetObjectTagging":           s3_constants.ACTION_TAGGING,
+	"GetObjectVersionTagging":    s3_constants.ACTION_TAGGING,
+	"PutObjectTagging":           s3_constants.ACTION_TAGGING,
+	"PutObjectVersionTagging":    s3_constants.ACTION_TAGGING,
+	"DeleteObjectTagging":        s3_constants.ACTION_TAGGING,
+	"DeleteObjectVersionTagging": s3_constants.ACTION_TAGGING,
+	"GetBucketTagging":           s3_constants.ACTION_TAGGING,
+	"PutBucketTagging":           s3_constants.ACTION_TAGGING,
+	"DeleteBucketTagging":        s3_constants.ACTION_TAGGING,
 	// ACL operations
 	"PutObjectAcl":        s3_constants.ACTION_WRITE_ACP,
 	"GetObjectAcl":        s3_constants.ACTION_READ_ACP,

weed/iamapi/iamapi_authz_test.go

@@ -2,10 +2,16 @@ package iamapi
 
 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -141,3 +147,69 @@ func TestIamManagementAllowedForAdmin(t *testing.T) {
 		"", "Action=CreateUser&UserName=new-user&Version=2010-05-08")
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 }
+
+// The implicit UserName must come from the identity authentication verified,
+// never from re-parsing the Authorization header: a request can carry presigned
+// credentials for one user alongside a junk SigV4 header naming another.
+func TestImplicitUserNameUsesVerifiedIdentity(t *testing.T) {
+	server := newAuthzTestServer(t)
+	expires := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10)
+	sts := "POST\n\napplication/x-www-form-urlencoded\n" + expires + "\n/"
+	mac := hmac.New(sha1.New, []byte("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))
+	mac.Write([]byte(sts))
+	sig := base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	q := url.Values{"AWSAccessKeyId": {"AKIAIOSFODNN7EXAMPLE"}, "Expires": {expires}, "Signature": {sig}}
+
+	post := func(t *testing.T, body string) (int, string) {
+		t.Helper()
+		req, err := http.NewRequest(http.MethodPost, server.URL+"/?"+q.Encode(), strings.NewReader(body))
+		require.NoError(t, err)
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIATESTFAKEADMIN0001/20261021/us-east-1/iam/aws4_request, SignedHeaders=host, Signature=00")
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		b, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("%d %s", resp.StatusCode, b)
+		return resp.StatusCode, string(b)
+	}
+
+	for _, action := range []string{"ListAccessKeys", "CreateAccessKey", "GetUser"} {
+		t.Run(action, func(t *testing.T) {
+			code, body := post(t, "Action="+action+"&Version=2010-05-08")
+			assert.Equal(t, http.StatusOK, code)
+			assert.NotContains(t, body, "iam_admin")
+			assert.Contains(t, body, "power_user")
+		})
+	}
+
+	// ListUserPolicies is not self-service: a caller without IAM rights is
+	// denied whichever name the implicit resolution lands on.
+	t.Run("ListUserPolicies", func(t *testing.T) {
+		code, body := post(t, "Action=ListUserPolicies&Version=2010-05-08")
+		assert.Equal(t, http.StatusForbidden, code)
+		assert.NotContains(t, body, "iam_admin")
+	})
+
+	// A write aimed at the header-named admin's key resolves to the verified
+	// caller instead: the key is not power_user's, so it is not found.
+	t.Run("UpdateAccessKey", func(t *testing.T) {
+		code, body := post(t, "Action=UpdateAccessKey&AccessKeyId=AKIATESTFAKEADMIN0001&Status=Inactive&Version=2010-05-08")
+		assert.Equal(t, http.StatusNotFound, code)
+		assert.Contains(t, body, "NoSuchEntity")
+	})
+
+	t.Run("DeleteAccessKey", func(t *testing.T) {
+		code, _ := post(t, "Action=DeleteAccessKey&AccessKeyId=AKIATESTFAKEADMIN0001&Version=2010-05-08")
+		assert.Equal(t, http.StatusOK, code)
+	})
+
+	// The admin key the junk header named is untouched: still present and active.
+	resp := postSignedIamAction(t, server.URL, "AKIATESTFAKEADMIN0001", "testAdminSecretFake0000000000000000000000",
+		"", "Action=ListAccessKeys&Version=2010-05-08")
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Contains(t, string(body), "AKIATESTFAKEADMIN0001")
+	assert.Contains(t, string(body), "<Status>Active</Status>")
+}

weed/iamapi/iamapi_management_handlers.go

@@ -4,6 +4,7 @@ package iamapi
 // Common IAM types and helpers are imported from the shared weed/iam package.
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -14,12 +15,14 @@ import (
 	"sync"
 
 	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
 )
@@ -1263,53 +1266,128 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
 	return resp
 }
 
-// handleImplicitUsername adds username who signs the request to values if 'username' is not specified.
-// According to AWS documentation: "If you do not specify a user name, IAM determines the user name
-// implicitly based on the Amazon Web Services access key ID signing the request."
-// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding
-// identity name in the credential store.
+// handleImplicitUsername adds the name of the authenticated caller to values
+// when 'UserName' is not specified, per AWS: "If you do not specify a user name,
+// IAM determines the user name implicitly based on the Amazon Web Services
+// access key ID signing the request." The name comes from the identity the
+// authentication middleware verified, which the request context carries; the
+// Authorization header itself is attacker-controlled and is never re-parsed.
 func (iama *IamApiServer) handleImplicitUsername(r *http.Request, values url.Values) {
-	if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" {
+	if values.Get("UserName") != "" {
 		return
 	}
-	// Log presence of auth header without exposing sensitive signature material
-	glog.V(4).Infof("Authorization header present, extracting access key")
-	// Parse AWS SigV4 Authorization header format:
-	// "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..."
-	s := strings.Split(r.Header["Authorization"][0], "Credential=")
-	if len(s) < 2 {
-		return
+	if name := s3_constants.GetIdentityNameFromContext(r); name != "" {
+		values.Set("UserName", name)
 	}
-	s = strings.Split(s[1], ",")
-	if len(s) < 1 {
-		return
-	}
-	s = strings.Split(s[0], "/")
-	if len(s) < 1 {
-		return
-	}
-	// s[0] is the AccessKeyId
-	accessKeyId := s[0]
-	if accessKeyId == "" {
-		return
-	}
-	// Nil-guard: ensure iam is initialized before lookup
+}
+
+// targetedCredentialManager returns the credential manager through which a change to the named
+// identity is written on its own instead of via SaveConfiguration, or nil when this server has
+// none, or the identity is static: static identities live outside the store and keep the
+// full-save path.
+func (iama *IamApiServer) targetedCredentialManager(userName string) *credential.CredentialManager {
 	if iama.iam == nil {
-		glog.V(4).Infof("IAM not initialized, cannot look up access key")
-		return
+		return nil
 	}
-	// Look up the identity by access key to get the username
-	identity, _, found := iama.iam.LookupByAccessKey(accessKeyId)
-	if !found {
-		// Mask access key in logs - show only first 4 chars
-		maskedKey := accessKeyId
-		if len(accessKeyId) > 4 {
-			maskedKey = accessKeyId[:4] + "***"
+	cm := iama.iam.GetCredentialManager()
+	if cm == nil || cm.IsStaticIdentity(userName) {
+		return nil
+	}
+	return cm
+}
+
+// findIdentityByName returns the identity named userName in s3cfg, or nil.
+func findIdentityByName(s3cfg *iam_pb.S3ApiConfiguration, userName string) *iam_pb.Identity {
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == userName {
+			return ident
 		}
-		glog.V(4).Infof("Access key %s not found in credential store", maskedKey)
-		return
 	}
-	values.Set("UserName", identity.Name)
+	return nil
+}
+
+// groupsWithMember returns the groups in s3cfg that list userName as a member. Taken before a
+// handler renames or removes the user, it names the group records that handler will change.
+func groupsWithMember(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.Group {
+	var groups []*iam_pb.Group
+	for _, g := range s3cfg.Groups {
+		for _, m := range g.Members {
+			if m == userName {
+				groups = append(groups, g)
+				break
+			}
+		}
+	}
+	return groups
+}
+
+// serviceAccountsOf returns the service accounts in s3cfg whose parent is userName.
+func serviceAccountsOf(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.ServiceAccount {
+	var accounts []*iam_pb.ServiceAccount
+	for _, sa := range s3cfg.ServiceAccounts {
+		if sa.ParentUser == userName {
+			accounts = append(accounts, sa)
+		}
+	}
+	return accounts
+}
+
+// persistIdentity writes the one identity a handler changed in s3cfg through the credential
+// manager. The store writes that identity's record and propagates it alone; the other
+// identities are not rewritten.
+func persistIdentity(ctx context.Context, cm *credential.CredentialManager, s3cfg *iam_pb.S3ApiConfiguration, userName string) error {
+	ident := findIdentityByName(s3cfg, userName)
+	if ident == nil {
+		return credential.ErrUserNotFound
+	}
+	return cm.UpdateUser(ctx, userName, ident)
+}
+
+// persistRenamedIdentity writes a user renamed from oldName to newName by UpdateUser through the
+// credential manager: the identity, the groups that listed the old name and the service accounts
+// it parented. UpdateUser under the old name moves the record in stores that treat a changed Name
+// as a rename (filer_etc, memory), and the propagating store propagates the new identity and the
+// old name's removal; UpdateUser already moved the inline policies through PutPolicies. A store
+// that did not move the record still holds the user under the old name; the caller is told so
+// (false) and keeps the full save, which is what renames such a store today.
+func persistRenamedIdentity(ctx context.Context, cm *credential.CredentialManager, s3cfg *iam_pb.S3ApiConfiguration, oldName, newName string, groups []*iam_pb.Group, accounts []*iam_pb.ServiceAccount) (bool, error) {
+	ident := findIdentityByName(s3cfg, newName)
+	if ident == nil {
+		return false, credential.ErrUserNotFound
+	}
+	if err := cm.UpdateUser(ctx, oldName, ident); err != nil {
+		return false, err
+	}
+	if _, err := cm.GetUser(ctx, newName); err != nil {
+		return false, nil
+	}
+	for _, g := range groups {
+		if err := cm.UpdateGroup(ctx, g); err != nil {
+			return false, fmt.Errorf("update group %s: %w", g.Name, err)
+		}
+	}
+	for _, sa := range accounts {
+		if err := cm.UpdateServiceAccount(ctx, sa.Id, sa); err != nil {
+			return false, fmt.Errorf("update service account %s: %w", sa.Id, err)
+		}
+	}
+	return true, nil
+}
+
+// deleteIdentity removes a user deleted by DeleteUser through the credential manager: its record
+// (the store propagates the one removal) and the groups that listed it, whose membership
+// DeleteUser already changed in s3cfg. DeleteUser already removed the inline policies through
+// PutPolicies.
+func deleteIdentity(ctx context.Context, cm *credential.CredentialManager, userName string, groups []*iam_pb.Group) error {
+	if err := cm.DeleteUser(ctx, userName); err != nil {
+		return err
+	}
+	for _, g := range groups {
+		if err := cm.UpdateGroup(ctx, g); err != nil {
+			return fmt.Errorf("update group %s: %w", g.Name, err)
+		}
+	}
+	return nil
 }
 
 func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
@@ -1333,6 +1411,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	glog.V(4).Infof("DoActions: %+v", iamlib.RedactSensitiveFormValues(values))
 	var response iamlib.RequestIDSetter
 	changed := true
+	// persisted is set by the actions that wrote one identity through the credential manager
+	// instead of SaveConfiguration; the in-memory identity maps are reloaded after them too.
+	persisted := false
 	switch r.Form.Get("Action") {
 	case "ListUsers":
 		response = iama.ListUsers(s3cfg, values)
@@ -1356,6 +1437,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	case "GetUser":
+		iama.handleImplicitUsername(r, values)
 		userName := values.Get("UserName")
 		var err *IamError
 		response, err = iama.GetUser(s3cfg, userName)
@@ -1365,40 +1447,107 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		}
 		changed = false
 	case "UpdateUser":
+		userName, newUserName := values.Get("UserName"), values.Get("NewUserName")
+		// Taken before the rename: the records the handler changes besides the identity.
+		groups := groupsWithMember(s3cfg, userName)
+		accounts := serviceAccountsOf(s3cfg, userName)
 		var err *IamError
 		response, err = iama.UpdateUser(s3cfg, values)
 		if err != nil {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		// Write the renamed identity, its groups and service accounts through the credential
+		// manager instead of rewriting every identity via SaveConfiguration. A request that
+		// renames nothing changed nothing, so there is nothing to write either way.
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			renamed := true
+			if newUserName != "" && newUserName != userName {
+				var err error
+				renamed, err = persistRenamedIdentity(r.Context(), cm, s3cfg, userName, newUserName, groups, accounts)
+				if err != nil {
+					writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+					return
+				}
+			}
+			changed, persisted = !renamed, renamed
+		}
 	case "DeleteUser":
 		userName := values.Get("UserName")
+		groups := groupsWithMember(s3cfg, userName)
 		var err *IamError
 		response, err = iama.DeleteUser(s3cfg, userName)
 		if err != nil {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		// Targeted delete: one record removed and one removal propagated, plus the groups whose
+		// membership changed, instead of rewriting every remaining identity.
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			if err := deleteIdentity(r.Context(), cm, userName, groups); err != nil {
+				writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+				return
+			}
+			changed, persisted = false, true
+		}
 	case "CreateAccessKey":
 		iama.handleImplicitUsername(r, values)
-		var err *IamError
-		response, err = iama.CreateAccessKey(s3cfg, values)
+		userName := values.Get("UserName")
+		resp, err := iama.CreateAccessKey(s3cfg, values)
 		if err != nil {
 			glog.Errorf("CreateAccessKey: %+v", err.Error)
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		response = resp
+		// Targeted write: credentialManager.CreateAccessKey adds the key to the one identity and
+		// propagates that identity alone. The handler creates the identity for a user it does
+		// not know; that case still needs the full save.
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			if _, known := cm.GetUser(r.Context(), userName); known == nil {
+				cred := &iam_pb.Credential{
+					AccessKey: *resp.CreateAccessKeyResult.AccessKey.AccessKeyId,
+					SecretKey: *resp.CreateAccessKeyResult.AccessKey.SecretAccessKey,
+					Status:    accessKeyStatusActive,
+				}
+				if err := cm.CreateAccessKey(r.Context(), userName, cred); err != nil {
+					writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+					return
+				}
+				changed, persisted = false, true
+			}
+		}
 	case "DeleteAccessKey":
 		iama.handleImplicitUsername(r, values)
+		userName, accessKeyId := values.Get("UserName"), values.Get("AccessKeyId")
 		response = iama.DeleteAccessKey(s3cfg, values)
+		// Targeted write: credentialManager.DeleteAccessKey removes the key from the one identity
+		// and propagates that identity alone. The handler answers 200 for a key or user it does
+		// not know, so a not-found from the store is the same no-op it always was.
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			err := cm.DeleteAccessKey(r.Context(), userName, accessKeyId)
+			if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !errors.Is(err, credential.ErrUserNotFound) {
+				writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+				return
+			}
+			changed, persisted = false, true
+		}
 	case "UpdateAccessKey":
 		iama.handleImplicitUsername(r, values)
+		userName := values.Get("UserName")
 		var err *IamError
 		response, err = iama.UpdateAccessKey(s3cfg, values)
 		if err != nil {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
+				writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+				return
+			}
+			changed, persisted = false, true
+		}
 	case "CreatePolicy":
 		var err *IamError
 		response, err = iama.CreatePolicy(s3cfg, values)
@@ -1410,6 +1559,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		// The `changed` flag is false because this does not modify the main s3cfg.Identities configuration.
 		changed = false
 	case "PutUserPolicy":
+		userName := values.Get("UserName")
 		var err *IamError
 		response, err = iama.PutUserPolicy(s3cfg, values)
 		if err != nil {
@@ -1417,6 +1567,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		// The policy document is already stored through PutPolicies; what remains is the
+		// identity's recomputed Actions, written and propagated for that identity alone.
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
+				writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+				return
+			}
+			changed, persisted = false, true
+		}
 	case "GetUserPolicy":
 		var err *IamError
 		response, err = iama.GetUserPolicy(s3cfg, values)
@@ -1426,12 +1585,20 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		}
 		changed = false
 	case "DeleteUserPolicy":
+		userName := values.Get("UserName")
 		var err *IamError
 		response, err = iama.DeleteUserPolicy(s3cfg, values)
 		if err != nil {
 			writeIamErrorResponse(w, r, reqID, err)
 			return
 		}
+		if cm := iama.targetedCredentialManager(userName); cm != nil {
+			if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
+				writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
+				return
+			}
+			changed, persisted = false, true
+		}
 	case "ListUserPolicies":
 		iama.handleImplicitUsername(r, values)
 		var err *IamError
@@ -1642,6 +1809,8 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			writeIamErrorResponse(w, r, reqID, &iamError)
 			return
 		}
+	}
+	if changed || persisted {
 		// Reload in-memory identity maps so subsequent LookupByAccessKey calls
 		// can see newly created or deleted keys immediately
 		if iama.iam != nil {

weed/iamapi/iamapi_targeted_writes_test.go

@@ -0,0 +1,302 @@
+package iamapi
+
+// Targeted-write regression tests: single-identity IAM actions must write only
+// the identity they changed through the credential manager, not every identity
+// via PutS3ApiConfiguration/SaveConfiguration.
+// See https://github.com/seaweedfs/seaweedfs/issues/11865.
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/credential/memory"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// targetedRecordingStore wraps a memory store and counts the write methods
+// the dispatcher used, so a test can assert the single-identity path was
+// taken. It embeds the concrete store (rather than the CredentialStore
+// interface) so optional interfaces like InlinePolicyStore stay visible
+// through the wrapper.
+type targetedRecordingStore struct {
+	*memory.MemoryStore
+	saves            int
+	createAccessKeys int
+	deleteAccessKeys int
+	updateUsers      int
+	deleteUsers      int
+	updateGroups     int
+}
+
+func (r *targetedRecordingStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
+	r.saves++
+	return r.MemoryStore.SaveConfiguration(ctx, config)
+}
+
+func (r *targetedRecordingStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error {
+	r.createAccessKeys++
+	return r.MemoryStore.CreateAccessKey(ctx, username, cred)
+}
+
+func (r *targetedRecordingStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
+	r.deleteAccessKeys++
+	return r.MemoryStore.DeleteAccessKey(ctx, username, accessKey)
+}
+
+func (r *targetedRecordingStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error {
+	r.updateUsers++
+	return r.MemoryStore.UpdateUser(ctx, username, identity)
+}
+
+func (r *targetedRecordingStore) DeleteUser(ctx context.Context, username string) error {
+	r.deleteUsers++
+	return r.MemoryStore.DeleteUser(ctx, username)
+}
+
+func (r *targetedRecordingStore) UpdateGroup(ctx context.Context, group *iam_pb.Group) error {
+	r.updateGroups++
+	return r.MemoryStore.UpdateGroup(ctx, group)
+}
+
+func targetedIamApiServer() (*IamApiServer, *credential.CredentialManager, *countingConfigSaver, *targetedRecordingStore) {
+	srv, cm, counter := iamApiServerWithCredentialManager()
+	rec := &targetedRecordingStore{MemoryStore: cm.Store.(*memory.MemoryStore)}
+	cm.Store = rec
+	return srv, cm, counter, rec
+}
+
+const targetedTestPolicy = `{
+	"Version": "2012-10-17",
+	"Statement": [{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::bench/*"]}]
+}`
+
+func TestIamApiCreateAccessKeyTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
+
+	req, _ := iam.New(session.New()).CreateAccessKeyRequest(&iam.CreateAccessKeyInput{UserName: aws.String("alice")})
+	_ = req.Build()
+	out := CreateAccessKeyResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "CreateAccessKey must not PutS3ApiConfiguration")
+	assert.Equal(t, 0, rec.saves)
+	assert.Equal(t, 1, rec.createAccessKeys)
+
+	ident, err := cm.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	require.Len(t, ident.Credentials, 1)
+	assert.Equal(t, *out.CreateAccessKeyResult.AccessKey.AccessKeyId, ident.Credentials[0].AccessKey)
+}
+
+// CreateAccessKey for a user the store does not know still needs the full save:
+// the handler creates the identity in the config, so there is no existing
+// record to write a key into.
+func TestIamApiCreateAccessKeyUnknownUserFallsBack(t *testing.T) {
+	srv, cm, counter, _ := targetedIamApiServer()
+	ctx := context.Background()
+
+	req, _ := iam.New(session.New()).CreateAccessKeyRequest(&iam.CreateAccessKeyInput{UserName: aws.String("ghost")})
+	_ = req.Build()
+	out := CreateAccessKeyResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 1, counter.putCalled)
+	ident, err := cm.GetUser(ctx, "ghost")
+	require.NoError(t, err)
+	assert.Len(t, ident.Credentials, 1)
+}
+
+func TestIamApiDeleteAccessKeyTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{
+		Name:        "alice",
+		Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake"}},
+	}))
+
+	req, _ := iam.New(session.New()).DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{
+		UserName:    aws.String("alice"),
+		AccessKeyId: aws.String("AKIATESTFAKEKEY00001"),
+	})
+	_ = req.Build()
+	out := DeleteAccessKeyResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "DeleteAccessKey must not PutS3ApiConfiguration")
+	assert.Equal(t, 1, rec.deleteAccessKeys)
+
+	ident, err := cm.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, ident.Credentials)
+
+	// Deleting a key that does not exist still answers 200 and stays targeted.
+	req, _ = iam.New(session.New()).DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{
+		UserName:    aws.String("alice"),
+		AccessKeyId: aws.String("AKIATESTFAKEKEY99999"),
+	})
+	_ = req.Build()
+	resp, err = executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+	assert.Equal(t, 0, counter.putCalled)
+}
+
+func TestIamApiUpdateAccessKeyTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{
+		Name:        "alice",
+		Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake", Status: "Active"}},
+	}))
+
+	req, _ := iam.New(session.New()).UpdateAccessKeyRequest(&iam.UpdateAccessKeyInput{
+		UserName:    aws.String("alice"),
+		AccessKeyId: aws.String("AKIATESTFAKEKEY00001"),
+		Status:      aws.String(iam.StatusTypeInactive),
+	})
+	_ = req.Build()
+	out := UpdateAccessKeyResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "UpdateAccessKey must not PutS3ApiConfiguration")
+	assert.Equal(t, 1, rec.updateUsers)
+
+	ident, err := cm.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	require.Len(t, ident.Credentials, 1)
+	assert.Equal(t, iam.StatusTypeInactive, ident.Credentials[0].Status)
+}
+
+func TestIamApiPutUserPolicyTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
+
+	req, _ := iam.New(session.New()).PutUserPolicyRequest(&iam.PutUserPolicyInput{
+		UserName:       aws.String("alice"),
+		PolicyName:     aws.String("read-bench"),
+		PolicyDocument: aws.String(targetedTestPolicy),
+	})
+	_ = req.Build()
+	out := PutUserPolicyResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "PutUserPolicy must not PutS3ApiConfiguration")
+	assert.Equal(t, 0, rec.saves)
+	assert.Equal(t, 1, rec.updateUsers)
+
+	ident, err := cm.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.NotEmpty(t, ident.Actions, "recomputed Actions must be persisted for the one identity")
+}
+
+func TestIamApiDeleteUserPolicyTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
+
+	putReq, _ := iam.New(session.New()).PutUserPolicyRequest(&iam.PutUserPolicyInput{
+		UserName:       aws.String("alice"),
+		PolicyName:     aws.String("read-bench"),
+		PolicyDocument: aws.String(targetedTestPolicy),
+	})
+	_ = putReq.Build()
+	putOut := PutUserPolicyResponse{}
+	resp, err := executeRequestWith(srv, putReq.HTTPRequest, &putOut)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	delReq, _ := iam.New(session.New()).DeleteUserPolicyRequest(&iam.DeleteUserPolicyInput{
+		UserName:   aws.String("alice"),
+		PolicyName: aws.String("read-bench"),
+	})
+	_ = delReq.Build()
+	delOut := DeleteUserPolicyResponse{}
+	resp, err = executeRequestWith(srv, delReq.HTTPRequest, &delOut)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "DeleteUserPolicy must not PutS3ApiConfiguration")
+	assert.Equal(t, 2, rec.updateUsers)
+
+	policies := Policies{}
+	require.NoError(t, srv.s3ApiConfig.GetPolicies(&policies))
+	assert.Empty(t, policies.InlinePolicies["alice"])
+}
+
+func TestIamApiUpdateUserRenameTargeted(t *testing.T) {
+	srv, cm, counter, _ := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
+	require.NoError(t, cm.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
+
+	req, _ := iam.New(session.New()).UpdateUserRequest(&iam.UpdateUserInput{
+		UserName:    aws.String("alice"),
+		NewUserName: aws.String("alice-renamed"),
+	})
+	_ = req.Build()
+	out := UpdateUserResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "UpdateUser rename must not PutS3ApiConfiguration")
+
+	_, err = cm.GetUser(ctx, "alice")
+	require.ErrorIs(t, err, credential.ErrUserNotFound)
+	_, err = cm.GetUser(ctx, "alice-renamed")
+	require.NoError(t, err)
+
+	g, err := cm.GetGroup(ctx, "devs")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"alice-renamed", "bob"}, g.Members)
+}
+
+func TestIamApiDeleteUserTargeted(t *testing.T) {
+	srv, cm, counter, rec := targetedIamApiServer()
+	ctx := context.Background()
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
+	require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
+	require.NoError(t, cm.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
+
+	req, _ := iam.New(session.New()).DeleteUserRequest(&iam.DeleteUserInput{UserName: aws.String("alice")})
+	_ = req.Build()
+	out := DeleteUserResponse{}
+	resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.Code)
+
+	assert.Equal(t, 0, counter.putCalled, "DeleteUser must not PutS3ApiConfiguration")
+	assert.Equal(t, 1, rec.deleteUsers)
+	assert.Equal(t, 1, rec.updateGroups)
+
+	_, err = cm.GetUser(ctx, "alice")
+	require.ErrorIs(t, err, credential.ErrUserNotFound)
+	g, err := cm.GetGroup(ctx, "devs")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"bob"}, g.Members)
+	_, err = cm.GetUser(ctx, "bob")
+	require.NoError(t, err, "other identities must be untouched")
+}

weed/iamapi/iamapi_test.go

@@ -17,9 +17,11 @@ import (
 	"github.com/jinzhu/copier"
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/credential/memory"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -523,32 +525,13 @@ func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorde
 }
 
 func TestHandleImplicitUsername(t *testing.T) {
-	// Create a mock IamApiServer with credential store
-	// The handleImplicitUsername function now looks up the username from the
-	// credential store based on AccessKeyId, not from the region field in the auth header.
-	// Note: Using obviously fake access keys to avoid CI secret scanner false positives
+	// The implicit name is the authenticated identity recorded in the request
+	// context by the auth middleware; request headers are never re-parsed.
+	iama := &IamApiServer{}
 
-	// Create IAM directly as struct literal (same pattern as other tests)
-	iam := &s3api.IdentityAccessManagement{}
-
-	// Load test credentials - map access key to identity name
-	testConfig := &iam_pb.S3ApiConfiguration{
-		Identities: []*iam_pb.Identity{
-			{
-				Name: "testuser1",
-				Credentials: []*iam_pb.Credential{
-					{AccessKey: "AKIATESTFAKEKEY000001", SecretKey: "testsecretfake"},
-				},
-			},
-		},
-	}
-	err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(t, testConfig))
-	if err != nil {
-		t.Fatalf("Failed to load test config: %v", err)
-	}
-
-	iama := &IamApiServer{
-		iam: iam,
+	withIdentity := func(name string) *http.Request {
+		r := &http.Request{Header: http.Header{}}
+		return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), name))
 	}
 
 	var tests = []struct {
@@ -556,14 +539,14 @@ func TestHandleImplicitUsername(t *testing.T) {
 		values   url.Values
 		userName string
 	}{
-		// No authorization header - should not set username
+		// No authenticated identity - should not set username
 		{&http.Request{}, url.Values{}, ""},
-		// Valid auth header with known access key - should look up and find "testuser1"
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"},
-		// Malformed auth header (no Credential=) - should not set username
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =AKIATESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
-		// Unknown access key - should not set username
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
+		// Authenticated identity in context - sets its name
+		{withIdentity("testuser1"), url.Values{}, "testuser1"},
+		// A client-controlled Authorization header naming another user is ignored
+		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
+		// An explicit UserName is left alone
+		{withIdentity("testuser1"), url.Values{"UserName": {"other"}}, "other"},
 	}
 
 	for i, test := range tests {
@@ -607,6 +590,7 @@ func iamApiServerWithCredentialManager() (*IamApiServer, *credential.CredentialM
 type countingConfigSaver struct {
 	cm        *credential.CredentialManager
 	putCalled int
+	policies  Policies
 }
 
 func (c *countingConfigSaver) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error {
@@ -616,6 +600,8 @@ func (c *countingConfigSaver) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigura
 	}
 	s3cfg.Identities = config.Identities
 	s3cfg.Policies = config.Policies
+	s3cfg.Groups = config.Groups
+	s3cfg.ServiceAccounts = config.ServiceAccounts
 	return nil
 }
 
@@ -625,10 +611,15 @@ func (c *countingConfigSaver) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigura
 }
 
 func (c *countingConfigSaver) GetPolicies(policies *Policies) error {
+	*policies = c.policies
+	if c.policies.Policies == nil && c.policies.InlinePolicies == nil {
+		return filer_pb.ErrNotFound
+	}
 	return nil
 }
 
 func (c *countingConfigSaver) PutPolicies(policies *Policies) error {
+	c.policies = *policies
 	return nil
 }
 

weed/pb/filer_pb/filer_client.go

@@ -244,7 +244,31 @@ func Mkdir(ctx context.Context, filerClient FilerClient, parentDirectoryPath str
 	})
 }
 
+// MkdirExclusive is Mkdir that never replaces an entry already at the path:
+// the create is O_EXCL and a taken name answers ErrEntryAlreadyExists. The
+// taken name leaves the callback as a result, not an error: failover counts a
+// callback error against the filer and retries on the next one.
+func MkdirExclusive(ctx context.Context, filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
+	var taken error
+	err := filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
+		err := doMkdir(ctx, client, parentDirectoryPath, dirName, true, fn)
+		if errors.Is(err, ErrEntryAlreadyExists) {
+			taken = err
+			return nil
+		}
+		return err
+	})
+	if err == nil {
+		err = taken
+	}
+	return err
+}
+
 func DoMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
+	return doMkdir(ctx, client, parentDirectoryPath, dirName, false, fn)
+}
+
+func doMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, dirName string, exclusive bool, fn func(entry *Entry)) error {
 	entry := &Entry{
 		Name:        dirName,
 		IsDirectory: true,
@@ -264,6 +288,7 @@ func DoMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath
 	request := &CreateEntryRequest{
 		Directory: parentDirectoryPath,
 		Entry:     entry,
+		OExcl:     exclusive,
 	}
 
 	glog.V(1).InfofCtx(ctx, "mkdir: %v", request)

weed/pb/master.proto

@@ -59,6 +59,8 @@ service Seaweed {
   }
   rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
   }
+  rpc ResumeVolumeGrowth (ResumeVolumeGrowthRequest) returns (ResumeVolumeGrowthResponse) {
+  }
   rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
   }
   rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
@@ -324,6 +326,13 @@ message VolumeGrowRequest {
   float grow_avg_fill_threshold = 10;
 }
 
+message ResumeVolumeGrowthRequest {
+}
+
+message ResumeVolumeGrowthResponse {
+  bool was_frozen = 1;
+}
+
 message AssignResponse {
   string fid = 1;
   uint64 count = 4;

weed/pb/master_pb/master_grpc.pb.go

@@ -44,6 +44,7 @@ const (
 	Seaweed_RaftAddServer_FullMethodName          = "/master_pb.Seaweed/RaftAddServer"
 	Seaweed_RaftRemoveServer_FullMethodName       = "/master_pb.Seaweed/RaftRemoveServer"
 	Seaweed_RaftLeadershipTransfer_FullMethodName = "/master_pb.Seaweed/RaftLeadershipTransfer"
+	Seaweed_ResumeVolumeGrowth_FullMethodName     = "/master_pb.Seaweed/ResumeVolumeGrowth"
 	Seaweed_VolumeGrow_FullMethodName             = "/master_pb.Seaweed/VolumeGrow"
 	Seaweed_CollectionStatistics_FullMethodName   = "/master_pb.Seaweed/CollectionStatistics"
 )
@@ -77,6 +78,7 @@ type SeaweedClient interface {
 	RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error)
 	RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error)
 	RaftLeadershipTransfer(ctx context.Context, in *RaftLeadershipTransferRequest, opts ...grpc.CallOption) (*RaftLeadershipTransferResponse, error)
+	ResumeVolumeGrowth(ctx context.Context, in *ResumeVolumeGrowthRequest, opts ...grpc.CallOption) (*ResumeVolumeGrowthResponse, error)
 	VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error)
 	CollectionStatistics(ctx context.Context, in *CollectionStatisticsRequest, opts ...grpc.CallOption) (*CollectionStatisticsResponse, error)
 }
@@ -357,6 +359,16 @@ func (c *seaweedClient) RaftLeadershipTransfer(ctx context.Context, in *RaftLead
 	return out, nil
 }
 
+func (c *seaweedClient) ResumeVolumeGrowth(ctx context.Context, in *ResumeVolumeGrowthRequest, opts ...grpc.CallOption) (*ResumeVolumeGrowthResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ResumeVolumeGrowthResponse)
+	err := c.cc.Invoke(ctx, Seaweed_ResumeVolumeGrowth_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedClient) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(VolumeGrowResponse)
@@ -406,6 +418,7 @@ type SeaweedServer interface {
 	RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error)
 	RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error)
 	RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error)
+	ResumeVolumeGrowth(context.Context, *ResumeVolumeGrowthRequest) (*ResumeVolumeGrowthResponse, error)
 	VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error)
 	CollectionStatistics(context.Context, *CollectionStatisticsRequest) (*CollectionStatisticsResponse, error)
 	mustEmbedUnimplementedSeaweedServer()
@@ -493,6 +506,9 @@ func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveS
 func (UnimplementedSeaweedServer) RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error) {
 	return nil, status.Error(codes.Unimplemented, "method RaftLeadershipTransfer not implemented")
 }
+func (UnimplementedSeaweedServer) ResumeVolumeGrowth(context.Context, *ResumeVolumeGrowthRequest) (*ResumeVolumeGrowthResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method ResumeVolumeGrowth not implemented")
+}
 func (UnimplementedSeaweedServer) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) {
 	return nil, status.Error(codes.Unimplemented, "method VolumeGrow not implemented")
 }
@@ -930,6 +946,24 @@ func _Seaweed_RaftLeadershipTransfer_Handler(srv interface{}, ctx context.Contex
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Seaweed_ResumeVolumeGrowth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ResumeVolumeGrowthRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).ResumeVolumeGrowth(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Seaweed_ResumeVolumeGrowth_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).ResumeVolumeGrowth(ctx, req.(*ResumeVolumeGrowthRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Seaweed_VolumeGrow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(VolumeGrowRequest)
 	if err := dec(in); err != nil {
@@ -1057,6 +1091,10 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "RaftLeadershipTransfer",
 			Handler:    _Seaweed_RaftLeadershipTransfer_Handler,
 		},
+		{
+			MethodName: "ResumeVolumeGrowth",
+			Handler:    _Seaweed_ResumeVolumeGrowth_Handler,
+		},
 		{
 			MethodName: "VolumeGrow",
 			Handler:    _Seaweed_VolumeGrow_Handler,

weed/remote_storage/gcs/gcs_storage_client.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/storage"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -348,6 +349,16 @@ func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocatio
 		wc.ContentType = entry.Attributes.Mime
 	}
 	wc.ContentEncoding = remote_storage.EntryContentEncoding(entry)
+	// A Writer reserves a buffer of ChunkSize bytes (16 MiB by default) so a
+	// failed request can be retried. An object below that size takes a buffer
+	// of its own size plus one 256 KiB unit, the SDK's granularity: a burst of
+	// small uploads in flight does not hold 16 MiB each, every upload keeps
+	// its retry buffer, and the object stays below ChunkSize so it goes in one
+	// request rather than a resumable session. The size counts the chunks,
+	// not only the attribute, which some write paths leave at zero.
+	if size := filer.FileSize(entry); size < uint64(googleapi.DefaultUploadChunkSize) {
+		wc.ChunkSize = int((size/googleapi.MinUploadChunkSize + 1) * googleapi.MinUploadChunkSize)
+	}
 	if _, err = io.Copy(wc, reader); err != nil {
 		return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
 	}

weed/s3api/auth_credentials.go

@@ -1670,6 +1670,7 @@ func (iam *IdentityAccessManagement) isConfiguredAccountID(id string) bool {
 
 func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		scrubInternalHeaders(r)
 		if !iam.isEnabled() {
 			f(w, r)
 			return
@@ -1695,6 +1696,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 // signature verification, which is mandatory for all PostPolicy uploads.
 func (iam *IdentityAccessManagement) AuthPostPolicy(f http.HandlerFunc, action Action) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		scrubInternalHeaders(r)
 		if !iam.isEnabled() {
 			f(w, r)
 			return
@@ -1780,19 +1782,33 @@ func (iam *IdentityAccessManagement) AuthenticateRequest(r *http.Request) (*Iden
 	return ident, err
 }
 
+// scrubInternalHeaders drops internal headers only the server may set, so
+// client-supplied values cannot spoof identity claims or reach the handlers'
+// internal consumers (putToFiler persists the object-lock and version-id ones
+// into authoritative metadata; the SSE ones carry key material between handler
+// stages of a request). The server sets them itself after this point.
+func scrubInternalHeaders(r *http.Request) {
+	r.Header.Del(s3_constants.SeaweedFSPrincipalHeader)
+	r.Header.Del(s3_constants.SeaweedFSSessionTokenHeader)
+	r.Header.Del(s3_constants.ExtObjectLockModeKey)
+	r.Header.Del(s3_constants.ExtRetentionUntilDateKey)
+	r.Header.Del(s3_constants.ExtLegalHoldKey)
+	r.Header.Del(s3_constants.ExtVersionIdKey)
+	r.Header.Del(s3_constants.SeaweedFSSSEKMSBaseIVHeader)
+	r.Header.Del(s3_constants.SeaweedFSSSES3BaseIVHeader)
+	r.Header.Del(s3_constants.SeaweedFSSSES3KeyDataHeader)
+}
+
 func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request) (*Identity, s3err.ErrorCode, authType) {
 	var identity *Identity
 	var s3Err s3err.ErrorCode
 	var found bool
 	var amzAuthType string
 
-	// SECURITY: Prevent clients from spoofing internal IAM headers.
-	// These headers are only set by the server after successful JWT authentication
-	// and are trusted by downstream authorization (including S3Tables IAM checks
-	// reached via AuthSignatureOnly). Clearing them here — the shared entry point
-	// for every auth path — prevents privilege escalation via header injection.
-	r.Header.Del(s3_constants.SeaweedFSPrincipalHeader)
-	r.Header.Del(s3_constants.SeaweedFSSessionTokenHeader)
+	// Clearing them here — the shared entry point for every auth path —
+	// prevents privilege escalation via header injection, including through
+	// the S3Tables checks reached via AuthSignatureOnly.
+	scrubInternalHeaders(r)
 	// Deferred so signature verification can still read a client-signed value.
 	defer r.Header.Del(s3_constants.AmzAccountId)
 

weed/s3api/auth_signature_v4.go

@@ -959,6 +959,12 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request, externalHost
 			extractedSignedHeaders[header] = []string{hostHeaderValue}
 			continue
 		}
+		// The Go server strips Content-Length into r.ContentLength; clients do
+		// sign it, so rebuild the value the same way the client saw it.
+		if strings.EqualFold(header, "content-length") && r.ContentLength >= 0 {
+			extractedSignedHeaders[header] = []string{strconv.FormatInt(r.ContentLength, 10)}
+			continue
+		}
 		// For all other headers we need to find them in the HTTP headers and copy them over.
 		// We skip non-existent headers to be compatible with AWS signatures.
 		if values, ok := reqHeaders[http.CanonicalHeaderKey(header)]; ok {

weed/s3api/auth_signature_v4_test.go

@@ -681,6 +681,22 @@ func TestExtractSignedHeadersCase(t *testing.T) {
 	}
 }
 
+// A client may sign content-length even when the request carries no body
+// (aws-cli's checksum HEAD does). The Go server strips the header into
+// r.ContentLength, so extraction rebuilds it rather than dropping it.
+func TestExtractSignedHeadersContentLength(t *testing.T) {
+	r, _ := http.NewRequest("HEAD", "http://bucket.domain.com/bucket/key", nil)
+	r.Host = "bucket.domain.com"
+	r.ContentLength = 0
+	extracted, errCode := extractSignedHeaders([]string{"host", "content-length"}, r, "")
+	if errCode != s3err.ErrNone {
+		t.Fatalf("extractSignedHeaders failed: %v", errCode)
+	}
+	if got := getCanonicalHeaders(extracted); got != "content-length:0\nhost:bucket.domain.com\n" {
+		t.Fatalf("unexpected canonical headers %q", got)
+	}
+}
+
 // A router that internally rewrites the request path (the Iceberg catalog
 // collapses a doubled /v1 prefix) must still verify the signature against
 // the path the client signed, which is kept in the request context.

weed/s3api/filer_util.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -22,6 +23,23 @@ func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn fun
 
 }
 
+// mkdirExclusive never replaces an entry already at the path: PutBucket uses
+// it so a plain bucket cannot replace a table bucket created in its
+// check-to-create window.
+func (s3a *S3ApiServer) mkdirExclusive(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
+
+	return filer_pb.MkdirExclusive(context.Background(), s3a, parentDirectoryPath, dirName, fn)
+
+}
+
+// concurrentEntryIsForeign: after PutBucket's exclusive mkdir met an entry,
+// whether that entry must be answered BucketAlreadyExists without looking at
+// its owner: a table bucket (it carries no S3 owner and must never read as
+// "owned by you"), or an entry that could not be read.
+func concurrentEntryIsForeign(entry *filer_pb.Entry, err error) bool {
+	return err != nil || entry == nil || s3tables.IsTableBucketEntry(entry)
+}
+
 func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {
 
 	err := filer_pb.MkFile(context.Background(), s3a, parentDirectoryPath, fileName, chunks, fn)

weed/s3api/iceberg/handlers_table.go

@@ -294,7 +294,10 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		if markerErr := s.writeStageCreateMarker(r.Context(), bucketName, namespace, tableName, tableUUID, location, stagedMetadataLocation); markerErr != nil {
 			glog.V(1).Infof("Iceberg: failed to persist stage-create marker for %s.%s: %v", flattenNamespacePath(namespace), tableName, markerErr)
 		}
-		config, storageCredentials := s.buildFileIOConfig(r, location)
+		// The staged table is not registered yet, so there is nothing to
+		// authorize an update against; the create gate above is the write
+		// authority for the prefix the staged table will occupy.
+		config, storageCredentials := s.buildFileIOConfig(r, location, func() bool { return true })
 		result := LoadTableResult{
 			MetadataLocation:   metadataLocation,
 			Metadata:           metadata,
@@ -395,7 +398,9 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", flattenNamespacePath(namespace), tableName, markerErr)
 	}
 
-	config, storageCredentials := s.buildFileIOConfig(r, location)
+	config, storageCredentials := s.buildFileIOConfig(r, location, func() bool {
+		return s.mayUpdateTable(r, bucketName, namespace, req.Name)
+	})
 	result := LoadTableResult{
 		MetadataLocation:   finalLocation,
 		Metadata:           metadata,
@@ -590,7 +595,9 @@ func (s *Server) buildLoadTableResult(r *http.Request, getResp s3tables.GetTable
 		return LoadTableResult{}, fmt.Errorf("build metadata for %s: %w", tableName, err)
 	}
 
-	config, storageCredentials := s.buildFileIOConfig(r, location)
+	config, storageCredentials := s.buildFileIOConfig(r, location, func() bool {
+		return s.mayUpdateTable(r, bucketName, namespace, tableName)
+	})
 	return LoadTableResult{
 		MetadataLocation:   getResp.MetadataLocation,
 		Metadata:           metadata,
@@ -646,7 +653,7 @@ func applySnapshotsParam(r *http.Request, metadata table.Metadata) (table.Metada
 // a region on attach don't need to be told it out-of-band. See issue #9103.
 // It also returns, for a client that asked for credential vending, the
 // credentials scoped to this table.
-func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Properties, []StorageCredential) {
+func (s *Server) buildFileIOConfig(r *http.Request, location string, canWrite func() bool) (iceberg.Properties, []StorageCredential) {
 	config := make(iceberg.Properties)
 	if s.s3Endpoint == "" {
 		return config, nil
@@ -654,7 +661,7 @@ func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Pr
 
 	vended := iceberg.Properties(nil)
 	if wantsVendedCredentials(r) {
-		vended = s.vendCredentials(r, location)
+		vended = s.vendCredentials(r, location, canWrite)
 		// A client asking for vended credentials builds its storage credential
 		// out of whatever comes back here and stops using the one it was
 		// configured with. With nothing to vend, an endpoint on its own leaves
@@ -687,7 +694,7 @@ func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Pr
 // nil when the deployment has no vending configured or the mint failed, which
 // leaves the caller withholding the endpoint so the client keeps its own
 // credentials rather than falling back to unsigned requests.
-func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Properties {
+func (s *Server) vendCredentials(r *http.Request, location string, canWrite func() bool) iceberg.Properties {
 	if s.credentialVendor == nil || location == "" {
 		return nil
 	}
@@ -699,7 +706,7 @@ func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Prope
 	}
 
 	principal := s3_constants.GetIdentityNameFromContext(r)
-	credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, prefix)
+	credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, prefix, canWrite == nil || !canWrite())
 	if err != nil {
 		glog.Warningf("Iceberg: failed to vend credentials for %s: %v", location, err)
 		return nil
@@ -719,6 +726,42 @@ func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Prope
 	return vended
 }
 
+// mayUpdateTable reports whether the caller may commit to the table the
+// credentials would cover. The session stays read-only when it may not, so
+// the vend carries no more authority than the caller holds at the catalog.
+func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace []string, tableName string) bool {
+	updateReq := &s3tables.UpdateTableRequest{
+		TableBucketARN: buildTableBucketARN(bucketName),
+		Namespace:      namespace,
+		Name:           tableName,
+	}
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
+	})
+	if err != nil {
+		glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err)
+	}
+	return err == nil
+}
+
+// mayUpdateView is mayUpdateTable for a view, whose commit path is UpdateView.
+func (s *Server) mayUpdateView(r *http.Request, bucketName string, namespace []string, viewName string) bool {
+	updateReq := &s3tables.UpdateViewRequest{
+		TableBucketARN: buildTableBucketARN(bucketName),
+		Namespace:      namespace,
+		Name:           viewName,
+	}
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s.tablesManager.AuthorizeUpdateView(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
+	})
+	if err != nil {
+		glog.V(1).Infof("Iceberg: UpdateView check for %s/%s failed, vending read-only: %v", bucketName, viewName, err)
+	}
+	return err == nil
+}
+
 // handleTableExists checks if a table exists.
 func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)

weed/s3api/iceberg/handlers_view.go

@@ -218,7 +218,9 @@ func (s *Server) handleCreateView(w http.ResponseWriter, r *http.Request) {
 	if finalLocation == "" {
 		finalLocation = metadataLocation
 	}
-	config, _ := s.buildFileIOConfig(r, location)
+	config, _ := s.buildFileIOConfig(r, location, func() bool {
+		return s.mayUpdateView(r, bucketName, namespace, req.Name)
+	})
 	writeLoadResult(w, http.StatusOK, ViewResponse{
 		MetadataLocation: finalLocation,
 		Metadata:         metadata,
@@ -349,7 +351,9 @@ func (s *Server) buildViewResponse(r *http.Request, getResp s3tables.GetViewResp
 	if err != nil {
 		return ViewResponse{}, fmt.Errorf("failed to parse view metadata: %w", err)
 	}
-	config, _ := s.buildFileIOConfig(r, tableLocationFromMetadataLocation(getResp.MetadataLocation))
+	config, _ := s.buildFileIOConfig(r, tableLocationFromMetadataLocation(getResp.MetadataLocation), func() bool {
+		return s.mayUpdateView(r, bucketName, namespace, viewName)
+	})
 	return ViewResponse{
 		MetadataLocation: getResp.MetadataLocation,
 		Metadata:         metadata,

weed/s3api/iceberg/handlers_view_update.go

@@ -133,7 +133,7 @@ func (s *Server) handleUpdateView(w http.ResponseWriter, r *http.Request) {
 			return s.tablesManager.Execute(r.Context(), mgrClient, "UpdateView", updateReq, nil, identityName)
 		})
 		if err == nil {
-			config, _ := s.buildFileIOConfig(r, location)
+			config, _ := s.buildFileIOConfig(r, location, func() bool { return true })
 			writeLoadResult(w, http.StatusOK, ViewResponse{
 				MetadataLocation: newMetadataLocation,
 				Metadata:         newMetadata,

weed/s3api/iceberg/iceberg_access_delegation_test.go

@@ -43,12 +43,12 @@ func TestBuildFileIOConfigWithholdsEndpointFromCredentialVendingClients(t *testi
 
 	r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
 	r.Header.Set(accessDelegationHeader, "vended-credentials")
-	if got, _ := s.buildFileIOConfig(r, "s3://warehouse/ns/t"); len(got) != 0 {
+	if got, _ := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil); len(got) != 0 {
 		t.Fatalf("buildFileIOConfig() = %v, want empty for a vended-credentials request", got)
 	}
 
 	plain := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
-	if got, _ := s.buildFileIOConfig(plain, "s3://warehouse/ns/t"); got["s3.endpoint"] != s.s3Endpoint {
+	if got, _ := s.buildFileIOConfig(plain, "s3://warehouse/ns/t", nil); got["s3.endpoint"] != s.s3Endpoint {
 		t.Fatalf("s3.endpoint = %q, want %q", got["s3.endpoint"], s.s3Endpoint)
 	}
 }

weed/s3api/iceberg/iceberg_credential_vending_test.go

@@ -14,10 +14,11 @@ type stubVendor struct {
 	gotBucket   string
 	gotPrefix   string
 	gotcaller   string
+	gotReadOnly bool
 }
 
-func (v *stubVendor) VendTableCredentials(_ context.Context, principal, bucket, prefix string) (*VendedCredentials, error) {
-	v.gotcaller, v.gotBucket, v.gotPrefix = principal, bucket, prefix
+func (v *stubVendor) VendTableCredentials(_ context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) {
+	v.gotcaller, v.gotBucket, v.gotPrefix, v.gotReadOnly = principal, bucket, prefix, readOnly
 	return v.credentials, v.err
 }
 
@@ -40,7 +41,7 @@ func TestBuildFileIOConfigVendingVendsScopedCredentials(t *testing.T) {
 	r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
 	r.Header.Set(accessDelegationHeader, "vended-credentials")
 
-	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
+	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
 
 	if vendor.gotBucket != "warehouse" || vendor.gotPrefix != "ns/t" {
 		t.Errorf("vendor called with bucket=%q prefix=%q, want warehouse and ns/t", vendor.gotBucket, vendor.gotPrefix)
@@ -70,7 +71,7 @@ func TestBuildFileIOConfigVendingWithoutVendorSaysNothing(t *testing.T) {
 	r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
 	r.Header.Set(accessDelegationHeader, "vended-credentials")
 
-	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
+	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
 	if len(config) != 0 || storageCredentials != nil {
 		t.Errorf("config = %v, storage-credentials = %v, want both empty", config, storageCredentials)
 	}
@@ -81,12 +82,35 @@ func TestBuildFileIOConfigVendingFallsBackWhenVendingFails(t *testing.T) {
 	r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
 	r.Header.Set(accessDelegationHeader, "vended-credentials")
 
-	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
+	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
 	if len(config) != 0 || storageCredentials != nil {
 		t.Errorf("config = %v, storage-credentials = %v, want both empty", config, storageCredentials)
 	}
 }
 
+// The vend stays read-only unless the caller proved write authority: a nil
+// or false canWrite must reach the vendor as readOnly.
+func TestBuildFileIOConfigVendingReadOnlyUnlessCanWrite(t *testing.T) {
+	vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
+	s := vendingServer(vendor)
+
+	r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
+	r.Header.Set(accessDelegationHeader, "vended-credentials")
+
+	s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
+	if !vendor.gotReadOnly {
+		t.Error("vendCredentials() passed readOnly=false for a nil canWrite, want readOnly=true")
+	}
+	s.buildFileIOConfig(r, "s3://warehouse/ns/t", func() bool { return false })
+	if !vendor.gotReadOnly {
+		t.Error("vendCredentials() passed readOnly=false for canWrite=false, want readOnly=true")
+	}
+	s.buildFileIOConfig(r, "s3://warehouse/ns/t", func() bool { return true })
+	if vendor.gotReadOnly {
+		t.Error("vendCredentials() passed readOnly=true for canWrite=true, want readOnly=false")
+	}
+}
+
 // A client that did not ask for delegation keeps the plain endpoint config and
 // must never be handed credentials.
 func TestBuildFileIOConfigVendingWithoutDelegationHeader(t *testing.T) {
@@ -94,7 +118,7 @@ func TestBuildFileIOConfigVendingWithoutDelegationHeader(t *testing.T) {
 	s := vendingServer(vendor)
 
 	r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
-	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
+	config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
 
 	if _, vended := config["s3.access-key-id"]; vended {
 		t.Errorf("credentials vended without the delegation header: %v", config)

weed/s3api/iceberg/iceberg_issue_9103_test.go

@@ -61,7 +61,7 @@ func TestBuildFileIOConfig(t *testing.T) {
 
 	t.Run("no endpoint configured yields empty config", func(t *testing.T) {
 		s := &Server{}
-		got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t")
+		got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t", nil)
 		if len(got) != 0 {
 			t.Fatalf("buildFileIOConfig() = %v, want empty", got)
 		}
@@ -69,7 +69,7 @@ func TestBuildFileIOConfig(t *testing.T) {
 
 	t.Run("endpoint is advertised with path-style-access and region", func(t *testing.T) {
 		s := &Server{s3Endpoint: "http://seaweed.example:8333"}
-		got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t")
+		got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t", nil)
 		if got["s3.endpoint"] != "http://seaweed.example:8333" {
 			t.Fatalf("s3.endpoint = %q, want %q", got["s3.endpoint"], "http://seaweed.example:8333")
 		}

weed/s3api/iceberg/server.go

@@ -45,10 +45,11 @@ type VendedCredentials struct {
 }
 
 // CredentialVendor mints credentials limited to a single table's prefix for a
-// caller the catalog has already authenticated and authorized. A nil result
-// with no error means the deployment has vending switched off.
+// caller the catalog has already authenticated and authorized, read-only when
+// readOnly is set. A nil result with no error means the deployment has vending
+// switched off.
 type CredentialVendor interface {
-	VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
+	VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error)
 }
 
 // Server implements the Iceberg REST Catalog API.

weed/s3api/lance/handlers_table.go

@@ -150,7 +150,9 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
 		glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
 	}
 
-	options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
+	// A Lance commit is a direct S3 write that never revisits the catalog, so
+	// the create authorization above is the write evidence for the dataset.
+	options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true })
 	if err != nil {
 		writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
 		return
@@ -210,7 +212,9 @@ func (s *Server) handleDescribeTable(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
+	options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool {
+		return s.mayUpdateTable(r, bucket, ns, name)
+	})
 	if err != nil {
 		writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
 		return

weed/s3api/lance/server.go

@@ -45,10 +45,11 @@ type VendedCredentials struct {
 }
 
 // CredentialVendor mints credentials limited to a single table's prefix for a
-// caller the catalog has already authenticated and authorized. A nil result with
-// no error means the deployment has vending switched off.
+// caller the catalog has already authenticated and authorized, read-only when
+// readOnly is set. A nil result with no error means the deployment has vending
+// switched off.
 type CredentialVendor interface {
-	VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
+	VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error)
 }
 
 // Server implements the Lance Namespace REST spec.

weed/s3api/lance/storage.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -135,7 +136,7 @@ func confineLocation(bucket, location string) error {
 // storageOptions builds the object_store settings a Lance client needs to reach
 // the dataset. The key names are the aws_-prefixed forms Lance clients pass
 // through to object_store.
-func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool) (map[string]string, error) {
+func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool, canWrite func() bool) (map[string]string, error) {
 	options := map[string]string{}
 	if s.s3Endpoint != "" {
 		options["aws_endpoint"] = s.s3Endpoint
@@ -154,7 +155,7 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
 	}
 
 	principal := s3_constants.GetIdentityNameFromContext(r)
-	credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location))
+	credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location), canWrite == nil || !canWrite())
 	if err != nil {
 		return nil, err
 	}
@@ -172,6 +173,25 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
 	return options, nil
 }
 
+// mayUpdateTable reports whether the caller may commit to the table the
+// credentials would cover, the same gate the Iceberg catalog applies so a
+// read-only caller never gets write-capable storage_options.
+func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool {
+	req := &s3tables.UpdateTableRequest{
+		TableBucketARN: bucketARN(bucket),
+		Namespace:      ns,
+		Name:           name,
+	}
+	identityName := s3_constants.GetIdentityNameFromContext(r)
+	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), req, identityName)
+	})
+	if err != nil {
+		glog.V(1).Infof("lance: UpdateTable check for %s/%s failed, vending read-only: %v", bucket, name, err)
+	}
+	return err == nil
+}
+
 // locationPrefix strips the s3://bucket/ part of a location, leaving the key
 // prefix a credential is scoped to.
 func locationPrefix(location string) string {

weed/s3api/s3_action_resolver.go

@@ -182,20 +182,33 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool)
 		}
 	}
 
-	// Tagging operations
+	// Tagging operations. The routes match ?tagging whenever the key is
+	// present (gorilla/mux does not compare the value), so any value counts.
+	// A versionId retargets the request at a specific version, so the
+	// version-tagging action applies, not the current-object one.
 	if query.Has("tagging") {
+		versioned := hasObject && query.Get("versionId") != ""
 		switch method {
 		case http.MethodGet:
+			if versioned {
+				return s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING
+			}
 			if hasObject {
 				return s3_constants.S3_ACTION_GET_OBJECT_TAGGING
 			}
 			return s3_constants.S3_ACTION_GET_BUCKET_TAGGING
 		case http.MethodPut:
+			if versioned {
+				return s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING
+			}
 			if hasObject {
 				return s3_constants.S3_ACTION_PUT_OBJECT_TAGGING
 			}
 			return s3_constants.S3_ACTION_PUT_BUCKET_TAGGING
 		case http.MethodDelete:
+			if versioned {
+				return s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING
+			}
 			if hasObject {
 				return s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING
 			}

weed/s3api/s3_action_resolver_test.go

@@ -114,6 +114,38 @@ func TestResolveS3Action_AttributesBeforeVersionId(t *testing.T) {
 	}
 }
 
+// A tagging request carrying versionId targets a specific version and must
+// resolve to the version-tagging action so a grant limited to current-object
+// tagging cannot reach historical versions.
+func TestResolveS3Action_VersionTagging(t *testing.T) {
+	tests := []struct {
+		name   string
+		method string
+		object string
+		query  string
+		want   string
+	}{
+		{"get current tags", http.MethodGet, "key", "tagging", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
+		{"put current tags", http.MethodPut, "key", "tagging", s3_constants.S3_ACTION_PUT_OBJECT_TAGGING},
+		{"delete current tags", http.MethodDelete, "key", "tagging", s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING},
+		{"get version tags", http.MethodGet, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING},
+		{"put version tags", http.MethodPut, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING},
+		{"delete version tags", http.MethodDelete, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING},
+		{"empty versionId stays current", http.MethodGet, "key", "tagging&versionId=", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
+		{"bucket tagging ignores versionId", http.MethodGet, "", "tagging&versionId=v1", s3_constants.S3_ACTION_GET_BUCKET_TAGGING},
+		{"non-empty tagging is still tagging", http.MethodGet, "key", "tagging=x&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING},
+		{"non-empty tagging plain", http.MethodGet, "key", "tagging=x", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil)
+			if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want {
+				t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
 // Bucket subresources registered with ACTION_ADMIN must resolve to their own
 // S3 actions so a policy granting one of them does not need s3:*, and so no
 // broader grant sweeps them in.

weed/s3api/s3_constants/s3_action_strings.go

@@ -20,6 +20,11 @@ const (
 	S3_ACTION_PUT_OBJECT_TAGGING    = "s3:PutObjectTagging"
 	S3_ACTION_DELETE_OBJECT_TAGGING = "s3:DeleteObjectTagging"
 
+	// Version-specific tagging operations (?tagging&versionId)
+	S3_ACTION_GET_OBJECT_VERSION_TAGGING    = "s3:GetObjectVersionTagging"
+	S3_ACTION_PUT_OBJECT_VERSION_TAGGING    = "s3:PutObjectVersionTagging"
+	S3_ACTION_DELETE_OBJECT_VERSION_TAGGING = "s3:DeleteObjectVersionTagging"
+
 	// Object retention and legal hold
 	S3_ACTION_GET_OBJECT_RETENTION  = "s3:GetObjectRetention"
 	S3_ACTION_PUT_OBJECT_RETENTION  = "s3:PutObjectRetention"

weed/s3api/s3api_bucket_handlers.go

@@ -330,7 +330,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 	// This ensures Object Lock configuration is set in the same CreateEntry call,
 	// preventing race conditions where the bucket exists without Object Lock enabled
 	var bucketCrtime int64
-	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
+	if err := s3a.mkdirExclusive(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
 		bucketCrtime = entry.Attributes.Crtime
 
 		// Set bucket owner
@@ -373,6 +373,16 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		// return the appropriate already-exists error instead of InternalError.
 		if exist, checkErr := s3a.exists(s3a.option.BucketsPath, bucket, true); checkErr == nil && exist {
 			glog.V(3).Infof("PutBucketHandler: bucket %s was created concurrently", bucket)
+			// From the entry itself, not the bucket registry: the pre-check
+			// above has just cached this name as not found, so in the race this
+			// branch exists for the registry would still say "no table bucket".
+			if concurrentEntryIsForeign(s3a.getEntry(s3a.option.BucketsPath, bucket)) {
+				s3err.WriteErrorResponse(w, r, s3err.ErrBucketAlreadyExists)
+				return
+			}
+			if s3a.bucketConfigCache != nil {
+				s3a.bucketConfigCache.RemoveNegativeCache(bucket)
+			}
 			errCode := s3a.existingBucketError(r, bucket, currentIdentityId, requestHasACL)
 			if errCode == s3err.ErrBucketAlreadyOwnedByYou {
 				s3a.healBucketOwnerIndex(bucket, currentIdentityId)
@@ -744,7 +754,7 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
 
 	identityId := s3_constants.GetIdentityNameFromContext(r)
 	var bucketCrtime int64
-	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
+	if err := s3a.mkdirExclusive(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
 		bucketCrtime = entry.Attributes.Crtime
 		setBucketOwner(r)(entry)
 	}); err != nil {
@@ -754,26 +764,35 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
 			glog.Warningf("autoCreateBucket: failed to check existence for bucket %s: %v", bucket, err2)
 			return fmt.Errorf("failed to auto-create bucket %s: %w", bucket, errors.Join(err, err2))
 		} else if exist {
+			entry, getErr := s3a.getEntry(s3a.option.BucketsPath, bucket)
+			if getErr != nil {
+				glog.Warningf("autoCreateBucket: failed to get entry for existing bucket %s: %v", bucket, getErr)
+				return fmt.Errorf("failed to auto-create bucket %s: %w", bucket, errors.Join(err, getErr))
+			}
+			if concurrentEntryIsForeign(entry, nil) {
+				// A table bucket carries no S3 owner and must not gain one —
+				// or a write intended for a missing plain bucket.
+				if s3a.bucketConfigCache != nil {
+					s3a.bucketConfigCache.RemoveNegativeCache(bucket)
+				}
+				return fmt.Errorf("auto-create bucket %s: name taken by a table bucket: %w", bucket, err)
+			}
 			// The bucket exists, which is fine. However, we should ensure it has an owner.
 			// If it was created by a concurrent request that didn't set an owner,
 			// we'll set it here to ensure consistency.
-			if entry, getErr := s3a.getEntry(s3a.option.BucketsPath, bucket); getErr == nil {
-				if entry.Extended == nil || len(entry.Extended[s3_constants.AmzIdentityId]) == 0 {
-					// No owner set, assign current admin as owner
-					setBucketOwner(r)(entry)
-					if updateErr := s3a.updateEntry(s3a.option.BucketsPath, entry); updateErr != nil {
-						glog.Warningf("autoCreateBucket: failed to set owner for existing bucket %s: %v", bucket, updateErr)
-					} else {
-						glog.V(1).Infof("Set owner for existing bucket %s (created by concurrent request)", bucket)
-						if owner := bucketEntryOwner(entry); owner != "" {
-							if indexErr := s3a.addBucketToOwnerIndex(owner, bucket, entry.Attributes.Crtime); indexErr != nil {
-								glog.Warningf("autoCreateBucket: owner index add %s/%s: %v", owner, bucket, indexErr)
-							}
+			if entry.Extended == nil || len(entry.Extended[s3_constants.AmzIdentityId]) == 0 {
+				// No owner set, assign current admin as owner
+				setBucketOwner(r)(entry)
+				if updateErr := s3a.updateEntry(s3a.option.BucketsPath, entry); updateErr != nil {
+					glog.Warningf("autoCreateBucket: failed to set owner for existing bucket %s: %v", bucket, updateErr)
+				} else {
+					glog.V(1).Infof("Set owner for existing bucket %s (created by concurrent request)", bucket)
+					if owner := bucketEntryOwner(entry); owner != "" {
+						if indexErr := s3a.addBucketToOwnerIndex(owner, bucket, entry.Attributes.Crtime); indexErr != nil {
+							glog.Warningf("autoCreateBucket: owner index add %s/%s: %v", owner, bucket, indexErr)
 						}
 					}
 				}
-			} else {
-				glog.Warningf("autoCreateBucket: failed to get entry for existing bucket %s: %v", bucket, getErr)
 			}
 			// Remove bucket from negative cache — it exists now
 			if s3a.bucketConfigCache != nil {

weed/s3api/s3api_embedded_iam.go

@@ -67,6 +67,136 @@ func (e *EmbeddedIamApi) refreshIAMConfiguration() error {
 	return nil
 }
 
+// canPersistIdentity reports whether a change to the named identity can be written through the
+// credential manager's single-identity methods instead of SaveConfiguration. It is false when
+// there is no credential manager, when the caller asked for no persistent write (skipPersist),
+// and for static identities, which live outside the store: those keep the full-save path.
+func (e *EmbeddedIamApi) canPersistIdentity(skipPersist bool, userName string) bool {
+	return e.credentialManager != nil && !skipPersist && !e.credentialManager.IsStaticIdentity(userName)
+}
+
+// groupsWithMember returns the groups in s3cfg that list userName as a member. Taken before a
+// handler renames or removes the user, it names the group records that handler will change.
+func groupsWithMember(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.Group {
+	var groups []*iam_pb.Group
+	for _, g := range s3cfg.Groups {
+		for _, m := range g.Members {
+			if m == userName {
+				groups = append(groups, g)
+				break
+			}
+		}
+	}
+	return groups
+}
+
+// serviceAccountsOf returns the service accounts in s3cfg whose parent is userName.
+func serviceAccountsOf(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.ServiceAccount {
+	var accounts []*iam_pb.ServiceAccount
+	for _, sa := range s3cfg.ServiceAccounts {
+		if sa.ParentUser == userName {
+			accounts = append(accounts, sa)
+		}
+	}
+	return accounts
+}
+
+// persistIdentity writes the one identity a handler changed in s3cfg through the credential
+// manager. The store writes that identity's record and propagates it alone; the other
+// identities are not rewritten.
+func (e *EmbeddedIamApi) persistIdentity(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration, userName string) error {
+	ident := findIdentityByName(s3cfg, userName)
+	if ident == nil {
+		return credential.ErrUserNotFound
+	}
+	return e.credentialManager.UpdateUser(ctx, userName, ident)
+}
+
+// persistRenamedIdentity writes a user renamed from oldName to newName by UpdateUser through the
+// credential manager: the identity, the groups that listed the old name and the service accounts
+// it parented. Where the store has RenameUser the handler already moved the record and its inline
+// policies, and the identity is written under the new name. Otherwise UpdateUser under the old
+// name moves the record in stores that treat a changed Name as a rename (filer_etc, memory), the
+// propagating store propagates the new identity and the old name's removal, and the inline policy
+// copies the handler left under the old name are removed. A store that did neither still holds
+// the user under the old name; the caller is told so (false) and keeps the full save, which is
+// what renames such a store today.
+func (e *EmbeddedIamApi) persistRenamedIdentity(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration, oldName, newName string, groups []*iam_pb.Group, accounts []*iam_pb.ServiceAccount) (bool, error) {
+	ident := findIdentityByName(s3cfg, newName)
+	if ident == nil {
+		return false, credential.ErrUserNotFound
+	}
+	storedAs := oldName
+	if _, err := e.credentialManager.GetUser(ctx, newName); err == nil {
+		storedAs = newName
+	}
+	if err := e.credentialManager.UpdateUser(ctx, storedAs, ident); err != nil {
+		return false, err
+	}
+	if storedAs == oldName {
+		if _, err := e.credentialManager.GetUser(ctx, newName); err != nil {
+			return false, nil
+		}
+		policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, oldName)
+		if err != nil {
+			return false, fmt.Errorf("list inline policies for %s: %w", oldName, err)
+		}
+		for _, policyName := range policyNames {
+			if err := e.credentialManager.DeleteUserInlinePolicy(ctx, oldName, policyName); err != nil {
+				return false, fmt.Errorf("remove inline policy %s from %s: %w", policyName, oldName, err)
+			}
+		}
+	}
+	for _, g := range groups {
+		if err := e.credentialManager.UpdateGroup(ctx, g); err != nil {
+			return false, fmt.Errorf("update group %s: %w", g.Name, err)
+		}
+	}
+	for _, sa := range accounts {
+		if err := e.credentialManager.UpdateServiceAccount(ctx, sa.Id, sa); err != nil {
+			return false, fmt.Errorf("update service account %s: %w", sa.Id, err)
+		}
+	}
+	return true, nil
+}
+
+// deleteIdentity removes a user deleted by DeleteUser through the credential manager: its inline
+// policies, its record (the store propagates the one removal) and the groups that listed it,
+// whose membership DeleteUser already changed in s3cfg.
+func (e *EmbeddedIamApi) deleteIdentity(ctx context.Context, userName string, groups []*iam_pb.Group) error {
+	policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, userName)
+	if err != nil {
+		return fmt.Errorf("list inline policies for %s: %w", userName, err)
+	}
+	for _, policyName := range policyNames {
+		if err := e.credentialManager.DeleteUserInlinePolicy(ctx, userName, policyName); err != nil {
+			return fmt.Errorf("remove inline policy %s from %s: %w", policyName, userName, err)
+		}
+	}
+	if err := e.credentialManager.DeleteUser(ctx, userName); err != nil {
+		return err
+	}
+	for _, g := range groups {
+		if err := e.credentialManager.UpdateGroup(ctx, g); err != nil {
+			return fmt.Errorf("update group %s: %w", g.Name, err)
+		}
+	}
+	return nil
+}
+
+// reloadsAfterTargetedWrite names the actions that persist through the credential manager
+// rather than SaveConfiguration; the in-memory identity maps are refreshed after them the way
+// they are after a full save.
+func reloadsAfterTargetedWrite(action string) bool {
+	switch action {
+	case "AttachUserPolicy", "DetachUserPolicy", "CreatePolicy", "CreatePolicyVersion", "DeletePolicy",
+		"CreateUser", "UpdateUser", "DeleteUser", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey",
+		"SetUserStatus", "PutUserPolicy", "DeleteUserPolicy", "PutGroupPolicy", "DeleteGroupPolicy":
+		return true
+	}
+	return false
+}
+
 // Constants for service account identifiers
 const (
 	ServiceAccountIDLength  = 12 // Length of the service account ID
@@ -2464,53 +2594,19 @@ func (e *EmbeddedIamApi) ListGroupPolicies(s3cfg *iam_pb.S3ApiConfiguration, val
 	return resp, nil
 }
 
-// handleImplicitUsername adds username who signs the request to values if 'username' is not specified.
-// According to AWS documentation: "If you do not specify a user name, IAM determines the user name
-// implicitly based on the Amazon Web Services access key ID signing the request."
-// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding
-// identity name in the credential store.
+// handleImplicitUsername adds the name of the authenticated caller to values
+// when 'UserName' is not specified, per AWS: "If you do not specify a user name,
+// IAM determines the user name implicitly based on the Amazon Web Services
+// access key ID signing the request." The name comes from the identity the
+// authentication middleware verified, which the request context carries; the
+// Authorization header itself is attacker-controlled and is never re-parsed.
 func (e *EmbeddedIamApi) handleImplicitUsername(r *http.Request, values url.Values) {
-	if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" {
+	if values.Get("UserName") != "" {
 		return
 	}
-	// Log presence of auth header without exposing sensitive signature material
-	glog.V(4).Infof("Authorization header present, extracting access key")
-	// Parse AWS SigV4 Authorization header format:
-	// "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..."
-	s := strings.Split(r.Header["Authorization"][0], "Credential=")
-	if len(s) < 2 {
-		return
+	if name := s3_constants.GetIdentityNameFromContext(r); name != "" {
+		values.Set("UserName", name)
 	}
-	s = strings.Split(s[1], ",")
-	if len(s) < 1 {
-		return
-	}
-	s = strings.Split(s[0], "/")
-	if len(s) < 1 {
-		return
-	}
-	// s[0] is the AccessKeyId
-	accessKeyId := s[0]
-	if accessKeyId == "" {
-		return
-	}
-	// Nil-guard: ensure iam is initialized before lookup
-	if e.iam == nil {
-		glog.V(4).Infof("IAM not initialized, cannot look up access key")
-		return
-	}
-	// Look up the identity by access key to get the username
-	identity, _, found := e.iam.LookupByAccessKey(accessKeyId)
-	if !found {
-		// Mask access key in logs - show only first 4 chars
-		maskedKey := accessKeyId
-		if len(accessKeyId) > 4 {
-			maskedKey = accessKeyId[:4] + "***"
-		}
-		glog.V(4).Infof("Access key %s not found in credential store", maskedKey)
-		return
-	}
-	values.Set("UserName", identity.Name)
 }
 
 // iamSelfServiceActions are actions that users can perform on their own resources without admin rights.
@@ -2735,27 +2831,79 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		}
 		changed = false
 	case "UpdateUser":
+		userName, newUserName := values.Get("UserName"), values.Get("NewUserName")
+		// Taken before the rename: the records the handler changes besides the identity.
+		groups := groupsWithMember(s3cfg, userName)
+		accounts := serviceAccountsOf(s3cfg, userName)
 		var iamErr *iamError
 		response, iamErr = e.UpdateUser(ctx, s3cfg, values)
 		if iamErr != nil {
 			return nil, iamErr
 		}
+		// Write the renamed identity, its groups and service accounts through the credential
+		// manager instead of rewriting every identity via SaveConfiguration. A request that
+		// renames nothing changed nothing, so there is nothing to write either way.
+		if e.canPersistIdentity(skipPersist, userName) {
+			renamed := true
+			if newUserName != "" && newUserName != userName {
+				var err error
+				renamed, err = e.persistRenamedIdentity(ctx, s3cfg, userName, newUserName, groups, accounts)
+				if err != nil {
+					return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+				}
+			}
+			changed = !renamed
+		}
 	case "DeleteUser":
 		userName := values.Get("UserName")
+		groups := groupsWithMember(s3cfg, userName)
 		var iamErr *iamError
 		response, iamErr = e.DeleteUser(s3cfg, userName)
 		if iamErr != nil {
 			return nil, iamErr
 		}
+		// Targeted delete: one record removed and one removal propagated, plus the groups whose
+		// membership changed, instead of rewriting every remaining identity.
+		if e.canPersistIdentity(skipPersist, userName) {
+			if err := e.deleteIdentity(ctx, userName, groups); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "CreateAccessKey":
-		var iamErr *iamError
-		response, iamErr = e.CreateAccessKey(s3cfg, values)
+		userName := values.Get("UserName")
+		resp, iamErr := e.CreateAccessKey(s3cfg, values)
 		if iamErr != nil {
 			glog.Errorf("CreateAccessKey: %+v", iamErr.Error)
 			return nil, iamErr
 		}
+		response = resp
+		// Targeted write: credentialManager.CreateAccessKey adds the key to the one identity
+		// and propagates that identity alone.
+		if e.canPersistIdentity(skipPersist, userName) {
+			cred := &iam_pb.Credential{
+				AccessKey: *resp.CreateAccessKeyResult.AccessKey.AccessKeyId,
+				SecretKey: *resp.CreateAccessKeyResult.AccessKey.SecretAccessKey,
+				Status:    iamAccessKeyStatusActive,
+			}
+			if err := e.credentialManager.CreateAccessKey(ctx, userName, cred); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "DeleteAccessKey":
+		userName, accessKeyId := values.Get("UserName"), values.Get("AccessKeyId")
 		response = e.DeleteAccessKey(s3cfg, values)
+		// Targeted write: credentialManager.DeleteAccessKey removes the key from the one identity
+		// and propagates that identity alone. The handler answers 200 for a key or user it does
+		// not know, so a not-found from the store is the same no-op it always was.
+		if e.canPersistIdentity(skipPersist, userName) {
+			err := e.credentialManager.DeleteAccessKey(ctx, userName, accessKeyId)
+			if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !errors.Is(err, credential.ErrUserNotFound) {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "CreatePolicy":
 		var iamErr *iamError
 		response, iamErr = e.CreatePolicy(ctx, values)
@@ -2773,12 +2921,21 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		}
 		changed = false
 	case "PutUserPolicy":
+		userName := values.Get("UserName")
 		var iamErr *iamError
 		response, iamErr = e.PutUserPolicy(s3cfg, values)
 		if iamErr != nil {
 			glog.Errorf("PutUserPolicy: %+v", iamErr.Error)
 			return nil, iamErr
 		}
+		// The policy document is already stored; what remains is the identity's recomputed
+		// Actions, written and propagated for that identity alone.
+		if e.canPersistIdentity(skipPersist, userName) {
+			if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "GetUserPolicy":
 		var iamErr *iamError
 		response, iamErr = e.GetUserPolicy(s3cfg, values)
@@ -2787,11 +2944,18 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		}
 		changed = false
 	case "DeleteUserPolicy":
+		userName := values.Get("UserName")
 		var iamErr *iamError
 		response, iamErr = e.DeleteUserPolicy(s3cfg, values)
 		if iamErr != nil {
 			return nil, iamErr
 		}
+		if e.canPersistIdentity(skipPersist, userName) {
+			if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "ListUserPolicies":
 		var iamErr *iamError
 		response, iamErr = e.ListUserPolicies(s3cfg, values)
@@ -2883,17 +3047,31 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 		}
 		changed = false
 	case "SetUserStatus":
+		userName := values.Get("UserName")
 		var iamErr *iamError
 		response, iamErr = e.SetUserStatus(s3cfg, values)
 		if iamErr != nil {
 			return nil, iamErr
 		}
+		if e.canPersistIdentity(skipPersist, userName) {
+			if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	case "UpdateAccessKey":
+		userName := values.Get("UserName")
 		var iamErr *iamError
 		response, iamErr = e.UpdateAccessKey(s3cfg, values)
 		if iamErr != nil {
 			return nil, iamErr
 		}
+		if e.canPersistIdentity(skipPersist, userName) {
+			if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
+				return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
+			}
+			changed = false
+		}
 	// Service Account actions
 	case "CreateServiceAccount":
 		createdBy := values.Get("CreatedBy")
@@ -3034,7 +3212,7 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
 			glog.Errorf("Failed to reload IAM configuration after mutation: %v", err)
 			// Don't fail the request since the persistent save succeeded
 		}
-	} else if action == "AttachUserPolicy" || action == "DetachUserPolicy" || action == "CreatePolicy" || action == "CreatePolicyVersion" || action == "DeletePolicy" || action == "CreateUser" || action == "PutGroupPolicy" || action == "DeleteGroupPolicy" {
+	} else if reloadsAfterTargetedWrite(action) {
 		// Even if changed=false (persisted via credentialManager), we should still reload
 		// if we are utilizing the local in-memory cache for speed
 		if err := e.ReloadConfiguration(); err != nil {

weed/s3api/s3api_embedded_iam_targeted_test.go

@@ -0,0 +1,362 @@
+package s3api
+
+// Targeted-write regression tests: single-identity IAM actions must write and
+// propagate only the identity they changed, not every identity via
+// SaveConfiguration. See https://github.com/seaweedfs/seaweedfs/issues/11865.
+
+import (
+	"context"
+	"encoding/json"
+	"net/url"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/credential/memory"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingStore wraps a memory store and counts the write methods the
+// dispatcher used, so a test can assert the single-identity path was taken and
+// SaveConfiguration was not. It embeds the concrete store (rather than the
+// CredentialStore interface) so optional interfaces like InlinePolicyStore and
+// UserRenamer stay visible through the wrapper.
+type recordingStore struct {
+	*memory.MemoryStore
+	saves            int
+	createAccessKeys int
+	deleteAccessKeys int
+	updateUsers      int
+	deleteUsers      int
+	updateGroups     int
+}
+
+func (r *recordingStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
+	r.saves++
+	return r.MemoryStore.SaveConfiguration(ctx, config)
+}
+
+func (r *recordingStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error {
+	r.createAccessKeys++
+	return r.MemoryStore.CreateAccessKey(ctx, username, cred)
+}
+
+func (r *recordingStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
+	r.deleteAccessKeys++
+	return r.MemoryStore.DeleteAccessKey(ctx, username, accessKey)
+}
+
+func (r *recordingStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error {
+	r.updateUsers++
+	return r.MemoryStore.UpdateUser(ctx, username, identity)
+}
+
+func (r *recordingStore) DeleteUser(ctx context.Context, username string) error {
+	r.deleteUsers++
+	return r.MemoryStore.DeleteUser(ctx, username)
+}
+
+func (r *recordingStore) UpdateGroup(ctx context.Context, group *iam_pb.Group) error {
+	r.updateGroups++
+	return r.MemoryStore.UpdateGroup(ctx, group)
+}
+
+// newTargetedWriteTestApi seeds cfg into the store and wraps the store in a
+// recordingStore. The initial sync save happens before the recorder is
+// installed so it does not count.
+func newTargetedWriteTestApi(t *testing.T, cfg *iam_pb.S3ApiConfiguration) (*EmbeddedIamApiForTest, *recordingStore) {
+	t.Helper()
+	api := NewEmbeddedIamApiForTest()
+	api.mockConfig = cfg
+	// Trigger the one-time mockConfig -> store seed through a read action.
+	_, iamErr := api.ExecuteAction(context.Background(), url.Values{"Action": {"ListUsers"}}, false, "")
+	require.Nil(t, iamErr)
+	rec := &recordingStore{MemoryStore: api.credentialManager.Store.(*memory.MemoryStore)}
+	api.credentialManager.Store = rec
+	return api, rec
+}
+
+func executeAction(t *testing.T, api *EmbeddedIamApiForTest, vals url.Values) {
+	t.Helper()
+	_, iamErr := api.ExecuteAction(context.Background(), vals, false, "")
+	require.Nil(t, iamErr)
+}
+
+func TestEmbeddedIamCreateAccessKeyTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
+	})
+	ctx := context.Background()
+
+	resp, iamErr := api.ExecuteAction(ctx, url.Values{
+		"Action":   {"CreateAccessKey"},
+		"UserName": {"alice"},
+	}, false, "")
+	require.Nil(t, iamErr)
+	require.Equal(t, 0, rec.saves, "CreateAccessKey must not SaveConfiguration")
+	require.Equal(t, 1, rec.createAccessKeys)
+
+	out := resp.(*iamCreateAccessKeyResponse)
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	require.Len(t, ident.Credentials, 1)
+	assert.Equal(t, *out.CreateAccessKeyResult.AccessKey.AccessKeyId, ident.Credentials[0].AccessKey)
+
+	// The new key resolves in the store.
+	byKey, err := api.credentialManager.GetUserByAccessKey(ctx, *out.CreateAccessKeyResult.AccessKey.AccessKeyId)
+	require.NoError(t, err)
+	assert.Equal(t, "alice", byKey.Name)
+}
+
+func TestEmbeddedIamDeleteAccessKeyTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{
+			Name:        "alice",
+			Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake"}},
+		}},
+	})
+	ctx := context.Background()
+
+	executeAction(t, api, url.Values{
+		"Action":      {"DeleteAccessKey"},
+		"UserName":    {"alice"},
+		"AccessKeyId": {"AKIATESTFAKEKEY00001"},
+	})
+	require.Equal(t, 0, rec.saves, "DeleteAccessKey must not SaveConfiguration")
+	require.Equal(t, 1, rec.deleteAccessKeys)
+
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, ident.Credentials)
+	_, err = api.credentialManager.GetUserByAccessKey(ctx, "AKIATESTFAKEKEY00001")
+	require.ErrorIs(t, err, credential.ErrAccessKeyNotFound)
+}
+
+func TestEmbeddedIamUpdateAccessKeyTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{
+			Name: "alice",
+			Credentials: []*iam_pb.Credential{
+				{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake", Status: iamAccessKeyStatusActive},
+			},
+		}},
+	})
+	ctx := context.Background()
+
+	executeAction(t, api, url.Values{
+		"Action":      {"UpdateAccessKey"},
+		"UserName":    {"alice"},
+		"AccessKeyId": {"AKIATESTFAKEKEY00001"},
+		"Status":      {iamAccessKeyStatusInactive},
+	})
+	require.Equal(t, 0, rec.saves, "UpdateAccessKey must not SaveConfiguration")
+	require.Equal(t, 1, rec.updateUsers)
+
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	require.Len(t, ident.Credentials, 1)
+	assert.Equal(t, iamAccessKeyStatusInactive, ident.Credentials[0].Status)
+}
+
+const iamTargetedTestPolicy = `{
+	"Version": "2012-10-17",
+	"Statement": [{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::bench/*"]}]
+}`
+
+func TestEmbeddedIamPutUserPolicyTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
+	})
+	ctx := context.Background()
+
+	executeAction(t, api, url.Values{
+		"Action":         {"PutUserPolicy"},
+		"UserName":       {"alice"},
+		"PolicyName":     {"read-bench"},
+		"PolicyDocument": {iamTargetedTestPolicy},
+	})
+	require.Equal(t, 0, rec.saves, "PutUserPolicy must not SaveConfiguration")
+	require.Equal(t, 1, rec.updateUsers)
+
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.NotEmpty(t, ident.Actions, "identity Actions must be recomputed and persisted")
+
+	doc, err := api.credentialManager.GetUserInlinePolicy(ctx, "alice", "read-bench")
+	require.NoError(t, err)
+	require.NotNil(t, doc)
+}
+
+func TestEmbeddedIamDeleteUserPolicyTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	})
+	ctx := context.Background()
+
+	executeAction(t, api, url.Values{
+		"Action":         {"PutUserPolicy"},
+		"UserName":       {"alice"},
+		"PolicyName":     {"read-bench"},
+		"PolicyDocument": {iamTargetedTestPolicy},
+	})
+	require.Equal(t, 1, rec.updateUsers)
+
+	executeAction(t, api, url.Values{
+		"Action":     {"DeleteUserPolicy"},
+		"UserName":   {"alice"},
+		"PolicyName": {"read-bench"},
+	})
+	require.Equal(t, 0, rec.saves, "DeleteUserPolicy must not SaveConfiguration")
+	require.Equal(t, 2, rec.updateUsers)
+
+	policies, err := api.credentialManager.ListUserInlinePolicies(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, policies)
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, ident.Actions)
+}
+
+func TestEmbeddedIamSetUserStatusTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	})
+	ctx := context.Background()
+
+	executeAction(t, api, url.Values{
+		"Action":   {"SetUserStatus"},
+		"UserName": {"alice"},
+		"Status":   {iamAccessKeyStatusInactive},
+	})
+	require.Equal(t, 0, rec.saves, "SetUserStatus must not SaveConfiguration")
+	require.Equal(t, 1, rec.updateUsers)
+
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.True(t, ident.Disabled)
+}
+
+func TestEmbeddedIamUpdateUserRenameTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
+		Groups: []*iam_pb.Group{
+			{Name: "devs", Members: []string{"alice", "bob"}},
+			{Name: "ops", Members: []string{"bob"}},
+		},
+	})
+	ctx := context.Background()
+	// Groups in the seed config must also be created in the store directly:
+	// the memory store only surfaces groups written via CreateGroup.
+	require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
+	require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "ops", Members: []string{"bob"}}))
+	require.NoError(t, api.credentialManager.PutUserInlinePolicy(ctx, "alice", "p1", mustParseIamTargetedTestPolicy(t)))
+
+	executeAction(t, api, url.Values{
+		"Action":      {"UpdateUser"},
+		"UserName":    {"alice"},
+		"NewUserName": {"alice-renamed"},
+	})
+	require.Equal(t, 0, rec.saves, "UpdateUser rename must not SaveConfiguration")
+
+	_, err := api.credentialManager.GetUser(ctx, "alice")
+	require.ErrorIs(t, err, credential.ErrUserNotFound)
+	_, err = api.credentialManager.GetUser(ctx, "alice-renamed")
+	require.NoError(t, err)
+
+	// Group membership and inline policies follow the rename.
+	g, err := api.credentialManager.GetGroup(ctx, "devs")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"alice-renamed", "bob"}, g.Members)
+	g, err = api.credentialManager.GetGroup(ctx, "ops")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"bob"}, g.Members)
+
+	policies, err := api.credentialManager.ListUserInlinePolicies(ctx, "alice-renamed")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"p1"}, policies)
+	policies, err = api.credentialManager.ListUserInlinePolicies(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, policies)
+}
+
+func TestEmbeddedIamDeleteUserTargeted(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
+		Groups:     []*iam_pb.Group{{Name: "devs", Members: []string{"alice", "bob"}}},
+	})
+	ctx := context.Background()
+	require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
+
+	executeAction(t, api, url.Values{
+		"Action":   {"DeleteUser"},
+		"UserName": {"alice"},
+	})
+	require.Equal(t, 0, rec.saves, "DeleteUser must not SaveConfiguration")
+	require.Equal(t, 1, rec.deleteUsers)
+	require.Equal(t, 1, rec.updateGroups)
+
+	_, err := api.credentialManager.GetUser(ctx, "alice")
+	require.ErrorIs(t, err, credential.ErrUserNotFound)
+	g, err := api.credentialManager.GetGroup(ctx, "devs")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"bob"}, g.Members)
+	_, err = api.credentialManager.GetUser(ctx, "bob")
+	require.NoError(t, err, "other identities must be untouched")
+}
+
+// TestEmbeddedIamTargetedSkipPersist: skipPersist must suppress the targeted
+// write exactly like it suppresses the full save.
+func TestEmbeddedIamTargetedSkipPersist(t *testing.T) {
+	api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{{Name: "alice"}},
+	})
+	ctx := context.Background()
+
+	for _, vals := range []url.Values{
+		{"Action": {"CreateAccessKey"}, "UserName": {"alice"}},
+		{"Action": {"SetUserStatus"}, "UserName": {"alice"}, "Status": {iamAccessKeyStatusInactive}},
+		{"Action": {"PutUserPolicy"}, "UserName": {"alice"}, "PolicyName": {"p"}, "PolicyDocument": {iamTargetedTestPolicy}},
+	} {
+		_, iamErr := api.ExecuteAction(ctx, vals, true /*skipPersist*/, "")
+		require.Nil(t, iamErr, "action %v", vals.Get("Action"))
+	}
+	assert.Equal(t, 0, rec.saves)
+	assert.Equal(t, 0, rec.createAccessKeys)
+	assert.Equal(t, 0, rec.updateUsers)
+
+	ident, err := api.credentialManager.GetUser(ctx, "alice")
+	require.NoError(t, err)
+	assert.Empty(t, ident.Credentials)
+	assert.False(t, ident.Disabled)
+}
+
+// TestEmbeddedIamStaticIdentityKeepsFullSave: static identities live outside
+// the store, so a change to one still takes the full-save path.
+func TestEmbeddedIamStaticIdentityKeepsFullSave(t *testing.T) {
+	api := NewEmbeddedIamApiForTest()
+	api.mockConfig = &iam_pb.S3ApiConfiguration{}
+	api.credentialManager.SetStaticIdentities([]*iam_pb.Identity{{Name: "static-admin"}})
+	rec := &recordingStore{MemoryStore: api.credentialManager.Store.(*memory.MemoryStore)}
+	api.credentialManager.Store = rec
+
+	// Seed via a read action; static identities are merged into the loaded config.
+	_, iamErr := api.ExecuteAction(context.Background(), url.Values{"Action": {"ListUsers"}}, false, "")
+	require.Nil(t, iamErr)
+	rec.saves = 0
+
+	executeAction(t, api, url.Values{
+		"Action":   {"SetUserStatus"},
+		"UserName": {"static-admin"},
+		"Status":   {iamAccessKeyStatusInactive},
+	})
+	assert.Equal(t, 1, rec.saves, "static identity changes keep the full-save path")
+	assert.Equal(t, 0, rec.updateUsers)
+}
+
+func mustParseIamTargetedTestPolicy(t *testing.T) (doc policy_engine.PolicyDocument) {
+	t.Helper()
+	require.NoError(t, json.Unmarshal([]byte(iamTargetedTestPolicy), &doc))
+	return doc
+}

weed/s3api/s3api_embedded_iam_test.go

@@ -372,13 +372,13 @@ func TestEmbeddedIamGetUserImplicitUsername(t *testing.T) {
 		t.Fatalf("failed to load iam config: %v", err)
 	}
 
-	// GetUser request with NO UserName, but signed by accessKey.
+	// GetUser request with NO UserName, carrying the identity authentication
+	// verified for accessKey, the way the middleware records it.
 	form := url.Values{}
 	form.Set("Action", "GetUser")
 	req, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode()))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+
-		"/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesig")
+	req = req.WithContext(SetIdentityNameInContext(req.Context(), "TestUser"))
 
 	out := iamGetUserResponse{}
 	rr, err := executeEmbeddedIamRequest(api, req, &out)
@@ -1578,29 +1578,15 @@ func TestEmbeddedIamDeleteAccessKey(t *testing.T) {
 	assert.Len(t, api.mockConfig.Identities[0].Credentials, 0)
 }
 
-// TestEmbeddedIamHandleImplicitUsername tests implicit username extraction from authorization header
+// TestEmbeddedIamHandleImplicitUsername checks that the implicit user name is
+// taken from the authenticated identity in the request context, never from
+// re-parsing the Authorization header.
 func TestEmbeddedIamHandleImplicitUsername(t *testing.T) {
-	// Create IAM with test credentials - the handleImplicitUsername function now looks
-	// up the username from the credential store based on AccessKeyId
-	// Note: Using obviously fake access keys to avoid secret scanner false positives
-	iam := &IdentityAccessManagement{}
-	testConfig := &iam_pb.S3ApiConfiguration{
-		Identities: []*iam_pb.Identity{
-			{
-				Name: "testuser1",
-				Credentials: []*iam_pb.Credential{
-					{AccessKey: UserAccessKeyPrefix + "TESTFAKEKEY000001", SecretKey: "testsecretfake"},
-				},
-			},
-		},
-	}
-	err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(testConfig))
-	if err != nil {
-		t.Fatalf("Failed to load test config: %v", err)
-	}
+	embeddedApi := &EmbeddedIamApi{}
 
-	embeddedApi := &EmbeddedIamApi{
-		iam: iam,
+	withIdentity := func(name string) *http.Request {
+		r := &http.Request{Header: http.Header{}}
+		return r.WithContext(SetIdentityNameInContext(r.Context(), name))
 	}
 
 	var tests = []struct {
@@ -1608,14 +1594,14 @@ func TestEmbeddedIamHandleImplicitUsername(t *testing.T) {
 		values   url.Values
 		userName string
 	}{
-		// No authorization header - should not set username
+		// No authenticated identity - should not set username
 		{&http.Request{}, url.Values{}, ""},
-		// Valid auth header with known access key - should look up and find "testuser1"
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"},
-		// Malformed auth header (no Credential=) - should not set username
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
-		// Unknown access key - should not set username
-		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
+		// Authenticated identity in context - sets its name
+		{withIdentity("testuser1"), url.Values{}, "testuser1"},
+		// A client-controlled Authorization header naming another user is ignored
+		{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
+		// An explicit UserName is left alone
+		{withIdentity("testuser1"), url.Values{"UserName": {"other"}}, "other"},
 	}
 
 	for i, test := range tests {

weed/s3api/s3api_iceberg_credentials.go

@@ -29,11 +29,12 @@ type VendedCredentials struct {
 	Expiration      time.Time
 }
 
-// VendTableCredentials mints credentials that can read and write exactly one
-// table's prefix, for a caller the Iceberg catalog has already authenticated
-// and authorized. Returns nil when no role is configured, which leaves the
-// catalog telling the client to keep using its own credentials.
-func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error) {
+// VendTableCredentials mints credentials that can read exactly one table's
+// prefix, and write it too unless readOnly, for a caller the Iceberg catalog
+// has already authenticated and authorized. Returns nil when no role is
+// configured, which leaves the catalog telling the client to keep using its
+// own credentials.
+func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) {
 	if s3a.icebergCredentialRole == "" {
 		return nil, nil
 	}
@@ -49,7 +50,7 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc
 		return nil, fmt.Errorf("STS is not configured")
 	}
 
-	policy, err := tablePrefixSessionPolicy(bucket, prefix)
+	policy, err := tablePrefixSessionPolicy(bucket, prefix, readOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -98,8 +99,9 @@ func (s3a *S3ApiServer) stsService() *sts.STSService {
 }
 
 // tablePrefixSessionPolicy narrows a session to one table's files plus the
-// bucket listing that clients need to resolve them.
-func tablePrefixSessionPolicy(bucket, prefix string) (string, error) {
+// bucket listing that clients need to resolve them. A read-only session drops
+// the write verbs: the catalog vends no more authority than the caller holds.
+func tablePrefixSessionPolicy(bucket, prefix string, readOnly bool) (string, error) {
 	prefix = strings.Trim(prefix, "/")
 	// A resource pattern is matched with wildcards, so a location carrying one
 	// would widen the session to sibling tables. Refuse rather than escape it:
@@ -114,12 +116,17 @@ func tablePrefixSessionPolicy(bucket, prefix string) (string, error) {
 		return "", fmt.Errorf("refusing to scope credentials to a whole bucket: s3://%s", bucket)
 	}
 
+	objectActions := []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"}
+	if readOnly {
+		objectActions = []string{"s3:GetObject"}
+	}
+
 	policy := map[string]interface{}{
 		"Version": "2012-10-17",
 		"Statement": []map[string]interface{}{
 			{
 				"Effect":   "Allow",
-				"Action":   []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"},
+				"Action":   objectActions,
 				"Resource": []string{fmt.Sprintf("arn:aws:s3:::%s/%s/*", bucket, prefix)},
 			},
 			{

weed/s3api/s3api_iceberg_credentials_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestTablePrefixSessionPolicyScopesToTheTable(t *testing.T) {
-	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
+	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
 	if err != nil {
 		t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
 	}
@@ -42,13 +42,43 @@ func TestTablePrefixSessionPolicyScopesToTheTable(t *testing.T) {
 	}
 }
 
+// A caller the catalog would not let commit gets a session that can read the
+// table's files and nothing more.
+func TestTablePrefixSessionPolicyReadOnly(t *testing.T) {
+	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", true)
+	if err != nil {
+		t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
+	}
+
+	var policy struct {
+		Statement []struct {
+			Action []string `json:"Action"`
+		} `json:"Statement"`
+	}
+	if err := json.Unmarshal([]byte(raw), &policy); err != nil {
+		t.Fatalf("policy is not valid JSON: %v", err)
+	}
+
+	for _, statement := range policy.Statement {
+		for _, action := range statement.Action {
+			switch action {
+			case "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts":
+				t.Errorf("read-only session policy grants %s", action)
+			}
+		}
+	}
+	if got := policy.Statement[0].Action; len(got) != 1 || got[0] != "s3:GetObject" {
+		t.Errorf("object actions = %v, want [s3:GetObject]", got)
+	}
+}
+
 // A table at the bucket root has no prefix to scope to, and vending would hand
 // out read and write over every other table in the bucket.
 func TestTablePrefixSessionPolicyRefusesAWholeBucket(t *testing.T) {
-	if _, err := tablePrefixSessionPolicy("warehouse", ""); err == nil {
+	if _, err := tablePrefixSessionPolicy("warehouse", "", false); err == nil {
 		t.Error("tablePrefixSessionPolicy() error = nil, want a refusal for an empty prefix")
 	}
-	if _, err := tablePrefixSessionPolicy("warehouse", "/"); err == nil {
+	if _, err := tablePrefixSessionPolicy("warehouse", "/", false); err == nil {
 		t.Error("a prefix of only separators was accepted")
 	}
 }
@@ -56,7 +86,7 @@ func TestTablePrefixSessionPolicyRefusesAWholeBucket(t *testing.T) {
 // GetBucketLocation carries no prefix, so conditioning it would deny location
 // discovery to every vended credential.
 func TestTablePrefixSessionPolicyLeavesBucketLocationUnconditioned(t *testing.T) {
-	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
+	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
 	if err != nil {
 		t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
 	}
@@ -101,7 +131,7 @@ func TestIcebergSessionNameIsBounded(t *testing.T) {
 // telling clients to use their own credentials.
 func TestVendTableCredentialsDisabledWithoutRole(t *testing.T) {
 	s3a := &S3ApiServer{}
-	credentials, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t")
+	credentials, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t", false)
 	if err != nil {
 		t.Fatalf("VendTableCredentials() error = %v, want nil", err)
 	}
@@ -114,7 +144,7 @@ func TestVendTableCredentialsNeedsSTS(t *testing.T) {
 	s3a := &S3ApiServer{}
 	s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600)
 
-	if _, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t"); err == nil {
+	if _, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t", false); err == nil {
 		t.Error("VendTableCredentials() error = nil, want an error when STS is not configured")
 	}
 }
@@ -122,7 +152,7 @@ func TestVendTableCredentialsNeedsSTS(t *testing.T) {
 // Listing is granted on the bucket, so the prefix condition is what keeps a
 // vended credential from enumerating other tables.
 func TestTablePrefixSessionPolicyRestrictsListing(t *testing.T) {
-	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
+	raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
 	if err != nil {
 		t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
 	}
@@ -168,11 +198,11 @@ func TestTablePrefixSessionPolicyRestrictsListing(t *testing.T) {
 // tables, so it is refused instead of vended.
 func TestTablePrefixSessionPolicyRefusesWildcards(t *testing.T) {
 	for _, prefix := range []string{"analytics/*", "analytics/ev?nts", "*"} {
-		if _, err := tablePrefixSessionPolicy("warehouse", prefix); err == nil {
+		if _, err := tablePrefixSessionPolicy("warehouse", prefix, false); err == nil {
 			t.Errorf("tablePrefixSessionPolicy(%q) error = nil, want a refusal", prefix)
 		}
 	}
-	if _, err := tablePrefixSessionPolicy("ware*house", "analytics/events"); err == nil {
+	if _, err := tablePrefixSessionPolicy("ware*house", "analytics/events", false); err == nil {
 		t.Error("a wildcard in the bucket name was accepted")
 	}
 }

weed/s3api/s3api_object_anonymous_cache_test.go

@@ -174,7 +174,7 @@ func TestAnonymousObjectACLVersionedEmptyDirectoryMarker(t *testing.T) {
 					require.Equal(t, "null", rr.Header().Get("x-amz-version-id"))
 				}
 				rr := serveAnonymousRead(s3a, method, "folder", "", nil)
-				require.Equal(t, 404, rr.Code, "the bare prefix is still not an object")
+				require.Equal(t, 403, rr.Code, "the bare prefix is not an object, and the miss stays masked for a non-listable bucket")
 				marker.Extended[s3_constants.ExtAmzAclKey] = []byte(`[]`)
 				rr = serveAnonymousRead(s3a, method, "folder/", "", nil)
 				require.Equal(t, 403, rr.Code, rr.Body.String())
@@ -238,8 +238,12 @@ func TestAnonymousObjectACLVersionedDirectoryIdentity(t *testing.T) {
 							f.entries["/buckets/b/folder/.versions"] = &filer_pb.Entry{IsDirectory: true, Extended: map[string][]byte{s3_constants.ExtLatestVersionIdKey: []byte("")}}
 						}
 						// Equivalent filer-path spellings must not recover a different object's data.
-						for _, object := range []string{"folder/", "folder//", "folder\\"} {
-							assertMissing(serveAnonymousRead(s3a, method, object, route.query, headers))
+						assertMissing(serveAnonymousRead(s3a, method, "folder/", route.query, headers))
+						// Spellings the policy's folder/ grant does not cover are
+						// masked: a miss must be indistinguishable from a denied read.
+						for _, object := range []string{"folder//", "folder\\"} {
+							rr := serveAnonymousRead(s3a, method, object, route.query, headers)
+							require.Equal(t, http.StatusForbidden, rr.Code, rr.Body.String())
 						}
 						if kind == "regular file" || kind == "prefix object" || kind == "empty prefix object" {
 							// A slash-only policy must not authorize data belonging to the bare key.

weed/s3api/s3api_object_anonymous_read.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
@@ -110,3 +112,70 @@ func (s3a *S3ApiServer) authorizeAnonymousObjectRead(r *http.Request, bucket, ob
 	}
 	return s3err.ErrAccessDenied
 }
+
+// authorizeAnonymousBucketList reports whether an anonymous caller may learn
+// which keys exist in the bucket: bucket listing via policy or identity, or a
+// public-read bucket. Anything else masks the miss as AccessDenied.
+func (s3a *S3ApiServer) authorizeAnonymousBucketList(r *http.Request, bucket string) s3err.ErrorCode {
+	identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity)
+	object := mux.Vars(r)["object"]
+	// The caller's query parameters would otherwise retarget these checks
+	// (?policy resolves to GetBucketPolicy) or satisfy a prefix condition on an
+	// unrelated subtree, so each runs on a request with a fixed shape: a
+	// ListBucket scoped to the probed key, and a plain read of the key itself.
+	// A caller that could learn the key's existence either way — by listing or
+	// by getting it — gets the real miss.
+	listReq := r.Clone(r.Context())
+	listReq.URL = &url.URL{Path: "/" + bucket, RawQuery: url.Values{"prefix": {object}}.Encode()}
+	// versionId must survive: a read of a specific version resolves to
+	// s3:GetObjectVersion, and dropping it would let a GetObject grant answer
+	// for versions the caller may not read.
+	readQuery := url.Values{}
+	if v := r.URL.Query().Get("versionId"); v != "" {
+		readQuery.Set("versionId", v)
+	}
+	readReq := r.Clone(r.Context())
+	readReq.URL = &url.URL{Path: "/" + bucket + "/" + object, RawQuery: readQuery.Encode()}
+
+	principal := buildPrincipalARN(identity, r)
+	policyAllowed := false
+	code, allowed := s3a.checkPolicyWithEntry(listReq, bucket, "", string(s3_constants.ACTION_LIST), principal, nil)
+	if code != s3err.ErrNone {
+		return code
+	}
+	policyAllowed = allowed
+	if !policyAllowed {
+		code, allowed = s3a.checkPolicyWithEntry(readReq, bucket, object, string(s3_constants.ACTION_READ), principal, nil)
+		if code != s3err.ErrNone {
+			return code
+		}
+		policyAllowed = allowed
+	}
+	if s3a.iam.isActionExplicitlyDeniedByApplicablePolicies(listReq, identity, s3_constants.ACTION_LIST, bucket, "") ||
+		s3a.iam.isActionExplicitlyDeniedByApplicablePolicies(readReq, identity, s3_constants.ACTION_READ, bucket, object) {
+		return s3err.ErrAccessDenied
+	}
+	if policyAllowed {
+		return s3err.ErrNone
+	}
+	if identity != nil &&
+		(s3a.iam.VerifyActionPermission(listReq, identity, s3_constants.ACTION_LIST, bucket, "") == s3err.ErrNone ||
+			s3a.iam.VerifyActionPermission(readReq, identity, s3_constants.ACTION_READ, bucket, object) == s3err.ErrNone) {
+		return s3err.ErrNone
+	}
+	if config, code := s3a.getBucketConfig(bucket); code == s3err.ErrNone && config.IsPublicRead {
+		return s3err.ErrNone
+	}
+	return s3err.ErrAccessDenied
+}
+
+// writeAnonymousAwareNotFound keeps missing-bucket and missing-key misses
+// indistinguishable from denied reads for anonymous callers: a 404 only
+// reaches the wire when the bucket is anonymously listable.
+func (s3a *S3ApiServer) writeAnonymousAwareNotFound(w http.ResponseWriter, r *http.Request, bucket string, code s3err.ErrorCode) {
+	if isAnonymousObjectRead(r) && s3a.authorizeAnonymousBucketList(r, bucket) != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
+		return
+	}
+	s3err.WriteErrorResponse(w, r, code)
+}

weed/s3api/s3api_object_handlers.go

@@ -663,7 +663,7 @@ func (s3a *S3ApiServer) handleDirectoryObjectRequest(w http.ResponseWriter, r *h
 	} else if isDirectoryObject {
 		// Directory object but doesn't exist
 		glog.V(2).Infof("%s: directory object %s/%s not found", handlerName, bucket, object)
-		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+		s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 		return true // Request was handled (with not found)
 	}
 
@@ -770,7 +770,11 @@ func (s3a *S3ApiServer) processConditionalHeaders(w http.ResponseWriter, r *http
 			w.Header().Set("ETag", result.ETag)
 		}
 
-		s3err.WriteErrorResponse(w, r, result.ErrorCode)
+		if result.ErrorCode == s3err.ErrNoSuchKey || result.ErrorCode == s3err.ErrNoSuchBucket {
+			s3a.writeAnonymousAwareNotFound(w, r, bucket, result.ErrorCode)
+		} else {
+			s3err.WriteErrorResponse(w, r, result.ErrorCode)
+		}
 		return result, true // request handled
 	}
 	return result, false // request not handled
@@ -841,7 +845,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	versioningConfigured, err = s3a.isVersioningConfigured(bucket)
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
-			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchBucket)
 			return
 		}
 		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
@@ -860,7 +864,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 			if err != nil {
 				glog.Errorf("Failed to get specific version %s: %v", versionId, err)
 				if errors.Is(err, filer_pb.ErrNotFound) {
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchVersion)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchVersion)
 				} else {
 					s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 				}
@@ -886,7 +890,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 				entry, err = s3a.getLatestObjectVersion(bucket, object)
 				if err != nil {
 					glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			} else if errors.Is(versionsErr, filer_pb.ErrNotFound) {
@@ -900,7 +904,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 				} else {
 					// No object at regular path either - object doesn't exist
 					glog.V(3).Infof("GetObject: object not found at regular path or .versions for %s/%s", bucket, object)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			} else {
@@ -909,7 +913,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 				entry, err = s3a.getLatestObjectVersion(bucket, object)
 				if err != nil {
 					glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			}
@@ -930,7 +934,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 		// Check if this is a delete marker
 		if entry.Extended != nil {
 			if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
-				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 				return
 			}
 		}
@@ -975,7 +979,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 				return
 			}
 			if objectEntryForSSE == nil {
-				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 				return
 			}
 		}
@@ -992,7 +996,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	// A bare directory path is not an object: 404 like AWS instead of an empty 200,
 	// so Hadoop-style clients fall back to LIST discovery. HEAD and ranged GET already 404.
 	if !strings.HasSuffix(object, "/") && isBareDirectory(objectEntryForSSE) {
-		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+		s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 		return
 	}
 
@@ -2518,7 +2522,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	versioningConfigured, err = s3a.isVersioningConfigured(bucket)
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
-			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchBucket)
 			return
 		}
 		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
@@ -2536,7 +2540,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 			if err != nil {
 				glog.Errorf("Failed to get specific version %s for %s/%s: %v", versionId, bucket, object, err)
 				if errors.Is(err, filer_pb.ErrNotFound) {
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchVersion)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchVersion)
 				} else {
 					s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 				}
@@ -2562,7 +2566,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 				entry, err = s3a.getLatestObjectVersion(bucket, object)
 				if err != nil {
 					glog.Errorf("HeadObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			} else if errors.Is(versionsErr, filer_pb.ErrNotFound) {
@@ -2576,7 +2580,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 				} else {
 					// No object at regular path either - object doesn't exist
 					glog.V(3).Infof("HeadObject: object not found at regular path or .versions for %s/%s", bucket, object)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			} else {
@@ -2585,7 +2589,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 				entry, err = s3a.getLatestObjectVersion(bucket, object)
 				if err != nil {
 					glog.Errorf("HeadObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			}
@@ -2606,7 +2610,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 		// Check if this is a delete marker
 		if entry.Extended != nil {
 			if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
-				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 				return
 			}
 		}
@@ -2644,7 +2648,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 				return
 			}
 			if objectEntryForSSE == nil {
-				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 				return
 			}
 		}
@@ -2710,7 +2714,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 			isZeroByteFile := objectEntryForSSE.Attributes.FileSize == 0 && !objectEntryForSSE.IsDirectory
 			// A directory with data (a promoted file) is retrievable; empty directories 404 for LIST fallback.
 			if isBareDirectory(objectEntryForSSE) {
-				s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+				s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 				return
 			}
 			if isZeroByteFile {
@@ -2718,7 +2722,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 				if s3a.hasChildren(r.Context(), bucket, object) {
 					// This is an implicit directory with children
 					// Return 404 to force clients (like s3fs) to use LIST-based discovery
-					s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
+					s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
 					return
 				}
 			}

weed/s3api/s3api_object_handlers_conditional_write_test.go

@@ -0,0 +1,58 @@
+package s3api
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// A conditional write with If-Match has nothing to compare against when the key has
+// no current version; AWS answers 404 there, not 412.
+func TestValidateConditionalHeadersMissingObject(t *testing.T) {
+	s3a := &S3ApiServer{}
+
+	existing := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
+		Extended:   map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
+	}
+	deleteMarker := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
+		Extended:   map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
+	}
+
+	testCases := []struct {
+		name   string
+		header string
+		value  string
+		entry  *filer_pb.Entry
+		want   s3err.ErrorCode
+	}{
+		{"if-match on missing object", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", nil, s3err.ErrNoSuchKey},
+		{"if-match star on missing object", s3_constants.IfMatch, "*", nil, s3err.ErrNoSuchKey},
+		{"if-match on delete marker", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", deleteMarker, s3err.ErrNoSuchKey},
+		{"if-match mismatch on existing object", s3_constants.IfMatch, "0000", existing, s3err.ErrPreconditionFailed},
+		{"if-match hit on existing object", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", existing, s3err.ErrNone},
+		{"if-none-match star on missing object", s3_constants.IfNoneMatch, "*", nil, s3err.ErrNone},
+		{"if-none-match star on delete marker", s3_constants.IfNoneMatch, "*", deleteMarker, s3err.ErrNone},
+		{"if-none-match star on existing object", s3_constants.IfNoneMatch, "*", existing, s3err.ErrPreconditionFailed},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := httptest.NewRequest(http.MethodPut, "/bucket/object", nil)
+			r.Header.Set(tc.header, tc.value)
+			headers, errCode := parseConditionalHeaders(r)
+			if errCode != s3err.ErrNone {
+				t.Fatalf("parseConditionalHeaders: %v", errCode)
+			}
+			if got := s3a.validateConditionalHeaders(r, headers, tc.entry, "bucket", "object"); got != tc.want {
+				t.Errorf("got %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

weed/s3api/s3api_object_handlers_list.go

@@ -19,6 +19,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 type OptionalString struct {
@@ -565,6 +566,8 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
 					// plain key. The traversal never descends into it under a delimiter,
 					// so the CommonPrefix its nested keys fold into is added here too.
 					if entry.IsPrefixObject() {
+						keyOnly := cursor.prefixObjectKeyOnly
+						cursor.prefixObjectKeyOnly = false
 						key := fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):]
 						if strings.HasPrefix(key, originalPrefix) {
 							if folded := prefixForKey(dir, entry.Name); folded != "" {
@@ -579,8 +582,10 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
 						// whole, so a page ending between them cannot resume at the prefix:
 						// a marker of "<key>/" means the subtree is done, not that it is
 						// next. The page runs one item over maxKeys instead of dropping it.
-						if childPrefix := prefixForKey(dir, entry.Name+"/"); childPrefix != "" && s3a.hasChildren(ctx, bucket, key) {
-							addCommonPrefix(childPrefix)
+						if !keyOnly {
+							if childPrefix := prefixForKey(dir, entry.Name+"/"); childPrefix != "" && s3a.hasChildren(ctx, bucket, key) {
+								addCommonPrefix(childPrefix)
+							}
 						}
 						return
 					}
@@ -804,6 +809,10 @@ type ListingCursor struct {
 	// emitDirAsCommonPrefix marks the next directory entry passed to the entry
 	// callback for reporting as a CommonPrefix rather than a Contents key.
 	emitDirAsCommonPrefix bool
+	// prefixObjectKeyOnly limits the next prefix-object entry passed to the entry
+	// callback to its plain key; its "<name>/" CommonPrefix is deferred to the
+	// directory's own block position.
+	prefixObjectKeyOnly bool
 }
 
 // excludedMarkerKey returns the key an exclusive marker names that the walk's cutoff no
@@ -950,35 +959,20 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 		return
 	}
 
-	// markerParentWalked tracks whether marker now names a directory whose
-	// subtree this call already walked: after the descent below it is the
-	// parent directory, and seeing its entry again must not re-list it.
-	markerParentWalked := false
-	if strings.Contains(marker, "/") {
-		subDir, subMarker := toParentAndDescendants(marker)
-		// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker)
-		subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + subDir, marker: subMarker, delimiter: delimiter, bucket: bucket}, cursor, eachEntryFn)
-		if subErr != nil {
-			err = subErr
-			return
+	// A directory's keys sort at "<name>/" among this level's children, not at
+	// the name itself: siblings like "a-c" and "a.txt" sort before "a/b" while
+	// "a0" sorts after. Start the window at the first child whose block can
+	// still hold keys past the marker — the shortest prefix of the marker
+	// followed by a byte no greater than "/" — and ask for it inclusively, so
+	// a marker-named directory's subtree is reached at all.
+	startName := marker
+	for i := 0; i < len(marker); i++ {
+		if marker[i] <= '/' {
+			startName = marker[:i]
+			break
 		}
-		nextMarker = subDir + "/" + subNextMarker
-		// finished processing this subdirectory
-		marker = subDir
-		markerParentWalked = true
-	}
-	if cursor.isTruncated {
-		return
-	}
-
-	// Ask for the marker's own entry: an exclusive window never delivers the
-	// directory a marker names, and its children sort after the marker key.
-	// Delimited listings keep exclusive windows — a marker equal to a
-	// directory there is the resume position of an already-emitted common
-	// prefix, not a subtree to walk.
-	if marker != "" && delimiter == "" {
-		inclusiveStartFrom = true
 	}
+	inclusiveStartFrom = startName != ""
 
 	// now marker is also a direct child of dir
 	ctx, cancel := context.WithCancel(ctx)
@@ -988,6 +982,152 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 	// each request window inside the page.
 	pageMarker := marker
 
+	// Defer each directory until a child past its "<name>/" block arrives, so
+	// emission follows S3 byte order rather than the filer's name order.
+	type pendingDir struct {
+		e                *filer_pb.Entry
+		matchedPrefixDir bool
+	}
+	var pendingDirs []pendingDir
+	stopped := false
+	// hasEntries reports whether the filer directory holds at least one entry,
+	// probed on the request's own client; errors answer conservatively as true.
+	hasEntries := func(path string) bool {
+		probeCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		stream, lerr := client.ListEntries(probeCtx, &filer_pb.ListEntriesRequest{Directory: path, Limit: 1})
+		if lerr != nil {
+			return true
+		}
+		_, recvErr := stream.Recv()
+		return recvErr == nil
+	}
+	emitDir := func(e *filer_pb.Entry, matchedPrefixDir bool) error {
+		blockKey := e.Name + "/"
+		var subMarker string
+		if strings.HasPrefix(marker, blockKey) {
+			if delimiter == "/" {
+				// this block collapses to one common prefix, already emitted
+				return nil
+			}
+			subMarker = marker[len(blockKey):]
+		} else if blockKey <= marker {
+			return nil
+		}
+		// A prefix object's plain key is emitted at the name position or not at
+		// all; the block emits only its "<name>/" fold, so the callback sees a
+		// plain directory.
+		emitEntry := e
+		if e.IsPrefixObject() {
+			emitEntry = proto.Clone(e).(*filer_pb.Entry)
+			delete(emitEntry.Extended, s3_constants.SeaweedFSPrefixObject)
+		}
+		isKeyObject := e.IsDirectoryKeyObject() && !e.IsPrefixObject()
+		if delimiter != "/" || cursor.prefixEndsOnDelimiter {
+			// A trailing-slash prefix (e.g. "logs/") names one directory and asks
+			// whether it exists, so a real but empty directory must be reported for
+			// that probe.
+			explicitDirProbe := cursor.prefixEndsOnDelimiter
+			if cursor.prefixEndsOnDelimiter {
+				cursor.prefixEndsOnDelimiter = false
+			}
+			if isKeyObject && blockKey > marker {
+				// Directory key objects (created via PutObject with trailing "/")
+				// appear as regular keys in recursive listing mode.
+				eachEntryFn(dir, emitEntry)
+			}
+			// Recurse into subdirectory to list any children, noting whether the
+			// subtree produced any entries.
+			childEmitted := false
+			subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + e.Name, marker: subMarker, delimiter: delimiter, bucket: bucket}, cursor, func(d string, ent *filer_pb.Entry) {
+				childEmitted = true
+				eachEntryFn(d, ent)
+			})
+			if subErr != nil {
+				return fmt.Errorf("doListFilerEntries2: %w", subErr)
+			}
+			nextMarker = e.Name + "/" + subNextMarker
+			if cursor.maxKeys <= 0 && subNextMarker == "" && hasEntries(dir+"/"+e.Name) {
+				// The recursion starved on quota before listing the subtree;
+				// keep a resume position that re-enters it.
+				cursor.isTruncated = true
+			}
+			// A real but empty directory (created out of band via mount, mkdir or
+			// the filer API, so it carries no MIME) is otherwise invisible to S3
+			// clients that detect directories by listing it under its own "<dir>/"
+			// prefix. Answer that explicit probe so tools like hadoop-aws can find
+			// the directory: a delimited listing reports it as a CommonPrefix --
+			// a real key in Contents would look like an object a client can read --
+			// while a recursive listing keeps the directory marker key it used.
+			// Plain listings are left untouched, so empty directories left behind
+			// by deleted objects are not shown as keys.
+			// A directory that still holds version history no longer names anything,
+			// so it gets no marker either.
+			if explicitDirProbe && !isKeyObject && !childEmitted && !cursor.isTruncated && e.Attributes != nil &&
+				!s3a.dirHoldsOnlyHiddenEntries(ctx, client, bucket, dir+"/"+e.Name, cursor) {
+				if delimiter == "/" {
+					cursor.emitDirAsCommonPrefix = true
+				} else {
+					emitEntry.Attributes.Mime = s3_constants.FolderMimeType
+				}
+				eachEntryFn(dir, emitEntry)
+			}
+			// println("doListFilerEntries2 dir", dir+"/"+e.Name, "subNextMarker", subNextMarker)
+			if cursor.isTruncated || matchedPrefixDir {
+				stopped = true
+			}
+			// println("doListFilerEntries2 nextMarker", nextMarker)
+		} else {
+			switch {
+			case isKeyObject && blockKey > marker:
+				eachEntryFn(dir, emitEntry)
+			case e.IsPrefixObject():
+				// The plain key already listed at the name position; the
+				// deferred block emits the "<name>/" common prefix only when
+				// children fold under it.
+				if hasEntries(dir + "/" + e.Name) {
+					eachEntryFn(dir, emitEntry)
+				}
+			case !s3a.dirHoldsOnlyHiddenEntries(ctx, client, bucket, dir+"/"+e.Name, cursor):
+				eachEntryFn(dir, emitEntry)
+			}
+			// Whatever the block emitted sits at its "<name>/" position, so the
+			// resume marker is the block boundary: the bare name sorts inside the
+			// (name, name/) band and would reopen the block for a second emission.
+			nextMarker = e.Name + "/"
+		}
+		return nil
+	}
+	// pendingDirs is kept ordered by "<name>/" rather than by name: "a-c/" and
+	// "a./" sort before "a/" even though "a" is the smaller name.
+	flushDirs := func(upTo string) error {
+		for len(pendingDirs) > 0 && (upTo == "" || pendingDirs[0].e.Name+"/" < upTo) {
+			if cursor.maxKeys <= 0 {
+				cursor.isTruncated = true
+				return nil
+			}
+			d := pendingDirs[0]
+			pendingDirs = pendingDirs[1:]
+			if err := emitDir(d.e, d.matchedPrefixDir); err != nil {
+				return err
+			}
+			if stopped || cursor.isTruncated {
+				return nil
+			}
+		}
+		return nil
+	}
+	queueDir := func(pd pendingDir) {
+		blockKey := pd.e.Name + "/"
+		i := 0
+		for i < len(pendingDirs) && pendingDirs[i].e.Name+"/" < blockKey {
+			i++
+		}
+		pendingDirs = append(pendingDirs, pendingDir{})
+		copy(pendingDirs[i+1:], pendingDirs[i:])
+		pendingDirs[i] = pd
+	}
+
 	// Entries that emit nothing (empty directories, the .uploads folder, the marker
 	// echo) consume the request window without consuming maxKeys, so one window may
 	// end before maxKeys is satisfied. Keep requesting from the last received entry
@@ -997,7 +1137,7 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 			Directory:          dir,
 			Prefix:             prefix,
 			Limit:              uint32(cursor.maxKeys) + 2, // bucket root directory needs to skip additional s3_constants.MultipartUploadsFolder folder
-			StartFromFileName:  marker,
+			StartFromFileName:  startName,
 			InclusiveStartFrom: inclusiveStartFrom,
 		}
 
@@ -1028,47 +1168,21 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 			}
 			entriesReceived++
 			lastEntryName = entry.Name
-			// listFilerEntries always calls doListFilerEntries with inclusiveStartFrom=false
-			// (S3 marker semantics are exclusive), but keep the guard explicit to preserve
-			// behavior if inclusive callers are introduced in the future.
-			// A versioned object lives in a "<key>.versions" directory, so the marker also
-			// has to be matched against the object name that directory stands for.
-			markerName := entry.Name
-			if entry.IsDirectory {
-				markerName = strings.TrimSuffix(markerName, s3_constants.VersionsFolder)
+			// A directory whose block sorts before this entry emits now — every
+			// key it yields still sorts before this entry's name.
+			if err = flushDirs(entry.Name); err != nil || stopped || cursor.isTruncated {
+				return
 			}
-			if marker != "" && (entry.Name == marker || markerName == marker) {
-				// The marker excludes only the key it names. A directory under
-				// that name sorts its children after the marker, so a recursive
-				// listing still walks the subtree — from the start. A directory
-				// key object lists as "<marker>/" — a different key that also
-				// sorts after — so it is emitted; a prefix object would repeat
-				// the marker itself and is not. Internal folders keep their
-				// exclusion, .versions dirs fold into the marker key, and a
-				// delimited listing keeps skipping — the same marker resumes an
-				// already-emitted common prefix there.
-				walkableMarkerDir := delimiter == "" && entry.IsDirectory && entry.Name == marker &&
-					!markerParentWalked && !strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) &&
-					entry.Name != s3_constants.MultipartUploadsFolder
-				if !walkableMarkerDir {
+			// A directory block still contributes past a marker that lands inside
+			// its "<name>/" band; only a block entirely behind it is skipped.
+			if marker != "" {
+				if entry.IsDirectory {
+					if entry.Name+"/" <= marker && !strings.HasPrefix(marker, entry.Name+"/") {
+						continue
+					}
+				} else if entry.Name <= marker {
 					continue
 				}
-				if entry.IsDirectoryKeyObject() && !entry.IsPrefixObject() {
-					eachEntryFn(dir, entry)
-					if cursor.isTruncated {
-						return
-					}
-				}
-				subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + entry.Name, delimiter: delimiter, bucket: bucket}, cursor, eachEntryFn)
-				if subErr != nil {
-					err = fmt.Errorf("doListFilerEntries: %w", subErr)
-					return
-				}
-				nextMarker = entry.Name + "/" + subNextMarker
-				if cursor.isTruncated {
-					return
-				}
-				continue
 			}
 
 			// The .versions sibling of the key just emitted still decides that key's
@@ -1105,6 +1219,25 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 			}
 			if entry.IsDirectory {
 				// glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
+				// A marker inside a skipped subtree still descends — the
+				// internal folder is not listed, its contents may be.
+				if strings.HasPrefix(marker, entry.Name+"/") {
+					if delimiter == "/" {
+						// The whole block already collapsed into its common
+						// prefix; nothing under it is listed again.
+						continue
+					}
+					subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + entry.Name, marker: marker[len(entry.Name)+1:], delimiter: delimiter, bucket: bucket}, cursor, eachEntryFn)
+					if subErr != nil {
+						err = fmt.Errorf("doListFilerEntries: %w", subErr)
+						return
+					}
+					nextMarker = entry.Name + "/" + subNextMarker
+					if cursor.isTruncated {
+						return
+					}
+					continue
+				}
 				if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
 					continue
 				}
@@ -1119,6 +1252,10 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 					}
 					// Extract object name from .versions directory name
 					baseObjectName := strings.TrimSuffix(entry.Name, s3_constants.VersionsFolder)
+					// A marker at or past the folded key already covered it.
+					if marker != "" && baseObjectName <= marker {
+						continue
+					}
 					// A page resuming from a marker inside the base key's extension
 					// region ("k.bak" sorts between "k" and "k.versions") means an
 					// earlier page already listed and settled the base null object;
@@ -1155,76 +1292,32 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.
 					continue
 				}
 
-				if delimiter != "/" || cursor.prefixEndsOnDelimiter {
-					// A trailing-slash prefix (e.g. "logs/") names one directory and asks
-					// whether it exists, so a real but empty directory must be reported for
-					// that probe.
-					explicitDirProbe := cursor.prefixEndsOnDelimiter
-					if cursor.prefixEndsOnDelimiter {
-						cursor.prefixEndsOnDelimiter = false
-					}
-					isKeyObject := entry.IsDirectoryKeyObject()
-					if isKeyObject {
-						// Directory key objects (created via PutObject with trailing "/")
-						// must appear as regular keys in recursive listing mode.
-						eachEntryFn(dir, entry)
-					}
-					// Recurse into subdirectory to list any children, noting whether the
-					// subtree produced any entries.
-					childEmitted := false
-					subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + entry.Name, delimiter: delimiter, bucket: bucket}, cursor, func(d string, e *filer_pb.Entry) {
-						childEmitted = true
-						eachEntryFn(d, e)
-					})
-					if subErr != nil {
-						err = fmt.Errorf("doListFilerEntries2: %w", subErr)
-						return
-					}
-					// A real but empty directory (created out of band via mount, mkdir or
-					// the filer API, so it carries no MIME) is otherwise invisible to S3
-					// clients that detect directories by listing it under its own "<dir>/"
-					// prefix. Answer that explicit probe so tools like hadoop-aws can find
-					// the directory: a delimited listing reports it as a CommonPrefix --
-					// a real key in Contents would look like an object a client can read --
-					// while a recursive listing keeps the directory marker key it used.
-					// Plain listings are left untouched, so empty directories left behind
-					// by deleted objects are not shown as keys.
-					// A directory that still holds version history no longer names anything,
-					// so it gets no marker either.
-					if explicitDirProbe && !isKeyObject && !childEmitted && !cursor.isTruncated && entry.Attributes != nil &&
-						!s3a.dirHoldsOnlyHiddenEntries(ctx, client, bucket, dir+"/"+entry.Name, cursor) {
-						if delimiter == "/" {
-							cursor.emitDirAsCommonPrefix = true
-						} else {
-							entry.Attributes.Mime = s3_constants.FolderMimeType
-						}
-						eachEntryFn(dir, entry)
-					}
-					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "subNextMarker", subNextMarker)
-					nextMarker = entry.Name + "/" + subNextMarker
-					if cursor.isTruncated {
-						return
-					}
-					if matchedPrefixDir {
-						return
-					}
-					// println("doListFilerEntries2 nextMarker", nextMarker)
-				} else if entry.IsDirectoryKeyObject() || !s3a.dirHoldsOnlyHiddenEntries(ctx, client, bucket, dir+"/"+entry.Name, cursor) {
+				pd := pendingDir{e: entry, matchedPrefixDir: matchedPrefixDir}
+				// A prefix object lists as the plain key "<name>", which sorts
+				// at the name, not at the "<name>/" block its children occupy; the
+				// deferred block emits the common prefix at its own position.
+				if entry.IsPrefixObject() && entry.Name > marker {
+					cursor.prefixObjectKeyOnly = true
 					eachEntryFn(dir, entry)
 				}
-			} else {
-				eachEntryFn(dir, entry)
-				// glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
+				queueDir(pd)
+				continue
 			}
+			eachEntryFn(dir, entry)
+			// glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
 			if cursor.prefixEndsOnDelimiter {
 				cursor.prefixEndsOnDelimiter = false
 			}
 		}
 
-		if cursor.isTruncated || entriesReceived < request.Limit {
+		if cursor.isTruncated || stopped {
 			return
 		}
-		marker = lastEntryName
+		if entriesReceived < request.Limit {
+			err = flushDirs("")
+			return
+		}
+		startName = lastEntryName
 		inclusiveStartFrom = false
 	}
 }
@@ -1482,6 +1575,15 @@ func adjustMarkerForDelimiter(marker, prefix, delimiter string) string {
 		return marker
 	}
 
+	// A marker ending on "/" already names the bound past that directory's
+	// block — the walk reads "boo/" as "the boo/ block is done". Trimming it
+	// to the bare name would sort inside the (name, name/) band instead and
+	// reopen the block. Other delimiters have no matching directory boundary,
+	// so their markers still need the trim below.
+	if delimiter == "/" {
+		return marker
+	}
+
 	// Remove the trailing delimiter
 	// This ensures we skip all entries under the prefix but don't skip
 	// potential directory entries that start with a similar prefix

weed/s3api/s3api_object_handlers_list_test.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"io"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/stretchr/testify/assert"
 	grpc "google.golang.org/grpc"
@@ -98,18 +100,31 @@ func (c *markerEchoFilerClient) ListEntries(ctx context.Context, in *filer_pb.Li
 		return &testListEntriesStream{entries: entries}, nil
 	}
 
-	for i, e := range entries {
-		if e.Name == in.StartFromFileName {
-			// Emulate buggy backend behavior: return marker again even when exclusive.
-			if c.returnFollowing {
-				echoAndFollowing := entries[i:]
-				return &testListEntriesStream{entries: echoAndFollowing}, nil
-			}
-			return &testListEntriesStream{entries: []*filer_pb.Entry{e}}, nil
+	filtered := make([]*filer_pb.Entry, 0, len(entries))
+	for _, e := range entries {
+		if e.Name > in.StartFromFileName || (in.InclusiveStartFrom && e.Name == in.StartFromFileName) {
+			filtered = append(filtered, e)
 		}
 	}
-
-	return &testListEntriesStream{entries: nil}, nil
+	// Emulate buggy backend behavior: echo the start entry again even when the
+	// request did not ask for it.
+	if !in.InclusiveStartFrom {
+		var echo *filer_pb.Entry
+		for _, e := range entries {
+			if e.Name == in.StartFromFileName {
+				echo = e
+				break
+			}
+		}
+		if echo != nil {
+			if c.returnFollowing {
+				filtered = append([]*filer_pb.Entry{echo}, filtered...)
+			} else {
+				filtered = []*filer_pb.Entry{echo}
+			}
+		}
+	}
+	return &testListEntriesStream{entries: filtered}, nil
 }
 
 func ensureEntryAttributes(entries []*filer_pb.Entry) {
@@ -1143,3 +1158,323 @@ func TestListObjectsV2_PrefixEndingWithSlash_WithDelimiter(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []string{"fileA"}, results, "Should only return files under directory '1', not '1000'")
 }
+
+// A marker landing between a directory's name and its trailing slash must
+// still produce keys inside the directory in byte order: the marker's own
+// block band, not just its name, decides what follows it.
+func TestDoListFilerEntries_MarkerBetweenDirAndSlash(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dir := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b/o":     {dir("a"), file("a-c"), file("a.txt"), file("a0"), file("ab")},
+			"/buckets/b/o/a":   {file("b"), dir("c")},
+			"/buckets/b/o/a/c": {file("d")},
+		},
+	}
+
+	collect := func(marker string) []string {
+		cursor := &ListingCursor{maxKeys: 1000}
+		var seen []string
+		_, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b/o", marker: marker, bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+			seen = append(seen, strings.TrimPrefix(dir, "/buckets/b/")+"/"+entry.Name)
+			cursor.maxKeys--
+		})
+		assert.NoError(t, err)
+		return seen
+	}
+
+	// byte order: o/a-c, o/a.txt, o/a/b, o/a/c/d, o/a0, o/ab
+	assert.Equal(t, []string{
+		"o/a-c", "o/a.txt", "o/a/b", "o/a/c/d", "o/a0", "o/ab",
+	}, collect(""))
+
+	assert.Equal(t, []string{
+		"o/a/b", "o/a/c/d", "o/a0", "o/ab",
+	}, collect("a.txt"))
+
+	assert.Equal(t, []string{
+		"o/a/c/d", "o/a0", "o/ab",
+	}, collect("a/b"))
+
+	assert.Equal(t, []string{
+		"o/ab",
+	}, collect("a0"))
+
+	// a marker that sorts inside the "a" band before its slash still walks a:
+	assert.Equal(t, []string{
+		"o/a.txt", "o/a/b", "o/a/c/d", "o/a0", "o/ab",
+	}, collect("a-c"))
+}
+
+// Directory blocks order by "<name>/", not by name: "a-/" sorts before "a/".
+func TestDoListFilerEntries_DirBlocksSortBySlashName(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dir := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":    {dir("a"), dir("a-"), file("a.txt")},
+			"/buckets/b/a":  {file("b")},
+			"/buckets/b/a-": {file("x")},
+		},
+	}
+
+	cursor := &ListingCursor{maxKeys: 1000}
+	var seen []string
+	_, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, strings.TrimPrefix(dir, "/buckets/b")+"/"+entry.Name)
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"/a-/x", "/a.txt", "/a/b"}, seen)
+}
+
+// A full page must report truncation while deferred directories still hold keys.
+func TestDoListFilerEntries_FullPageTruncatesWithPendingDir(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dir := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dir("a"), file("a-c")},
+			"/buckets/b/a": {file("x")},
+		},
+	}
+
+	cursor := &ListingCursor{maxKeys: 1}
+	var seen []string
+	nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, entry.Name)
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"a-c"}, seen)
+	assert.True(t, cursor.isTruncated, "deferred directory still holds a key")
+	assert.Equal(t, "a-c", nextMarker)
+
+	// the next page re-enters the deferred block through the marker band
+	cursor = &ListingCursor{maxKeys: 1000}
+	seen = nil
+	_, err = s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: nextMarker, bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, dir+"/"+entry.Name)
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"/buckets/b/a/x"}, seen)
+}
+
+// A directory that is also a plain object lists "a" at the name position and
+// does not repeat it when the marker is the object itself.
+func TestDoListFilerEntries_PrefixObjectKeepsNamePosition(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dirObj := func(name string) *filer_pb.Entry {
+		e := &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{FileMode: uint32(os.ModeDir | 0644)}}
+		e.MarkPrefixObject()
+		return e
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dirObj("a"), file("a.txt")},
+			"/buckets/b/a": {file("b")},
+		},
+	}
+
+	collect := func(marker string) []string {
+		cursor := &ListingCursor{maxKeys: 1000}
+		var seen []string
+		_, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: marker, bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+			seen = append(seen, dir+"/"+entry.Name)
+			cursor.maxKeys--
+		})
+		assert.NoError(t, err)
+		return seen
+	}
+
+	assert.Equal(t, []string{"/buckets/b/a", "/buckets/b/a.txt", "/buckets/b/a/b"}, collect(""))
+	// the marker is the object "a" itself: excluded, but "a.txt" and the
+	// "a/" block still follow
+	assert.Equal(t, []string{"/buckets/b/a.txt", "/buckets/b/a/b"}, collect("a"))
+}
+
+// A marker that lands inside a directory's "<name>/" block — the continuation
+// token a delimited page returns for it — must not re-list the block's folded
+// contents under delimiter "/".
+func TestDoListFilerEntries_DelimiterSkipsResumedDirBlock(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dir := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dir("a"), file("b")},
+			"/buckets/b/a": {file("x")},
+		},
+	}
+
+	collect := func(marker string) []string {
+		cursor := &ListingCursor{maxKeys: 1000}
+		var seen []string
+		_, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: marker, delimiter: "/", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+			seen = append(seen, strings.TrimPrefix(dir, "/buckets/b")+"/"+entry.Name)
+			cursor.maxKeys--
+		})
+		assert.NoError(t, err)
+		return seen
+	}
+
+	// The "a/" block already collapsed into a common prefix on the earlier
+	// page, so neither "a/" nor anything under it is emitted again.
+	assert.Equal(t, []string{"/b"}, collect("a/"))
+	assert.Equal(t, []string{"/b"}, collect("a/x"))
+}
+
+// A delimited listing resumes a directory at its "<name>/" block boundary, not
+// the bare name: the name alone sorts inside the (name, name/) band and makes
+// the next pass re-emit the same common prefix until the quota runs out. The
+// marker also survives a round trip through the request path unchanged —
+// adjustMarkerForDelimiter must not trim the trailing slash off it.
+func TestDoListFilerEntries_DelimitedDirResumesAtBlockBoundary(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dir := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dir("a"), file("b")},
+			"/buckets/b/a": {file("x")},
+		},
+	}
+
+	// The outer listFilerEntries loop re-enters with the returned marker until
+	// a pass comes back empty; every entry must be emitted exactly once.
+	var seen []string
+	marker := ""
+	for pass := 0; pass < 4; pass++ {
+		cursor := &ListingCursor{maxKeys: 1000}
+		emitted := 0
+		nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: marker, delimiter: "/", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+			seen = append(seen, strings.TrimPrefix(dir, "/buckets/b")+"/"+entry.Name)
+			emitted++
+		})
+		assert.NoError(t, err)
+		if emitted == 0 {
+			break
+		}
+		marker = adjustMarkerForDelimiter(nextMarker, "", "/")
+	}
+	assert.Equal(t, []string{"/a", "/b"}, seen)
+}
+
+// A prefix object's "<name>/" common prefix belongs to the block position, not
+// the name position: it must not outrank "a.txt" on a full page nor repeat on
+// the next one.
+func TestDoListFilerEntries_PrefixObjectDefersCommonPrefix(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dirObj := func(name string) *filer_pb.Entry {
+		e := &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{FileMode: uint32(os.ModeDir | 0644)}}
+		e.MarkPrefixObject()
+		return e
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dirObj("a"), file("a.txt")},
+			"/buckets/b/a": {file("b")},
+		},
+	}
+
+	// Page one holds the plain key "a" and "a.txt"; the "a/" block sort after
+	// both, so the page truncates at "a.txt" without touching it.
+	cursor := &ListingCursor{maxKeys: 2}
+	var seen []string
+	var sawPrefixObject bool
+	nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", delimiter: "/", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, entry.Name)
+		sawPrefixObject = sawPrefixObject || entry.IsPrefixObject()
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"a", "a.txt"}, seen)
+	assert.True(t, sawPrefixObject, "the name-position emission is the plain object")
+	assert.True(t, cursor.isTruncated)
+	assert.Equal(t, "a.txt", nextMarker)
+
+	// Page two emits the deferred block once, as a plain directory fold.
+	cursor = &ListingCursor{maxKeys: 2}
+	seen = nil
+	sawPrefixObject = false
+	_, err = s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: nextMarker, delimiter: "/", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, entry.Name)
+		sawPrefixObject = sawPrefixObject || entry.IsPrefixObject()
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"a"}, seen)
+	assert.False(t, sawPrefixObject, "the block emission is the \"<name>/\" fold, not the plain key again")
+	assert.False(t, cursor.isTruncated)
+}
+
+// A directory key object that spends the page's last slot still reports
+// truncation so the next page re-enters its subtree.
+func TestDoListFilerEntries_DirKeyObjectBoundaryTruncates(t *testing.T) {
+	s3a := &S3ApiServer{}
+	dirKeyObj := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mime: s3_constants.FolderMimeType}}
+	}
+	file := func(name string) *filer_pb.Entry {
+		return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}
+	}
+	client := &testFilerClient{
+		entriesByDir: map[string][]*filer_pb.Entry{
+			"/buckets/b":   {dirKeyObj("a")},
+			"/buckets/b/a": {file("x")},
+		},
+	}
+
+	cursor := &ListingCursor{maxKeys: 1}
+	var seen []string
+	nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, entry.Name)
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"a"}, seen)
+	assert.True(t, cursor.isTruncated, "the subtree may still hold unlisted keys")
+	assert.Equal(t, "a/", nextMarker)
+
+	// The next page resumes inside the "a/" block and lists the child.
+	cursor = &ListingCursor{maxKeys: 1000}
+	seen = nil
+	_, err = s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/b", marker: nextMarker, bucket: "b"}, cursor, func(dir string, entry *filer_pb.Entry) {
+		seen = append(seen, strings.TrimPrefix(dir, "/buckets/b")+"/"+entry.Name)
+		cursor.maxKeys--
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"/a/x"}, seen)
+	assert.False(t, cursor.isTruncated)
+}

weed/s3api/s3api_object_handlers_put.go

@@ -2796,8 +2796,9 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
 
 	// 1. Check If-Match
 	if headers.ifMatch != "" {
+		// AWS answers 404 when there is no current version (or it is a delete marker)
 		if !objectExists {
-			return s3err.ErrPreconditionFailed
+			return s3err.ErrNoSuchKey
 		}
 		// If `ifMatch` is "*", the condition is met if the object exists.
 		// Otherwise, we need to check the ETag.

weed/s3api/s3api_object_lock_actions_authz_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gorilla/mux"
 	iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/stretchr/testify/require"
 )
 
@@ -227,3 +228,83 @@ func TestObjectLockActionsRoundTrip(t *testing.T) {
 		})
 	}
 }
+
+// Anonymous misses must not disclose bucket or key existence: 404 is only
+// served when anonymous listing is allowed.
+func TestAnonymousBucketListAuthorization(t *testing.T) {
+	s3a := &S3ApiServer{iam: &IdentityAccessManagement{}, bucketConfigCache: NewBucketConfigCache(time.Minute)}
+	s3a.bucketConfigCache.Set("open", &BucketConfig{Name: "open", IsPublicRead: true})
+	s3a.bucketConfigCache.Set("private", &BucketConfig{Name: "private"})
+
+	req := httptest.NewRequest(http.MethodGet, "http://localhost:8333/open/key", nil)
+	if got := s3a.authorizeAnonymousBucketList(req, "open"); got != s3err.ErrNone {
+		t.Errorf("listable bucket: %v, want ErrNone", got)
+	}
+	if got := s3a.authorizeAnonymousBucketList(req, "private"); got != s3err.ErrAccessDenied {
+		t.Errorf("private bucket: %v, want AccessDenied", got)
+	}
+}
+
+// Internal Seaweed-X-Amz-* headers carry server-validated object lock values
+// into entry metadata; client-supplied ones must be dropped on entry or the
+// dedicated-permission check on x-amz-object-lock-* is bypassed.
+func TestInternalObjectLockHeadersStrippedOnAuth(t *testing.T) {
+	iam := newTestIAM()
+
+	internal := []string{
+		s3_constants.ExtObjectLockModeKey,
+		s3_constants.ExtRetentionUntilDateKey,
+		s3_constants.ExtLegalHoldKey,
+		s3_constants.ExtVersionIdKey,
+		s3_constants.SeaweedFSSSEKMSBaseIVHeader,
+		s3_constants.SeaweedFSSSES3BaseIVHeader,
+		s3_constants.SeaweedFSSSES3KeyDataHeader,
+	}
+	req := httptest.NewRequest(http.MethodPut, "http://localhost:8333/worm/doc.txt", nil)
+	for _, k := range internal {
+		req.Header.Set(k, "injected")
+	}
+	require.NoError(t, signRawHTTPRequest(context.Background(), req,
+		"AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1"))
+
+	_, _, _ = iam.authenticateRequestInternal(req)
+
+	for _, k := range internal {
+		if got := req.Header.Get(k); got != "" {
+			t.Errorf("client-supplied %s survived authentication: %q", k, got)
+		}
+	}
+}
+
+// With IAM disabled the Auth wrapper calls the handler directly; internal
+// headers must still be stripped so they never reach putToFiler.
+func TestInternalHeadersStrippedWithoutAuth(t *testing.T) {
+	iam := &IdentityAccessManagement{}
+
+	internal := []string{
+		s3_constants.ExtObjectLockModeKey,
+		s3_constants.ExtRetentionUntilDateKey,
+		s3_constants.ExtLegalHoldKey,
+		s3_constants.ExtVersionIdKey,
+		s3_constants.SeaweedFSSSEKMSBaseIVHeader,
+		s3_constants.SeaweedFSSSES3BaseIVHeader,
+		s3_constants.SeaweedFSSSES3KeyDataHeader,
+	}
+	var leaked []string
+	handler := iam.Auth(func(w http.ResponseWriter, r *http.Request) {
+		for _, k := range internal {
+			if v := r.Header.Get(k); v != "" {
+				leaked = append(leaked, k)
+			}
+		}
+	}, s3_constants.ACTION_WRITE)
+
+	req := httptest.NewRequest(http.MethodPut, "http://localhost:8333/worm/doc.txt", nil)
+	for _, k := range internal {
+		req.Header.Set(k, "injected")
+	}
+	handler(httptest.NewRecorder(), req)
+	if len(leaked) > 0 {
+		t.Errorf("client-supplied internal headers reached handler with auth disabled: %v", leaked)
+	}
+}

weed/s3api/s3tables/filer_ops.go

@@ -27,10 +27,23 @@ const extendedAttributeUpdateAttempts = 5
 
 // createDirectory creates a new directory at the specified path
 func (h *S3TablesHandler) createDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
+	return h.createDirectoryEntry(ctx, client, path, false, nil)
+}
+
+// createDirectoryExclusive creates a directory entry together with its
+// extended attributes in one CreateEntry that fails with
+// filer_pb.ErrEntryAlreadyExists when an entry is already there: nothing is
+// replaced and no reader sees the directory without its attributes.
+func (h *S3TablesHandler) createDirectoryExclusive(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, extended map[string][]byte) error {
+	return h.createDirectoryEntry(ctx, client, path, true, extended)
+}
+
+func (h *S3TablesHandler) createDirectoryEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, exclusive bool, extended map[string][]byte) error {
 	dir, name := splitPath(path)
 	now := time.Now().Unix()
 	return filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
 		Directory: dir,
+		OExcl:     exclusive,
 		Entry: &filer_pb.Entry{
 			Name:        name,
 			IsDirectory: true,
@@ -39,6 +52,7 @@ func (h *S3TablesHandler) createDirectory(ctx context.Context, client filer_pb.S
 				Crtime:   now,
 				FileMode: uint32(0755 | os.ModeDir),
 			},
+			Extended: extended,
 		},
 	})
 }
@@ -248,6 +262,14 @@ func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer
 	return data, nil
 }
 
+// absentEntry reports whether err means there is no usable catalog entry at
+// the path: nothing is there, or what is there carries none of the attributes
+// the operation keys on (a plain S3 bucket holding the name, a directory left
+// without metadata).
+func absentEntry(err error) bool {
+	return errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrAttributeNotFound)
+}
+
 // loadNamespaceMetadata resolves a namespace to its metadata. A directory that
 // carries no namespace metadata is not a namespace, so a missing attribute
 // reports the same absence as a missing entry and every caller tests one

weed/s3api/s3tables/filer_ops_race_test.go

@@ -0,0 +1,89 @@
+package s3tables
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// competingCreateClient lands a foreign entry where the handler's existence
+// check just answered "not there": the create that follows must lose to it
+// rather than replace it, the way a second caller's create wins the window in
+// a check-then-create race.
+type competingCreateClient struct {
+	filer_pb.SeaweedFilerClient
+	f     *s3tablestest.MemFiler
+	dir   string
+	name  string
+	plant bool
+}
+
+func (c *competingCreateClient) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	resp, err := c.SeaweedFilerClient.LookupDirectoryEntry(ctx, req, opts...)
+	if c.plant && err != nil && req.Directory == c.dir && req.Name == c.name {
+		c.plant = false
+		c.f.Put(c.dir, c.name, map[string][]byte{
+			ExtendedKeyTableBucket: []byte("true"),
+			ExtendedKeyMetadata:    []byte(`{"OwnerAccountID":"competitor"}`),
+		})
+	}
+	return resp, err
+}
+
+func runHandler(t *testing.T, m *Manager, client filer_pb.SeaweedFilerClient, operation string, input interface{}) (int, S3TablesError) {
+	t.Helper()
+	body, err := json.Marshal(input)
+	require.NoError(t, err)
+	req, err := newManagerRequest(context.Background(), operation, body, "admin")
+	require.NoError(t, err)
+	recorder := httptest.NewRecorder()
+	m.handler.HandleRequest(recorder, req, NewManagerClient(client))
+	result := recorder.Result()
+	defer result.Body.Close()
+	var response S3TablesError
+	json.NewDecoder(result.Body).Decode(&response)
+	return result.StatusCode, response
+}
+
+func TestCreateTableBucketLostRaceKeepsTheFirstCreate(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	m := NewManager()
+	client := &competingCreateClient{SeaweedFilerClient: fs.Client, f: fs, dir: TablesPath, name: "bucket", plant: true}
+
+	status, resp := runHandler(t, m, client, "CreateTableBucket", &CreateTableBucketRequest{Name: "bucket"})
+	assert.Equal(t, http.StatusConflict, status)
+	assert.Equal(t, ErrCodeBucketAlreadyExists, resp.Type)
+
+	survivor := fs.Get(TablesPath, "bucket")
+	require.NotNil(t, survivor)
+	assert.Equal(t, `{"OwnerAccountID":"competitor"}`, string(survivor.Extended[ExtendedKeyMetadata]))
+}
+
+func TestCreateNamespaceLostRaceKeepsTheFirstCreate(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	m := NewManager()
+	bucketMetadata, _ := json.Marshal(&tableBucketMetadata{Name: "bucket", OwnerAccountID: DefaultAccountID})
+	fs.Put(TablesPath, "bucket", map[string][]byte{ExtendedKeyTableBucket: []byte("true"), ExtendedKeyMetadata: bucketMetadata})
+
+	arn, err := BuildBucketARN(DefaultRegion, DefaultAccountID, "bucket")
+	require.NoError(t, err)
+	client := &competingCreateClient{SeaweedFilerClient: fs.Client, f: fs, dir: GetTableBucketPath("bucket"), name: "ns", plant: true}
+	status, resp := runHandler(t, m, client, "CreateNamespace", &CreateNamespaceRequest{
+		TableBucketARN: arn,
+		Namespace:      []string{"ns"},
+	})
+	assert.Equal(t, http.StatusConflict, status)
+	assert.Equal(t, ErrCodeNamespaceAlreadyExists, resp.Type)
+
+	survivor := fs.Get(GetTableBucketPath("bucket"), "ns")
+	require.NotNil(t, survivor)
+	assert.Equal(t, `{"OwnerAccountID":"competitor"}`, string(survivor.Extended[ExtendedKeyMetadata]))
+}

weed/s3api/s3tables/handler.go

@@ -459,6 +459,20 @@ func isAuthError(err error) bool {
 	return errors.As(err, &authErr) || errors.Is(err, ErrAccessDenied)
 }
 
+// mayLearnAbsence reports whether the caller may be shown that an operation's
+// target is absent rather than the operation's denial: the admin, a trusted
+// handler, or a grant the request itself carries (an identity-level action
+// grant, or the open default-allow context).
+func (h *S3TablesHandler) mayLearnAbsence(r *http.Request, operation, bucketName string) bool {
+	if h.getAccountID(r) == s3_constants.AccountAdminId || h.defaultAllowFor(r) {
+		return true
+	}
+	return hasIdentityPermission(operation, &PolicyContext{
+		TableBucketName: bucketName,
+		IdentityActions: getIdentityActions(r),
+	})
+}
+
 func (h *S3TablesHandler) readTags(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (map[string]string, error) {
 	data, err := h.getExtendedAttribute(ctx, client, path, ExtendedKeyTags)
 	if err != nil {

weed/s3api/s3tables/handler_absent_target_test.go

@@ -0,0 +1,246 @@
+package s3tables
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func runAbsentOp(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, operation string, input interface{}, name string, actions []string) s3TablesHTTPError {
+	t.Helper()
+	body, err := json.Marshal(input)
+	require.NoError(t, err)
+	identity := &testIdentity{Name: name, Account: &testIdentityAccount{Id: name}, Actions: actions}
+	req, err := newManagerRequest(s3_constants.SetIdentityInContext(context.Background(), identity), operation, body, name)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	m.handler.HandleRequest(recorder, req, NewManagerClient(fs.Client))
+	result := recorder.Result()
+	defer result.Body.Close()
+
+	var response S3TablesError
+	if result.StatusCode >= 400 {
+		require.NoError(t, json.NewDecoder(result.Body).Decode(&response))
+	}
+	return s3TablesHTTPError{status: result.StatusCode, body: response}
+}
+
+func absentBucketARN(t *testing.T, name string) string {
+	t.Helper()
+	arn, err := BuildBucketARN(DefaultRegion, DefaultAccountID, name)
+	require.NoError(t, err)
+	return arn
+}
+
+func absentTableARN(t *testing.T, bucket, namespace, table string) string {
+	t.Helper()
+	arn, err := BuildTableARN(DefaultRegion, DefaultAccountID, bucket, namespace, table)
+	require.NoError(t, err)
+	return arn
+}
+
+// An outsider gets the operation's own denial for an absent name, a name a
+// plain S3 bucket holds, and a name it is denied on, so none of them can be
+// told apart.
+func TestAbsentTargetsAnswerLikeDenials(t *testing.T) {
+	fs, m := startRenameManager(t)
+	m.SetTrusted(false)
+	m.SetDefaultAllow(false)
+
+	ghostARN := absentBucketARN(t, "ghostbkt")
+	plainARN := absentBucketARN(t, "plainbkt")
+	fs.Put(TablesPath, "plainbkt", nil) // a plain S3 bucket holds the name
+
+	outsider := []string{"Read"}
+
+	t.Run("ListTables", func(t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			req  *ListTablesRequest
+		}{
+			{"denied namespace", &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}},
+			{"missing namespace", &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}},
+			{"namespace in missing bucket", &ListTablesRequest{TableBucketARN: ghostARN, Namespace: []string{"ns"}}},
+			{"namespace in plain bucket", &ListTablesRequest{TableBucketARN: plainARN, Namespace: []string{"ns"}}},
+			{"denied bucket", &ListTablesRequest{TableBucketARN: mustBucketARN(t)}},
+			{"missing bucket", &ListTablesRequest{TableBucketARN: ghostARN}},
+			{"plain bucket", &ListTablesRequest{TableBucketARN: plainARN}},
+		} {
+			got := runAbsentOp(t, m, fs, "ListTables", tc.req, "attacker", outsider)
+			assert.Equal(t, 403, got.status, "%s: %s", tc.name, got.body.Message)
+			assert.Equal(t, ErrCodeAccessDenied, got.body.Type, tc.name)
+		}
+	})
+
+	t.Run("ListViews", func(t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			req  *ListViewsRequest
+		}{
+			{"denied namespace", &ListViewsRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}},
+			{"missing namespace", &ListViewsRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}},
+			{"missing bucket", &ListViewsRequest{TableBucketARN: ghostARN, Namespace: []string{"ns"}}},
+			{"plain bucket", &ListViewsRequest{TableBucketARN: plainARN, Namespace: []string{"ns"}}},
+		} {
+			got := runAbsentOp(t, m, fs, "ListViews", tc.req, "attacker", outsider)
+			assert.Equal(t, 403, got.status, "%s: %s", tc.name, got.body.Message)
+			assert.Equal(t, ErrCodeAccessDenied, got.body.Type, tc.name)
+		}
+	})
+
+	t.Run("CreateNamespace", func(t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			arn  string
+		}{
+			{"denied bucket", mustBucketARN(t)},
+			{"missing bucket", ghostARN},
+			{"plain bucket", plainARN},
+		} {
+			got := runAbsentOp(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{TableBucketARN: tc.arn, Namespace: []string{"ns"}}, "attacker", outsider)
+			assert.Equal(t, 403, got.status, "%s: %s", tc.name, got.body.Message)
+			assert.Equal(t, ErrCodeAccessDenied, got.body.Type, tc.name)
+		}
+	})
+
+	t.Run("GetTableBucket and ListNamespaces", func(t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			arn  string
+		}{
+			{"denied bucket", mustBucketARN(t)},
+			{"missing bucket", ghostARN},
+			{"plain bucket", plainARN},
+		} {
+			for _, op := range []string{"GetTableBucket", "ListNamespaces"} {
+				var req interface{} = &GetTableBucketRequest{TableBucketARN: tc.arn}
+				if op == "ListNamespaces" {
+					req = &ListNamespacesRequest{TableBucketARN: tc.arn}
+				}
+				got := runAbsentOp(t, m, fs, op, req, "attacker", outsider)
+				assert.Equal(t, 404, got.status, "%s %s: %s", op, tc.name, got.body.Message)
+				assert.Equal(t, ErrCodeNoSuchBucket, got.body.Type, "%s %s", op, tc.name)
+			}
+		}
+	})
+
+	t.Run("tag operations", func(t *testing.T) {
+		missingTableARN := absentTableARN(t, renameTestBucket, "ns", "ghost")
+		deniedTableARN := absentTableARN(t, renameTestBucket, "ns", "t")
+		for _, tc := range []struct {
+			name string
+			op   string
+			req  interface{}
+		}{
+			{"TagResource denied table", "TagResource", &TagResourceRequest{ResourceARN: deniedTableARN, Tags: map[string]string{"k": "v"}}},
+			{"TagResource missing table", "TagResource", &TagResourceRequest{ResourceARN: missingTableARN, Tags: map[string]string{"k": "v"}}},
+			{"TagResource missing bucket", "TagResource", &TagResourceRequest{ResourceARN: ghostARN, Tags: map[string]string{"k": "v"}}},
+			{"ListTagsForResource denied table", "ListTagsForResource", &ListTagsForResourceRequest{ResourceARN: deniedTableARN}},
+			{"ListTagsForResource missing table", "ListTagsForResource", &ListTagsForResourceRequest{ResourceARN: missingTableARN}},
+			{"ListTagsForResource missing bucket", "ListTagsForResource", &ListTagsForResourceRequest{ResourceARN: ghostARN}},
+			{"UntagResource denied table", "UntagResource", &UntagResourceRequest{ResourceARN: deniedTableARN, TagKeys: []string{"k"}}},
+			{"UntagResource missing table", "UntagResource", &UntagResourceRequest{ResourceARN: missingTableARN, TagKeys: []string{"k"}}},
+			{"UntagResource missing bucket", "UntagResource", &UntagResourceRequest{ResourceARN: ghostARN, TagKeys: []string{"k"}}},
+		} {
+			got := runAbsentOp(t, m, fs, tc.op, tc.req, "attacker", outsider)
+			assert.Equal(t, 403, got.status, "%s: %s", tc.name, got.body.Message)
+			assert.Equal(t, ErrCodeAccessDenied, got.body.Type, tc.name)
+		}
+	})
+}
+
+// A caller the operation would have authorized — the bucket owner, a bucket-
+// policy grantee, an identity-level grant, the admin or a trusted handler —
+// may still be told the name is absent: the empty list or the not-found stays.
+func TestGrantedCallersLearnAbsence(t *testing.T) {
+	fs, m := startRenameManager(t)
+	m.SetTrusted(false)
+	m.SetDefaultAllow(false)
+
+	ghostARN := absentBucketARN(t, "ghostbkt")
+	missingTableARN := absentTableARN(t, renameTestBucket, "ns", "ghost")
+	adminActions := []string{s3_constants.ACTION_ADMIN}
+
+	// The bucket owner, and a grantee the bucket policy names, learn absence.
+	bucketGrant := fmt.Sprintf(`{"Statement":[{"Effect":"Allow","Principal":"%s","Action":"s3tables:*","Resource":"*"}]}`, "grantee")
+	fs.Get(TablesPath, renameTestBucket).Extended[ExtendedKeyPolicy] = []byte(bucketGrant)
+
+	t.Run("bucket grant learns an absent namespace", func(t *testing.T) {
+		req := &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}
+		for _, name := range []string{DefaultAccountID, "grantee"} {
+			got := runAbsentOp(t, m, fs, "ListTables", req, name, nil)
+			assert.Equal(t, 200, got.status, "%s", name)
+		}
+		// And on a namespace that exists they still see the table.
+		got := runAbsentOp(t, m, fs, "ListTables", &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}, "grantee", nil)
+		assert.Equal(t, 200, got.status)
+	})
+
+	t.Run("identity grant learns an absent bucket", func(t *testing.T) {
+		got := runAbsentOp(t, m, fs, "ListTables", &ListTablesRequest{TableBucketARN: ghostARN}, "lister", []string{"s3tables:ListTables"})
+		assert.Equal(t, 200, got.status)
+		got = runAbsentOp(t, m, fs, "ListViews", &ListViewsRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}, "lister", []string{"s3tables:ListViews"})
+		assert.Equal(t, 200, got.status)
+	})
+
+	t.Run("admin learns an absent bucket", func(t *testing.T) {
+		got := runAbsentOp(t, m, fs, "ListTables", &ListTablesRequest{TableBucketARN: ghostARN}, s3_constants.AccountAdminId, adminActions)
+		assert.Equal(t, 200, got.status)
+		got = runAbsentOp(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{TableBucketARN: ghostARN, Namespace: []string{"ns"}}, s3_constants.AccountAdminId, adminActions)
+		assert.Equal(t, 404, got.status)
+		assert.Equal(t, ErrCodeNoSuchBucket, got.body.Type)
+	})
+
+	t.Run("tag operations answer 404 to granted callers", func(t *testing.T) {
+		got := runAbsentOp(t, m, fs, "TagResource", &TagResourceRequest{ResourceARN: missingTableARN, Tags: map[string]string{"k": "v"}}, "grantee", nil)
+		assert.Equal(t, 404, got.status, got.body.Message)
+		assert.Equal(t, ErrCodeNoSuchTable, got.body.Type)
+		got = runAbsentOp(t, m, fs, "ListTagsForResource", &ListTagsForResourceRequest{ResourceARN: missingTableARN}, DefaultAccountID, nil)
+		assert.Equal(t, 404, got.status)
+		got = runAbsentOp(t, m, fs, "UntagResource", &UntagResourceRequest{ResourceARN: missingTableARN, TagKeys: []string{"k"}}, s3_constants.AccountAdminId, adminActions)
+		assert.Equal(t, 404, got.status)
+	})
+
+	t.Run("a trusted handler still lists an absent namespace", func(t *testing.T) {
+		m.SetTrusted(true)
+		got := runAbsentOp(t, m, fs, "ListTables", &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}, "svc", nil)
+		assert.Equal(t, 200, got.status)
+	})
+}
+
+// A bucket-policy Deny still vetoes the operation on an absent name: a denied
+// caller must not trade the denial for a not-found or an empty list.
+func TestAbsentResourceBucketDenyStillDenies(t *testing.T) {
+	fs, m := startRenameManager(t)
+	m.SetTrusted(false)
+	m.SetDefaultAllow(false)
+
+	deny := `{"Statement":[{"Effect":"Deny","Principal":"lister","Action":"s3tables:*","Resource":"*"}]}`
+	fs.Get(TablesPath, renameTestBucket).Extended[ExtendedKeyPolicy] = []byte(deny)
+
+	got := runAbsentOp(t, m, fs, "TagResource",
+		&TagResourceRequest{ResourceARN: absentTableARN(t, renameTestBucket, "ns", "ghost"), Tags: map[string]string{"k": "v"}},
+		"lister", []string{"s3tables:TagResource"})
+	assert.Equal(t, 403, got.status, got.body.Message)
+	assert.Equal(t, ErrCodeAccessDenied, got.body.Type)
+
+	for _, tc := range []struct {
+		op  string
+		req interface{}
+	}{
+		{"ListTables", &ListTablesRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}},
+		{"ListTables", &ListTablesRequest{TableBucketARN: mustBucketARN(t)}},
+		{"ListViews", &ListViewsRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ghost"}}},
+	} {
+		got := runAbsentOp(t, m, fs, tc.op, tc.req, "lister", []string{"s3tables:ListTables", "s3tables:ListViews"})
+		assert.Equal(t, 403, got.status, "%s: %s", tc.op, got.body.Message)
+	}
+}

weed/s3api/s3tables/handler_bucket_create.go

@@ -140,6 +140,7 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
 		return err
 	}
 
+	var taken error
 	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		// Ensure root tables directory exists
 		if !h.entryExists(r.Context(), client, TablesPath) {
@@ -148,35 +149,35 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
 			}
 		}
 
-		// Create bucket directory
-		if err := h.createDirectory(r.Context(), client, bucketPath); err != nil {
-			return err
+		extended := map[string][]byte{
+			ExtendedKeyTableBucket: []byte("true"),
+			ExtendedKeyMetadata:    metadataBytes,
 		}
-
-		// Mark as a table bucket
-		if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTableBucket, []byte("true")); err != nil {
-			return err
-		}
-
-		// Set metadata as extended attribute
-		if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
-			return err
-		}
-
-		// Set tags if provided
 		if len(req.Tags) > 0 {
 			tagsBytes, err := json.Marshal(req.Tags)
 			if err != nil {
 				return fmt.Errorf("failed to marshal tags: %w", err)
 			}
-			if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTags, tagsBytes); err != nil {
-				return err
+			extended[ExtendedKeyTags] = tagsBytes
+		}
+		if err := h.createDirectoryExclusive(r.Context(), client, bucketPath, extended); err != nil {
+			if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
+				taken = err // an answer, not a filer failure
+				return nil
 			}
+			return err
 		}
 
 		return nil
 	})
+	if err == nil && taken != nil {
+		err = taken
+	}
 
+	if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
+		h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("table bucket %s already exists", req.Name))
+		return err
+	}
 	if err != nil {
 		glog.Errorf("S3Tables: failed to create table bucket %s: %v", req.Name, err)
 		h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create table bucket")

weed/s3api/s3tables/handler_bucket_get_list_delete.go

@@ -58,7 +58,7 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
 	})
 
 	if err != nil {
-		if errors.Is(err, filer_pb.ErrNotFound) {
+		if absentEntry(err) {
 			h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
 		} else {
 			h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket: %v", err))

weed/s3api/s3tables/handler_namespace.go

@@ -11,8 +11,29 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )
 
+// errOwnerlessBucket marks a table bucket that has the marker but no
+// metadata: damaged, and answered with the operation's own denial rather than
+// letting the caller name itself the bucket's owner before any check runs.
+var errOwnerlessBucket = fmt.Errorf("table bucket has no owner metadata")
+
+// mayRepairOwnerlessBucket reports whether the caller may repair a bucket in
+// that state: the admin or a trusted internal handler. hasAdminAction also
+// answers s3tables:*, an operation grant, not administrator status.
+func (h *S3TablesHandler) mayRepairOwnerlessBucket(r *http.Request) bool {
+	if h.trusted || h.getAccountID(r) == s3_constants.AccountAdminId {
+		return true
+	}
+	for _, action := range getIdentityActions(r) {
+		if action == s3_constants.ACTION_ADMIN {
+			return true
+		}
+	}
+	return false
+}
+
 // handleCreateNamespace creates a new namespace in a table bucket
 func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
 	glog.Errorf("S3Tables: handleCreateNamespace called")
@@ -66,16 +87,29 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
 				if entryResp.Entry == nil || !IsTableBucketEntry(entryResp.Entry) {
 					return filer_pb.ErrNotFound
 				}
-				bucketMetadata = tableBucketMetadata{
-					Name:           bucketName,
-					CreatedAt:      time.Now(),
-					OwnerAccountID: ownerAccountID,
+				if !h.mayRepairOwnerlessBucket(r) {
+					return errOwnerlessBucket
 				}
-				metadataBytes, err := json.Marshal(&bucketMetadata)
-				if err != nil {
-					return fmt.Errorf("failed to marshal bucket metadata: %w", err)
-				}
-				if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
+				// Repair the marker-only entry under the entry lock so a
+				// creator finishing its own create in the meantime keeps its
+				// owner: adopt whatever metadata is there, write only when it
+				// is still absent.
+				if err := h.mutateEntryExtended(r.Context(), client, bucketPath, func(extended map[string][]byte) error {
+					if data, ok := extended[ExtendedKeyMetadata]; ok {
+						return json.Unmarshal(data, &bucketMetadata)
+					}
+					bucketMetadata = tableBucketMetadata{
+						Name:           bucketName,
+						CreatedAt:      time.Now(),
+						OwnerAccountID: ownerAccountID,
+					}
+					metadataBytes, err := json.Marshal(&bucketMetadata)
+					if err != nil {
+						return fmt.Errorf("failed to marshal bucket metadata: %w", err)
+					}
+					extended[ExtendedKeyMetadata] = metadataBytes
+					return nil
+				}); err != nil {
 					return err
 				}
 			} else {
@@ -101,7 +135,17 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
 	})
 
 	if err != nil {
+		if errors.Is(err, errOwnerlessBucket) {
+			h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
+			return ErrAccessDenied
+		}
 		if errors.Is(err, filer_pb.ErrNotFound) {
+			// An absent name and a denied one must read alike; the caller is
+			// told the bucket is missing only when it may learn that.
+			if !h.mayLearnAbsence(r, "CreateNamespace", bucketName) {
+				h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
+				return ErrAccessDenied
+			}
 			h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
 		} else {
 			h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
@@ -159,20 +203,22 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
 		return fmt.Errorf("failed to marshal metadata: %w", err)
 	}
 
+	var taken error
 	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		// Create namespace directory
-		if err := h.createDirectory(r.Context(), client, namespacePath); err != nil {
-			return err
+		err := h.createDirectoryExclusive(r.Context(), client, namespacePath, map[string][]byte{ExtendedKeyMetadata: metadataBytes})
+		if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
+			taken = err // an answer, not a filer failure
+			return nil
 		}
-
-		// Set metadata as extended attribute
-		if err := h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes); err != nil {
-			return err
-		}
-
-		return nil
+		return err
 	})
-
+	if err == nil && taken != nil {
+		err = taken
+	}
+	if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
+		h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName))
+		return err
+	}
 	if err != nil {
 		h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace")
 		return err
@@ -441,7 +487,7 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re
 	})
 
 	if err != nil {
-		if errors.Is(err, filer_pb.ErrNotFound) {
+		if absentEntry(err) {
 			h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
 		} else {
 			h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))

weed/s3api/s3tables/handler_ownerless_test.go

@@ -0,0 +1,88 @@
+package s3tables
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A table bucket with the marker but no metadata is damaged, not ownerless for
+// the taking: an ordinary caller is answered with the operation's own denial,
+// and the admin or a trusted handler may repair it.
+func TestCreateNamespaceOnAnOwnerlessBucket(t *testing.T) {
+	const bucket = "broken"
+	plant := func(fs *s3tablestest.MemFiler) {
+		fs.Put(TablesPath, bucket, map[string][]byte{ExtendedKeyTableBucket: []byte("true")})
+	}
+	arn, err := BuildBucketARN(DefaultRegion, DefaultAccountID, bucket)
+	require.NoError(t, err)
+	request := &CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{"ns"}}
+
+	t.Run("an outsider gets the denial and writes nothing", func(t *testing.T) {
+		fs := s3tablestest.Start(t)
+		plant(fs)
+		m := NewManager()
+		body, err := json.Marshal(request)
+		require.NoError(t, err)
+		identity := &testIdentity{Name: "outsider", Account: &testIdentityAccount{Id: "outsider"}, Actions: []string{"Read"}}
+		req, err := newManagerRequest(s3_constants.SetIdentityInContext(context.Background(), identity), "CreateNamespace", body, "outsider")
+		require.NoError(t, err)
+
+		recorder := httptest.NewRecorder()
+		m.handler.HandleRequest(recorder, req, NewManagerClient(fs.Client))
+		assert.Equal(t, http.StatusForbidden, recorder.Code, recorder.Body.String())
+		entry := fs.Get(TablesPath, bucket)
+		require.NotNil(t, entry)
+		assert.Empty(t, entry.Extended[ExtendedKeyMetadata], "the denied caller wrote itself in as the bucket owner")
+		assert.Nil(t, fs.Get(GetTableBucketPath(bucket), "ns"))
+	})
+
+	t.Run("a trusted handler repairs it and owns it", func(t *testing.T) {
+		fs := s3tablestest.Start(t)
+		plant(fs)
+		m := NewManager()
+		m.SetTrusted(true)
+		body, err := json.Marshal(request)
+		require.NoError(t, err)
+		identity := &testIdentity{Name: "svc", Account: &testIdentityAccount{Id: "svc"}, Actions: []string{"Read"}}
+		req, err := newManagerRequest(s3_constants.SetIdentityInContext(context.Background(), identity), "CreateNamespace", body, "svc")
+		require.NoError(t, err)
+
+		recorder := httptest.NewRecorder()
+		m.handler.HandleRequest(recorder, req, NewManagerClient(fs.Client))
+		assert.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
+		entry := fs.Get(TablesPath, bucket)
+		require.NotNil(t, entry)
+		var metadata tableBucketMetadata
+		require.NoError(t, json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata))
+		assert.Equal(t, "svc", metadata.OwnerAccountID)
+		require.NotNil(t, fs.Get(GetTableBucketPath(bucket), "ns"))
+	})
+
+	t.Run("an untrusted admin repairs it too", func(t *testing.T) {
+		fs := s3tablestest.Start(t)
+		plant(fs)
+		m := NewManager()
+		body, err := json.Marshal(request)
+		require.NoError(t, err)
+		identity := &testIdentity{Name: "admin", Account: &testIdentityAccount{Id: s3_constants.AccountAdminId}, Actions: []string{"Admin"}}
+		req, err := newManagerRequest(s3_constants.SetIdentityInContext(context.Background(), identity), "CreateNamespace", body, "admin")
+		require.NoError(t, err)
+
+		recorder := httptest.NewRecorder()
+		m.handler.HandleRequest(recorder, req, NewManagerClient(fs.Client))
+		assert.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
+		entry := fs.Get(TablesPath, bucket)
+		require.NotNil(t, entry)
+		var metadata tableBucketMetadata
+		require.NoError(t, json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata))
+		assert.Equal(t, s3_constants.AccountAdminId, metadata.OwnerAccountID)
+	})
+}

weed/s3api/s3tables/handler_policy.go

@@ -659,7 +659,16 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque
 		// Read metadata for ownership check
 		data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
 		if err != nil {
-			return err
+			if !absentEntry(err) {
+				return err
+			}
+			ctx := &PolicyContext{RequestTags: req.Tags, TagKeys: requestTagKeys}
+			if rType == ResourceTypeTable {
+				if _, ctx.Namespace, ctx.TableName, err = parseTableFromARN(req.ResourceARN); err != nil {
+					return err
+				}
+			}
+			return h.authorizeAbsentResource(r, client, "TagResource", req.ResourceARN, resourcePath, "not authorized to tag resource", ctx)
 		}
 
 		ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
@@ -793,7 +802,16 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht
 		// Read metadata for ownership check
 		data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
 		if err != nil {
-			return err
+			if !absentEntry(err) {
+				return err
+			}
+			ctx := &PolicyContext{}
+			if rType == ResourceTypeTable {
+				if _, ctx.Namespace, ctx.TableName, err = parseTableFromARN(req.ResourceARN); err != nil {
+					return err
+				}
+			}
+			return h.authorizeAbsentResource(r, client, "ListTagsForResource", req.ResourceARN, resourcePath, "not authorized to list tags for resource", ctx)
 		}
 
 		ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
@@ -913,7 +931,16 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
 		// Read metadata for ownership check
 		data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
 		if err != nil {
-			return err
+			if !absentEntry(err) {
+				return err
+			}
+			ctx := &PolicyContext{TagKeys: req.TagKeys}
+			if rType == ResourceTypeTable {
+				if _, ctx.Namespace, ctx.TableName, err = parseTableFromARN(req.ResourceARN); err != nil {
+					return err
+				}
+			}
+			return h.authorizeAbsentResource(r, client, "UntagResource", req.ResourceARN, resourcePath, "not authorized to untag resource", ctx)
 		}
 
 		ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
@@ -1017,6 +1044,62 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
 	return nil
 }
 
+// authorizeAbsentResource answers a tag operation whose target carries no
+// readable metadata. A caller holding a grant on the bucket that owns the
+// name, or an identity-level or admin grant, is told the resource is absent;
+// anyone else is refused with the operation's denial.
+func (h *S3TablesHandler) authorizeAbsentResource(r *http.Request, client filer_pb.SeaweedFilerClient, operation, resourceARN, resourcePath, message string, ctx *PolicyContext) error {
+	parts := strings.Split(strings.Trim(resourcePath, "/"), "/")
+	if len(parts) < 2 {
+		return filer_pb.ErrNotFound
+	}
+	bucketName := parts[1]
+	bucketPath := GetTableBucketPath(bucketName)
+	ctx.TableBucketName = bucketName
+
+	var bucketMeta tableBucketMetadata
+	var bucketPolicy string
+	if data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata); err == nil {
+		if err := json.Unmarshal(data, &bucketMeta); err != nil {
+			return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
+		}
+	} else if !absentEntry(err) {
+		return err
+	}
+	if policy, err := h.readPolicy(r, client, bucketPath); err == nil {
+		bucketPolicy = policy
+	} else if !absentEntry(err) {
+		return err
+	}
+	if tags, err := h.readTags(r.Context(), client, bucketPath); err == nil {
+		if tags != nil {
+			ctx.TableBucketTags = tags
+		}
+	} else if !absentEntry(err) {
+		return err
+	}
+
+	principal := h.getAccountID(r)
+	ctx.TrustedIssuers = h.trustedIssuers(r)
+	ctx.IsLocalAccountID = h.localAccountIDs
+	ctx.IdentityActions = getIdentityActions(r)
+	ctx.DefaultAllow = h.defaultAllowFor(r)
+	// Evaluate against the canonical ARN, not the request's: the ARN parser
+	// accepts any account, so a crafted ARN could otherwise dodge a Deny's
+	// resource match.
+	if ctx.TableName != "" {
+		resourceARN = h.generateTableARN(bucketMeta.OwnerAccountID, bucketName, ctx.Namespace+"/"+ctx.TableName)
+	} else {
+		resourceARN = h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
+	}
+	if deniedByPolicy(operation, principal, bucketPolicy, resourceARN, ctx) ||
+		(!CheckPermissionWithContext(operation, principal, bucketMeta.OwnerAccountID, bucketPolicy, resourceARN, ctx) &&
+			!h.mayLearnAbsence(r, operation, bucketName)) {
+		return NewAuthError(operation, principal, message)
+	}
+	return filer_pb.ErrNotFound
+}
+
 // resolveResourcePath determines the resource path and extended attribute key from a resource ARN
 func (h *S3TablesHandler) resolveResourcePath(resourceARN string) (path string, key string, rType ResourceType, err error) {
 	// Try parsing as table ARN first

weed/s3api/s3tables/handler_table.go

@@ -514,59 +514,69 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
 			var namespacePolicy, bucketPolicy string
 			bucketTags := map[string]string{}
 
-			// Fetch namespace metadata and policy
+			// Fetch namespace metadata and policy; a missing or foreign entry
+			// leaves them empty and lets the check below answer.
 			data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
-			if err != nil {
-				return err // Not Found handled by caller
-			}
-			if err := json.Unmarshal(data, &nsMeta); err != nil {
+			nsFound := err == nil
+			if nsFound {
+				if err := json.Unmarshal(data, &nsMeta); err != nil {
+					return err
+				}
+				policyData, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyPolicy)
+				if err == nil {
+					namespacePolicy = string(policyData)
+				} else if !errors.Is(err, ErrAttributeNotFound) {
+					return fmt.Errorf("failed to fetch namespace policy: %w", err)
+				}
+			} else if !absentEntry(err) {
 				return err
 			}
 
-			// Fetch namespace policy if it exists
-			policyData, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyPolicy)
-			if err == nil {
-				namespacePolicy = string(policyData)
-			} else if !errors.Is(err, ErrAttributeNotFound) {
-				return fmt.Errorf("failed to fetch namespace policy: %w", err)
-			}
-
 			// Fetch bucket metadata and policy
 			data, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
 			if err == nil {
 				if err := json.Unmarshal(data, &bucketMeta); err != nil {
 					return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
 				}
-			} else if !errors.Is(err, ErrAttributeNotFound) {
+			} else if !absentEntry(err) {
 				return fmt.Errorf("failed to fetch bucket metadata: %w", err)
 			}
 
-			policyData, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
+			policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
 			if err == nil {
 				bucketPolicy = string(policyData)
-			} else if !errors.Is(err, ErrAttributeNotFound) {
+			} else if !absentEntry(err) {
 				return fmt.Errorf("failed to fetch bucket policy: %w", err)
 			}
 			if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
-				return fmt.Errorf("failed to read bucket tags: %w", err)
+				if !absentEntry(err) {
+					return fmt.Errorf("failed to read bucket tags: %w", err)
+				}
 			} else if tags != nil {
 				bucketTags = tags
 			}
 
 			bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
 			identityActions := getIdentityActions(r)
+			policyCtx := &PolicyContext{
+				TrustedIssuers:   h.trustedIssuers(r),
+				IsLocalAccountID: h.localAccountIDs,
+				TableBucketName:  bucketName,
+				Namespace:        namespaceName,
+				TableBucketTags:  bucketTags,
+				IdentityActions:  identityActions,
+				DefaultAllow:     h.defaultAllowFor(r),
+			}
 			if !CheckTableOrBucketPermission("ListTables", accountID,
 				nsMeta.OwnerAccountID, namespacePolicy, bucketARN,
-				bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
-					TrustedIssuers:   h.trustedIssuers(r),
-					IsLocalAccountID: h.localAccountIDs,
-					TableBucketName:  bucketName,
-					Namespace:        namespaceName,
-					TableBucketTags:  bucketTags,
-					IdentityActions:  identityActions,
-					DefaultAllow:     h.defaultAllowFor(r),
-				}) {
-				return ErrAccessDenied
+				bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, policyCtx) {
+				// The target is absent: an empty list would reveal that, so
+				// only a caller that may learn absence proceeds to it, and a
+				// resource-policy Deny vetoes even that.
+				if nsFound || !h.mayLearnAbsence(r, "ListTables", bucketName) ||
+					deniedByPolicy("ListTables", accountID, bucketPolicy, bucketARN, policyCtx) {
+					return ErrAccessDenied
+				}
 			}
 
 			tables, paginationToken, err = h.listTablesInNamespaceWithClient(r, client, bucketName, namespaceName, req.Prefix, req.ContinuationToken, maxTables)
@@ -580,39 +590,47 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
 			var bucketPolicy string
 			bucketTags := map[string]string{}
 
-			// Fetch bucket metadata and policy
+			// Fetch bucket metadata and policy; a missing or foreign entry
+			// leaves them empty and lets the check below answer.
 			data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
-			if err != nil {
+			bucketFound := err == nil
+			if bucketFound {
+				if err := json.Unmarshal(data, &bucketMeta); err != nil {
+					return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
+				}
+			} else if !absentEntry(err) {
 				return fmt.Errorf("failed to fetch bucket metadata: %w", err)
 			}
-			if err := json.Unmarshal(data, &bucketMeta); err != nil {
-				return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
-			}
 
-			// Fetch bucket policy if it exists
 			policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
 			if err == nil {
 				bucketPolicy = string(policyData)
-			} else if !errors.Is(err, ErrAttributeNotFound) {
+			} else if !absentEntry(err) {
 				return fmt.Errorf("failed to fetch bucket policy: %w", err)
 			}
 			if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
-				return fmt.Errorf("failed to read bucket tags: %w", err)
+				if !absentEntry(err) {
+					return fmt.Errorf("failed to read bucket tags: %w", err)
+				}
 			} else if tags != nil {
 				bucketTags = tags
 			}
 
 			bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
 			identityActions := getIdentityActions(r)
-			if !CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
+			policyCtx := &PolicyContext{
 				TrustedIssuers:   h.trustedIssuers(r),
 				IsLocalAccountID: h.localAccountIDs,
 				TableBucketName:  bucketName,
 				TableBucketTags:  bucketTags,
 				IdentityActions:  identityActions,
 				DefaultAllow:     h.defaultAllowFor(r),
-			}) {
-				return ErrAccessDenied
+			}
+			if !CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, policyCtx) {
+				if bucketFound || !h.mayLearnAbsence(r, "ListTables", bucketName) ||
+					deniedByPolicy("ListTables", accountID, bucketPolicy, bucketARN, policyCtx) {
+					return ErrAccessDenied
+				}
 			}
 
 			tables, paginationToken, err = h.listTablesInAllNamespaces(r, client, bucketName, req.Prefix, req.ContinuationToken, maxTables)
@@ -1465,40 +1483,51 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ
 	return nil
 }
 
-// handleUpdateTable updates table metadata
-func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
-	var req UpdateTableRequest
-	if err := h.readRequestBody(r, &req); err != nil {
-		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
-	}
+// updateTableTarget is the table state an update was authorized against: the
+// stored values the optimistic-concurrency check compares and the fields the
+// response echoes back.
+type updateTableTarget struct {
+	bucketName     string
+	namespaceName  string
+	tableName      string
+	tablePath      string
+	tableARN       string
+	metadata       tableMetadataInternal
+	storedMetadata []byte
+	storedPolicy   []byte
+}
 
+// authorizeUpdateTable validates the names an update names, loads the table
+// and the policies governing it, and checks the caller may update it, writing
+// the error response itself. Credential vending asks the same question without
+// updating anything, so it runs this gate through Manager.AuthorizeUpdateTable.
+func (h *S3TablesHandler) authorizeUpdateTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient, req *UpdateTableRequest) (*updateTableTarget, error) {
 	if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
-		return fmt.Errorf("missing required parameters")
+		return nil, fmt.Errorf("missing required parameters")
 	}
 
 	namespaceName, err := validateNamespace(req.Namespace)
 	if err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
 	if err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	tableName, err := validateTableName(req.Name)
 	if err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	tablePath := GetTablePath(bucketName, namespaceName, tableName)
@@ -1567,7 +1596,7 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 		} else {
 			h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
 		}
-		return err
+		return nil, err
 	}
 
 	// Authorization Check
@@ -1596,9 +1625,35 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 		} else {
 			h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, "table not found")
 		}
-		return NewAuthError("UpdateTable", principal, "not authorized to update table")
+		return nil, NewAuthError("UpdateTable", principal, "not authorized to update table")
 	}
 
+	return &updateTableTarget{
+		bucketName:     bucketName,
+		namespaceName:  namespaceName,
+		tableName:      tableName,
+		tablePath:      tablePath,
+		tableARN:       tableARN,
+		metadata:       metadata,
+		storedMetadata: storedMetadata,
+		storedPolicy:   storedPolicy,
+	}, nil
+}
+
+// handleUpdateTable updates table metadata
+func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
+	var req UpdateTableRequest
+	if err := h.readRequestBody(r, &req); err != nil {
+		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
+		return err
+	}
+
+	target, err := h.authorizeUpdateTable(w, r, filerClient, &req)
+	if err != nil {
+		return err
+	}
+	metadata := target.metadata
+
 	// Check version token if provided
 	if req.VersionToken != "" && req.VersionToken != metadata.VersionToken {
 		h.writeError(w, http.StatusConflict, ErrCodeConflict, "Version token mismatch")
@@ -1645,15 +1700,15 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 	// snapshot. mutateEntryExtended retries on a changed entry, so the check
 	// lives in the mutation, where it sees the value that is current now.
 	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		return h.mutateEntryExtended(r.Context(), client, tablePath, func(extended map[string][]byte) error {
-			if !bytes.Equal(extended[ExtendedKeyMetadata], storedMetadata) {
+		return h.mutateEntryExtended(r.Context(), client, target.tablePath, func(extended map[string][]byte) error {
+			if !bytes.Equal(extended[ExtendedKeyMetadata], target.storedMetadata) {
 				return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyMetadata)
 			}
 			// The policy this request was authorized against must still be the
 			// one in force: an administrator restricting it mid-commit should
 			// send the caller back through authorization, not have its decision
 			// applied afterwards.
-			if !bytes.Equal(extended[ExtendedKeyPolicy], storedPolicy) {
+			if !bytes.Equal(extended[ExtendedKeyPolicy], target.storedPolicy) {
 				return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy)
 			}
 			extended[ExtendedKeyMetadata] = metadataBytes
@@ -1671,7 +1726,7 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 	}
 
 	h.writeJSON(w, http.StatusOK, &UpdateTableResponse{
-		TableARN:         tableARN,
+		TableARN:         target.tableARN,
 		MetadataLocation: metadata.MetadataLocation,
 		VersionToken:     metadata.VersionToken,
 	})

weed/s3api/s3tables/handler_view.go

@@ -278,10 +278,12 @@ func (h *S3TablesHandler) handleListViews(w http.ResponseWriter, r *http.Request
 		bucketPath := GetTableBucketPath(bucketName)
 		var nsMeta namespaceMetadata
 		data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
-		if err != nil {
-			return err
-		}
-		if err := json.Unmarshal(data, &nsMeta); err != nil {
+		nsFound := err == nil
+		if nsFound {
+			if err := json.Unmarshal(data, &nsMeta); err != nil {
+				return err
+			}
+		} else if !absentEntry(err) {
 			return err
 		}
 
@@ -290,8 +292,23 @@ func (h *S3TablesHandler) handleListViews(w http.ResponseWriter, r *http.Request
 			return err
 		}
 		bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
-		if !h.authorizeViewOp(r, "ListViews", h.getAccountID(r), nsMeta.OwnerAccountID, bucketMeta.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, "", bucketTags) {
-			return ErrAccessDenied
+		accountID := h.getAccountID(r)
+		if !h.authorizeViewOp(r, "ListViews", accountID, nsMeta.OwnerAccountID, bucketMeta.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, "", bucketTags) {
+			// An empty list would reveal the namespace is absent; only a
+			// caller that may learn absence proceeds to it, and a
+			// resource-policy Deny vetoes even that.
+			if nsFound || !h.mayLearnAbsence(r, "ListViews", bucketName) ||
+				deniedByPolicy("ListViews", accountID, bucketPolicy, bucketARN, &PolicyContext{
+					TrustedIssuers:   h.trustedIssuers(r),
+					IsLocalAccountID: h.localAccountIDs,
+					TableBucketName:  bucketName,
+					Namespace:        namespaceName,
+					TableBucketTags:  bucketTags,
+					IdentityActions:  getIdentityActions(r),
+					DefaultAllow:     h.defaultAllowFor(r),
+				}) {
+				return ErrAccessDenied
+			}
 		}
 
 		views, paginationToken, err = h.listViewsInNamespace(r, client, bucketName, namespaceName, req.Prefix, req.ContinuationToken, maxViews)
@@ -396,23 +413,31 @@ func (h *S3TablesHandler) listViewsInNamespace(r *http.Request, client filer_pb.
 	return views, lastFileName, nil
 }
 
-// handleUpdateView replaces the metadata pointer for a view, mirroring UpdateTable.
-func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
-	var req UpdateViewRequest
-	if err := h.readRequestBody(r, &req); err != nil {
-		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
-	}
+// updateViewTarget is the view state an update was authorized against: the
+// metadata the version-token check reads and the fields the response echoes.
+type updateViewTarget struct {
+	bucketName    string
+	namespaceName string
+	viewName      string
+	viewPath      string
+	viewARN       string
+	metadata      tableMetadataInternal
+}
 
+// authorizeUpdateView validates the names an update names, loads the view and
+// the policies governing it, and checks the caller may update it, writing the
+// error response itself. Credential vending asks the same question without
+// updating anything, so it runs this gate through Manager.AuthorizeUpdateView.
+func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient, req *UpdateViewRequest) (*updateViewTarget, error) {
 	bucketName, namespaceName, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
 	if err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
 		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
-		return err
+		return nil, err
 	}
 
 	viewPath := GetTablePath(bucketName, namespaceName, viewName)
@@ -437,21 +462,46 @@ func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Reques
 		} else {
 			h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
 		}
-		return err
+		return nil, err
 	}
 
 	accountID := h.getAccountID(r)
 	viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
 	if err != nil {
 		h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
-		return err
+		return nil, err
 	}
 	viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
 	if !h.authorizeViewOp(r, "UpdateView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespaceName, viewName, bucketTags) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to update view")
-		return NewAuthError("UpdateView", accountID, "not authorized to update view")
+		return nil, NewAuthError("UpdateView", accountID, "not authorized to update view")
 	}
 
+	return &updateViewTarget{
+		bucketName:    bucketName,
+		namespaceName: namespaceName,
+		viewName:      viewName,
+		viewPath:      viewPath,
+		viewARN:       viewARN,
+		metadata:      metadata,
+	}, nil
+}
+
+// handleUpdateView replaces the metadata pointer for a view, mirroring UpdateTable.
+func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
+	var req UpdateViewRequest
+	if err := h.readRequestBody(r, &req); err != nil {
+		h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
+		return err
+	}
+
+	target, err := h.authorizeUpdateView(w, r, filerClient, &req)
+	if err != nil {
+		return err
+	}
+	metadata := target.metadata
+	viewPath := target.viewPath
+
 	if req.VersionToken != "" && req.VersionToken != metadata.VersionToken {
 		h.writeError(w, http.StatusConflict, ErrCodeConflict, "Version token mismatch")
 		return ErrVersionTokenMismatch
@@ -499,7 +549,7 @@ func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Reques
 	}
 
 	h.writeJSON(w, http.StatusOK, &UpdateViewResponse{
-		ViewARN:          viewARN,
+		ViewARN:          target.viewARN,
 		MetadataLocation: metadata.MetadataLocation,
 		VersionToken:     metadata.VersionToken,
 	})
@@ -604,25 +654,27 @@ func (h *S3TablesHandler) loadAuthContext(r *http.Request, filerClient FilerClie
 			if e := json.Unmarshal(data, &bucketMetadata); e != nil {
 				return fmt.Errorf("failed to unmarshal bucket metadata: %w", e)
 			}
-		} else if !errors.Is(e, ErrAttributeNotFound) {
+		} else if !absentEntry(e) {
 			return fmt.Errorf("failed to fetch bucket metadata: %w", e)
 		}
 
 		policyData, e := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyPolicy)
 		if e == nil {
 			resourcePolicy = string(policyData)
-		} else if !errors.Is(e, ErrAttributeNotFound) {
+		} else if !absentEntry(e) {
 			return fmt.Errorf("failed to fetch resource policy: %w", e)
 		}
 
 		policyData, e = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
 		if e == nil {
 			bucketPolicy = string(policyData)
-		} else if !errors.Is(e, ErrAttributeNotFound) {
+		} else if !absentEntry(e) {
 			return fmt.Errorf("failed to fetch bucket policy: %w", e)
 		}
 		if tags, e := h.readTags(r.Context(), client, bucketPath); e != nil {
-			return e
+			if !absentEntry(e) {
+				return e
+			}
 		} else if tags != nil {
 			bucketTags = tags
 		}

weed/s3api/s3tables/iceberg_layout.go

@@ -15,6 +15,7 @@ import (
 // - metadata/v*.metadata.json (table metadata)
 // - metadata/snap-*.avro (snapshot manifest lists)
 // - metadata/*.avro (manifest files)
+// - metadata/*.stats, metadata/*.puffin (statistics files)
 // - data/*.parquet, *.orc, *.avro, *.puffin (data files)
 
 const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
@@ -68,6 +69,9 @@ var (
 		// whose naming we can't anticipate across engines and versions.
 		regexp.MustCompile(`^[A-Za-z0-9._-]+\.avro$`),
 		regexp.MustCompile(`^[A-Za-z0-9._-]+\.metadata\.json$`),
+		// Table and partition statistics files listed in the table metadata
+		// are Puffin files; engines name them *.stats or *.puffin.
+		regexp.MustCompile(`^[A-Za-z0-9._-]+\.puffin$`),
 	}
 
 	// Patterns for valid data files

weed/s3api/s3tables/iceberg_layout_test.go

@@ -45,6 +45,10 @@ func TestIcebergLayoutValidator_AcceptsRealWorldManifestNames(t *testing.T) {
 			"trino/iceberg stats file",
 			"metadata/table.stats",
 		},
+		{
+			"puffin statistics file",
+			"metadata/20260928-1-0199a3c2-7d1e-7c55-9a0e-3b1f2c4d5e6f.puffin",
+		},
 	}
 
 	v := NewIcebergLayoutValidator()
@@ -70,6 +74,8 @@ func TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames(t *testing.T) {
 		{"subdirectory under metadata is not allowed", "metadata/sub/file.avro"},
 		{"top-level dir other than metadata or data", "garbage/file.avro"},
 		{"catch-all anchors must reject trailing extension", "metadata/file.avro.txt"},
+		{"puffin with a trailing extension", "metadata/stats.puffin.txt"},
+		{"puffin in a subdirectory under metadata", "metadata/sub/stats.puffin"},
 	}
 
 	v := NewIcebergLayoutValidator()
@@ -137,3 +143,31 @@ func TestValidateFilePathAcceptsLanceLayout(t *testing.T) {
 		}
 	}
 }
+
+// Statistics files are referenced from the table metadata and kept by orphan
+// cleanup; the upload door has to admit them in the table's metadata/, and
+// only there.
+func TestTableBucketFileValidator_PuffinStatisticsOnlyInTableMetadata(t *testing.T) {
+	v := NewTableBucketFileValidator()
+	accepted := []string{
+		TablesPath + "/cat/ns/t/metadata/stats-1.puffin",
+		TablesPath + "/cat/ns/t/metadata/table.stats",
+		TablesPath + "/cat/ns/t/data/0199a3c2-deletes.puffin",
+	}
+	for _, p := range accepted {
+		if err := v.ValidateTableBucketUpload(p); err != nil {
+			t.Errorf("ValidateTableBucketUpload(%q) = %v, want nil", p, err)
+		}
+	}
+	rejected := []string{
+		TablesPath + "/cat/ns/t/stats-1.puffin",
+		TablesPath + "/cat/ns/t/other/stats-1.puffin",
+		TablesPath + "/cat/ns/t/metadata/sub/stats-1.puffin",
+		TablesPath + "/cat/ns/t/metadata/notes.txt",
+	}
+	for _, p := range rejected {
+		if err := v.ValidateTableBucketUpload(p); err == nil {
+			t.Errorf("ValidateTableBucketUpload(%q) = nil, want an error", p)
+		}
+	}
+}

weed/s3api/s3tables/manager.go

@@ -81,6 +81,43 @@ func (m *Manager) AuthorizeCreateTable(ctx context.Context, filerClient FilerCli
 	return authErr
 }
 
+// AuthorizeUpdateTable checks that identity may update the table the request
+// describes, without updating anything. Credential vending gates on the same
+// permission a commit needs.
+func (m *Manager) AuthorizeUpdateTable(ctx context.Context, filerClient FilerClient, req *UpdateTableRequest, identity string) error {
+	httpReq, err := newManagerRequest(ctx, "UpdateTable", nil, identity)
+	if err != nil {
+		return err
+	}
+	recorder := httptest.NewRecorder()
+	_, authErr := m.handler.authorizeUpdateTable(recorder, httpReq, filerClient, req)
+	if authErr == nil {
+		return nil
+	}
+	if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
+		return decoded
+	}
+	return authErr
+}
+
+// AuthorizeUpdateView checks that identity may update the view the request
+// describes, without updating anything.
+func (m *Manager) AuthorizeUpdateView(ctx context.Context, filerClient FilerClient, req *UpdateViewRequest, identity string) error {
+	httpReq, err := newManagerRequest(ctx, "UpdateView", nil, identity)
+	if err != nil {
+		return err
+	}
+	recorder := httptest.NewRecorder()
+	_, authErr := m.handler.authorizeUpdateView(recorder, httpReq, filerClient, req)
+	if authErr == nil {
+		return nil
+	}
+	if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
+		return decoded
+	}
+	return authErr
+}
+
 func newManagerRequest(ctx context.Context, operation string, body []byte, identity string) (*http.Request, error) {
 	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/", bytes.NewReader(body))
 	if err != nil {

weed/server/master_grpc_server.go

@@ -184,6 +184,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 
 		ms.Topo.Sequence.SetMax(heartbeat.MaxFileKey)
 		ms.Topo.UpAdjustMaxVolumeId(needle.VolumeId(heartbeat.MaxVolumeIdSeen))
+		if len(heartbeat.Volumes) > 0 {
+			ms.Topo.CheckReportedTopologyId(heartbeat.LastSeenTopologyId)
+		}
 		if dn == nil {
 			// Skip delta heartbeat for volume server versions  better than 3.28 https://github.com/seaweedfs/seaweedfs/pull/3630
 			if heartbeat.Ip == "" {

weed/server/master_grpc_server_digest_test.go

@@ -132,7 +132,11 @@ func TestDigestCheckAsksForTheListWhenTheLookupIndexDrifts(t *testing.T) {
 // preallocate is a bare bool with no way to tell "off" from "not mentioned". A
 // response that left it out would turn preallocation off until reconnect.
 func TestHeartbeatResponsesCarryTheVolumeOptions(t *testing.T) {
-	ms := &MasterServer{option: &MasterOption{VolumeSizeLimitMB: 1024}, preallocateSize: 1}
+	ms := &MasterServer{
+		option:          &MasterOption{VolumeSizeLimitMB: 1024},
+		preallocateSize: 1,
+		Topo:            topology.NewTopology("test", sequence.NewMemorySequencer(), 32*1024, 5, false),
+	}
 
 	resend := ms.heartbeatResponse()
 	resend.ResendFullVolumeList = true

weed/server/master_grpc_server_volume.go

@@ -471,6 +471,13 @@ func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.V
 	return resp, nil
 }
 
+func (ms *MasterServer) ResumeVolumeGrowth(ctx context.Context, req *master_pb.ResumeVolumeGrowthRequest) (*master_pb.ResumeVolumeGrowthResponse, error) {
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+	return &master_pb.ResumeVolumeGrowthResponse{WasFrozen: ms.Topo.ResumeVolumeGrowth()}, nil
+}
+
 func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGrowRequest) (*master_pb.VolumeGrowResponse, error) {
 	if !ms.Topo.IsLeader() {
 		return nil, raft.NotLeaderError

weed/server/master_server.go

@@ -15,7 +15,9 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/telemetry"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -28,6 +30,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/sequence"
 	"github.com/seaweedfs/seaweedfs/weed/shell"
@@ -122,6 +125,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	v.SetDefault("master.volume_growth.reservation_timeout", "5m")
 	v.SetDefault("master.volume_growth.disable", false)
 	option.VolumeGrowthDisabled = v.GetBool("master.volume_growth.disable")
+	v.SetDefault("master.volume_growth.freeze_on_topology_mismatch", true)
+	v.SetDefault("master.volume_growth.state_loss_recovery_wait", "0s")
 
 	topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1")
 	topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2")
@@ -157,6 +162,25 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 		glog.Fatalf("create sequencer failed.")
 	}
 	ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
+	ms.Topo.SetGrowthFreezeConfig(
+		v.GetBool("master.volume_growth.freeze_on_topology_mismatch"),
+		v.GetDuration("master.volume_growth.state_loss_recovery_wait"))
+	ms.Topo.QuarantineVolumeFn = func(dn *topology.DataNode, vid needle.VolumeId, readOnly bool) error {
+		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			if readOnly {
+				_, err := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: uint32(vid), CanDelete: true})
+				return err
+			}
+			_, err := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: uint32(vid)})
+			return err
+		})
+		if err != nil {
+			glog.Warningf("quarantine volume %d on %s readOnly=%v: %v", vid, dn.Id(), readOnly, err)
+		}
+		return err
+	}
 	ms.vg = topology.NewDefaultVolumeGrowth()
 	glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")
 

weed/server/raft_common.go

@@ -30,9 +30,15 @@ func recoverTopologyIdFromState(fsmState []byte, topo *topology.Topology) {
 // It uses the provided checkLeaderFn to verify leadership and persistFn to save the new ID.
 func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persistFn func(string) error) {
 	if topo.GetTopologyId() != "" {
+		// The id arrived through raft replay or the mirror file; reports
+		// stashed while it was unknown can now be checked.
+		topo.SetTopologyInitPending(false)
 		return
 	}
 
+	topo.SetTopologyInitPending(true)
+	defer topo.SetTopologyInitPending(false)
+
 	topologyId := uuid.New().String()
 	for {
 		if !checkLeaderFn() {
@@ -59,6 +65,7 @@ func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persis
 		} else {
 			glog.V(0).Infof("TopologyId generated: %s", topologyId)
 		}
+		topo.DeferAllocation(topo.WarmupDuration())
 		break
 	}
 }

weed/server/raft_server.go

@@ -60,6 +60,10 @@ func (s StateMachine) Save() ([]byte, error) {
 		MaxVolumeId: s.topo.GetMaxVolumeId(),
 		TopologyId:  s.topo.GetTopologyId(),
 	}
+	if s.topo.IsGrowthFrozen() {
+		state.GrowthFrozen = new(bool)
+		*state.GrowthFrozen = true
+	}
 	glog.V(1).Infof("Save raft state %+v", state)
 	return json.Marshal(state)
 }
@@ -76,6 +80,9 @@ func (s StateMachine) Recovery(data []byte) error {
 		s.topo.SetTopologyId(state.TopologyId)
 		glog.V(0).Infof("Recovered TopologyId: %s", state.TopologyId)
 	}
+	if state.GrowthFrozen != nil {
+		s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+	}
 	if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
 		glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
 	}
@@ -98,6 +105,9 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
 			glog.V(0).Infof("Set TopologyId from raft log: %s", state.TopologyId)
 		}
 	}
+	if state.GrowthFrozen != nil {
+		s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+	}
 	if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
 		glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
 	}
@@ -107,10 +117,13 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
 }
 
 func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
-	return &topology.MaxVolumeIdCommand{
-		MaxVolumeId: s.topo.GetMaxVolumeId(),
-		TopologyId:  s.topo.GetTopologyId(),
-	}, nil
+	frozen := s.topo.IsGrowthFrozen()
+	state := &topology.MaxVolumeIdCommand{
+		MaxVolumeId:  s.topo.GetMaxVolumeId(),
+		TopologyId:   s.topo.GetTopologyId(),
+		GrowthFrozen: &frozen,
+	}
+	return state, nil
 }
 
 func (s *StateMachine) Restore(r io.ReadCloser) error {

weed/shell/command_volume_check_duplicate_ids.go

@@ -0,0 +1,140 @@
+package shell
+
+import (
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeCheckDuplicateIds{})
+}
+
+type commandVolumeCheckDuplicateIds struct {
+}
+
+func (c *commandVolumeCheckDuplicateIds) Name() string {
+	return "volume.check.duplicate_ids"
+}
+
+func (c *commandVolumeCheckDuplicateIds) Help() string {
+	return `find volume ids registered under more than one layout
+
+	volume.check.duplicate_ids
+
+	Volume ids are global. The same id under two collections, replica
+	placements, TTLs, or disk types means the allocator reused an id, for
+	example after master state loss. See volume.growth.resume.
+
+`
+}
+
+func (c *commandVolumeCheckDuplicateIds) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandVolumeCheckDuplicateIds) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	// vid -> layout key -> locations; ec shards tracked per collection since
+	// shards plus regular copies of the same encoded volume are legitimate
+	seen := make(map[needle.VolumeId]map[string][]string)
+	ecSeen := make(map[needle.VolumeId]map[string][]string)
+	collections := make(map[needle.VolumeId]map[string]bool)
+	ecCollections := make(map[needle.VolumeId]map[string]bool)
+	add := func(m map[needle.VolumeId]map[string][]string, id uint32, key, location string) {
+		layouts := m[needle.VolumeId(id)]
+		if layouts == nil {
+			layouts = make(map[string][]string)
+			m[needle.VolumeId(id)] = layouts
+		}
+		layouts[key] = append(layouts[key], location)
+	}
+	addCollection := func(m map[needle.VolumeId]map[string]bool, id uint32, collection string) {
+		if m[needle.VolumeId(id)] == nil {
+			m[needle.VolumeId(id)] = make(map[string]bool)
+		}
+		m[needle.VolumeId(id)][collection] = true
+	}
+	for _, dc := range topoInfo.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				for diskKey, di := range dn.DiskInfos {
+					for _, vi := range di.VolumeInfos {
+						rp, _ := super_block.NewReplicaPlacementFromByte(byte(vi.ReplicaPlacement))
+						add(seen, vi.Id, fmt.Sprintf("collection=%q replication=%s ttl=%s disk=%s",
+							vi.Collection, rp.String(), needle.LoadTTLFromUint32(vi.Ttl), vi.DiskType),
+							fmt.Sprintf("%s %s", dn.Id, diskKey))
+						addCollection(collections, vi.Id, vi.Collection)
+					}
+					for _, si := range di.EcShardInfos {
+						add(ecSeen, si.Id, fmt.Sprintf("collection=%q ec_shards", si.Collection),
+							fmt.Sprintf("%s %s disk=%s", dn.Id, diskKey, si.DiskType))
+						addCollection(ecCollections, si.Id, si.Collection)
+					}
+				}
+			}
+		}
+	}
+
+	vids := make(map[needle.VolumeId]struct{}, len(seen)+len(ecSeen))
+	for vid := range seen {
+		vids[vid] = struct{}{}
+	}
+	for vid := range ecSeen {
+		vids[vid] = struct{}{}
+	}
+
+	duplicates := 0
+	for _, vid := range sortedKeys(vids) {
+		layouts := seen[vid]
+		shardLayouts := ecSeen[vid]
+		ecForeign := false
+		if len(collections[vid]) > 0 {
+			for collection := range ecCollections[vid] {
+				if !collections[vid][collection] {
+					ecForeign = true
+				}
+			}
+		}
+		if len(layouts) < 2 && len(shardLayouts) < 2 && !ecForeign {
+			continue
+		}
+		duplicates++
+		fmt.Fprintf(writer, "volume %d is registered under %d layouts:\n", vid, len(layouts)+len(shardLayouts))
+		for _, key := range sortedKeysString(layouts) {
+			fmt.Fprintf(writer, "  %-60s %v\n", key, layouts[key])
+		}
+		for _, key := range sortedKeysString(shardLayouts) {
+			fmt.Fprintf(writer, "  %-60s %v\n", key, shardLayouts[key])
+		}
+	}
+	if duplicates == 0 {
+		fmt.Fprintf(writer, "no duplicate volume ids found\n")
+	}
+	return nil
+}
+
+func sortedKeys[V any](m map[needle.VolumeId]V) []needle.VolumeId {
+	keys := make([]needle.VolumeId, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func sortedKeysString[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}

weed/shell/command_volume_growth_resume.go

@@ -0,0 +1,52 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeGrowthResume{})
+}
+
+type commandVolumeGrowthResume struct {
+}
+
+func (c *commandVolumeGrowthResume) Name() string {
+	return "volume.growth.resume"
+}
+
+func (c *commandVolumeGrowthResume) Help() string {
+	return `resume volume growth frozen after detected master state loss
+
+	volume.growth.resume
+
+	Volume growth freezes automatically when a volume server reports a
+	topology id that differs from this cluster's, which means the master
+	state was rebuilt and max volume id knowledge may be incomplete. Run
+	this after verifying the cluster inventory so new ids are not reused.
+
+`
+}
+
+func (c *commandVolumeGrowthResume) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandVolumeGrowthResume) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	return commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.ResumeVolumeGrowth(context.Background(), &master_pb.ResumeVolumeGrowthRequest{})
+		if err != nil {
+			return err
+		}
+		if resp.WasFrozen {
+			fmt.Fprintf(writer, "volume growth resumed\n")
+		} else {
+			fmt.Fprintf(writer, "volume growth was not frozen\n")
+		}
+		return nil
+	})
+}

weed/stats/metrics.go

@@ -101,6 +101,22 @@ var (
 			Help:      "admin lock",
 		}, []string{"client"})
 
+	MasterVolumeGrowthFrozen = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: subsystemMaster,
+			Name:      "volume_growth_frozen",
+			Help:      "1 while volume growth is frozen after detected master state loss",
+		})
+
+	MasterDuplicateVolumeIds = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystemMaster,
+			Name:      "duplicate_volume_ids",
+			Help:      "volume id registered under two different layouts",
+		}, []string{"collection", "volumeId"})
+
 	MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -991,6 +1007,8 @@ func init() {
 	Gather.MustRegister(MasterClientConnectCounter)
 	Gather.MustRegister(MasterRaftIsleader)
 	Gather.MustRegister(MasterAdminLock)
+	Gather.MustRegister(MasterVolumeGrowthFrozen)
+	Gather.MustRegister(MasterDuplicateVolumeIds)
 	Gather.MustRegister(MasterReceivedHeartbeatCounter)
 	Gather.MustRegister(MasterLeaderChangeCounter)
 	Gather.MustRegister(MasterReplicaPlacementMismatch)

weed/storage/store.go

@@ -913,6 +913,8 @@ func (s *Store) MarkVolumeReadonly(i needle.VolumeId, canDelete bool, persist bo
 	}
 	// Wait for in-flight appends so a mark acked mid-write cannot race an
 	// encode that reads the index and deletes the replica the write lands on.
+	v.dataFileSyncLock.Lock()
+	defer v.dataFileSyncLock.Unlock()
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
 	if canDelete && !v.HasRemoteFile() {
@@ -1207,6 +1209,7 @@ func (s *Store) deleteVolumeGuarded(i needle.VolumeId, onlyEmpty bool, onlyGarba
 	unlockAll := func() {
 		for _, v := range lockedVolumes {
 			v.dataFileAccessLock.Unlock()
+			v.dataFileSyncLock.Unlock()
 		}
 		for _, location := range lockedLocations {
 			location.volumesLock.Unlock()
@@ -1217,6 +1220,7 @@ func (s *Store) deleteVolumeGuarded(i needle.VolumeId, onlyEmpty bool, onlyGarba
 	for _, location := range s.Locations {
 		location.volumesLock.Lock()
 		if v, ok := location.volumes[i]; ok {
+			v.dataFileSyncLock.Lock()
 			v.dataFileAccessLock.Lock()
 			lockedLocations = append(lockedLocations, location)
 			lockedVolumes = append(lockedVolumes, v)

weed/storage/volume.go

@@ -43,7 +43,12 @@ type Volume struct {
 
 	super_block.SuperBlock
 
-	dataFileAccessLock   sync.RWMutex
+	dataFileAccessLock sync.RWMutex
+	// Serializes a data-file mutation with the sync that makes it durable:
+	// write, delete, lifecycle and compaction paths hold it for their whole
+	// critical section, while the fsync itself runs with dataFileAccessLock
+	// released so readers of this volume do not wait on it.
+	dataFileSyncLock     sync.Mutex
 	superBlockAccessLock sync.Mutex
 
 	// The batch worker exists only once the volume takes a durable write. Most
@@ -129,6 +134,8 @@ func (v *Volume) FileName(ext string) (fileName string) {
 // gone. A no-op when the index already lives in newIdxDir. A derived .ldb is
 // not moved: the reload rebuilds it in newIdxDir from the .idx.
 func (v *Volume) RelocateIndexTo(newIdxDir string) error {
+	v.dataFileSyncLock.Lock()
+	defer v.dataFileSyncLock.Unlock()
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
 
@@ -290,32 +297,41 @@ func (v *Volume) DiskType() types.DiskType {
 }
 
 func (v *Volume) SyncToDisk() {
+	v.dataFileSyncLock.Lock()
+	defer v.dataFileSyncLock.Unlock()
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
-	if v.nm != nil {
-		if err := v.nm.Sync(); err != nil {
+	// Flush with the access lock dropped so readers are not stalled by the
+	// fsync; writers and file swaps stay excluded by dataFileSyncLock.
+	nm, backend := v.nm, v.DataBackend
+	v.dataFileAccessLock.Unlock()
+	if nm != nil {
+		if err := nm.Sync(); err != nil {
 			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
 		}
 	}
-	if v.DataBackend != nil {
-		if err := v.DataBackend.Sync(); err != nil {
+	if backend != nil {
+		if err := backend.Sync(); err != nil {
 			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
 		}
 	}
+	v.dataFileAccessLock.Lock()
 }
 
 // Close cleanly shuts down this volume
 func (v *Volume) Close() {
 	// Wait for any in-progress compaction to finish and claim the flag so no
 	// new compaction can start. This must happen BEFORE acquiring
-	// dataFileAccessLock to avoid deadlocking with CommitCompact which holds
-	// the flag while waiting for the lock.
+	// dataFileSyncLock and dataFileAccessLock to avoid deadlocking with
+	// CommitCompact which holds the flag while waiting for the locks.
 	for !v.isCompactionInProgress.CompareAndSwap(false, true) {
 		time.Sleep(521 * time.Millisecond)
 		glog.Warningf("Volume Close wait for compaction %d", v.Id)
 	}
 	defer v.isCompactionInProgress.Store(false)
 
+	v.dataFileSyncLock.Lock()
+	defer v.dataFileSyncLock.Unlock()
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
 
@@ -331,6 +347,8 @@ func (v *Volume) Close() {
 // stale-false flag would make doDeleteRequest skip the new .dat's tombstones and
 // disable the phantom-.dat guard.
 func (v *Volume) SwapDataBackend(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
+	v.dataFileSyncLock.Lock()
+	defer v.dataFileSyncLock.Unlock()
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
 	v.swapDataBackendLocked(newBackend, hasRemoteFile)