From 413739824fbd784f83a7dd0cce8dfd4b01dee4bb Mon Sep 17 00:00:00 2001
From: Ernest Zaslavsky <ernest.zaslavsky@scylladb.com>
Date: Thu, 9 Oct 2025 10:01:29 +0300
Subject: [PATCH] s3_client: track memory starvation in background filling
 fiber

Introduce a counter metric to monitor instances where the background
filling fiber is blocked due to insufficient memory in the S3 client.

Closes scylladb/scylladb#26466
---
 utils/s3/client.cc | 7 +++++++
 utils/s3/client.hh | 1 +
 2 files changed, 8 insertions(+)

diff --git a/utils/s3/client.cc b/utils/s3/client.cc
index 1e9fd61807..479c707c8b 100644
--- a/utils/s3/client.cc
+++ b/utils/s3/client.cc
@@ -235,6 +235,11 @@ void client::group_client::register_metrics(std::string class_name, std::string
                 sm::description("Total time spend writing data to objects"), {ep_label, sg_label}),
         sm::make_counter("total_read_prefetch_bytes", [this] { return prefetch_bytes; },
                 sm::description("Total number of bytes requested from object"), {ep_label, sg_label}),
+         sm::make_counter("downloads_blocked_on_memory",
+                          [this] { return downloads_blocked_on_memory; },
+                          sm::description("Counts the number of times S3 client downloads were delayed due to insufficient memory availability"),
+                          {ep_label, sg_label})
+
     });
 }
 
@@ -1147,6 +1152,8 @@ class client::chunked_download_source final : public seastar::data_source_impl {
                 }
 
                 if (auto units = try_get_units(_client->_memory, _socket_buff_size); !_is_finished && !_buffers.empty() && !units) {
+                    auto& gc = _client->find_or_create_client();
+                    ++gc.downloads_blocked_on_memory;
                     co_await _bg_fiber_cv.when([this] {
                         return _is_finished || _buffers.empty() || try_get_units(_client->_memory, _socket_buff_size);
                     });
diff --git a/utils/s3/client.hh b/utils/s3/client.hh
index d2ef097d3e..e13965a2d0 100644
--- a/utils/s3/client.hh
+++ b/utils/s3/client.hh
@@ -131,6 +131,7 @@ class client : public enable_shared_from_this<client> {
         io_stats read_stats;
         io_stats write_stats;
         uint64_t prefetch_bytes = 0;
+        uint64_t downloads_blocked_on_memory = 0;
         seastar::metrics::metric_groups metrics;
         group_client(std::unique_ptr<http::experimental::connection_factory> f, unsigned max_conn, const aws::retry_strategy& retry_strategy);
         void register_metrics(std::string class_name, std::string host);