diff --git a/database.cc b/database.cc index 625f78ac40..b1cd4a14fd 100644 --- a/database.cc +++ b/database.cc @@ -456,6 +456,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {user_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _read_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {user_label_instance}), + sm::make_gauge("active_reads", [this] { return max_count_streaming_concurrent_reads - _streaming_concurrency_sem.available_resources().count; }, sm::description("Holds the number of currently active read operations issued on behalf of streaming "), {streaming_label_instance}), @@ -481,6 +486,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {streaming_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _streaming_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {streaming_label_instance}), + sm::make_gauge("active_reads", [this] { return max_count_system_concurrent_reads - _system_read_concurrency_sem.available_resources().count; }, sm::description("Holds the number of currently active read operations from \"system\" keyspace tables. "), {system_label_instance}), @@ -505,6 +515,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {system_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _system_read_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {system_label_instance}), + sm::make_gauge("total_result_bytes", [this] { return get_result_memory_limiter().total_used_memory(); }, sm::description("Holds the current amount of memory used for results.")), diff --git a/reader_concurrency_semaphore.cc b/reader_concurrency_semaphore.cc index 9b2024022e..d30abf0594 100644 --- a/reader_concurrency_semaphore.cc +++ b/reader_concurrency_semaphore.cc @@ -436,6 +436,7 @@ bool reader_concurrency_semaphore::may_proceed(const resources& r) const { future reader_concurrency_semaphore::do_wait_admission(reader_permit permit, size_t memory, db::timeout_clock::time_point timeout) { if (_wait_list.size() >= _max_queue_length) { + _stats.total_reads_shed_due_to_overload++; if (_prethrow_action) { _prethrow_action(); } diff --git a/reader_concurrency_semaphore.hh b/reader_concurrency_semaphore.hh index 65c908ba43..22ec10ba7f 100644 --- a/reader_concurrency_semaphore.hh +++ b/reader_concurrency_semaphore.hh @@ -91,6 +91,8 @@ public: uint64_t total_successful_reads = 0; // Total number of failed reads executed through this semaphore. uint64_t total_failed_reads = 0; + // Total number of reads rejected because the admission queue reached its max capacity + uint64_t total_reads_shed_due_to_overload = 0; }; struct permit_list;