From 3ce7848bdfd59a28400f47ff1f4e9fa3534099b9 Mon Sep 17 00:00:00 2001 From: Piotr Sarna Date: Tue, 10 Nov 2020 16:32:39 +0100 Subject: [PATCH] reader_concurrency_semaphore: add metrics for shed reads When the admission queue capacity reaches its limits, excessive reads are shed in order to avoid overload. Each such operation now bumps the metrics, which can help the user judge if a replica is overloaded. --- database.cc | 15 +++++++++++++++ reader_concurrency_semaphore.cc | 1 + reader_concurrency_semaphore.hh | 2 ++ 3 files changed, 18 insertions(+) diff --git a/database.cc b/database.cc index 625f78ac40..b1cd4a14fd 100644 --- a/database.cc +++ b/database.cc @@ -456,6 +456,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {user_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _read_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {user_label_instance}), + sm::make_gauge("active_reads", [this] { return max_count_streaming_concurrent_reads - _streaming_concurrency_sem.available_resources().count; }, sm::description("Holds the number of currently active read operations issued on behalf of streaming "), {streaming_label_instance}), @@ -481,6 +486,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {streaming_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _streaming_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {streaming_label_instance}), + sm::make_gauge("active_reads", [this] { return max_count_system_concurrent_reads - _system_read_concurrency_sem.available_resources().count; }, sm::description("Holds the number of currently active read operations from \"system\" keyspace tables. "), {system_label_instance}), @@ -505,6 +515,11 @@ database::setup_metrics() { " to be able to admit new ones, if there is a shortage of permits."), {system_label_instance}), + sm::make_derive("reads_shed_due_to_overload", _system_read_concurrency_sem.get_stats().total_reads_shed_due_to_overload, + sm::description("The number of reads shed because the admission queue reached its max capacity." + " When the queue is full, excessive reads are shed to avoid overload."), + {system_label_instance}), + sm::make_gauge("total_result_bytes", [this] { return get_result_memory_limiter().total_used_memory(); }, sm::description("Holds the current amount of memory used for results.")), diff --git a/reader_concurrency_semaphore.cc b/reader_concurrency_semaphore.cc index 9b2024022e..d30abf0594 100644 --- a/reader_concurrency_semaphore.cc +++ b/reader_concurrency_semaphore.cc @@ -436,6 +436,7 @@ bool reader_concurrency_semaphore::may_proceed(const resources& r) const { future reader_concurrency_semaphore::do_wait_admission(reader_permit permit, size_t memory, db::timeout_clock::time_point timeout) { if (_wait_list.size() >= _max_queue_length) { + _stats.total_reads_shed_due_to_overload++; if (_prethrow_action) { _prethrow_action(); } diff --git a/reader_concurrency_semaphore.hh b/reader_concurrency_semaphore.hh index 65c908ba43..22ec10ba7f 100644 --- a/reader_concurrency_semaphore.hh +++ b/reader_concurrency_semaphore.hh @@ -91,6 +91,8 @@ public: uint64_t total_successful_reads = 0; // Total number of failed reads executed through this semaphore. uint64_t total_failed_reads = 0; + // Total number of reads rejected because the admission queue reached its max capacity + uint64_t total_reads_shed_due_to_overload = 0; }; struct permit_list;