mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-06-09 00:13:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

cql3/statements/describe_statement: use chunked_vector to prevent oversized allocations

Browse Source 
Running the 5000 tables scenario using tablets following Scylla warnings appeared:
```
2026-02-23T23:18:31.903 schema-scale-tablets-5000t-2026-1-db-node-77930459-4 !WARNING | scylla[5208]  [shard  1:sl:d] seastar_memory - oversized allocation: 655360 bytes. This is non-fatal, but could lead to latency and/or fragmentation issues. Please report: at 0x320cf9f 0x320cba0 0x1826a28 0x2fb8f97 0x180340e 0x447855e 0x4461c5a 0x161c3c6 0x161c4b3 0x161e9b7 0x551f43c 0x54df6ca /opt/scylladb/libreloc/libc.so.6+0x72463 /opt/scylladb/libreloc/libc.so.6+0xf55ab
seastar::current_backtrace_tasklocal() at ./build/release/seastar/./seastar/include/seastar/util/backtrace.hh:85
seastar::current_tasktrace() at ./build/release/seastar/./build/release/seastar/./seastar/src/util/backtrace.cc:136
seastar::current_backtrace() at ./build/release/seastar/./build/release/seastar/./seastar/src/util/backtrace.cc:169
seastar::memory::cpu_pages::warn_large_allocation(unsigned long) at ./build/release/seastar/./build/release/seastar/./seastar/src/core/memory.cc:865
seastar::memory::allocate_slowpath(unsigned long) at ./build/release/seastar/./build/release/seastar/./seastar/src/core/memory.cc:928
cql3::statements::(anonymous namespace)::tables(data_dictionary::database const&, seastar::lw_shared_ptr<data_dictionary::keyspace_metadata> const&, std::optional<bool>) [clone .resume] at ././seastar/src/core/memory.cc:1727
std::__n4861::coroutine_handle<seastar::internal::coroutine_traits_base<std::vector<cql3::description, std::allocator<cql3::description> > >::promise_type>::resume() const at /usr/lib/gcc/x86_64-redhat-linux/15/../../../../include/c++/15/coroutine:247
```

This patch replaces the use of `std::vector<description>` with `utils::chunked_vector`
to prevent the large allocation.

Fixes: SCYLLADB-2388

Signed-off-by: Benny Halevy <bhalevy@scylladb.com>

Closes scylladb/scylladb#30146

(cherry picked from commit d4d43213f6)

Closes scylladb/scylladb#30218
This commit is contained in:
Benny Halevy
2026-05-29 13:57:14 +03:00
committed by Avi Kivity
parent 452c246476
commit d9f721baf3
1 changed files with 25 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
cql3/statements/describe_statement.cc
View File

@@ -51,6 +51,7 @@
#include "utils/overloaded_functor.hh"
#include "db/system_keyspace.hh"
#include "db/extensions.hh"
#include "utils/chunked_vector.hh"
#include "utils/sorting.hh"
#include "replica/database.hh"
#include "cql3/description.hh"
@@ -72,8 +73,8 @@ namespace {
template <typename Range, typename Describer>
requires std::is_invocable_r_v<description, Describer, std::ranges::range_value_t<Range>>
future<std::vector<description>> generate_descriptions(Range&& range, const Describer& describer, bool sort_by_name = true) {
std::vector<description> result{};
future<utils::chunked_vector<description>> generate_descriptions(Range&& range, const Describer& describer, bool sort_by_name = true) {
utils::chunked_vector<description> result{};
if constexpr (std::ranges::sized_range<Range>) {
result.reserve(std::ranges::size(range));
}
@@ -93,8 +94,8 @@ future<std::vector<description>> generate_descriptions(Range&& range, const Desc
// `cql3::description` is a move-only type at the moment, and so we cannot initialize a vector passing
// its instances as elements of an initializer list. This function serves as an intermediary to avoid
// unnecessarily bloating some parts of the code below.
std::vector<description> wrap_in_vector(description desc) {
std::vector<description> result{};
utils::chunked_vector<description> wrap_in_vector(description desc) {
utils::chunked_vector<description> result{};
result.reserve(1);
result.push_back(std::move(desc));
return result;
@@ -151,7 +152,7 @@ future<std::vector<user_type>> get_sorted_types(const lw_shared_ptr<keyspace_met
co_return co_await utils::topological_sort(all_udts, adjacency);
}
future<std::vector<description>> types(replica::database& db, const lw_shared_ptr<keyspace_metadata>& ks, with_create_statement with_stmt) {
future<utils::chunked_vector<description>> types(replica::database& db, const lw_shared_ptr<keyspace_metadata>& ks, with_create_statement with_stmt) {
auto udts = co_await get_sorted_types(ks);
auto describer = [with_stmt] (const user_type udt) -> cql3::description {
return udt->describe(with_stmt);
@@ -159,7 +160,7 @@ future<std::vector<description>> types(replica::database& db, const lw_shared_pt
co_return co_await generate_descriptions(udts, describer, false);
}
future<std::vector<description>> function(replica::database& db, const sstring& ks, const sstring& name) {
future<utils::chunked_vector<description>> function(replica::database& db, const sstring& ks, const sstring& name) {
auto fs = functions::instance().find(functions::function_name(ks, name));
if (fs.empty()) {
throw exceptions::invalid_request_exception(format("Function '{}' not found in keyspace '{}'", name, ks));
@@ -181,7 +182,7 @@ future<std::vector<description>> function(replica::database& db, const sstring&
co_return co_await generate_descriptions(udfs, describer, true);
}
future<std::vector<description>> functions(replica::database& db,const sstring& ks, with_create_statement with_stmt) {
future<utils::chunked_vector<description>> functions(replica::database& db,const sstring& ks, with_create_statement with_stmt) {
auto udfs = cql3::functions::instance().get_user_functions(ks);
auto describer = [with_stmt] (shared_ptr<const functions::user_function> udf) {
return udf->describe(with_stmt);
@@ -189,7 +190,7 @@ future<std::vector<description>> functions(replica::database& db,const sstring&
co_return co_await generate_descriptions(udfs, describer, true);
}
future<std::vector<description>> aggregate(replica::database& db, const sstring& ks, const sstring& name) {
future<utils::chunked_vector<description>> aggregate(replica::database& db, const sstring& ks, const sstring& name) {
auto fs = functions::instance().find(functions::function_name(ks, name));
if (fs.empty()) {
throw exceptions::invalid_request_exception(format("Aggregate '{}' not found in keyspace '{}'", name, ks));
@@ -210,7 +211,7 @@ future<std::vector<description>> aggregate(replica::database& db, const sstring&
co_return co_await generate_descriptions(udas, describer, true);
}
future<std::vector<description>> aggregates(replica::database& db, const sstring& ks, with_create_statement with_stmt) {
future<utils::chunked_vector<description>> aggregates(replica::database& db, const sstring& ks, with_create_statement with_stmt) {
auto udas = functions::instance().get_user_aggregates(ks);
auto describer = [with_stmt] (shared_ptr<const functions::user_aggregate> uda) {
return uda->describe(with_stmt);
@@ -298,7 +299,7 @@ std::optional<description> describe_cdc_log_table(const data_dictionary::databas
return schema_desc;
}
future<std::vector<description>> table(const data_dictionary::database& db, const sstring& ks, const sstring& name, bool with_internals) {
future<utils::chunked_vector<description>> table(const data_dictionary::database& db, const sstring& ks, const sstring& name, bool with_internals) {
auto table = db.try_find_table(ks, name);
if (!table) {
throw exceptions::invalid_request_exception(format("Table '{}' not found in keyspace '{}'", name, ks));
@@ -312,7 +313,7 @@ future<std::vector<description>> table(const data_dictionary::database& db, cons
auto schema = table->schema();
auto idxs = table->get_index_manager().list_indexes();
auto views = table->views();
std::vector<description> result;
utils::chunked_vector<description> result;
// table
auto table_desc = schema->describe(replica::make_schema_describe_helper(schema, db), with_internals ? describe_option::STMTS_AND_INTERNALS : describe_option::STMTS);
@@ -374,7 +375,7 @@ future<std::vector<description>> table(const data_dictionary::database& db, cons
co_return result;
}
future<std::vector<description>> tables(const data_dictionary::database& db, const lw_shared_ptr<keyspace_metadata>& ks, std::optional<bool> with_internals = std::nullopt) {
future<utils::chunked_vector<description>> tables(const data_dictionary::database& db, const lw_shared_ptr<keyspace_metadata>& ks, std::optional<bool> with_internals = std::nullopt) {
auto& replica_db = db.real_database();
auto tables = ks->tables() | std::views::filter([&replica_db] (const schema_ptr& s) {
return !cdc::is_log_for_some_table(replica_db, s->ks_name(), s->cf_name()) && !service::paxos::paxos_store::try_get_base_table(s->cf_name());
@@ -382,7 +383,7 @@ future<std::vector<description>> tables(const data_dictionary::database& db, con
std::ranges::sort(tables, std::ranges::less(), std::mem_fn(&schema::cf_name));
if (with_internals) {
std::vector<description> result;
utils::chunked_vector<description> result;
for (const auto& t: tables) {
auto tables_desc = co_await table(db, ks->name(), t->cf_name(), *with_internals);
result.insert(result.end(), std::make_move_iterator(tables_desc.begin()), std::make_move_iterator(tables_desc.end()));
@@ -394,7 +395,7 @@ future<std::vector<description>> tables(const data_dictionary::database& db, con
co_return tables | std::views::transform([&db] (auto&& t) {
return t->describe(replica::make_schema_describe_helper(t, db), describe_option::NO_STMTS);
}) | std::ranges::to<std::vector>();
}) | std::ranges::to<utils::chunked_vector<description>>();
}
// DESCRIBE UTILITY
@@ -422,7 +423,7 @@ lw_shared_ptr<keyspace_metadata> get_keyspace_metadata(const data_dictionary::da
Descriptions don't contain create_statements.
* @throw `invalid_request_exception` if there is no such keyspace
*/
future<std::vector<description>> list_elements(const data_dictionary::database& db, const sstring& ks, element_type element) {
future<utils::chunked_vector<description>> list_elements(const data_dictionary::database& db, const sstring& ks, element_type element) {
auto ks_meta = get_keyspace_metadata(db, ks);
auto& replica_db = db.real_database();
@@ -446,7 +447,7 @@ future<std::vector<description>> list_elements(const data_dictionary::database&
Description contains create_statement.
* @throw `invalid_request_exception` if there is no such keyspace or there is no element with given name
*/
future<std::vector<description>> describe_element(const data_dictionary::database& db, const sstring& ks, element_type element, const sstring& name, bool with_internals) {
future<utils::chunked_vector<description>> describe_element(const data_dictionary::database& db, const sstring& ks, element_type element, const sstring& name, bool with_internals) {
auto ks_meta = get_keyspace_metadata(db, ks);
auto& replica_db = db.real_database();
@@ -470,12 +471,12 @@ future<std::vector<description>> describe_element(const data_dictionary::databas
Descriptions contain create_statements.
* @throw `invalid_request_exception` if there is no such keyspace or there is no element with given name
*/
future<std::vector<description>> describe_all_keyspace_elements(const data_dictionary::database& db, const sstring& ks, bool with_internals) {
future<utils::chunked_vector<description>> describe_all_keyspace_elements(const data_dictionary::database& db, const sstring& ks, bool with_internals) {
auto ks_meta = get_keyspace_metadata(db, ks);
auto& replica_db = db.real_database();
std::vector<description> result;
utils::chunked_vector<description> result;
auto inserter = [&result] (std::vector<description>&& elements) mutable {
auto inserter = [&result] (utils::chunked_vector<description>&& elements) mutable {
result.insert(result.end(), std::make_move_iterator(elements.begin()), std::make_move_iterator(elements.end()));
};
@@ -506,7 +507,7 @@ std::vector<lw_shared_ptr<column_specification>> get_element_column_specificatio
return col_specs;
}
std::vector<std::vector<managed_bytes_opt>> serialize_descriptions(std::vector<description>&& descs, bool serialize_create_statement = true) {
std::vector<std::vector<managed_bytes_opt>> serialize_descriptions(utils::chunked_vector<description>&& descs, bool serialize_create_statement = true) {
return descs | std::views::as_rvalue | std::views::transform([serialize_create_statement] (description desc) {
return std::move(desc).serialize(serialize_create_statement);
}) | std::ranges::to<std::vector>();
@@ -651,7 +652,7 @@ future<std::vector<std::vector<managed_bytes_opt>>> schema_describe_statement::d
auto db = qp.db();
auto result = co_await std::visit(overloaded_functor{
[&] (const schema_desc& config) -> future<std::vector<description>> {
[&] (const schema_desc& config) -> future<utils::chunked_vector<description>> {
auto& auth_service = *client_state.get_auth_service();
if (config.with_hashed_passwords) {
@@ -662,7 +663,7 @@ future<std::vector<std::vector<managed_bytes_opt>>> schema_describe_statement::d
}
auto keyspaces = config.full_schema ? db.get_all_keyspaces() : db.get_user_keyspaces();
std::vector<description> schema_result;
utils::chunked_vector<description> schema_result;
for (auto&& ks: keyspaces) {
if (!config.full_schema && db.extensions().is_extension_internal_keyspace(ks)) {
@@ -686,7 +687,7 @@ future<std::vector<std::vector<managed_bytes_opt>>> schema_describe_statement::d
co_return schema_result;
},
[&] (const keyspace_desc& config) -> future<std::vector<description>> {
[&] (const keyspace_desc& config) -> future<utils::chunked_vector<description>> {
auto ks = client_state.get_raw_keyspace();
if (config.keyspace) {
ks = *config.keyspace;
@@ -730,7 +731,7 @@ future<std::vector<std::vector<managed_bytes_opt>>> listing_describe_statement::
std::ranges::sort(keyspaces);
}
std::vector<description> result;
utils::chunked_vector<description> result;
for (auto&& ks: keyspaces) {
auto ks_result = co_await list_elements(db, ks, _element);
result.insert(result.end(), std::make_move_iterator(ks_result.begin()), std::make_move_iterator(ks_result.end()));