mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-05-12 19:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

db: large_data_handler: dynamically update config thresholds

Browse Source 
make the various large data thresholds live-updateable
and construct the observers and updaters in
cql_table_large_data_handler to dynamically update
the base large_data_handler class threshold members.

Fixes #11685

Signed-off-by: Benny Halevy <bhalevy@scylladb.com>
This commit is contained in:
Benny Halevy
2022-10-02 16:42:34 +03:00
parent 6d582054c0
commit 2c4ff71d2b
4 changed files with 43 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
db/config.cc
View File

@@ -374,15 +374,15 @@ db::config::config(std::shared_ptr<db::extensions> exts)
, compaction_throughput_mb_per_sec(this, "compaction_throughput_mb_per_sec", liveness::LiveUpdate, value_status::Used, 0,
"Throttles compaction to the specified total throughput across the entire system. The faster you insert data, the faster you need to compact in order to keep the SSTable count down. The recommended Value is 16 to 32 times the rate of write throughput (in MBs/second). Setting the value to 0 disables compaction throttling.\n"
"Related information: Configuring compaction")
, compaction_large_partition_warning_threshold_mb(this, "compaction_large_partition_warning_threshold_mb", value_status::Used, 1000,
, compaction_large_partition_warning_threshold_mb(this, "compaction_large_partition_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 1000,
"Log a warning when writing partitions larger than this value")
, compaction_large_row_warning_threshold_mb(this, "compaction_large_row_warning_threshold_mb", value_status::Used, 10,
, compaction_large_row_warning_threshold_mb(this, "compaction_large_row_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 10,
"Log a warning when writing rows larger than this value")
, compaction_large_cell_warning_threshold_mb(this, "compaction_large_cell_warning_threshold_mb", value_status::Used, 1,
, compaction_large_cell_warning_threshold_mb(this, "compaction_large_cell_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 1,
"Log a warning when writing cells larger than this value")
, compaction_rows_count_warning_threshold(this, "compaction_rows_count_warning_threshold", value_status::Used, 100000,
, compaction_rows_count_warning_threshold(this, "compaction_rows_count_warning_threshold", liveness::LiveUpdate, value_status::Used, 100000,
"Log a warning when writing a number of rows larger than this value")
, compaction_collection_elements_count_warning_threshold(this, "compaction_collection_elements_count_warning_threshold", value_status::Used, 10000,
, compaction_collection_elements_count_warning_threshold(this, "compaction_collection_elements_count_warning_threshold", liveness::LiveUpdate, value_status::Used, 10000,
"Log a warning when writing a collection containing more elements than this value")
/* Common memtable settings */
, memtable_total_space_in_mb(this, "memtable_total_space_in_mb", value_status::Invalid, 0,

13
db/large_data_handler.cc
View File

@@ -105,8 +105,12 @@ future<> large_data_handler::maybe_delete_large_data_entries(sstables::shared_ss
}
cql_table_large_data_handler::cql_table_large_data_handler(gms::feature_service& feat,
uint64_t partition_threshold_bytes, uint64_t row_threshold_bytes, uint64_t cell_threshold_bytes, uint64_t rows_count_threshold, uint64_t collection_elements_count_threshold)
: large_data_handler(partition_threshold_bytes, row_threshold_bytes, cell_threshold_bytes, rows_count_threshold, collection_elements_count_threshold)
utils::updateable_value<uint32_t> partition_threshold_mb,
utils::updateable_value<uint32_t> row_threshold_mb,
utils::updateable_value<uint32_t> cell_threshold_mb,
utils::updateable_value<uint32_t> rows_count_threshold,
utils::updateable_value<uint32_t> collection_elements_count_threshold)
: large_data_handler(partition_threshold_mb() * MB, row_threshold_mb() * MB, cell_threshold_mb() * MB, rows_count_threshold(), collection_elements_count_threshold())
, _feat(feat)
, _record_large_cells([this] (const sstables::sstable& sst, const sstables::key& pk, const clustering_key_prefix* ck, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) {
return internal_record_large_cells(sst, pk, ck, cdef, cell_size, collection_elements);
@@ -117,6 +121,11 @@ cql_table_large_data_handler::cql_table_large_data_handler(gms::feature_service&
return internal_record_large_cells_and_collections(sst, pk, ck, cdef, cell_size, collection_elements);
};
}))
, _partition_threshold_mb_updater(_partition_threshold_bytes, std::move(partition_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; })
, _row_threshold_mb_updater(_row_threshold_bytes, std::move(row_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; })
, _cell_threshold_mb_updater(_cell_threshold_bytes, std::move(cell_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; })
, _rows_count_threshold_updater(_rows_count_threshold, std::move(rows_count_threshold))
, _collection_elements_count_threshold_updater(_collection_elements_count_threshold, std::move(collection_elements_count_threshold))
{}
template <typename... Args>

26
db/large_data_handler.hh
View File

@@ -12,6 +12,7 @@
#include "schema_fwd.hh"
#include "system_keyspace.hh"
#include "sstables/shared_sstable.hh"
#include "utils/updateable_value.hh"
namespace sstables {
class sstable;
@@ -50,11 +51,15 @@ private:
}
bool _running = false;
protected:
uint64_t _partition_threshold_bytes;
uint64_t _row_threshold_bytes;
uint64_t _cell_threshold_bytes;
uint64_t _rows_count_threshold;
uint64_t _collection_elements_count_threshold;
private:
mutable large_data_handler::stats _stats;
public:
@@ -131,11 +136,24 @@ protected:
class cql_table_large_data_handler : public large_data_handler {
gms::feature_service& _feat;
std::function<future<> (const sstables::sstable& sst, const sstables::key& partition_key,
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items)> _record_large_cells;
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements)> _record_large_cells;
std::optional<std::any> _feat_listener;
static constexpr uint64_t MB = 1024 * 1024;
using threshold_updater = utils::transforming_value_updater<uint64_t, uint32_t>;
threshold_updater _partition_threshold_mb_updater;
threshold_updater _row_threshold_mb_updater;
threshold_updater _cell_threshold_mb_updater;
threshold_updater _rows_count_threshold_updater;
threshold_updater _collection_elements_count_threshold_updater;
public:
explicit cql_table_large_data_handler(gms::feature_service& feat,
uint64_t partition_threshold_bytes, uint64_t row_threshold_bytes, uint64_t cell_threshold_bytes, uint64_t rows_count_threshold, uint64_t collection_elements_count_threshold);
utils::updateable_value<uint32_t> partition_threshold_mb,
utils::updateable_value<uint32_t> row_threshold_mb,
utils::updateable_value<uint32_t> cell_threshold_mb,
utils::updateable_value<uint32_t> rows_count_threshold,
utils::updateable_value<uint32_t> collection_elements_count_threshold);
protected:
virtual future<> record_large_partitions(const sstables::sstable& sst, const sstables::key& partition_key, uint64_t partition_size, uint64_t rows) const override;
@@ -146,9 +164,9 @@ protected:
private:
future<> internal_record_large_cells(const sstables::sstable& sst, const sstables::key& partition_key,
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items) const;
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) const;
future<> internal_record_large_cells_and_collections(const sstables::sstable& sst, const sstables::key& partition_key,
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items) const;
const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) const;
};
class nop_large_data_handler : public large_data_handler {

10
replica/database.cc
View File

@@ -349,11 +349,11 @@ database::database(const db::config& cfg, database_config dbcfg, service::migrat
, _compaction_manager(cm)
, _enable_incremental_backups(cfg.incremental_backups())
, _large_data_handler(std::make_unique<db::cql_table_large_data_handler>(feat,
_cfg.compaction_large_partition_warning_threshold_mb()*1024*1024,
_cfg.compaction_large_row_warning_threshold_mb()*1024*1024,
_cfg.compaction_large_cell_warning_threshold_mb()*1024*1024,
_cfg.compaction_rows_count_warning_threshold(),
_cfg.compaction_collection_elements_count_warning_threshold()))
_cfg.compaction_large_partition_warning_threshold_mb,
_cfg.compaction_large_row_warning_threshold_mb,
_cfg.compaction_large_cell_warning_threshold_mb,
_cfg.compaction_rows_count_warning_threshold,
_cfg.compaction_collection_elements_count_warning_threshold))
, _nop_large_data_handler(std::make_unique<db::nop_large_data_handler>())
, _user_sstables_manager(std::make_unique<sstables::sstables_manager>(*_large_data_handler, _cfg, feat, _row_cache_tracker, dbcfg.available_memory))
, _system_sstables_manager(std::make_unique<sstables::sstables_manager>(*_nop_large_data_handler, _cfg, feat, _row_cache_tracker, dbcfg.available_memory))