diff --git a/db/config.cc b/db/config.cc index b91e330644..3399960458 100644 --- a/db/config.cc +++ b/db/config.cc @@ -374,15 +374,15 @@ db::config::config(std::shared_ptr exts) , compaction_throughput_mb_per_sec(this, "compaction_throughput_mb_per_sec", liveness::LiveUpdate, value_status::Used, 0, "Throttles compaction to the specified total throughput across the entire system. The faster you insert data, the faster you need to compact in order to keep the SSTable count down. The recommended Value is 16 to 32 times the rate of write throughput (in MBs/second). Setting the value to 0 disables compaction throttling.\n" "Related information: Configuring compaction") - , compaction_large_partition_warning_threshold_mb(this, "compaction_large_partition_warning_threshold_mb", value_status::Used, 1000, + , compaction_large_partition_warning_threshold_mb(this, "compaction_large_partition_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 1000, "Log a warning when writing partitions larger than this value") - , compaction_large_row_warning_threshold_mb(this, "compaction_large_row_warning_threshold_mb", value_status::Used, 10, + , compaction_large_row_warning_threshold_mb(this, "compaction_large_row_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 10, "Log a warning when writing rows larger than this value") - , compaction_large_cell_warning_threshold_mb(this, "compaction_large_cell_warning_threshold_mb", value_status::Used, 1, + , compaction_large_cell_warning_threshold_mb(this, "compaction_large_cell_warning_threshold_mb", liveness::LiveUpdate, value_status::Used, 1, "Log a warning when writing cells larger than this value") - , compaction_rows_count_warning_threshold(this, "compaction_rows_count_warning_threshold", value_status::Used, 100000, + , compaction_rows_count_warning_threshold(this, "compaction_rows_count_warning_threshold", liveness::LiveUpdate, value_status::Used, 100000, "Log a warning when writing a number of rows larger than this value") - , compaction_collection_elements_count_warning_threshold(this, "compaction_collection_elements_count_warning_threshold", value_status::Used, 10000, + , compaction_collection_elements_count_warning_threshold(this, "compaction_collection_elements_count_warning_threshold", liveness::LiveUpdate, value_status::Used, 10000, "Log a warning when writing a collection containing more elements than this value") /* Common memtable settings */ , memtable_total_space_in_mb(this, "memtable_total_space_in_mb", value_status::Invalid, 0, diff --git a/db/large_data_handler.cc b/db/large_data_handler.cc index 2ec6977bf3..526a7bb920 100644 --- a/db/large_data_handler.cc +++ b/db/large_data_handler.cc @@ -105,8 +105,12 @@ future<> large_data_handler::maybe_delete_large_data_entries(sstables::shared_ss } cql_table_large_data_handler::cql_table_large_data_handler(gms::feature_service& feat, - uint64_t partition_threshold_bytes, uint64_t row_threshold_bytes, uint64_t cell_threshold_bytes, uint64_t rows_count_threshold, uint64_t collection_elements_count_threshold) - : large_data_handler(partition_threshold_bytes, row_threshold_bytes, cell_threshold_bytes, rows_count_threshold, collection_elements_count_threshold) + utils::updateable_value partition_threshold_mb, + utils::updateable_value row_threshold_mb, + utils::updateable_value cell_threshold_mb, + utils::updateable_value rows_count_threshold, + utils::updateable_value collection_elements_count_threshold) + : large_data_handler(partition_threshold_mb() * MB, row_threshold_mb() * MB, cell_threshold_mb() * MB, rows_count_threshold(), collection_elements_count_threshold()) , _feat(feat) , _record_large_cells([this] (const sstables::sstable& sst, const sstables::key& pk, const clustering_key_prefix* ck, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) { return internal_record_large_cells(sst, pk, ck, cdef, cell_size, collection_elements); @@ -117,6 +121,11 @@ cql_table_large_data_handler::cql_table_large_data_handler(gms::feature_service& return internal_record_large_cells_and_collections(sst, pk, ck, cdef, cell_size, collection_elements); }; })) + , _partition_threshold_mb_updater(_partition_threshold_bytes, std::move(partition_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; }) + , _row_threshold_mb_updater(_row_threshold_bytes, std::move(row_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; }) + , _cell_threshold_mb_updater(_cell_threshold_bytes, std::move(cell_threshold_mb), [] (uint32_t threshold_mb) { return uint64_t(threshold_mb) * MB; }) + , _rows_count_threshold_updater(_rows_count_threshold, std::move(rows_count_threshold)) + , _collection_elements_count_threshold_updater(_collection_elements_count_threshold, std::move(collection_elements_count_threshold)) {} template diff --git a/db/large_data_handler.hh b/db/large_data_handler.hh index 7686221267..a0522123f1 100644 --- a/db/large_data_handler.hh +++ b/db/large_data_handler.hh @@ -12,6 +12,7 @@ #include "schema_fwd.hh" #include "system_keyspace.hh" #include "sstables/shared_sstable.hh" +#include "utils/updateable_value.hh" namespace sstables { class sstable; @@ -50,11 +51,15 @@ private: } bool _running = false; + +protected: uint64_t _partition_threshold_bytes; uint64_t _row_threshold_bytes; uint64_t _cell_threshold_bytes; uint64_t _rows_count_threshold; uint64_t _collection_elements_count_threshold; + +private: mutable large_data_handler::stats _stats; public: @@ -131,11 +136,24 @@ protected: class cql_table_large_data_handler : public large_data_handler { gms::feature_service& _feat; std::function (const sstables::sstable& sst, const sstables::key& partition_key, - const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items)> _record_large_cells; + const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements)> _record_large_cells; std::optional _feat_listener; + + static constexpr uint64_t MB = 1024 * 1024; + + using threshold_updater = utils::transforming_value_updater; + threshold_updater _partition_threshold_mb_updater; + threshold_updater _row_threshold_mb_updater; + threshold_updater _cell_threshold_mb_updater; + threshold_updater _rows_count_threshold_updater; + threshold_updater _collection_elements_count_threshold_updater; public: explicit cql_table_large_data_handler(gms::feature_service& feat, - uint64_t partition_threshold_bytes, uint64_t row_threshold_bytes, uint64_t cell_threshold_bytes, uint64_t rows_count_threshold, uint64_t collection_elements_count_threshold); + utils::updateable_value partition_threshold_mb, + utils::updateable_value row_threshold_mb, + utils::updateable_value cell_threshold_mb, + utils::updateable_value rows_count_threshold, + utils::updateable_value collection_elements_count_threshold); protected: virtual future<> record_large_partitions(const sstables::sstable& sst, const sstables::key& partition_key, uint64_t partition_size, uint64_t rows) const override; @@ -146,9 +164,9 @@ protected: private: future<> internal_record_large_cells(const sstables::sstable& sst, const sstables::key& partition_key, - const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items) const; + const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) const; future<> internal_record_large_cells_and_collections(const sstables::sstable& sst, const sstables::key& partition_key, - const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_items) const; + const clustering_key_prefix* clustering_key, const column_definition& cdef, uint64_t cell_size, uint64_t collection_elements) const; }; class nop_large_data_handler : public large_data_handler { diff --git a/replica/database.cc b/replica/database.cc index f2ebe670d4..a563847f05 100644 --- a/replica/database.cc +++ b/replica/database.cc @@ -349,11 +349,11 @@ database::database(const db::config& cfg, database_config dbcfg, service::migrat , _compaction_manager(cm) , _enable_incremental_backups(cfg.incremental_backups()) , _large_data_handler(std::make_unique(feat, - _cfg.compaction_large_partition_warning_threshold_mb()*1024*1024, - _cfg.compaction_large_row_warning_threshold_mb()*1024*1024, - _cfg.compaction_large_cell_warning_threshold_mb()*1024*1024, - _cfg.compaction_rows_count_warning_threshold(), - _cfg.compaction_collection_elements_count_warning_threshold())) + _cfg.compaction_large_partition_warning_threshold_mb, + _cfg.compaction_large_row_warning_threshold_mb, + _cfg.compaction_large_cell_warning_threshold_mb, + _cfg.compaction_rows_count_warning_threshold, + _cfg.compaction_collection_elements_count_warning_threshold)) , _nop_large_data_handler(std::make_unique()) , _user_sstables_manager(std::make_unique(*_large_data_handler, _cfg, feat, _row_cache_tracker, dbcfg.available_memory)) , _system_sstables_manager(std::make_unique(*_nop_large_data_handler, _cfg, feat, _row_cache_tracker, dbcfg.available_memory))