From 83fc59402f69b381bba8bb2fcc1ef8da87606a69 Mon Sep 17 00:00:00 2001 From: "Raphael S. Carvalho" Date: Fri, 29 Oct 2021 22:00:17 -0300 Subject: [PATCH] LCS: reduce table dependency for task estimation let's reduce table dependency from LCS task estimation, to make it easier to switch to table_state. Signed-off-by: Raphael S. Carvalho --- compaction/leveled_compaction_strategy.cc | 3 +-- compaction/leveled_manifest.hh | 32 ++++++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/compaction/leveled_compaction_strategy.cc b/compaction/leveled_compaction_strategy.cc index a524132f98..2bbf1b20d3 100644 --- a/compaction/leveled_compaction_strategy.cc +++ b/compaction/leveled_compaction_strategy.cc @@ -137,8 +137,7 @@ int64_t leveled_compaction_strategy::estimated_pending_compactions(column_family for (auto all_sstables = cf.get_sstables(); auto& entry : *all_sstables) { sstables.push_back(entry); } - leveled_manifest manifest = leveled_manifest::create(cf, sstables, _max_sstable_size_in_mb, _stcs_options); - return manifest.get_estimated_tasks(); + return leveled_manifest::get_estimated_tasks(leveled_manifest::get_levels(sstables), _max_sstable_size_in_mb * 1024 * 1024); } compaction_descriptor diff --git a/compaction/leveled_manifest.hh b/compaction/leveled_manifest.hh index 4b9001a7c2..e0fdaabcb4 100644 --- a/compaction/leveled_manifest.hh +++ b/compaction/leveled_manifest.hh @@ -97,6 +97,19 @@ private: _generations.resize(MAX_LEVELS); } public: + static std::vector> get_levels(const std::vector& sstables) { + std::vector> levels; + levels.resize(MAX_LEVELS); + for (auto& sstable : sstables) { + uint32_t level = sstable->get_sstable_level(); + if (level >= levels.size()) { + throw std::runtime_error(format("Invalid level {:d} out of {:d}", level, (levels.size() - 1))); + } + levels[level].push_back(sstable); + } + return levels; + } + static leveled_manifest create(column_family& cf, std::vector& sstables, int max_sstable_size_in_mb, const sstables::size_tiered_compaction_strategy_options& stcs_options) { leveled_manifest manifest = leveled_manifest(cf, max_sstable_size_in_mb, stcs_options); @@ -104,14 +117,7 @@ public: // ensure all SSTables are in the manifest // FIXME: there can be tens of thousands of sstables. we can avoid this potentially expensive procedure if // partitioned_sstable_set keeps track of a list for each level. - for (auto& sstable : sstables) { - uint32_t level = sstable->get_sstable_level(); - if (level >= manifest._generations.size()) { - throw std::runtime_error(format("Invalid level {:d} out of {:d}", level, (manifest._generations.size() - 1))); - } - logger.debug("Adding {} to L{}", sstable->get_filename(), level); - manifest._generations[level].push_back(sstable); - } + manifest._generations = get_levels(sstables); return manifest; } @@ -524,19 +530,19 @@ public: return _generations[level]; } - int64_t get_estimated_tasks() const { + static int64_t get_estimated_tasks(const std::vector>& levels, uint64_t max_sstable_size_in_bytes) { int64_t tasks = 0; - for (int i = static_cast(_generations.size()) - 1; i >= 0; i--) { - const auto& sstables = _generations[i]; + for (int i = static_cast(levels.size()) - 1; i >= 0; i--) { + const auto& sstables = levels[i]; uint64_t total_bytes_for_this_level = get_total_bytes(sstables); - uint64_t max_bytes_for_this_level = max_bytes_for_level(i); + uint64_t max_bytes_for_this_level = max_bytes_for_level(i, max_sstable_size_in_bytes); if (total_bytes_for_this_level < max_bytes_for_this_level) { continue; } // If there is 1 byte over TBL - (MBL * 1.001), there is still a task left, so we need to round up. - tasks += std::ceil(float(total_bytes_for_this_level - max_bytes_for_this_level*TARGET_SCORE) / _max_sstable_size_in_bytes); + tasks += std::ceil(float(total_bytes_for_this_level - max_bytes_for_this_level*TARGET_SCORE) / max_sstable_size_in_bytes); } return tasks; }