From 84b5d6d6d07365a315139d0e330a161ad93bddc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Botond=20D=C3=A9nes?= Date: Thu, 2 Jul 2020 11:53:12 +0300 Subject: [PATCH 1/2] sstables: make_partitioned_sstable_set(): return an sstable_set Instead of an `std::unique_ptr`. The latter doesn't have a publicly available destructor, so it can only be called from withing `sstables/compaction_strategy.cc` where its definition resides. Thus it is not really usable as a public function in its current form, which shows as it has no users either. This patch makes it usable by returning an `sstable_set`. That is what potential callers would want anyway. In fact this patch prepares the ground for the next one, which wishes to use this function for just that but can't in its current form. --- sstables/compaction_strategy.cc | 4 ++-- sstables/sstable_set.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sstables/compaction_strategy.cc b/sstables/compaction_strategy.cc index f875d32841..29d9b09fee 100644 --- a/sstables/compaction_strategy.cc +++ b/sstables/compaction_strategy.cc @@ -438,8 +438,8 @@ std::unique_ptr leveled_compaction_strategy::make_sstable_set( return std::make_unique(std::move(schema)); } -std::unique_ptr make_partitioned_sstable_set(schema_ptr schema, bool use_level_metadata) { - return std::make_unique(std::move(schema), use_level_metadata); +sstable_set make_partitioned_sstable_set(schema_ptr schema, lw_shared_ptr all, bool use_level_metadata) { + return sstables::sstable_set(std::make_unique(schema, use_level_metadata), schema, std::move(all)); } compaction_descriptor compaction_strategy_impl::get_major_compaction_job(column_family& cf, std::vector candidates) { diff --git a/sstables/sstable_set.hh b/sstables/sstable_set.hh index fe52670ea7..b42ccf294f 100644 --- a/sstables/sstable_set.hh +++ b/sstables/sstable_set.hh @@ -101,7 +101,7 @@ public: incremental_selector make_incremental_selector() const; }; -std::unique_ptr make_partitioned_sstable_set(schema_ptr schema, bool use_level_metadata = true); +sstable_set make_partitioned_sstable_set(schema_ptr schema, lw_shared_ptr all, bool use_level_metadata = true); std::ostream& operator<<(std::ostream& os, const sstables::sstable_run& run); From 62c6859b6901ae7e4c4752b7f809393041856175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Botond=20D=C3=A9nes?= Date: Thu, 2 Jul 2020 11:56:17 +0300 Subject: [PATCH 2/2] db/view: view_update_generator: use partitioned sstable set And pass it to `make_range_sstable_reader()` when creating the reader, thus allowing the incremental selector created therein to exploit the fact that staging sstables are disjoint (in the case of repair and streaming at least). This should reduce the memory consumption of the staging reader considerably when reading from a lot of sstables. --- db/view/view_update_generator.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/view/view_update_generator.cc b/db/view/view_update_generator.cc index 11c42ef2fd..5c38a7ded5 100644 --- a/db/view/view_update_generator.cc +++ b/db/view/view_update_generator.cc @@ -51,10 +51,10 @@ future<> view_update_generator::start() { auto sstables = std::exchange(_sstables_with_tables[t], {}); try { - // temporary: need an sstable set for the flat mutation reader, but the - // compaction_descriptor takes a vector. Soon this will become a compaction - // so the transformation to the SSTable set will not be needed. - auto ssts = make_lw_shared(t->get_compaction_strategy().make_sstable_set(s)); + // Exploit the fact that sstables in the staging directory + // are usually non-overlapping and use a partitioned set for + // the read. + auto ssts = make_lw_shared(sstables::make_partitioned_sstable_set(s, make_lw_shared(sstable_list{}), false)); for (auto& sst : sstables) { ssts->insert(sst); }