treewide: Move compaction code into a new top-level compaction dir

Since compaction is layered on top of sstables, let's move all compaction code
into a new top-level directory.
This change will give me extra motivation to remove all layer violations, like
sstable calling compaction-specific code, and compaction entanglement with
other components like table and storage service.

Next steps:
- remove all layer violations
- move compaction code in sstables namespace into a new one for compaction.
- move compaction unit tests into its own file

Signed-off-by: Raphael S. Carvalho <raphaelsc@scylladb.com>
Message-Id: <20210707194058.87060-1-raphaelsc@scylladb.com>

api/compaction_manager.cc

@@ -20,7 +20,7 @@
  */
 
 #include "compaction_manager.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "api/api-doc/compaction_manager.json.hh"
 #include "db/system_keyspace.hh"
 #include "column_family.hh"

api/storage_service.cc

@@ -42,7 +42,7 @@
 #include "column_family.hh"
 #include "log.hh"
 #include "release.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/sstables.hh"
 #include "database.hh"
 #include "db/extensions.hh"

compaction/compaction.cc

@@ -53,8 +53,8 @@
 #include <seastar/core/scheduling.hh>
 #include <seastar/util/closeable.hh>
 
-#include "sstables.hh"
-#include "sstable_writer.hh"
+#include "sstables/sstables.hh"
+#include "sstables/sstable_writer.hh"
 #include "sstables/progress_monitor.hh"
 #include "sstables/sstables_manager.hh"
 #include "compaction.hh"

compaction/compaction.hh

@@ -23,8 +23,8 @@
 #pragma once
 
 #include "database_fwd.hh"
-#include "shared_sstable.hh"
-#include "sstables/compaction_descriptor.hh"
+#include "sstables/shared_sstable.hh"
+#include "compaction/compaction_descriptor.hh"
 #include "gc_clock.hh"
 #include "compaction_weight_registration.hh"
 #include "service/priority_manager.hh"

compaction/compaction_backlog_manager.hh

@@ -24,7 +24,7 @@
 #include <unordered_set>
 #include <memory>
 #include <seastar/core/shared_ptr.hh>
-#include "shared_sstable.hh"
+#include "sstables/shared_sstable.hh"
 #include "sstables/progress_monitor.hh"
 #include "timestamp.hh"
 

compaction/compaction_descriptor.hh

@@ -27,8 +27,8 @@
 #include <variant>
 #include <seastar/core/smp.hh>
 #include <seastar/core/file.hh>
-#include "shared_sstable.hh"
-#include "sstable_set.hh"
+#include "sstables/shared_sstable.hh"
+#include "sstables/sstable_set.hh"
 #include "utils/UUID.hh"
 #include "dht/i_partitioner.hh"
 #include "compaction_weight_registration.hh"

compaction/compaction_manager.cc

@@ -27,7 +27,7 @@
 #include "database.hh"
 #include "service/storage_service.hh"
 #include <seastar/core/metrics.hh>
-#include "exceptions.hh"
+#include "sstables/exceptions.hh"
 #include "locator/abstract_replication_strategy.hh"
 #include <cmath>
 

compaction/compaction_manager.hh

@@ -37,7 +37,7 @@
 #include <list>
 #include <functional>
 #include <algorithm>
-#include "sstables/compaction.hh"
+#include "compaction.hh"
 #include "compaction_weight_registration.hh"
 #include "compaction_backlog_manager.hh"
 #include "backlog_controller.hh"

compaction/compaction_strategy.cc

@@ -40,13 +40,13 @@
 #include <vector>
 #include <chrono>
 #include <seastar/core/shared_ptr.hh>
-#include "sstables.hh"
+#include "sstables/sstables.hh"
 #include "compaction.hh"
 #include "database.hh"
 #include "compaction_strategy.hh"
 #include "compaction_strategy_impl.hh"
 #include "schema.hh"
-#include "sstable_set.hh"
+#include "sstables/sstable_set.hh"
 #include <boost/range/algorithm/find.hpp>
 #include <boost/range/algorithm/remove_if.hpp>
 #include <boost/range/adaptors.hpp>
@@ -55,9 +55,9 @@
 #include "date_tiered_compaction_strategy.hh"
 #include "leveled_compaction_strategy.hh"
 #include "time_window_compaction_strategy.hh"
-#include "sstables/compaction_backlog_manager.hh"
-#include "sstables/size_tiered_backlog_tracker.hh"
-#include "sstables/leveled_manifest.hh"
+#include "compaction_backlog_manager.hh"
+#include "size_tiered_backlog_tracker.hh"
+#include "leveled_manifest.hh"
 
 logging::logger date_tiered_manifest::logger = logging::logger("DateTieredCompactionStrategy");
 logging::logger leveled_manifest::logger("LeveledManifest");

compaction/date_tiered_compaction_strategy.hh

@@ -46,11 +46,11 @@
 #include <algorithm>
 #include <vector>
 #include <iterator>
-#include "sstables.hh"
+#include "sstables/sstables.hh"
 #include "compaction.hh"
 #include "timestamp.hh"
 #include "cql3/statements/property_definitions.hh"
-#include "sstables/compaction_strategy_impl.hh"
+#include "compaction_strategy_impl.hh"
 
 static constexpr double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
 static constexpr int64_t DEFAULT_BASE_TIME_SECONDS = 60;

compaction/leveled_compaction_strategy.cc

@@ -20,7 +20,7 @@
  */
 
 #include "leveled_compaction_strategy.hh"
-#include "sstables/leveled_manifest.hh"
+#include "leveled_manifest.hh"
 #include <algorithm>
 #include <ranges>
 

compaction/leveled_compaction_strategy.hh

@@ -30,9 +30,9 @@
 
 #include "compaction_strategy_type.hh"
 #include "database_fwd.hh"
-#include "sstables/size_tiered_compaction_strategy.hh"
-#include "sstables/compaction_strategy_impl.hh"
-#include "sstables/compaction_backlog_manager.hh"
+#include "size_tiered_compaction_strategy.hh"
+#include "compaction_strategy_impl.hh"
+#include "compaction_backlog_manager.hh"
 #include "sstables/shared_sstable.hh"
 
 class leveled_manifest;

compaction/leveled_manifest.hh

@@ -41,7 +41,7 @@
 
 #pragma once
 
-#include "sstables.hh"
+#include "sstables/sstables.hh"
 #include "compaction.hh"
 #include "size_tiered_compaction_strategy.hh"
 #include "range.hh"

compaction/size_tiered_backlog_tracker.hh

@@ -19,7 +19,7 @@
  */
 
 #pragma once
-#include "sstables/compaction_backlog_manager.hh"
+#include "compaction_backlog_manager.hh"
 #include <cmath>
 #include <ctgmath>
 

compaction/time_window_compaction_strategy.cc

@@ -19,8 +19,8 @@
  * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sstables/time_window_compaction_strategy.hh"
-#include "sstables/leveled_manifest.hh"
+#include "time_window_compaction_strategy.hh"
+#include "leveled_manifest.hh"
 #include "mutation_writer/timestamp_based_splitting_writer.hh"
 #include "mutation_source_metadata.hh"
 

configure.py

@@ -710,12 +710,12 @@ scylla_core = (['database.cc',
                 'sstables/sstable_version.cc',
                 'sstables/compress.cc',
                 'sstables/sstable_mutation_reader.cc',
-                'sstables/compaction.cc',
-                'sstables/compaction_strategy.cc',
-                'sstables/size_tiered_compaction_strategy.cc',
-                'sstables/leveled_compaction_strategy.cc',
-                'sstables/time_window_compaction_strategy.cc',
-                'sstables/compaction_manager.cc',
+                'compaction/compaction.cc',
+                'compaction/compaction_strategy.cc',
+                'compaction/size_tiered_compaction_strategy.cc',
+                'compaction/leveled_compaction_strategy.cc',
+                'compaction/time_window_compaction_strategy.cc',
+                'compaction/compaction_manager.cc',
                 'sstables/integrity_checked_file_impl.cc',
                 'sstables/prepended_input_stream.cc',
                 'sstables/m_format_read_helpers.cc',

database.cc

@@ -39,7 +39,7 @@
 #include <boost/algorithm/string/classification.hpp>
 #include "sstables/sstables.hh"
 #include "sstables/sstables_manager.hh"
-#include "sstables/compaction.hh"
+#include "compaction/compaction.hh"
 #include <boost/range/adaptor/map.hpp>
 #include <boost/algorithm/cxx11/any_of.hpp>
 #include <boost/range/algorithm/find_if.hpp>
@@ -53,7 +53,7 @@
 #include "cell_locking.hh"
 #include "view_info.hh"
 #include "db/schema_tables.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "gms/feature_service.hh"
 
 #include "utils/human_readable.hh"

distributed_loader.cc

@@ -26,8 +26,8 @@
 #include "db/system_distributed_keyspace.hh"
 #include "db/schema_tables.hh"
 #include "lister.hh"
-#include "sstables/compaction.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/sstables.hh"
 #include "sstables/sstables_manager.hh"
 #include "sstables/sstable_directory.hh"

distributed_loader.hh

@@ -30,7 +30,7 @@
 #include <functional>
 #include <filesystem>
 #include "seastarx.hh"
-#include "sstables/compaction_descriptor.hh"
+#include "compaction/compaction_descriptor.hh"
 
 class database;
 class table;

main.cc

@@ -70,7 +70,7 @@
 
 #include "db/view/view_update_generator.hh"
 #include "service/cache_hitrate_calculator.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/sstables.hh"
 #include "gms/feature_service.hh"
 #include "distributed_loader.hh"

service/storage_service.cc

@@ -73,7 +73,7 @@
 #include "utils/exceptions.hh"
 #include "message/messaging_service.hh"
 #include "supervisor.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/sstables.hh"
 #include "db/config.hh"
 #include "db/schema_tables.hh"

sstables/sstable_directory.cc

@@ -22,7 +22,7 @@
 #include <boost/range/adaptor/map.hpp>
 #include "sstables/sstable_directory.hh"
 #include "sstables/sstables.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "log.hh"
 #include "sstable_directory.hh"
 #include "lister.hh"

sstables/sstable_directory.hh

@@ -32,7 +32,7 @@
 #include "seastarx.hh"
 #include "sstables/shared_sstable.hh"            // sstables::shared_sstable
 #include "sstables/version.hh"                   // sstable versions
-#include "sstables/compaction_descriptor.hh"     // for compaction_sstable_creator_fn
+#include "compaction/compaction_descriptor.hh"     // for compaction_sstable_creator_fn
 #include "sstables/open_info.hh"                 // for entry_descriptor and foreign_sstable_open_info, chunked_vector wants to know if they are move constructible
 #include "utils/chunked_vector.hh"
 #include "utils/phased_barrier.hh"

sstables/sstable_set.cc

@@ -25,9 +25,9 @@
 #include <boost/range/algorithm/sort.hpp>
 
 #include "compatible_ring_position.hh"
-#include "compaction_strategy_impl.hh"
-#include "leveled_compaction_strategy.hh"
-#include "time_window_compaction_strategy.hh"
+#include "compaction/compaction_strategy_impl.hh"
+#include "compaction/leveled_compaction_strategy.hh"
+#include "compaction/time_window_compaction_strategy.hh"
 
 #include "sstable_set_impl.hh"
 

table.cc

@@ -34,7 +34,7 @@
 #include "view_info.hh"
 #include "db/data_listeners.hh"
 #include "memtable-sstable.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/sstable_directory.hh"
 #include "db/system_keyspace.hh"
 #include "db/query_context.hh"

test/boost/cql_query_large_test.cc

@@ -40,7 +40,7 @@
 #include "types/list.hh"
 #include "types/set.hh"
 #include "db/config.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "test/lib/exception_utils.hh"
 #include "schema_builder.hh"
 

test/boost/cql_query_test.cc

@@ -46,7 +46,7 @@
 #include "types/set.hh"
 #include "db/config.hh"
 #include "cql3/cql_config.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "test/lib/exception_utils.hh"
 #include "utils/rjson.hh"
 #include "schema_builder.hh"

test/boost/gossip_test.cc

@@ -40,7 +40,7 @@
 #include "database.hh"
 #include "db/system_distributed_keyspace.hh"
 #include "db/config.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 
 namespace db::view {
 class view_update_generator;

test/boost/mutation_test.cc

@@ -45,7 +45,7 @@
 #include "partition_slice_builder.hh"
 #include "test/lib/tmpdir.hh"
 #include "test/lib/reader_permit.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 
 #include <seastar/testing/test_case.hh>
 #include <seastar/testing/thread_test_case.hh>

test/boost/sstable_3_x_test.cc

@@ -32,7 +32,7 @@
 #include <seastar/util/closeable.hh>
 
 #include "sstables/sstables.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "cell_locking.hh"
 #include "compress.hh"
 #include "counters.hh"

test/boost/sstable_datafile_test.cc

@@ -28,28 +28,28 @@
 #include "sstables/sstables.hh"
 #include "sstables/key.hh"
 #include "sstables/compress.hh"
-#include "sstables/compaction.hh"
+#include "compaction/compaction.hh"
 #include <seastar/testing/test_case.hh>
 #include <seastar/testing/thread_test_case.hh>
 #include "schema.hh"
 #include "schema_builder.hh"
 #include "database.hh"
-#include "sstables/leveled_manifest.hh"
+#include "compaction/leveled_manifest.hh"
 #include "sstables/metadata_collector.hh"
 #include "sstables/sstable_writer.hh"
 #include <memory>
 #include "test/boost/sstable_test.hh"
 #include <seastar/core/seastar.hh>
 #include <seastar/core/do_with.hh>
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "test/lib/tmpdir.hh"
 #include "dht/i_partitioner.hh"
 #include "dht/murmur3_partitioner.hh"
 #include "range.hh"
 #include "partition_slice_builder.hh"
-#include "sstables/compaction_strategy_impl.hh"
-#include "sstables/date_tiered_compaction_strategy.hh"
-#include "sstables/time_window_compaction_strategy.hh"
+#include "compaction/compaction_strategy_impl.hh"
+#include "compaction/date_tiered_compaction_strategy.hh"
+#include "compaction/time_window_compaction_strategy.hh"
 #include "test/lib/mutation_assertions.hh"
 #include "counters.hh"
 #include "cell_locking.hh"

test/boost/sstable_resharding_test.cc

@@ -13,7 +13,7 @@
 #include "schema.hh"
 #include "database.hh"
 #include "dht/murmur3_partitioner.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "mutation_reader.hh"
 #include "test/boost/sstable_test.hh"
 #include "test/lib/tmpdir.hh"

test/boost/sstable_test.cc

@@ -28,7 +28,7 @@
 #include <seastar/util/closeable.hh>
 
 #include "sstables/sstables.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "sstables/key.hh"
 #include "test/lib/sstable_utils.hh"
 #include <seastar/testing/test_case.hh>

test/boost/view_complex_test.cc

@@ -24,7 +24,7 @@
 
 #include "database.hh"
 #include "db/view/view_builder.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 
 #include <seastar/testing/test_case.hh>
 #include "test/lib/cql_test_env.hh"

test/lib/cql_test_env.cc

@@ -41,7 +41,7 @@
 #include <seastar/core/coroutine.hh>
 #include "utils/UUID_gen.hh"
 #include "service/migration_manager.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "message/messaging_service.hh"
 #include "service/raft/raft_group_registry.hh"
 #include "service/storage_service.hh"

test/lib/sstable_run_based_compaction_strategy_for_tests.hh

@@ -21,9 +21,9 @@
 
 #include <vector>
 #include <map>
-#include "sstables/compaction_strategy_impl.hh"
+#include "compaction/compaction_strategy_impl.hh"
 #include "sstables/sstable_set.hh"
-#include "sstables/compaction.hh"
+#include "compaction/compaction.hh"
 #include "database.hh"
 
 namespace sstables {

test/lib/test_services.hh

@@ -30,7 +30,7 @@
 #include "row_cache.hh"
 #include "database.hh"
 #include "cell_locking.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "db/large_data_handler.hh"
 #include "sstables/sstables_manager.hh"
 

test/perf/perf_fast_forward.cc

@@ -42,7 +42,7 @@
 #include <seastar/core/units.hh>
 #include <seastar/testing/test_runner.hh>
 #include <seastar/util/closeable.hh>
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "transport/messages/result_message.hh"
 #include "sstables/partition_index_cache.hh"
 #include "linux-perf-event.hh"

test/perf/perf_sstable.hh

@@ -24,7 +24,7 @@
 #include <seastar/util/closeable.hh>
 
 #include "sstables/sstables.hh"
-#include "sstables/compaction_manager.hh"
+#include "compaction/compaction_manager.hh"
 #include "cell_locking.hh"
 #include "mutation_reader.hh"
 #include "test/lib/sstable_utils.hh"