From 3e74dd4df3e85a3f1a3425f47bacea81ed665739 Mon Sep 17 00:00:00 2001 From: Tomasz Grabiec Date: Thu, 6 Feb 2020 11:53:47 +0100 Subject: [PATCH 1/5] sstables: Move all_sstable_versions to version.hh --- sstables/version.hh | 6 ++++++ test/lib/sstable_utils.hh | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sstables/version.hh b/sstables/version.hh index 52ef52fec6..423fa1d8e3 100644 --- a/sstables/version.hh +++ b/sstables/version.hh @@ -30,6 +30,12 @@ namespace sstables { enum class sstable_version_types { ka, la, mc }; enum class sstable_format_types { big }; +inline std::array all_sstable_versions = { + sstable_version_types::ka, + sstable_version_types::la, + sstable_version_types::mc, +}; + inline sstable_version_types from_string(const seastar::sstring& format) { if (format == "ka") { return sstable_version_types::ka; diff --git a/test/lib/sstable_utils.hh b/test/lib/sstable_utils.hh index 6422261523..257ed08188 100644 --- a/test/lib/sstable_utils.hh +++ b/test/lib/sstable_utils.hh @@ -234,12 +234,6 @@ inline auto replacer_fn_no_op() { return [](sstables::compaction_completion_desc desc) -> void {}; } -inline std::array all_sstable_versions = { - sstables::sstable::version_types::ka, - sstables::sstable::version_types::la, - sstables::sstable::version_types::mc, -}; - template GCC6_CONCEPT( requires requires (AsyncAction aa, sstables::sstable::version_types& c) { { aa(c) } -> future<>; } ) inline From 04c093cbecab8240ca4c89fcf3133be632a4b607 Mon Sep 17 00:00:00 2001 From: Tomasz Grabiec Date: Thu, 6 Feb 2020 11:55:18 +0100 Subject: [PATCH 2/5] test: memory_footprint: Calculate sstable size for each format version --- test/perf/memory_footprint_test.cc | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/perf/memory_footprint_test.cc b/test/perf/memory_footprint_test.cc index 19d1a2d5ed..1e45f55d7f 100644 --- a/test/perf/memory_footprint_test.cc +++ b/test/perf/memory_footprint_test.cc @@ -161,7 +161,7 @@ static mutation make_mutation(mutation_settings settings) { struct sizes { size_t memtable; size_t cache; - size_t sstable; + std::map sstable; size_t frozen; size_t canonical; size_t query_result; @@ -189,14 +189,16 @@ static sizes calculate_sizes(const mutation& m) { tmpdir sstable_dir; sstables::test_env env; - auto sst = env.make_sstable(s, - sstable_dir.path().string(), - 1 /* generation */, - sstables::sstable::version_types::la, - sstables::sstable::format_types::big); - write_memtable_to_sstable_for_test(*mt, sst).get(); - sst->load().get(); - result.sstable = sst->data_size(); + for (auto v : sstables::all_sstable_versions) { + auto sst = env.make_sstable(s, + sstable_dir.path().string(), + 1 /* generation */, + v, + sstables::sstable::format_types::big); + write_memtable_to_sstable_for_test(*mt, sst).get(); + sst->load().get(); + result.sstable[v] = sst->data_size(); + } return result; } @@ -233,7 +235,10 @@ int main(int argc, char** argv) { std::cout << "mutation footprint:" << "\n"; std::cout << " - in cache: " << sizes.cache << "\n"; std::cout << " - in memtable: " << sizes.memtable << "\n"; - std::cout << " - in sstable: " << sizes.sstable << "\n"; + std::cout << " - in sstable:\n"; + for (auto v : sizes.sstable) { + std::cout << " " << sstables::to_string(v.first) << ": " << v.second << "\n"; + } std::cout << " - frozen: " << sizes.frozen << "\n"; std::cout << " - canonical: " << sizes.canonical << "\n"; std::cout << " - query result: " << sizes.query_result << "\n"; From 7c2f6dd75ec674691ce21b9dd8ded0994aa99b83 Mon Sep 17 00:00:00 2001 From: Tomasz Grabiec Date: Thu, 6 Feb 2020 11:55:40 +0100 Subject: [PATCH 3/5] test: memory_footprint: Run under a cql_test_env --- test/perf/memory_footprint_test.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/perf/memory_footprint_test.cc b/test/perf/memory_footprint_test.cc index 1e45f55d7f..0322483972 100644 --- a/test/perf/memory_footprint_test.cc +++ b/test/perf/memory_footprint_test.cc @@ -36,6 +36,7 @@ #include "test/lib/sstable_utils.hh" #include "test/lib/test_services.hh" #include "test/lib/sstable_test_env.hh" +#include "test/lib/cql_test_env.hh" class size_calculator { class nest { @@ -167,11 +168,10 @@ struct sizes { size_t query_result; }; -static sizes calculate_sizes(const mutation& m) { +static sizes calculate_sizes(cache_tracker& tracker, const mutation& m) { sizes result; auto s = m.schema(); auto mt = make_lw_shared(s); - cache_tracker tracker; row_cache cache(s, make_empty_snapshot_source(), tracker); auto cache_initial_occupancy = tracker.region().occupancy().used_space(); @@ -218,9 +218,7 @@ int main(int argc, char** argv) { if (smp::count != 1) { throw std::runtime_error("This test has to be run with -c1"); } - return seastar::async([&] { - storage_service_for_tests ssft; - + return do_with_cql_env_thread([&](cql_test_env& env) { mutation_settings settings; settings.column_count = app.configuration()["column-count"].as(); settings.column_name_size = app.configuration()["column-name-size"].as(); @@ -230,7 +228,8 @@ int main(int argc, char** argv) { settings.data_size = app.configuration()["data-size"].as(); auto m = make_mutation(settings); - auto sizes = calculate_sizes(m); + auto& tracker = env.local_db().find_column_family("system", "local").get_row_cache().get_cache_tracker(); + auto sizes = calculate_sizes(tracker, m); std::cout << "mutation footprint:" << "\n"; std::cout << " - in cache: " << sizes.cache << "\n"; From 1df63b60c359d7f2690a2a5ccbcbc1a6b001646b Mon Sep 17 00:00:00 2001 From: Tomasz Grabiec Date: Thu, 6 Feb 2020 13:14:54 +0100 Subject: [PATCH 4/5] test: memory_footprint: Introduce --partition-count option --- test/perf/memory_footprint_test.cc | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/test/perf/memory_footprint_test.cc b/test/perf/memory_footprint_test.cc index 0322483972..9a3fa4eceb 100644 --- a/test/perf/memory_footprint_test.cc +++ b/test/perf/memory_footprint_test.cc @@ -130,12 +130,13 @@ struct mutation_settings { size_t column_count; size_t column_name_size; size_t row_count; + size_t partition_count; size_t partition_key_size; size_t clustering_key_size; size_t data_size; }; -static mutation make_mutation(mutation_settings settings) { +static schema_ptr make_schema(const mutation_settings& settings) { auto builder = schema_builder("ks", "cf") .with_column("pk", bytes_type, column_kind::partition_key) .with_column("ck", bytes_type, column_kind::clustering_key); @@ -144,8 +145,10 @@ static mutation make_mutation(mutation_settings settings) { builder.with_column(to_bytes(random_string(settings.column_name_size)), bytes_type); } - auto s = builder.build(); + return builder.build(); +} +static mutation make_mutation(schema_ptr s, mutation_settings settings) { mutation m(s, partition_key::from_single_value(*s, bytes_type->decompose(data_value(random_bytes(settings.partition_key_size))))); for (size_t i = 0; i < settings.row_count; ++i) { @@ -168,9 +171,9 @@ struct sizes { size_t query_result; }; -static sizes calculate_sizes(cache_tracker& tracker, const mutation& m) { +static sizes calculate_sizes(cache_tracker& tracker, const mutation_settings& settings) { sizes result; - auto s = m.schema(); + auto s = make_schema(settings); auto mt = make_lw_shared(s); row_cache cache(s, make_empty_snapshot_source(), tracker); @@ -178,9 +181,14 @@ static sizes calculate_sizes(cache_tracker& tracker, const mutation& m) { assert(mt->occupancy().used_space() == 0); - mt->apply(m); - cache.populate(m); + std::vector muts; + for (size_t i = 0; i < settings.partition_count; ++i) { + muts.emplace_back(make_mutation(s, settings)); + mt->apply(muts.back()); + cache.populate(muts.back()); + } + mutation& m = muts[0]; result.memtable = mt->occupancy().used_space(); result.cache = tracker.region().occupancy().used_space() - cache_initial_occupancy; result.frozen = freeze(m).representation().size(); @@ -195,7 +203,9 @@ static sizes calculate_sizes(cache_tracker& tracker, const mutation& m) { 1 /* generation */, v, sstables::sstable::format_types::big); - write_memtable_to_sstable_for_test(*mt, sst).get(); + auto mt2 = make_lw_shared(s); + mt2->apply(*mt).get(); + write_memtable_to_sstable_for_test(*mt2, sst).get(); sst->load().get(); result.sstable[v] = sst->data_size(); } @@ -210,6 +220,7 @@ int main(int argc, char** argv) { ("column-count", bpo::value()->default_value(5), "column count") ("column-name-size", bpo::value()->default_value(2), "column name size") ("row-count", bpo::value()->default_value(1), "row count") + ("partition-count", bpo::value()->default_value(1), "partition count") ("partition-key-size", bpo::value()->default_value(10), "partition key size") ("clustering-key-size", bpo::value()->default_value(10), "clustering key size") ("data-size", bpo::value()->default_value(32), "cell data size"); @@ -223,13 +234,13 @@ int main(int argc, char** argv) { settings.column_count = app.configuration()["column-count"].as(); settings.column_name_size = app.configuration()["column-name-size"].as(); settings.row_count = app.configuration()["row-count"].as(); + settings.partition_count = app.configuration()["partition-count"].as(); settings.partition_key_size = app.configuration()["partition-key-size"].as(); settings.clustering_key_size = app.configuration()["clustering-key-size"].as(); settings.data_size = app.configuration()["data-size"].as(); - auto m = make_mutation(settings); auto& tracker = env.local_db().find_column_family("system", "local").get_row_cache().get_cache_tracker(); - auto sizes = calculate_sizes(tracker, m); + auto sizes = calculate_sizes(tracker, settings); std::cout << "mutation footprint:" << "\n"; std::cout << " - in cache: " << sizes.cache << "\n"; From 92771e904a5bbda2167df58cd90c66146bb738bc Mon Sep 17 00:00:00 2001 From: Tomasz Grabiec Date: Thu, 6 Feb 2020 13:15:08 +0100 Subject: [PATCH 5/5] test: memory_footprint: Silence logging by default --- test/perf/memory_footprint_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/perf/memory_footprint_test.cc b/test/perf/memory_footprint_test.cc index 9a3fa4eceb..444c996153 100644 --- a/test/perf/memory_footprint_test.cc +++ b/test/perf/memory_footprint_test.cc @@ -217,6 +217,7 @@ int main(int argc, char** argv) { namespace bpo = boost::program_options; app_template app; app.add_options() + ("verbose", "Enable info-level logging") ("column-count", bpo::value()->default_value(5), "column count") ("column-name-size", bpo::value()->default_value(2), "column name size") ("row-count", bpo::value()->default_value(1), "row count") @@ -229,6 +230,11 @@ int main(int argc, char** argv) { if (smp::count != 1) { throw std::runtime_error("This test has to be run with -c1"); } + + if (!app.configuration().count("verbose")) { + logging::logger_registry().set_all_loggers_level(seastar::log_level::warn); + } + return do_with_cql_env_thread([&](cql_test_env& env) { mutation_settings settings; settings.column_count = app.configuration()["column-count"].as();