From 3577fa2bb8261c571609d02bc6180da079cdee1a Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Thu, 5 Mar 2020 10:40:43 +0300 Subject: [PATCH] test: Split sstable_mutation_test Detach test_schema_changes and test_sstable_conforms_to_mutation_source into individual files. These two take ~10 minutes each, what's left in origin finishes within 4 minutes alltogether. Signed-off-by: Pavel Emelyanov --- configure.py | 4 + test/boost/schema_changes_test.cc | 90 +++++++++++++++ ...stable_conforms_to_mutation_source_test.cc | 66 +++++++++++ test/boost/sstable_mutation_test.cc | 103 ------------------ test/lib/sstable_utils.cc | 26 +++++ test/lib/sstable_utils.hh | 9 ++ 6 files changed, 195 insertions(+), 103 deletions(-) create mode 100644 test/boost/schema_changes_test.cc create mode 100644 test/boost/sstable_conforms_to_mutation_source_test.cc diff --git a/configure.py b/configure.py index 014a3d601e..2d26ebebd2 100755 --- a/configure.py +++ b/configure.py @@ -356,6 +356,8 @@ scylla_tests = set([ 'test/boost/sstable_3_x_test', 'test/boost/sstable_datafile_test', 'test/boost/sstable_mutation_test', + 'test/boost/schema_changes_test', + 'test/boost/sstable_conforms_to_mutation_source_test', 'test/boost/sstable_resharding_test', 'test/boost/sstable_test', 'test/boost/storage_proxy_test', @@ -967,6 +969,8 @@ for t in perf_tests: deps['test/boost/sstable_test'] += ['test/lib/sstable_utils.cc', 'test/lib/normalizing_reader.cc'] deps['test/boost/sstable_datafile_test'] += ['test/lib/sstable_utils.cc', 'test/lib/normalizing_reader.cc'] deps['test/boost/mutation_reader_test'] += ['test/lib/sstable_utils.cc'] +deps['test/boost/sstable_mutation_test'] += ['test/lib/sstable_utils.cc'] +deps['test/boost/sstable_conforms_to_mutation_source_test'] += ['test/lib/sstable_utils.cc'] deps['test/boost/bytes_ostream_test'] = [ "test/boost/bytes_ostream_test.cc", diff --git a/test/boost/schema_changes_test.cc b/test/boost/schema_changes_test.cc new file mode 100644 index 0000000000..edebfbe0c4 --- /dev/null +++ b/test/boost/schema_changes_test.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2015 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + + +#include +#include +#include +#include "test/boost/sstable_test.hh" +#include +#include "sstables/sstables.hh" +#include "timestamp.hh" +#include "schema_builder.hh" +#include "mutation_reader.hh" +#include "test/lib/mutation_source_test.hh" +#include "test/lib/flat_mutation_reader_assertions.hh" +#include "test/lib/sstable_utils.hh" + +using namespace sstables; +using namespace std::chrono_literals; + +SEASTAR_THREAD_TEST_CASE(test_schema_changes) { + auto dir = tmpdir(); + storage_service_for_tests ssft; + auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); }); + int gen = 1; + + std::map, std::tuple> cache; + for_each_schema_change([&] (schema_ptr base, const std::vector& base_mutations, + schema_ptr changed, const std::vector& changed_mutations) { + for (auto version : all_sstable_versions) { + auto it = cache.find(std::tuple { version, base }); + + shared_sstable created_with_base_schema; + shared_sstable created_with_changed_schema; + sstables::test_env env; + if (it == cache.end()) { + auto mt = make_lw_shared(base); + for (auto& m : base_mutations) { + mt->apply(m); + } + created_with_base_schema = env.make_sstable(base, dir.path().string(), gen, version, sstables::sstable::format_types::big); + created_with_base_schema->write_components(mt->make_flat_reader(base), base_mutations.size(), base, test_sstables_manager.configure_writer(), mt->get_encoding_stats()).get(); + created_with_base_schema->load().get(); + + created_with_changed_schema = env.make_sstable(changed, dir.path().string(), gen, version, sstables::sstable::format_types::big); + created_with_changed_schema->load().get(); + + cache.emplace(std::tuple { version, base }, std::tuple { created_with_base_schema, gen }); + gen++; + } else { + created_with_base_schema = std::get(it->second); + + created_with_changed_schema = env.make_sstable(changed, dir.path().string(), std::get(it->second), version, sstables::sstable::format_types::big); + created_with_changed_schema->load().get(); + } + + auto mr = assert_that(created_with_base_schema->as_mutation_source() + .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice())); + for (auto& m : changed_mutations) { + mr.produces(m); + } + mr.produces_end_of_stream(); + + mr = assert_that(created_with_changed_schema->as_mutation_source() + .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice())); + for (auto& m : changed_mutations) { + mr.produces(m); + } + mr.produces_end_of_stream(); + } + }); +} diff --git a/test/boost/sstable_conforms_to_mutation_source_test.cc b/test/boost/sstable_conforms_to_mutation_source_test.cc new file mode 100644 index 0000000000..3c87da2bfc --- /dev/null +++ b/test/boost/sstable_conforms_to_mutation_source_test.cc @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2015 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + + +#include +#include +#include +#include "test/boost/sstable_test.hh" +#include +#include "sstables/sstables.hh" +#include "test/lib/mutation_source_test.hh" +#include "test/lib/sstable_utils.hh" + +using namespace sstables; +using namespace std::chrono_literals; + +static +mutation_source make_sstable_mutation_source(sstables::test_env& env, schema_ptr s, sstring dir, std::vector mutations, + sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) { + return as_mutation_source(make_sstable(env, s, dir, std::move(mutations), cfg, version, query_time)); +} + +// Must be run in a seastar thread +static +void test_mutation_source(sstables::test_env& env, sstable_writer_config cfg, sstables::sstable::version_types version) { + std::vector dirs; + run_mutation_source_tests([&env, &dirs, &cfg, version] (schema_ptr s, const std::vector& partitions, + gc_clock::time_point query_time) -> mutation_source { + dirs.emplace_back(); + return make_sstable_mutation_source(env, s, dirs.back().path().string(), partitions, cfg, version, query_time); + }); +} + + +SEASTAR_TEST_CASE(test_sstable_conforms_to_mutation_source) { + return seastar::async([] { + auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); }); + storage_service_for_tests ssft; + sstables::test_env env; + for (auto version : all_sstable_versions) { + for (auto index_block_size : {1, 128, 64*1024}) { + sstable_writer_config cfg = test_sstables_manager.configure_writer(); + cfg.promoted_index_block_size = index_block_size; + test_mutation_source(env, cfg, version); + } + } + }); +} diff --git a/test/boost/sstable_mutation_test.cc b/test/boost/sstable_mutation_test.cc index 82a193b262..7c7e3c6dec 100644 --- a/test/boost/sstable_mutation_test.cc +++ b/test/boost/sstable_mutation_test.cc @@ -393,62 +393,12 @@ SEASTAR_THREAD_TEST_CASE(read_partial_range_2) { }).get(); } -static -shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir, std::vector mutations, - sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) { - auto sst = env.make_sstable(s, - dir, - 1 /* generation */, - version, - sstables::sstable::format_types::big, - default_sstable_buffer_size, - query_time); - - auto mt = make_lw_shared(s); - - for (auto&& m : mutations) { - mt->apply(m); - } - - sst->write_components(mt->make_flat_reader(s), mutations.size(), s, cfg, mt->get_encoding_stats()).get(); - sst->load().get(); - - return sst; -} - static mutation_source make_sstable_mutation_source(sstables::test_env& env, schema_ptr s, sstring dir, std::vector mutations, sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) { return as_mutation_source(make_sstable(env, s, dir, std::move(mutations), cfg, version, query_time)); } -// Must be run in a seastar thread -static -void test_mutation_source(sstables::test_env& env, sstable_writer_config cfg, sstables::sstable::version_types version) { - std::vector dirs; - run_mutation_source_tests([&env, &dirs, &cfg, version] (schema_ptr s, const std::vector& partitions, - gc_clock::time_point query_time) -> mutation_source { - dirs.emplace_back(); - return make_sstable_mutation_source(env, s, dirs.back().path().string(), partitions, cfg, version, query_time); - }); -} - - -SEASTAR_TEST_CASE(test_sstable_conforms_to_mutation_source) { - return seastar::async([] { - auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); }); - storage_service_for_tests ssft; - sstables::test_env env; - for (auto version : all_sstable_versions) { - for (auto index_block_size : {1, 128, 64*1024}) { - sstable_writer_config cfg = test_sstables_manager.configure_writer(); - cfg.promoted_index_block_size = index_block_size; - test_mutation_source(env, cfg, version); - } - } - }); -} - SEASTAR_TEST_CASE(test_sstable_can_write_and_read_range_tombstone) { return seastar::async([] { auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); }); @@ -1568,59 +1518,6 @@ SEASTAR_THREAD_TEST_CASE(test_large_index_pages_do_not_cause_large_allocations) BOOST_REQUIRE_EQUAL(large_allocs_after - large_allocs_before, 0); } -SEASTAR_THREAD_TEST_CASE(test_schema_changes) { - auto dir = tmpdir(); - storage_service_for_tests ssft; - auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); }); - int gen = 1; - - std::map, std::tuple> cache; - for_each_schema_change([&] (schema_ptr base, const std::vector& base_mutations, - schema_ptr changed, const std::vector& changed_mutations) { - for (auto version : all_sstable_versions) { - auto it = cache.find(std::tuple { version, base }); - - shared_sstable created_with_base_schema; - shared_sstable created_with_changed_schema; - sstables::test_env env; - if (it == cache.end()) { - auto mt = make_lw_shared(base); - for (auto& m : base_mutations) { - mt->apply(m); - } - created_with_base_schema = env.make_sstable(base, dir.path().string(), gen, version, sstables::sstable::format_types::big); - created_with_base_schema->write_components(mt->make_flat_reader(base), base_mutations.size(), base, test_sstables_manager.configure_writer(), mt->get_encoding_stats()).get(); - created_with_base_schema->load().get(); - - created_with_changed_schema = env.make_sstable(changed, dir.path().string(), gen, version, sstables::sstable::format_types::big); - created_with_changed_schema->load().get(); - - cache.emplace(std::tuple { version, base }, std::tuple { created_with_base_schema, gen }); - gen++; - } else { - created_with_base_schema = std::get(it->second); - - created_with_changed_schema = env.make_sstable(changed, dir.path().string(), std::get(it->second), version, sstables::sstable::format_types::big); - created_with_changed_schema->load().get(); - } - - auto mr = assert_that(created_with_base_schema->as_mutation_source() - .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice())); - for (auto& m : changed_mutations) { - mr.produces(m); - } - mr.produces_end_of_stream(); - - mr = assert_that(created_with_changed_schema->as_mutation_source() - .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice())); - for (auto& m : changed_mutations) { - mr.produces(m); - } - mr.produces_end_of_stream(); - } - }); -} - SEASTAR_THREAD_TEST_CASE(test_reading_serialization_header) { auto dir = tmpdir(); storage_service_for_tests ssft; diff --git a/test/lib/sstable_utils.cc b/test/lib/sstable_utils.cc index 3c0bc63584..9c97a6eacc 100644 --- a/test/lib/sstable_utils.cc +++ b/test/lib/sstable_utils.cc @@ -28,6 +28,9 @@ #include #include "test/lib/flat_mutation_reader_assertions.hh" +using namespace sstables; +using namespace std::chrono_literals; + sstables::shared_sstable make_sstable_containing(std::function sst_factory, std::vector muts) { auto sst = sst_factory(); schema_ptr s = muts[0].schema(); @@ -66,3 +69,26 @@ sstables::shared_sstable make_sstable_containing(std::function mutations, + sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time) { + auto sst = env.make_sstable(s, + dir, + 1 /* generation */, + version, + sstables::sstable::format_types::big, + default_sstable_buffer_size, + query_time); + + auto mt = make_lw_shared(s); + + for (auto&& m : mutations) { + mt->apply(m); + } + + sst->write_components(mt->make_flat_reader(s), mutations.size(), s, cfg, mt->get_encoding_stats()).get(); + sst->load().get(); + + return sst; +} + diff --git a/test/lib/sstable_utils.hh b/test/lib/sstable_utils.hh index 885d9ac62e..0f9b96e4ea 100644 --- a/test/lib/sstable_utils.hh +++ b/test/lib/sstable_utils.hh @@ -22,11 +22,17 @@ #pragma once #include "sstables/sstables.hh" +#include "sstables/shared_sstable.hh" #include "memtable-sstable.hh" #include "dht/i_partitioner.hh" #include #include #include "test/lib/test_services.hh" +#include "test/lib/sstable_test_env.hh" +#include "gc_clock.hh" + +using namespace sstables; +using namespace std::chrono_literals; struct local_shard_only_tag { }; using local_shard_only = bool_class; @@ -76,3 +82,6 @@ inline sstring make_local_key(const schema_ptr& s, size_t min_key_size = 1) { inline std::vector make_keys(unsigned n, const schema_ptr& s, size_t min_key_size = 1) { return do_make_keys(n, s, min_key_size, local_shard_only::no); } + +shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir, std::vector mutations, + sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now());