test: Split sstable_mutation_test

Detach test_schema_changes and test_sstable_conforms_to_mutation_source
into individual files. These two take ~10 minutes each, what's left in
origin finishes within 4 minutes alltogether.

Signed-off-by: Pavel Emelyanov <xemul@scylladb.com>

configure.py

@@ -356,6 +356,8 @@ scylla_tests = set([
     'test/boost/sstable_3_x_test',
     'test/boost/sstable_datafile_test',
     'test/boost/sstable_mutation_test',
+    'test/boost/schema_changes_test',
+    'test/boost/sstable_conforms_to_mutation_source_test',
     'test/boost/sstable_resharding_test',
     'test/boost/sstable_test',
     'test/boost/storage_proxy_test',
@@ -967,6 +969,8 @@ for t in perf_tests:
 deps['test/boost/sstable_test'] += ['test/lib/sstable_utils.cc', 'test/lib/normalizing_reader.cc']
 deps['test/boost/sstable_datafile_test'] += ['test/lib/sstable_utils.cc', 'test/lib/normalizing_reader.cc']
 deps['test/boost/mutation_reader_test'] += ['test/lib/sstable_utils.cc']
+deps['test/boost/sstable_mutation_test'] += ['test/lib/sstable_utils.cc']
+deps['test/boost/sstable_conforms_to_mutation_source_test'] += ['test/lib/sstable_utils.cc']
 
 deps['test/boost/bytes_ostream_test'] = [
     "test/boost/bytes_ostream_test.cc",

test/boost/schema_changes_test.cc

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <boost/test/unit_test.hpp>
+#include <seastar/testing/test_case.hh>
+#include <seastar/testing/thread_test_case.hh>
+#include "test/boost/sstable_test.hh"
+#include <seastar/core/thread.hh>
+#include "sstables/sstables.hh"
+#include "timestamp.hh"
+#include "schema_builder.hh"
+#include "mutation_reader.hh"
+#include "test/lib/mutation_source_test.hh"
+#include "test/lib/flat_mutation_reader_assertions.hh"
+#include "test/lib/sstable_utils.hh"
+
+using namespace sstables;
+using namespace std::chrono_literals;
+
+SEASTAR_THREAD_TEST_CASE(test_schema_changes) {
+    auto dir = tmpdir();
+    storage_service_for_tests ssft;
+    auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); });
+    int gen = 1;
+
+    std::map<std::tuple<sstables::sstable::version_types, schema_ptr>, std::tuple<shared_sstable, int>> cache;
+    for_each_schema_change([&] (schema_ptr base, const std::vector<mutation>& base_mutations,
+                                schema_ptr changed, const std::vector<mutation>& changed_mutations) {
+        for (auto version : all_sstable_versions) {
+            auto it = cache.find(std::tuple { version, base });
+
+            shared_sstable created_with_base_schema;
+            shared_sstable created_with_changed_schema;
+            sstables::test_env env;
+            if (it == cache.end()) {
+                auto mt = make_lw_shared<memtable>(base);
+                for (auto& m : base_mutations) {
+                    mt->apply(m);
+                }
+                created_with_base_schema = env.make_sstable(base, dir.path().string(), gen, version, sstables::sstable::format_types::big);
+                created_with_base_schema->write_components(mt->make_flat_reader(base), base_mutations.size(), base, test_sstables_manager.configure_writer(), mt->get_encoding_stats()).get();
+                created_with_base_schema->load().get();
+
+                created_with_changed_schema = env.make_sstable(changed, dir.path().string(), gen, version, sstables::sstable::format_types::big);
+                created_with_changed_schema->load().get();
+
+                cache.emplace(std::tuple { version, base }, std::tuple { created_with_base_schema, gen });
+                gen++;
+            } else {
+                created_with_base_schema = std::get<shared_sstable>(it->second);
+
+                created_with_changed_schema = env.make_sstable(changed, dir.path().string(), std::get<int>(it->second), version, sstables::sstable::format_types::big);
+                created_with_changed_schema->load().get();
+            }
+
+            auto mr = assert_that(created_with_base_schema->as_mutation_source()
+                        .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice()));
+            for (auto& m : changed_mutations) {
+                mr.produces(m);
+            }
+            mr.produces_end_of_stream();
+
+            mr = assert_that(created_with_changed_schema->as_mutation_source()
+                    .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice()));
+            for (auto& m : changed_mutations) {
+                mr.produces(m);
+            }
+            mr.produces_end_of_stream();
+        }
+    });
+}

test/boost/sstable_conforms_to_mutation_source_test.cc

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <boost/test/unit_test.hpp>
+#include <seastar/testing/test_case.hh>
+#include <seastar/testing/thread_test_case.hh>
+#include "test/boost/sstable_test.hh"
+#include <seastar/core/thread.hh>
+#include "sstables/sstables.hh"
+#include "test/lib/mutation_source_test.hh"
+#include "test/lib/sstable_utils.hh"
+
+using namespace sstables;
+using namespace std::chrono_literals;
+
+static
+mutation_source make_sstable_mutation_source(sstables::test_env& env, schema_ptr s, sstring dir, std::vector<mutation> mutations,
+        sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) {
+    return as_mutation_source(make_sstable(env, s, dir, std::move(mutations), cfg, version, query_time));
+}
+
+// Must be run in a seastar thread
+static
+void test_mutation_source(sstables::test_env& env, sstable_writer_config cfg, sstables::sstable::version_types version) {
+    std::vector<tmpdir> dirs;
+    run_mutation_source_tests([&env, &dirs, &cfg, version] (schema_ptr s, const std::vector<mutation>& partitions,
+                gc_clock::time_point query_time) -> mutation_source {
+        dirs.emplace_back();
+        return make_sstable_mutation_source(env, s, dirs.back().path().string(), partitions, cfg, version, query_time);
+    });
+}
+
+
+SEASTAR_TEST_CASE(test_sstable_conforms_to_mutation_source) {
+    return seastar::async([] {
+        auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); });
+        storage_service_for_tests ssft;
+        sstables::test_env env;
+        for (auto version : all_sstable_versions) {
+            for (auto index_block_size : {1, 128, 64*1024}) {
+                sstable_writer_config cfg = test_sstables_manager.configure_writer();
+                cfg.promoted_index_block_size = index_block_size;
+                test_mutation_source(env, cfg, version);
+            }
+        }
+    });
+}

test/boost/sstable_mutation_test.cc

@@ -393,62 +393,12 @@ SEASTAR_THREAD_TEST_CASE(read_partial_range_2) {
     }).get();
 }
 
-static
-shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir, std::vector<mutation> mutations,
-        sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) {
-    auto sst = env.make_sstable(s,
-        dir,
-        1 /* generation */,
-        version,
-        sstables::sstable::format_types::big,
-        default_sstable_buffer_size,
-        query_time);
-
-    auto mt = make_lw_shared<memtable>(s);
-
-    for (auto&& m : mutations) {
-        mt->apply(m);
-    }
-
-    sst->write_components(mt->make_flat_reader(s), mutations.size(), s, cfg, mt->get_encoding_stats()).get();
-    sst->load().get();
-
-    return sst;
-}
-
 static
 mutation_source make_sstable_mutation_source(sstables::test_env& env, schema_ptr s, sstring dir, std::vector<mutation> mutations,
         sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now()) {
     return as_mutation_source(make_sstable(env, s, dir, std::move(mutations), cfg, version, query_time));
 }
 
-// Must be run in a seastar thread
-static
-void test_mutation_source(sstables::test_env& env, sstable_writer_config cfg, sstables::sstable::version_types version) {
-    std::vector<tmpdir> dirs;
-    run_mutation_source_tests([&env, &dirs, &cfg, version] (schema_ptr s, const std::vector<mutation>& partitions,
-                gc_clock::time_point query_time) -> mutation_source {
-        dirs.emplace_back();
-        return make_sstable_mutation_source(env, s, dirs.back().path().string(), partitions, cfg, version, query_time);
-    });
-}
-
-
-SEASTAR_TEST_CASE(test_sstable_conforms_to_mutation_source) {
-    return seastar::async([] {
-        auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); });
-        storage_service_for_tests ssft;
-        sstables::test_env env;
-        for (auto version : all_sstable_versions) {
-            for (auto index_block_size : {1, 128, 64*1024}) {
-                sstable_writer_config cfg = test_sstables_manager.configure_writer();
-                cfg.promoted_index_block_size = index_block_size;
-                test_mutation_source(env, cfg, version);
-            }
-        }
-    });
-}
-
 SEASTAR_TEST_CASE(test_sstable_can_write_and_read_range_tombstone) {
     return seastar::async([] {
         auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); });
@@ -1568,59 +1518,6 @@ SEASTAR_THREAD_TEST_CASE(test_large_index_pages_do_not_cause_large_allocations)
     BOOST_REQUIRE_EQUAL(large_allocs_after - large_allocs_before, 0);
 }
 
-SEASTAR_THREAD_TEST_CASE(test_schema_changes) {
-    auto dir = tmpdir();
-    storage_service_for_tests ssft;
-    auto wait_bg = seastar::defer([] { sstables::await_background_jobs().get(); });
-    int gen = 1;
-
-    std::map<std::tuple<sstables::sstable::version_types, schema_ptr>, std::tuple<shared_sstable, int>> cache;
-    for_each_schema_change([&] (schema_ptr base, const std::vector<mutation>& base_mutations,
-                                schema_ptr changed, const std::vector<mutation>& changed_mutations) {
-        for (auto version : all_sstable_versions) {
-            auto it = cache.find(std::tuple { version, base });
-
-            shared_sstable created_with_base_schema;
-            shared_sstable created_with_changed_schema;
-            sstables::test_env env;
-            if (it == cache.end()) {
-                auto mt = make_lw_shared<memtable>(base);
-                for (auto& m : base_mutations) {
-                    mt->apply(m);
-                }
-                created_with_base_schema = env.make_sstable(base, dir.path().string(), gen, version, sstables::sstable::format_types::big);
-                created_with_base_schema->write_components(mt->make_flat_reader(base), base_mutations.size(), base, test_sstables_manager.configure_writer(), mt->get_encoding_stats()).get();
-                created_with_base_schema->load().get();
-
-                created_with_changed_schema = env.make_sstable(changed, dir.path().string(), gen, version, sstables::sstable::format_types::big);
-                created_with_changed_schema->load().get();
-
-                cache.emplace(std::tuple { version, base }, std::tuple { created_with_base_schema, gen });
-                gen++;
-            } else {
-                created_with_base_schema = std::get<shared_sstable>(it->second);
-
-                created_with_changed_schema = env.make_sstable(changed, dir.path().string(), std::get<int>(it->second), version, sstables::sstable::format_types::big);
-                created_with_changed_schema->load().get();
-            }
-
-            auto mr = assert_that(created_with_base_schema->as_mutation_source()
-                        .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice()));
-            for (auto& m : changed_mutations) {
-                mr.produces(m);
-            }
-            mr.produces_end_of_stream();
-
-            mr = assert_that(created_with_changed_schema->as_mutation_source()
-                    .make_reader(changed, no_reader_permit(), dht::partition_range::make_open_ended_both_sides(), changed->full_slice()));
-            for (auto& m : changed_mutations) {
-                mr.produces(m);
-            }
-            mr.produces_end_of_stream();
-        }
-    });
-}
-
 SEASTAR_THREAD_TEST_CASE(test_reading_serialization_header) {
     auto dir = tmpdir();
     storage_service_for_tests ssft;

test/lib/sstable_utils.cc

@@ -28,6 +28,9 @@
 #include <boost/range/adaptor/map.hpp>
 #include "test/lib/flat_mutation_reader_assertions.hh"
 
+using namespace sstables;
+using namespace std::chrono_literals;
+
 sstables::shared_sstable make_sstable_containing(std::function<sstables::shared_sstable()> sst_factory, std::vector<mutation> muts) {
     auto sst = sst_factory();
     schema_ptr s = muts[0].schema();
@@ -66,3 +69,26 @@ sstables::shared_sstable make_sstable_containing(std::function<sstables::shared_
 
     return sst;
 }
+
+shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir, std::vector<mutation> mutations,
+        sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time) {
+    auto sst = env.make_sstable(s,
+        dir,
+        1 /* generation */,
+        version,
+        sstables::sstable::format_types::big,
+        default_sstable_buffer_size,
+        query_time);
+
+    auto mt = make_lw_shared<memtable>(s);
+
+    for (auto&& m : mutations) {
+        mt->apply(m);
+    }
+
+    sst->write_components(mt->make_flat_reader(s), mutations.size(), s, cfg, mt->get_encoding_stats()).get();
+    sst->load().get();
+
+    return sst;
+}
+

test/lib/sstable_utils.hh

@@ -22,11 +22,17 @@
 #pragma once
 
 #include "sstables/sstables.hh"
+#include "sstables/shared_sstable.hh"
 #include "memtable-sstable.hh"
 #include "dht/i_partitioner.hh"
 #include <boost/range/irange.hpp>
 #include <boost/range/adaptor/map.hpp>
 #include "test/lib/test_services.hh"
+#include "test/lib/sstable_test_env.hh"
+#include "gc_clock.hh"
+
+using namespace sstables;
+using namespace std::chrono_literals;
 
 struct local_shard_only_tag { };
 using local_shard_only = bool_class<local_shard_only_tag>;
@@ -76,3 +82,6 @@ inline sstring make_local_key(const schema_ptr& s, size_t min_key_size = 1) {
 inline std::vector<sstring> make_keys(unsigned n, const schema_ptr& s, size_t min_key_size = 1) {
     return do_make_keys(n, s, min_key_size, local_shard_only::no);
 }
+
+shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir, std::vector<mutation> mutations,
+        sstable_writer_config cfg, sstables::sstable::version_types version, gc_clock::time_point query_time = gc_clock::now());