mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-06-02 04:56:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Teach sstables tests not to use tempdir explicitly' from Pavel Emelyanov

Browse Source 
Many sstable test cases create tempdir on their own to create sstables with. Sometimes it's justified when the test needs to check files on disk by hand for some validation, but often all checks are fs-agnostic. The latter case(s) can be patched to work on top of any storage, in particular -- on top of object storage. To make it work tests should stop creating sstables explicitly in tempdir and this PR does exactly that.

All relevant occurrences of tempdir are removed from test cases, instead the sstable::test_env's tempdir is used. Next, the test_env::{create_sstable|reusable_sst} are patched not to accept the `fs::path dir` argument and pick the env's tempdir. Finally, the `make_sstable_easy` helper is patched to use path-less env methods too.

refs: #13015

Closes #13116

* github.com:scylladb/scylladb:
  test,sstables: Remove path from make_sstable_easy()
  test,lib: Remove wrapper over reusable_sst and move the comment
  test: Make "compact" test case use env dir
  test,compaction: Use env tempdir in some more cases
  test,compaction: Make check_compacted_sstables() use env's dir
  test: Relax making sstable with sequential generation
  test/sstable::test_env: Keep track of auto-incrementing generation
  test/lib: Add sstable maker helper without factory
  test: Remove last occurrence of test_env::do_with(rval, ...)
  test,sstables: Dont mess with tempdir where possible
  test/sstable::test_env: Add dir-less sstables making helpers
  test,sstables: Use sstables::test_env's tempdir with sweeper
  test,sstables: Use sstables::test_env's tempdir
  test/lib: Add tempdir sweeper
  test/lib: Open-code make_sstabl_easy into make_sstable
  test: Remove vector of mutation interposer from test_key_count_estimation
This commit is contained in:
Nadav Har'El
2023-03-12 10:14:26 +02:00
parent e7250e5a3f 47df084363
commit 1379d8330f
12 changed files with 368 additions and 517 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
test/boost/schema_changes_test.cc
View File

@@ -23,7 +23,6 @@ using namespace std::chrono_literals;
SEASTAR_TEST_CASE(test_schema_changes) {
return sstables::test_env::do_with_async([] (sstables::test_env& env) {
auto dir = tmpdir();
int gen = 1;
std::map<std::tuple<sstables::sstable::version_types, schema_ptr>, std::tuple<shared_sstable, int>> cache;
@@ -40,9 +39,9 @@ SEASTAR_TEST_CASE(test_schema_changes) {
mt->apply(m);
}
created_with_base_schema = make_sstable_easy(env, dir.path(), mt, env.manager().configure_writer(), gen, version, base_mutations.size());
created_with_base_schema = make_sstable_easy(env, mt, env.manager().configure_writer(), gen, version, base_mutations.size());
created_with_changed_schema = env.make_sstable(changed, dir.path().string(), gen, version, sstables::sstable::format_types::big);
created_with_changed_schema = env.make_sstable(changed, env.tempdir().path().string(), gen, version, sstables::sstable::format_types::big);
created_with_changed_schema->load().get();
cache.emplace(std::tuple { version, base }, std::tuple { created_with_base_schema, gen });
@@ -50,7 +49,7 @@ SEASTAR_TEST_CASE(test_schema_changes) {
} else {
created_with_base_schema = std::get<shared_sstable>(it->second);
created_with_changed_schema = env.make_sstable(changed, dir.path().string(), std::get<int>(it->second), version, sstables::sstable::format_types::big);
created_with_changed_schema = env.make_sstable(changed, env.tempdir().path().string(), std::get<int>(it->second), version, sstables::sstable::format_types::big);
created_with_changed_schema->load().get();
}

4
test/boost/sstable_3_x_test.cc
View File

@@ -5107,10 +5107,10 @@ SEASTAR_TEST_CASE(test_sstable_reader_on_unknown_column) {
auto mt = make_lw_shared<replica::memtable>(write_schema);
mt->apply(partition);
for (auto index_block_size : {1, 128, 64*1024}) {
tmpdir dir;
auto _ = env.tempdir().make_sweeper();
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = index_block_size;
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, 1, version);
auto sst = make_sstable_easy(env, mt, cfg, 1, version);
BOOST_REQUIRE_EXCEPTION(
assert_that(sst->make_reader(read_schema, env.make_reader_permit(), query::full_partition_range, read_schema->full_slice()))

350
test/boost/sstable_compaction_test.cc
View File

@@ -91,23 +91,13 @@ atomic_cell make_atomic_cell(data_type dt, bytes_view value, uint32_t ttl = 0, u
//////////////////////////////// Test basic compaction support
// open_sstable() opens the requested sstable for reading only (sstables are
// immutable, so an existing sstable cannot be opened for writing).
// It returns a future because opening requires reading from disk, and
// therefore may block. The future value is a shared sstable - a reference-
// counting pointer to an sstable - allowing for the returned handle to
// be passed around until no longer needed.
static future<sstables::shared_sstable> open_sstable(test_env& env, schema_ptr schema, sstring dir, unsigned long generation) {
return env.reusable_sst(std::move(schema), dir, generation);
}
// open_sstables() opens several generations of the same sstable, returning,
// after all the tables have been open, their vector.
static future<std::vector<sstables::shared_sstable>> open_sstables(test_env& env, schema_ptr s, sstring dir, std::vector<unsigned long> generations) {
return do_with(std::vector<sstables::shared_sstable>(),
[&env, dir = std::move(dir), generations = std::move(generations), s] (auto& ret) mutable {
return parallel_for_each(generations, [&env, &ret, &dir, s] (unsigned long generation) {
return open_sstable(env, s, dir, generation).then([&ret] (sstables::shared_sstable sst) {
return env.reusable_sst(s, dir, generation).then([&ret] (sstables::shared_sstable sst) {
ret.push_back(std::move(sst));
});
}).then([&ret] {
@@ -154,8 +144,7 @@ SEASTAR_TEST_CASE(compaction_manager_basic_test) {
auto s = make_shared_schema({}, some_keyspace, some_column_family,
{{"p1", utf8_type}}, {{"c1", utf8_type}}, {{"r1", int32_type}}, {}, utf8_type);
auto tmp = tmpdir();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto& cm = cf.get_compaction_manager();
auto close_cf = deferred_stop(cf);
cf->set_compaction_strategy(sstables::compaction_strategy_type::size_tiered);
@@ -176,7 +165,7 @@ SEASTAR_TEST_CASE(compaction_manager_basic_test) {
m.set_clustered_cell(c_key, r1_col, make_atomic_cell(int32_type, int32_type->decompose(1)));
mt->apply(std::move(m));
auto sst = env.make_sstable(s, tmp.path().string(), column_family_test::calculate_generation_for_new_table(*cf), sstables::get_highest_sstable_version(), big);
auto sst = env.make_sstable(s, column_family_test::calculate_generation_for_new_table(*cf), sstables::get_highest_sstable_version(), big);
write_memtable_to_sstable_for_test(*mt, sst).get();
sst->load().get();
@@ -221,15 +210,12 @@ SEASTAR_TEST_CASE(compact) {
auto s = builder.build();
table_for_tests cf(env.manager(), s);
auto close_cf = deferred_stop(cf);
tmpdir dir;
sstring tmpdir_path = dir.path().string();
open_sstables(env, s, "test/resource/sstables/compaction", {1,2,3}).then([&env, tmpdir_path, s, cf, generation] (auto sstables) mutable {
auto new_sstable = [&env, gen = make_lw_shared<unsigned>(generation), s, tmpdir_path] {
return env.make_sstable(s, tmpdir_path,
(*gen)++, sstables::get_highest_sstable_version(), sstables::sstable::format_types::big);
open_sstables(env, s, "test/resource/sstables/compaction", {1,2,3}).then([&env, s, cf, generation] (auto sstables) mutable {
auto new_sstable = [&env, gen = make_lw_shared<unsigned>(generation), s] {
return env.make_sstable(s, (*gen)++);
};
return compact_sstables(sstables::compaction_descriptor(std::move(sstables), default_priority_class()), cf, new_sstable).then([&env, s, cf, tmpdir_path] (auto) {
return compact_sstables(sstables::compaction_descriptor(std::move(sstables), default_priority_class()), cf, new_sstable).then([&env, s, cf] (auto) {
// Verify that the compacted sstable has the right content. We expect to see:
// name | age | height
// -------+-----+--------
@@ -237,7 +223,7 @@ SEASTAR_TEST_CASE(compact) {
// tom | 20 | 180
// john | 20 | deleted
// nadav - deleted partition
return open_sstable(env, s, tmpdir_path, generation).then([&env, s] (shared_sstable sst) {
return env.reusable_sst(s, generation).then([&env, s] (shared_sstable sst) {
auto reader = make_lw_shared<flat_mutation_reader_v2>(sstable_reader(sst, s, env.make_reader_permit())); // reader holds sst and s alive.
return read_mutation_from_flat_mutation_reader(*reader).then([reader, s] (mutation_opt m) {
BOOST_REQUIRE(m);
@@ -311,7 +297,7 @@ static std::vector<sstables::shared_sstable> get_candidates_for_leveled_strategy
}
// Return vector of sstables generated by compaction. Only relevant for leveled one.
static future<std::vector<unsigned long>> compact_sstables(test_env& env, sstring tmpdir_path, std::vector<unsigned long> generations_to_compact,
static future<std::vector<unsigned long>> compact_sstables(test_env& env, std::vector<unsigned long> generations_to_compact,
unsigned long new_generation, bool create_sstables, uint64_t min_sstable_size, compaction_strategy_type strategy) {
BOOST_REQUIRE(smp::count == 1);
schema_builder builder(make_shared_schema({}, some_keyspace, some_column_family,
@@ -328,16 +314,16 @@ static future<std::vector<unsigned long>> compact_sstables(test_env& env, sstrin
auto f = make_ready_future<>();
return f.then([&env, generations, sstables, s, create_sstables, min_sstable_size, tmpdir_path] () mutable {
return f.then([&env, generations, sstables, s, create_sstables, min_sstable_size] () mutable {
if (!create_sstables) {
return open_sstables(env, s, tmpdir_path, *generations).then([sstables] (auto opened_sstables) mutable {
return open_sstables(env, s, env.tempdir().path().native(), *generations).then([sstables] (auto opened_sstables) mutable {
for (auto& sst : opened_sstables) {
sstables->push_back(sst);
}
return make_ready_future<>();
});
}
return do_for_each(*generations, [&env, generations, sstables, s, min_sstable_size, tmpdir_path] (unsigned long generation) {
return do_for_each(*generations, [&env, generations, sstables, s, min_sstable_size] (unsigned long generation) {
auto mt = make_lw_shared<replica::memtable>(s);
const column_definition& r1_col = *s->get_column_definition("r1");
@@ -350,7 +336,7 @@ static future<std::vector<unsigned long>> compact_sstables(test_env& env, sstrin
m.set_clustered_cell(c_key, r1_col, make_atomic_cell(utf8_type, bytes(min_sstable_size, 'a')));
mt->apply(std::move(m));
auto sst = env.make_sstable(s, tmpdir_path, generation, sstables::get_highest_sstable_version(), big);
auto sst = env.make_sstable(s, generation, sstables::get_highest_sstable_version(), big);
return write_memtable_to_sstable_for_test(*mt, sst).then([mt, sst, s, sstables] {
return sst->load().then([sst, sstables] {
@@ -359,13 +345,12 @@ static future<std::vector<unsigned long>> compact_sstables(test_env& env, sstrin
});
});
});
}).then([&env, cf = *cf, sstables, new_generation, generations, strategy, created, min_sstable_size, s, tmpdir_path] () mutable {
}).then([&env, cf = *cf, sstables, new_generation, generations, strategy, created, min_sstable_size, s] () mutable {
auto generation = make_lw_shared<unsigned long>(new_generation);
auto new_sstable = [&env, generation, created, s, tmpdir_path] {
auto new_sstable = [&env, generation, created, s] {
auto gen = (*generation)++;
created->push_back(gen);
return env.make_sstable(s, tmpdir_path,
gen, sstables::get_highest_sstable_version(), sstables::sstable::format_types::big);
return env.make_sstable(s, gen, sstables::get_highest_sstable_version(), sstables::sstable::format_types::big);
};
// We must have opened at least all original candidates.
BOOST_REQUIRE(generations->size() == sstables->size());
@@ -409,9 +394,9 @@ static future<std::vector<unsigned long>> compact_sstables(test_env& env, sstrin
});
}
static future<> compact_sstables(test_env& env, sstring tmpdir_path, std::vector<unsigned long> generations_to_compact, unsigned long new_generation, bool create_sstables = true) {
static future<> compact_sstables(test_env& env, std::vector<unsigned long> generations_to_compact, unsigned long new_generation, bool create_sstables = true) {
uint64_t min_sstable_size = 50;
return compact_sstables(env, tmpdir_path, std::move(generations_to_compact), new_generation, create_sstables, min_sstable_size,
return compact_sstables(env, std::move(generations_to_compact), new_generation, create_sstables, min_sstable_size,
compaction_strategy_type::size_tiered).then([new_generation] (auto ret) {
// size tiered compaction will output at most one sstable, let's assert that.
BOOST_REQUIRE(ret.size() == 1);
@@ -420,13 +405,13 @@ static future<> compact_sstables(test_env& env, sstring tmpdir_path, std::vector
});
}
static future<> check_compacted_sstables(test_env& env, sstring tmpdir_path, unsigned long generation, std::vector<unsigned long> compacted_generations) {
static future<> check_compacted_sstables(test_env& env, unsigned long generation, std::vector<unsigned long> compacted_generations) {
auto s = make_shared_schema({}, some_keyspace, some_column_family,
{{"p1", utf8_type}}, {{"c1", utf8_type}}, {{"r1", utf8_type}}, {}, utf8_type);
auto generations = make_lw_shared<std::vector<unsigned long>>(std::move(compacted_generations));
return open_sstable(env, s, tmpdir_path, generation).then([&env, s, generations] (shared_sstable sst) {
return env.reusable_sst(s, generation).then([&env, s, generations] (shared_sstable sst) {
auto reader = sstable_reader(sst, s, env.make_reader_permit()); // reader holds sst and s alive.
auto keys = make_lw_shared<std::vector<partition_key>>();
@@ -465,29 +450,28 @@ SEASTAR_TEST_CASE(compact_02) {
// strategy algorithm that selects candidates for compaction.
return test_env::do_with([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
// Compact 4 sstables into 1 using size-tiered strategy to select sstables.
// E.g.: generations 18, 19, 20 and 21 will be compacted into generation 22.
return compact_sstables(env, tmpdir_path, { 18, 19, 20, 21 }, 22).then([&env, tmpdir_path] {
return compact_sstables(env, { 18, 19, 20, 21 }, 22).then([&env] {
// Check that generation 22 contains all keys of generations 18, 19, 20 and 21.
return check_compacted_sstables(env, tmpdir_path, 22, { 18, 19, 20, 21 });
}).then([&env, tmpdir_path] {
return compact_sstables(env, tmpdir_path, { 23, 24, 25, 26 }, 27).then([&env, tmpdir_path] {
return check_compacted_sstables(env, tmpdir_path, 27, { 23, 24, 25, 26 });
return check_compacted_sstables(env, 22, { 18, 19, 20, 21 });
}).then([&env] {
return compact_sstables(env, { 23, 24, 25, 26 }, 27).then([&env] {
return check_compacted_sstables(env, 27, { 23, 24, 25, 26 });
});
}).then([&env, tmpdir_path] {
return compact_sstables(env, tmpdir_path, { 28, 29, 30, 31 }, 32).then([&env, tmpdir_path] {
return check_compacted_sstables(env, tmpdir_path, 32, { 28, 29, 30, 31 });
}).then([&env] {
return compact_sstables(env, { 28, 29, 30, 31 }, 32).then([&env] {
return check_compacted_sstables(env, 32, { 28, 29, 30, 31 });
});
}).then([&env, tmpdir_path] {
return compact_sstables(env, tmpdir_path, { 33, 34, 35, 36 }, 37).then([&env, tmpdir_path] {
return check_compacted_sstables(env, tmpdir_path, 37, { 33, 34, 35, 36 });
}).then([&env] {
return compact_sstables(env, { 33, 34, 35, 36 }, 37).then([&env] {
return check_compacted_sstables(env, 37, { 33, 34, 35, 36 });
});
}).then([&env, tmpdir_path] {
}).then([&env] {
// In this step, we compact 4 compacted sstables.
return compact_sstables(env, tmpdir_path, { 22, 27, 32, 37 }, 38, false).then([&env, tmpdir_path] {
return compact_sstables(env, { 22, 27, 32, 37 }, 38, false).then([&env] {
// Check that the compacted sstable contains all keys.
return check_compacted_sstables(env, tmpdir_path, 38,
return check_compacted_sstables(env, 38,
{ 18, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30, 31, 33, 34, 35, 36 });
});
});
@@ -769,17 +753,15 @@ SEASTAR_TEST_CASE(leveled_04) {
SEASTAR_TEST_CASE(leveled_05) {
// NOTE: Generations from 48 to 51 are used here.
return test_env::do_with([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
// Check compaction code with leveled strategy. In this test, two sstables of level 0 will be created.
return compact_sstables(env, tmpdir_path, { 48, 49 }, 50, true, 1024*1024, compaction_strategy_type::leveled).then([tmpdir_path] (auto generations) {
return compact_sstables(env, { 48, 49 }, 50, true, 1024*1024, compaction_strategy_type::leveled).then([&env] (auto generations) {
BOOST_REQUIRE(generations.size() == 2);
BOOST_REQUIRE(generations[0] == 50);
BOOST_REQUIRE(generations[1] == 51);
return seastar::async([&, generations = std::move(generations), tmpdir_path] {
return seastar::async([&, generations = std::move(generations)] {
for (auto gen : generations) {
auto fname = sstable::filename(tmpdir_path, "ks", "cf", sstables::get_highest_sstable_version(), generation_from_value(gen), big, component_type::Data);
auto fname = sstable::filename(env.tempdir().path().native(), "ks", "cf", sstables::get_highest_sstable_version(), generation_from_value(gen), big, component_type::Data);
BOOST_REQUIRE(file_size(fname).get0() >= 1024*1024);
}
});
@@ -1000,9 +982,8 @@ SEASTAR_TEST_CASE(tombstone_purge_test) {
builder.set_gc_grace_seconds(0);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto compact = [&, s] (std::vector<shared_sstable> all, std::vector<shared_sstable> to_compact) -> std::vector<shared_sstable> {
@@ -1191,7 +1172,6 @@ SEASTAR_TEST_CASE(tombstone_purge_test) {
SEASTAR_TEST_CASE(sstable_rewrite) {
BOOST_REQUIRE(smp::count == 1);
return test_env::do_with([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
auto s = make_shared_schema({}, some_keyspace, some_column_family,
{{"p1", utf8_type}}, {{"c1", utf8_type}}, {{"r1", utf8_type}}, {}, utf8_type);
@@ -1208,13 +1188,13 @@ SEASTAR_TEST_CASE(sstable_rewrite) {
};
apply_key(key_for_this_shard[0]);
auto sst = env.make_sstable(s, tmpdir_path, 51, sstables::get_highest_sstable_version(), big);
return write_memtable_to_sstable_for_test(*mt, sst).then([&env, s, sst, tmpdir_path] {
return env.reusable_sst(s, tmpdir_path, 51);
}).then([&env, s, key = key_for_this_shard[0], tmpdir_path] (auto sstp) mutable {
auto sst = env.make_sstable(s, 51, sstables::get_highest_sstable_version(), big);
return write_memtable_to_sstable_for_test(*mt, sst).then([&env, s, sst] {
return env.reusable_sst(s, 51);
}).then([&env, s, key = key_for_this_shard[0]] (auto sstp) mutable {
auto new_tables = make_lw_shared<std::vector<sstables::shared_sstable>>();
auto creator = [&env, new_tables, s, tmpdir_path] {
auto sst = env.make_sstable(s, tmpdir_path, 52, sstables::get_highest_sstable_version(), big);
auto creator = [&env, new_tables, s] {
auto sst = env.make_sstable(s, 52, sstables::get_highest_sstable_version(), big);
new_tables->emplace_back(sst);
return sst;
};
@@ -1250,7 +1230,6 @@ SEASTAR_TEST_CASE(test_sstable_max_local_deletion_time_2) {
// Create sstable B with tombstone for column in sstable A with TTL 1000.
// Compact them and expect that maximum deletion time is that of column with TTL 100.
return test_env::do_with_async([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
for (auto version : writable_sstable_versions) {
schema_builder builder(some_keyspace, some_column_family);
builder.with_column("p1", utf8_type, column_kind::partition_key);
@@ -1269,10 +1248,10 @@ SEASTAR_TEST_CASE(test_sstable_max_local_deletion_time_2) {
make_atomic_cell(utf8_type, bytes(""), ttl, last_expiry));
mt->apply(std::move(m));
};
auto get_usable_sst = [&env, s, tmpdir_path, version](replica::memtable &mt, int64_t gen) -> future<sstable_ptr> {
auto sst = env.make_sstable(s, tmpdir_path, gen, version, big);
return write_memtable_to_sstable_for_test(mt, sst).then([&env, sst, gen, s, tmpdir_path, version] {
return env.reusable_sst(s, tmpdir_path, gen, version);
auto get_usable_sst = [&env, s, version](replica::memtable &mt, int64_t gen) -> future<sstable_ptr> {
auto sst = env.make_sstable(s, gen, version, big);
return write_memtable_to_sstable_for_test(mt, sst).then([&env, sst, gen, s, version] {
return env.reusable_sst(s, gen, version);
});
};
@@ -1292,7 +1271,7 @@ SEASTAR_TEST_CASE(test_sstable_max_local_deletion_time_2) {
auto sst2 = get_usable_sst(*mt, 55).get0();
BOOST_REQUIRE(now.time_since_epoch().count() == sst2->get_stats_metadata().max_local_deletion_time);
auto creator = [&env, s, tmpdir_path, version, gen = make_lw_shared<unsigned>(56)] { return env.make_sstable(s, tmpdir_path, (*gen)++, version, big); };
auto creator = [&env, s, version, gen = make_lw_shared<unsigned>(56)] { return env.make_sstable(s, (*gen)++, version, big); };
auto info = compact_sstables(sstables::compaction_descriptor({sst1, sst2}, default_priority_class()), cf, creator).get0();
BOOST_REQUIRE(info.new_sstables.size() == 1);
BOOST_REQUIRE(((now + gc_clock::duration(100)).time_since_epoch().count()) ==
@@ -1359,11 +1338,10 @@ SEASTAR_TEST_CASE(compaction_with_fully_expired_table) {
builder.set_gc_grace_seconds(0);
auto s = builder.build();
auto tmp = tmpdir();
auto key = partition_key::from_exploded(*s, {to_bytes("key1")});
auto c_key = clustering_key_prefix::from_exploded(*s, {to_bytes("c1")});
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto mt = make_lw_shared<replica::memtable>(s);
@@ -1373,7 +1351,7 @@ SEASTAR_TEST_CASE(compaction_with_fully_expired_table) {
mt->apply(std::move(m));
auto sst = sst_gen();
write_memtable_to_sstable_for_test(*mt, sst).get();
sst = env.reusable_sst(s, tmp.path().string(), 1).get0();
sst = env.reusable_sst(s, 1).get0();
table_for_tests cf(env.manager(), s);
auto close_cf = deferred_stop(cf);
@@ -1557,11 +1535,6 @@ SEASTAR_TEST_CASE(time_window_strategy_correctness_test) {
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("value", int32_type).build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto make_insert = [&] (partition_key key, api::timestamp_type t) {
mutation m(s, key);
m.set_clustered_cell(clustering_key::make_empty(), bytes("value"), data_value(int32_t(1)), t);
@@ -1577,14 +1550,14 @@ SEASTAR_TEST_CASE(time_window_strategy_correctness_test) {
for (api::timestamp_type t = 0; t < 3; t++) {
auto key = partition_key::from_exploded(*s, {to_bytes("key" + to_sstring(t))});
auto mut = make_insert(std::move(key), t);
sstables.push_back(make_sstable_containing(sst_gen, {std::move(mut)}));
sstables.push_back(make_sstable_containing(env.make_sstable(s), {std::move(mut)}));
}
// Decrement the timestamp to simulate a timestamp in the past hour
for (api::timestamp_type t = 3; t < 5; t++) {
// And add progressively more cells into each sstable
auto key = partition_key::from_exploded(*s, {to_bytes("key" + to_sstring(t))});
auto mut = make_insert(std::move(key), t);
sstables.push_back(make_sstable_containing(sst_gen, {std::move(mut)}));
sstables.push_back(make_sstable_containing(env.make_sstable(s), {std::move(mut)}));
}
std::map<sstring, sstring> options;
@@ -1631,7 +1604,7 @@ SEASTAR_TEST_CASE(time_window_strategy_correctness_test) {
for (int i = 0 ; i < r ; i++) {
mutations.push_back(make_insert(key, tstamp + r));
}
sstables.push_back(make_sstable_containing(sst_gen, std::move(mutations)));
sstables.push_back(make_sstable_containing(env.make_sstable(s), std::move(mutations)));
}
// Reset the buckets, overfill it now
@@ -1659,9 +1632,8 @@ SEASTAR_TEST_CASE(time_window_strategy_size_tiered_behavior_correctness) {
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("value", int32_type).build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto make_insert = [&] (partition_key key, api::timestamp_type t) {
@@ -1757,7 +1729,6 @@ SEASTAR_TEST_CASE(min_max_clustering_key_test_2) {
.build();
table_for_tests cf(env.manager(), s);
auto close_cf = deferred_stop(cf);
auto tmp = tmpdir();
auto mt = make_lw_shared<replica::memtable>(s);
const column_definition &r1_col = *s->get_column_definition("r1");
@@ -1770,9 +1741,9 @@ SEASTAR_TEST_CASE(min_max_clustering_key_test_2) {
}
mt->apply(std::move(m));
}
auto sst = env.make_sstable(s, tmp.path().string(), 1, version, big);
auto sst = env.make_sstable(s, 1, version, big);
write_memtable_to_sstable_for_test(*mt, sst).get();
sst = env.reusable_sst(s, tmp.path().string(), 1, version).get0();
sst = env.reusable_sst(s, 1, version).get0();
check_min_max_column_names(sst, {"0ck100"}, {"7ck149"});
mt = make_lw_shared<replica::memtable>(s);
@@ -1783,12 +1754,12 @@ SEASTAR_TEST_CASE(min_max_clustering_key_test_2) {
m.set_clustered_cell(c_key, r1_col, make_atomic_cell(int32_type, int32_type->decompose(1)));
}
mt->apply(std::move(m));
auto sst2 = env.make_sstable(s, tmp.path().string(), 2, version, big);
auto sst2 = env.make_sstable(s, 2, version, big);
write_memtable_to_sstable_for_test(*mt, sst2).get();
sst2 = env.reusable_sst(s, tmp.path().string(), 2, version).get0();
sst2 = env.reusable_sst(s, 2, version).get0();
check_min_max_column_names(sst2, {"9ck101"}, {"9ck298"});
auto creator = [&env, s, &tmp, version] { return env.make_sstable(s, tmp.path().string(), 3, version, big); };
auto creator = [&env, s, version] { return env.make_sstable(s, 3, version, big); };
auto info = compact_sstables(sstables::compaction_descriptor({sst, sst2}, default_priority_class()), cf, creator).get0();
BOOST_REQUIRE(info.new_sstables.size() == 1);
check_min_max_column_names(info.new_sstables.front(), {"0ck100"}, {"9ck298"});
@@ -1818,7 +1789,6 @@ SEASTAR_TEST_CASE(size_tiered_beyond_max_threshold_test) {
SEASTAR_TEST_CASE(sstable_expired_data_ratio) {
return test_env::do_with_async([] (test_env& env) {
auto tmp = tmpdir();
auto s = make_shared_schema({}, some_keyspace, some_column_family,
{{"p1", utf8_type}}, {{"c1", utf8_type}}, {{"r1", utf8_type}}, {}, utf8_type);
@@ -1849,9 +1819,9 @@ SEASTAR_TEST_CASE(sstable_expired_data_ratio) {
for (auto i = 0; i < remaining; i++) {
insert_key(to_bytes("key" + to_sstring(i)), 3600, expiration_time);
}
auto sst = env.make_sstable(s, tmp.path().string(), 1, sstables::get_highest_sstable_version(), big);
auto sst = env.make_sstable(s, 1, sstables::get_highest_sstable_version(), big);
write_memtable_to_sstable_for_test(*mt, sst).get();
sst = env.reusable_sst(s, tmp.path().string(), 1).get0();
sst = env.reusable_sst(s, 1).get0();
const auto& stats = sst->get_stats_metadata();
BOOST_REQUIRE(stats.estimated_tombstone_drop_time.bin.size() == sstables::TOMBSTONE_HISTOGRAM_BIN_SIZE);
auto gc_before = gc_clock::now() - s->gc_grace_seconds();
@@ -1865,7 +1835,7 @@ SEASTAR_TEST_CASE(sstable_expired_data_ratio) {
table_for_tests cf(env.manager(), s);
auto close_cf = deferred_stop(cf);
auto creator = [&, gen = make_lw_shared<unsigned>(2)] {
auto sst = env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst = env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
return sst;
};
auto info = compact_sstables(sstables::compaction_descriptor({ sst }, default_priority_class()), cf, creator).get0();
@@ -1930,9 +1900,8 @@ SEASTAR_TEST_CASE(compaction_correctness_with_partitioned_sstable_set) {
builder.set_compaction_strategy(sstables::compaction_strategy_type::leveled);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
return sst;
};
@@ -2038,9 +2007,8 @@ SEASTAR_TEST_CASE(sstable_cleanup_correctness_test) {
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("value", int32_type).build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto make_insert = [&] (dht::decorated_key key) {
@@ -2058,7 +2026,7 @@ SEASTAR_TEST_CASE(sstable_cleanup_correctness_test) {
auto sst = make_sstable_containing(sst_gen, mutations);
auto run_identifier = sst->run_identifier();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -2186,13 +2154,8 @@ SEASTAR_TEST_CASE(sstable_scrub_validate_mode_test) {
.with_column("v", int32_type).build();
auto permit = env.make_reader_permit();
auto tmp = tmpdir();
auto sst_gen = [&env, schema, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(schema, tmp.path().string(), (*gen)++);
};
auto scrubbed_mt = make_lw_shared<replica::memtable>(schema);
auto sst = sst_gen();
auto sst = env.make_sstable(schema);
testlog.info("Writing sstable {}", sst->get_filename());
@@ -2208,7 +2171,7 @@ SEASTAR_TEST_CASE(sstable_scrub_validate_mode_test) {
testlog.info("Loaded sstable {}", sst->get_filename());
table_for_tests table(env.manager(), schema, tmp.path().string());
table_for_tests table(env.manager(), schema, env.tempdir().path().string());
auto close_cf = deferred_stop(table);
table->start();
@@ -2384,13 +2347,8 @@ SEASTAR_TEST_CASE(sstable_scrub_skip_mode_test) {
.with_column("v", int32_type).build();
auto permit = env.make_reader_permit();
auto tmp = tmpdir();
auto sst_gen = [&env, schema, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(schema, tmp.path().string(), (*gen)++);
};
std::vector<mutation_fragment_v2> scrubbed_fragments;
auto sst = sst_gen();
auto sst = env.make_sstable(schema);
const auto corrupt_fragments = write_corrupt_sstable(env, *sst, permit, [&] (mutation_fragment_v2&& mf, bool is_corrupt) {
if (!is_corrupt) {
@@ -2404,7 +2362,7 @@ SEASTAR_TEST_CASE(sstable_scrub_skip_mode_test) {
testlog.info("Loaded sstable {}", sst->get_filename());
table_for_tests table(env.manager(), schema, tmp.path().string());
table_for_tests table(env.manager(), schema, env.tempdir().path().string());
auto close_cf = deferred_stop(table);
table->start();
auto& compaction_manager = table.get_compaction_manager();
@@ -2478,13 +2436,8 @@ SEASTAR_TEST_CASE(sstable_scrub_segregate_mode_test) {
.with_column("v", int32_type).build();
auto permit = env.make_reader_permit();
auto tmp = tmpdir();
auto sst_gen = [&env, schema, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(schema, tmp.path().string(), (*gen)++);
};
auto scrubbed_mt = make_lw_shared<replica::memtable>(schema);
auto sst = sst_gen();
auto sst = env.make_sstable(schema);
testlog.info("Writing sstable {}", sst->get_filename());
@@ -2500,7 +2453,7 @@ SEASTAR_TEST_CASE(sstable_scrub_segregate_mode_test) {
testlog.info("Loaded sstable {}", sst->get_filename());
table_for_tests table(env.manager(), schema, tmp.path().string());
table_for_tests table(env.manager(), schema, env.tempdir().path().string());
auto close_cf = deferred_stop(table);
table->start();
auto& compaction_manager = table.get_compaction_manager();
@@ -2589,13 +2542,8 @@ SEASTAR_TEST_CASE(sstable_scrub_quarantine_mode_test) {
.with_column("v", int32_type).build();
auto permit = env.make_reader_permit();
auto tmp = tmpdir();
auto sst_gen = [&env, schema, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(schema, tmp.path().string(), (*gen)++);
};
auto scrubbed_mt = make_lw_shared<replica::memtable>(schema);
auto sst = sst_gen();
auto sst = env.make_sstable(schema);
testlog.info("Writing sstable {}", sst->get_filename());
@@ -2611,7 +2559,7 @@ SEASTAR_TEST_CASE(sstable_scrub_quarantine_mode_test) {
testlog.info("Loaded sstable {}", sst->get_filename());
table_for_tests table(env.manager(), schema, tmp.path().string());
table_for_tests table(env.manager(), schema, env.tempdir().path().string());
auto close_cf = deferred_stop(table);
table->start();
auto& compaction_manager = table.get_compaction_manager();
@@ -2937,9 +2885,8 @@ SEASTAR_TEST_CASE(sstable_run_based_compaction_test) {
.with_column("value", int32_type);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
return sst;
};
@@ -3067,10 +3014,6 @@ SEASTAR_TEST_CASE(compaction_strategy_aware_major_compaction_test) {
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("value", int32_type).build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto make_insert = [&] (partition_key key) {
mutation m(s, key);
m.set_clustered_cell(clustering_key::make_empty(), bytes("value"), data_value(int32_t(1)), api::timestamp_type(0));
@@ -3078,9 +3021,9 @@ SEASTAR_TEST_CASE(compaction_strategy_aware_major_compaction_test) {
};
auto alpha = partition_key::from_exploded(*s, {to_bytes("alpha")});
auto sst = make_sstable_containing(sst_gen, {make_insert(alpha)});
auto sst = make_sstable_containing(env.make_sstable(s), {make_insert(alpha)});
sst->set_sstable_level(2);
auto sst2 = make_sstable_containing(sst_gen, {make_insert(alpha)});
auto sst2 = make_sstable_containing(env.make_sstable(s), {make_insert(alpha)});
sst2->set_sstable_level(3);
auto candidates = std::vector<sstables::shared_sstable>({ sst, sst2 });
@@ -3112,9 +3055,8 @@ SEASTAR_TEST_CASE(backlog_tracker_correctness_after_changing_compaction_strategy
.with_column("value", int32_type);
auto s = builder.build();
auto tmp = make_lw_shared<tmpdir>();
auto sst_gen = [&env, s, tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, tmp->path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
auto sst = env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
return sst;
};
@@ -3169,9 +3111,7 @@ SEASTAR_TEST_CASE(partial_sstable_run_filtered_out_test) {
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("value", int32_type).build();
auto tmp = tmpdir();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -3181,7 +3121,7 @@ SEASTAR_TEST_CASE(partial_sstable_run_filtered_out_test) {
sstable_writer_config sst_cfg = env.manager().configure_writer();
sst_cfg.run_identifier = partial_sstable_run_identifier;
auto partial_sstable_run_sst = make_sstable_easy(env, tmp.path(), make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), { std::move(mut) }), sst_cfg);
auto partial_sstable_run_sst = make_sstable_easy(env, make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), { std::move(mut) }), sst_cfg);
column_family_test(cf).add_sstable(partial_sstable_run_sst).get();
column_family_test::update_sstables_known_generation(*cf, generation_value(partial_sstable_run_sst->generation()));
@@ -3216,11 +3156,6 @@ SEASTAR_TEST_CASE(purged_tombstone_consumer_sstable_test) {
builder.set_gc_grace_seconds(0);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
};
class compacting_sstable_writer_test {
shared_sstable& _sst;
sstable_writer _writer;
@@ -3252,8 +3187,8 @@ SEASTAR_TEST_CASE(purged_tombstone_consumer_sstable_test) {
return max_purgeable_ts;
};
auto non_purged = sst_gen();
auto purged_only = sst_gen();
auto non_purged = env.make_sstable(s);
auto purged_only = env.make_sstable(s);
auto cr = compacting_sstable_writer_test(s, non_purged, env.manager());
auto purged_cr = compacting_sstable_writer_test(s, purged_only, env.manager());
@@ -3331,8 +3266,8 @@ SEASTAR_TEST_CASE(purged_tombstone_consumer_sstable_test) {
auto [mut3, mut3_tombstone] = make_delete(alpha);
std::vector<shared_sstable> sstables = {
make_sstable_containing(sst_gen, {mut1, mut2}),
make_sstable_containing(sst_gen, {mut3})
make_sstable_containing(env.make_sstable(s), {mut1, mut2}),
make_sstable_containing(env.make_sstable(s), {mut3})
};
forward_jump_clocks(std::chrono::seconds(ttl));
@@ -3373,9 +3308,8 @@ SEASTAR_TEST_CASE(incremental_compaction_data_resurrection_test) {
builder.set_gc_grace_seconds(0);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto next_timestamp = [] {
@@ -3420,7 +3354,7 @@ SEASTAR_TEST_CASE(incremental_compaction_data_resurrection_test) {
// make mut1_deletion gc'able.
forward_jump_clocks(std::chrono::seconds(ttl));
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
cf->set_compaction_strategy(sstables::compaction_strategy_type::null);
@@ -3492,9 +3426,8 @@ SEASTAR_TEST_CASE(twcs_major_compaction_test) {
.with_column("value", int32_type);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto next_timestamp = [] (auto step) {
@@ -3523,7 +3456,7 @@ SEASTAR_TEST_CASE(twcs_major_compaction_test) {
auto mut3 = make_insert(0ms);
auto mut4 = make_insert(1ms);
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
cf->set_compaction_strategy(sstables::compaction_strategy_type::time_window);
@@ -3546,8 +3479,7 @@ SEASTAR_TEST_CASE(autocompaction_control_test) {
.with_column("value", int32_type)
.build();
auto tmp = tmpdir();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto& cm = cf.get_compaction_manager();
auto close_cf = deferred_stop(cf);
cf->set_compaction_strategy(sstables::compaction_strategy_type::size_tiered);
@@ -3562,10 +3494,6 @@ SEASTAR_TEST_CASE(autocompaction_control_test) {
// check it is disabled
BOOST_REQUIRE(cf->is_auto_compaction_disabled_by_user());
// generate a few sstables
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto make_insert = [&] (const dht::decorated_key& key) {
mutation m(s, key);
m.set_clustered_cell(clustering_key::make_empty(), bytes("value"), data_value(int32_t(1)), 1 /* ts */);
@@ -3575,7 +3503,7 @@ SEASTAR_TEST_CASE(autocompaction_control_test) {
const auto keys = tests::generate_partition_keys(1, s);
for (auto i = 0; i < 2 * min_threshold; ++i) {
auto mut = make_insert(keys[0]);
auto sst = make_sstable_containing(sst_gen, {mut});
auto sst = make_sstable_containing(env.make_sstable(s), {mut});
cf->add_sstable_and_update_cache(sst).wait();
}
@@ -3606,7 +3534,6 @@ SEASTAR_TEST_CASE(autocompaction_control_test) {
//
SEASTAR_TEST_CASE(test_bug_6472) {
return test_env::do_with_async([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
auto builder = schema_builder("tests", "test_bug_6472")
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("cl", int32_type, column_kind::clustering_key)
@@ -3620,8 +3547,8 @@ SEASTAR_TEST_CASE(test_bug_6472) {
builder.set_gc_grace_seconds(0);
auto s = builder.build();
auto sst_gen = [&env, s, tmpdir_path, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmpdir_path, (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto next_timestamp = [] (auto step) {
@@ -3640,7 +3567,7 @@ SEASTAR_TEST_CASE(test_bug_6472) {
return m;
};
table_for_tests cf(env.manager(), s, tmpdir_path);
table_for_tests cf(env.manager(), s, env.tempdir().path().native());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -3715,7 +3642,6 @@ SEASTAR_TEST_CASE(sstable_needs_cleanup_test) {
SEASTAR_TEST_CASE(test_twcs_partition_estimate) {
return test_env::do_with_async([] (test_env& env) {
auto tmpdir_path = env.tempdir().path().native();
auto builder = schema_builder("tests", "test_bug_6472")
.with_column("id", utf8_type, column_kind::partition_key)
.with_column("cl", int32_type, column_kind::clustering_key)
@@ -3731,8 +3657,8 @@ SEASTAR_TEST_CASE(test_twcs_partition_estimate) {
const auto rows_per_partition = 200;
auto sst_gen = [&env, s, tmpdir_path, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmpdir_path, (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto next_timestamp = [] (int sstable_idx, int ck_idx) {
@@ -3756,7 +3682,7 @@ SEASTAR_TEST_CASE(test_twcs_partition_estimate) {
return make_sstable_containing(sst_gen, {m});
};
table_for_tests cf(env.manager(), s, tmpdir_path);
table_for_tests cf(env.manager(), s, env.tempdir().path().native());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -3872,8 +3798,8 @@ SEASTAR_TEST_CASE(test_twcs_interposer_on_memtable_flush) {
return m;
};
auto tmp = tmpdir();
table_for_tests cf(env.manager(), s, tmp.path().string());
auto _ = env.tempdir().make_sweeper();
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -3914,9 +3840,8 @@ SEASTAR_TEST_CASE(test_twcs_compaction_across_buckets) {
auto next_timestamp = [] (std::chrono::hours step = std::chrono::hours(0)) {
return (gc_clock::now().time_since_epoch() - std::chrono::duration_cast<std::chrono::microseconds>(step)).count();
};
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto pkey = tests::generate_partition_key(s);
@@ -4044,10 +3969,8 @@ SEASTAR_TEST_CASE(twcs_reshape_with_disjoint_set_test) {
return m;
};
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::sstable::version_types::md, big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, (*gen)++, sstables::sstable::version_types::md, big);
};
{
@@ -4181,10 +4104,8 @@ SEASTAR_TEST_CASE(stcs_reshape_overlapping_test) {
return m;
};
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::sstable::version_types::md, big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, (*gen)++, sstables::sstable::version_types::md, big);
};
{
@@ -4225,9 +4146,8 @@ SEASTAR_TEST_CASE(test_twcs_single_key_reader_filtering) {
builder.set_compaction_strategy(sstables::compaction_strategy_type::time_window);
auto s = builder.build();
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::sstable::version_types::md, big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, (*gen)++, sstables::sstable::version_types::md, big);
};
auto make_row = [&] (int32_t pk, int32_t ck) {
@@ -4240,7 +4160,7 @@ SEASTAR_TEST_CASE(test_twcs_single_key_reader_filtering) {
auto sst2 = make_sstable_containing(sst_gen, {make_row(0, 1)});
auto dkey = sst1->get_first_decorated_key();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
@@ -4304,7 +4224,6 @@ SEASTAR_TEST_CASE(max_ongoing_compaction_test) {
auto cm = compaction_manager_for_testing();
auto tmp = tmpdir();
auto cl_stats = make_lw_shared<cell_locker_stats>();
auto tracker = make_lw_shared<cache_tracker>();
@@ -4338,7 +4257,7 @@ SEASTAR_TEST_CASE(max_ongoing_compaction_test) {
schemas.push_back(s);
replica::column_family::config cfg = env.make_table_config();
cfg.datadir = tmp.path().string() + "/" + std::to_string(idx);
cfg.datadir = env.tempdir().path().string() + "/" + std::to_string(idx);
touch_directory(cfg.datadir).get();
cfg.enable_commitlog = false;
cfg.enable_incremental_backups = false;
@@ -4551,16 +4470,15 @@ SEASTAR_TEST_CASE(twcs_single_key_reader_through_compound_set_test) {
return m;
};
auto tmp = tmpdir();
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
cf->start();
auto set1 = make_lw_shared<sstable_set>(cs.make_sstable_set(s));
auto set2 = make_lw_shared<sstable_set>(cs.make_sstable_set(s));
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::sstable::version_types::md, big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)]() {
return env.make_sstable(s, (*gen)++, sstables::sstable::version_types::md, big);
};
// sstables with same key but belonging to different windows
@@ -4597,13 +4515,12 @@ SEASTAR_TEST_CASE(test_major_does_not_miss_data_in_memtable) {
.with_column("value", int32_type);
auto s = builder.build();
auto tmp = tmpdir();
auto pkey = tests::generate_partition_key(s);
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
auto sst_gen = [&env, &cf, s, &tmp] () mutable {
return env.make_sstable(s, tmp.path().string(), column_family_test::calculate_generation_for_new_table(*cf),
auto sst_gen = [&env, &cf, s] () mutable {
return env.make_sstable(s, column_family_test::calculate_generation_for_new_table(*cf),
sstables::get_highest_sstable_version(), big);
};
@@ -4769,13 +4686,13 @@ SEASTAR_TEST_CASE(test_compaction_strategy_cleanup_method) {
builder.set_compaction_strategy_options(std::move(strategy_options));
auto s = builder.build();
auto tmp = tmpdir();
auto _ = env.tempdir().make_sweeper();
auto keys = tests::generate_partition_keys(all_files, s);
table_for_tests cf(env.manager(), s, tmp.path().string());
table_for_tests cf(env.manager(), s, env.tempdir().path().string());
auto close_cf = deferred_stop(cf);
auto sst_gen = [&env, &cf, s, &tmp]() mutable {
return env.make_sstable(s, tmp.path().string(), column_family_test::calculate_generation_for_new_table(*cf),
auto sst_gen = [&env, &cf, s]() mutable {
return env.make_sstable(s, column_family_test::calculate_generation_for_new_table(*cf),
sstables::get_highest_sstable_version(), big);
};
@@ -4859,9 +4776,8 @@ SEASTAR_TEST_CASE(test_large_partition_splitting_on_compaction) {
auto next_timestamp = [] (std::chrono::seconds step = 0s) {
return (gc_clock::now().time_since_epoch() + duration_cast<microseconds>(step)).count();
};
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, tmp.path().string(), (*gen)++, sstables::get_highest_sstable_version(), big);
auto sst_gen = [&env, s, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(s, (*gen)++, sstables::get_highest_sstable_version(), big);
};
auto pkey = tests::generate_partition_key(s);
table_for_tests cf(env.manager(), s);
@@ -4931,7 +4847,7 @@ SEASTAR_TEST_CASE(test_large_partition_splitting_on_compaction) {
position_in_partition::tri_compare pos_tri_cmp(*s);
for (auto& sst : ret.new_sstables) {
sst = env.reusable_sst(s, tmp.path().string(), sst->generation().value()).get0();
sst = env.reusable_sst(s, sst->generation().value()).get0();
BOOST_REQUIRE(sst->may_have_partition_tombstones());
auto reader = sstable_reader(sst, s, env.make_reader_permit());
@@ -4987,14 +4903,10 @@ SEASTAR_TEST_CASE(check_table_sstable_set_includes_maintenance_sstables) {
simple_schema ss;
auto s = ss.schema();
auto pks = ss.make_pkeys(1);
auto tmp = tmpdir();
auto sst_gen = [&env, s, &tmp] () {
return env.make_sstable(s, tmp.path().string(), 1, sstables::get_highest_sstable_version(), big);
};
auto mut1 = mutation(s, pks[0]);
mut1.partition().apply_insert(*s, ss.make_ckey(0), ss.new_timestamp());
auto sst = make_sstable_containing(sst_gen, {std::move(mut1)});
auto sst = make_sstable_containing(env.make_sstable(s), {std::move(mut1)});
table_for_tests cf(env.manager(), s);
auto close_cf = deferred_stop(cf);

362
test/boost/sstable_datafile_test.cc
View File

File diff suppressed because it is too large Load Diff

70
test/boost/sstable_mutation_test.cc
View File

@@ -400,7 +400,6 @@ mutation_source make_sstable_mutation_source(sstables::test_env& env, schema_ptr
SEASTAR_TEST_CASE(test_sstable_can_write_and_read_range_tombstone) {
return test_env::do_with_async([] (test_env& env) {
auto dir = tmpdir();
auto s = make_shared_schema({}, "ks", "cf",
{{"p1", utf8_type}}, {{"c1", int32_type}}, {{"r1", int32_type}}, {}, utf8_type);
@@ -415,11 +414,7 @@ SEASTAR_TEST_CASE(test_sstable_can_write_and_read_range_tombstone) {
auto mt = make_lw_shared<replica::memtable>(s);
mt->apply(std::move(m));
auto sst = env.make_sstable(s,
dir.path().string(),
1 /* generation */,
sstables::get_highest_sstable_version(),
sstables::sstable::format_types::big);
auto sst = env.make_sstable(s);
write_memtable_to_sstable_for_test(*mt, sst).get();
sst->load().get();
auto mut = with_closeable(sst->make_reader(s, env.make_reader_permit(), query::full_partition_range, s->full_slice()), [] (auto& mr) {
@@ -842,7 +837,6 @@ SEASTAR_TEST_CASE(buffer_overflow) {
SEASTAR_TEST_CASE(test_non_compound_table_row_is_not_marked_as_static) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c", int32_type, column_kind::clustering_key);
@@ -859,11 +853,7 @@ SEASTAR_TEST_CASE(test_non_compound_table_row_is_not_marked_as_static) {
auto mt = make_lw_shared<replica::memtable>(s);
mt->apply(std::move(m));
auto sst = env.make_sstable(s,
dir.path().string(),
1 /* generation */,
version,
sstables::sstable::format_types::big);
auto sst = env.make_sstable(s, version);
write_memtable_to_sstable_for_test(*mt, sst).get();
sst->load().get();
auto mut = with_closeable(sst->make_reader(s, env.make_reader_permit(), query::full_partition_range, s->full_slice()), [] (auto& mr) {
@@ -877,7 +867,6 @@ SEASTAR_TEST_CASE(test_non_compound_table_row_is_not_marked_as_static) {
SEASTAR_TEST_CASE(test_has_partition_key) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c", int32_type, column_kind::clustering_key);
@@ -894,11 +883,7 @@ SEASTAR_TEST_CASE(test_has_partition_key) {
auto mt = make_lw_shared<replica::memtable>(s);
mt->apply(std::move(m));
auto sst = env.make_sstable(s,
dir.path().string(),
1 /* generation */,
version,
sstables::sstable::format_types::big);
auto sst = env.make_sstable(s, version);
write_memtable_to_sstable_for_test(*mt, sst).get();
auto hk = sstables::sstable::make_hashed_key(*s, dk.key());
sst->load().get();
@@ -922,7 +907,6 @@ static std::unique_ptr<index_reader> get_index_reader(shared_sstable sst, reader
SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic) {
return test_env::do_with_async([] (test_env& env) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c1", int32_type, column_kind::clustering_key);
@@ -959,14 +943,13 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic) {
cfg.promoted_index_block_size = 1;
cfg.promoted_index_auto_scale_threshold = 0; // disable auto-scaling
auto sst = make_sstable_easy(env, dir.path(), mt, cfg);
auto sst = make_sstable_easy(env, mt, cfg);
assert_that(get_index_reader(sst, env.make_reader_permit())).has_monotonic_positions(*s);
});
}
SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_with_auto_scaling) {
return test_env::do_with_async([] (test_env& env) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c1", int32_type, column_kind::clustering_key);
@@ -996,7 +979,7 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_with_auto_scaling) {
cfg.promoted_index_block_size = 1;
cfg.promoted_index_auto_scale_threshold = 100; // set to a low value to trigger auto-scaling
auto sst = make_sstable_easy(env, dir.path(), mt, cfg);
auto sst = make_sstable_easy(env, mt, cfg);
assert_that(get_index_reader(sst, env.make_reader_permit())).has_monotonic_positions(*s);
});
}
@@ -1004,7 +987,6 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_with_auto_scaling) {
SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_compound_dense) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c1", int32_type, column_kind::clustering_key);
@@ -1040,7 +1022,7 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_compound_dense) {
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = 1;
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, 1, version);
auto sst = make_sstable_easy(env, mt, cfg, 1, version);
{
assert_that(get_index_reader(sst, env.make_reader_permit())).has_monotonic_positions(*s);
@@ -1059,7 +1041,6 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_compound_dense) {
SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_non_compound_dense) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c1", int32_type, column_kind::clustering_key);
@@ -1091,7 +1072,7 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_non_compound_dense) {
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = 1;
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, 1, version);
auto sst = make_sstable_easy(env, mt, cfg, 1, version);
{
assert_that(get_index_reader(sst, env.make_reader_permit())).has_monotonic_positions(*s);
@@ -1110,7 +1091,6 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic_non_compound_dense) {
SEASTAR_TEST_CASE(test_promoted_index_repeats_open_tombstones) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
int id = 0;
for (auto& compact : { schema_builder::compact_storage::no, schema_builder::compact_storage::yes }) {
const auto generation = id++;
@@ -1139,7 +1119,7 @@ SEASTAR_TEST_CASE(test_promoted_index_repeats_open_tombstones) {
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = 1;
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, generation, version);
auto sst = make_sstable_easy(env, mt, cfg, generation, version);
{
auto slice = partition_slice_builder(*s).with_range(query::clustering_range::make_starting_with({ck})).build();
@@ -1155,7 +1135,6 @@ SEASTAR_TEST_CASE(test_promoted_index_repeats_open_tombstones) {
SEASTAR_TEST_CASE(test_range_tombstones_are_correctly_seralized_for_non_compound_dense_schemas) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("c", int32_type, column_kind::clustering_key);
@@ -1176,7 +1155,7 @@ SEASTAR_TEST_CASE(test_range_tombstones_are_correctly_seralized_for_non_compound
mt->apply(m);
sstable_writer_config cfg = env.manager().configure_writer();
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, 1, version);
auto sst = make_sstable_easy(env, mt, cfg, 1, version);
{
auto slice = partition_slice_builder(*s).build();
@@ -1191,7 +1170,6 @@ SEASTAR_TEST_CASE(test_range_tombstones_are_correctly_seralized_for_non_compound
SEASTAR_TEST_CASE(test_promoted_index_is_absent_for_schemas_without_clustering_key) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
schema_builder builder("ks", "cf");
builder.with_column("p", utf8_type, column_kind::partition_key);
builder.with_column("v", int32_type);
@@ -1208,7 +1186,7 @@ SEASTAR_TEST_CASE(test_promoted_index_is_absent_for_schemas_without_clustering_k
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = 1;
auto sst = make_sstable_easy(env, dir.path(), mt, cfg, 1, version);
auto sst = make_sstable_easy(env, mt, cfg, 1, version);
assert_that(get_index_reader(sst, env.make_reader_permit())).is_empty(*s);
}
});
@@ -1217,7 +1195,6 @@ SEASTAR_TEST_CASE(test_promoted_index_is_absent_for_schemas_without_clustering_k
SEASTAR_TEST_CASE(test_writing_combined_stream_with_tombstones_at_the_same_position) {
return test_env::do_with_async([] (test_env& env) {
for (const auto version : writable_sstable_versions) {
auto dir = tmpdir();
simple_schema ss;
auto s = ss.schema();
@@ -1247,7 +1224,7 @@ SEASTAR_TEST_CASE(test_writing_combined_stream_with_tombstones_at_the_same_posit
auto combined_permit = env.make_reader_permit();
auto mr = make_combined_reader(s, combined_permit,
mt1->make_flat_reader(s, combined_permit), mt2->make_flat_reader(s, combined_permit));
auto sst = make_sstable_easy(env, dir.path(), std::move(mr), env.manager().configure_writer(), 1, version);
auto sst = make_sstable_easy(env, std::move(mr), env.manager().configure_writer(), 1, version);
assert_that(sst->as_mutation_source().make_reader_v2(s, env.make_reader_permit()))
.produces(m1 + m2)
@@ -1309,15 +1286,15 @@ SEASTAR_TEST_CASE(test_key_count_estimation) {
int count = 10'000;
std::vector<dht::decorated_key> all_pks = tests::generate_partition_keys(count + 2, s, local_shard_only::yes, tests::key_size{8, 8});
std::vector<dht::decorated_key> pks(all_pks.begin() + 1, all_pks.end() - 1);
std::vector<mutation> muts;
auto mt = make_lw_shared<replica::memtable>(s);
for (auto pk : pks) {
mutation m(s, pk);
m.set_clustered_cell(clustering_key::make_empty(), bytes("v"), data_value(int32_t(1)), 1 /* ts */);
muts.push_back(std::move(m));
mt->apply(m);
}
tmpdir dir;
shared_sstable sst = make_sstable(env, s, dir.path().string(), muts, env.manager().configure_writer(), version);
auto _ = env.tempdir().make_sweeper();
shared_sstable sst = make_sstable_easy(env, mt, env.manager().configure_writer(), 1, version, pks.size());
auto max_est = sst->get_estimated_key_count();
testlog.trace("count = {}", count);
@@ -1368,7 +1345,6 @@ SEASTAR_TEST_CASE(test_large_index_pages_do_not_cause_large_allocations) {
return test_env::do_with_async([] (test_env& env) {
// We create a sequence of partitions such that first we have a partition with a very long key, then
// series of partitions with small keys. This should result in large index page.
auto dir = tmpdir();
simple_schema ss;
auto s = ss.schema();
@@ -1418,7 +1394,7 @@ SEASTAR_TEST_CASE(test_large_index_pages_do_not_cause_large_allocations) {
}
sstable_writer_config cfg = env.manager().configure_writer();
auto sst = make_sstable_easy(env, dir.path(), mt, cfg);
auto sst = make_sstable_easy(env, mt, cfg);
auto pr = dht::partition_range::make_singular(small_keys[0]);
@@ -1443,8 +1419,6 @@ SEASTAR_TEST_CASE(test_large_index_pages_do_not_cause_large_allocations) {
SEASTAR_TEST_CASE(test_reading_serialization_header) {
return test_env::do_with_async([] (test_env& env) {
auto dir = tmpdir();
auto random_int32_value = [] {
return int32_type->decompose(tests::random::get_int<int32_t>());
};
@@ -1485,11 +1459,11 @@ SEASTAR_TEST_CASE(test_reading_serialization_header) {
// SSTable class has way too many responsibilities. In particular, it mixes the reading and
// writting parts. Let's use a separate objects for writing and reading to ensure that nothing
// carries over that wouldn't normally be read from disk.
auto sst = env.make_sstable(s, dir.path().string(), 1, sstable::version_types::mc, sstables::sstable::format_types::big);
auto sst = env.make_sstable(s, 1, sstable::version_types::mc, sstables::sstable::format_types::big);
sst->write_components(mt->make_flat_reader(s, env.make_reader_permit()), 2, s, env.manager().configure_writer(), mt->get_encoding_stats()).get();
}
auto sst = env.make_sstable(s, dir.path().string(), 1, sstable::version_types::mc, sstables::sstable::format_types::big);
auto sst = env.make_sstable(s, 1, sstable::version_types::mc, sstables::sstable::format_types::big);
sst->load().get();
auto hdr = sst->get_serialization_header();
@@ -1549,8 +1523,6 @@ SEASTAR_THREAD_TEST_CASE(test_merging_encoding_stats) {
// Reproducer for #4206
SEASTAR_TEST_CASE(test_counter_header_size) {
return test_env::do_with_async([] (test_env& env) {
auto dir = tmpdir();
auto s = schema_builder("ks", "counter_test")
.with_column("pk", int32_type, column_kind::partition_key)
.with_column("ck", int32_type, column_kind::clustering_key)
@@ -1579,7 +1551,7 @@ SEASTAR_TEST_CASE(test_counter_header_size) {
mt->apply(m);
for (const auto version : writable_sstable_versions) {
auto sst = make_sstable_easy(env, dir.path(), mt, env.manager().configure_writer(), 1, version);
auto sst = make_sstable_easy(env, mt, env.manager().configure_writer(), 1, version);
assert_that(sst->as_mutation_source().make_reader_v2(s, env.make_reader_permit()))
.produces(m)
.produces_end_of_stream();
@@ -1671,8 +1643,8 @@ SEASTAR_TEST_CASE(writer_handles_subsequent_range_tombstone_changes_without_tomb
flat_mutation_reader_v2 input_reader = make_flat_mutation_reader_from_fragments(s, p, std::move(fragments));
deferred_close dc1{input_reader};
sstable_writer_config cfg = env.manager().configure_writer();
tmpdir dir;
shared_sstable sstable = env.make_sstable(s, dir.path().string(), 0);
auto _ = env.tempdir().make_sweeper();
shared_sstable sstable = env.make_sstable(s, 0);
encoding_stats es;
sstable->write_components(std::move(input_reader), 1, s, cfg, es).get();
sstable->load().get();

10
test/boost/sstable_set_test.cc
View File

@@ -31,7 +31,6 @@ SEASTAR_TEST_CASE(test_sstables_sstable_set_read_modify_write) {
return test_env::do_with_async([] (test_env& env) {
simple_schema ss;
auto s = ss.schema();
fs::path tmp = env.tempdir().path();
auto pk = tests::generate_partition_key(s);
auto mut = mutation(s, pk);
@@ -40,14 +39,14 @@ SEASTAR_TEST_CASE(test_sstables_sstable_set_read_modify_write) {
auto mr = make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), {mut});
sstable_writer_config cfg = env.manager().configure_writer("");
auto sst1 = make_sstable_easy(env, tmp, std::move(mr), cfg, gen++);
auto sst1 = make_sstable_easy(env, std::move(mr), cfg, gen++);
auto ss1 = make_lw_shared<sstables::sstable_set>(make_sstable_set(ss.schema(), make_lw_shared<sstable_list>({sst1})));
BOOST_REQUIRE_EQUAL(ss1->all()->size(), 1);
// Test that a random sstable_origin is stored and retrieved properly.
mr = make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), {mut});
auto sst2 = make_sstable_easy(env, tmp, std::move(mr), cfg, gen++);
auto sst2 = make_sstable_easy(env, std::move(mr), cfg, gen++);
auto ss2 = make_lw_shared<sstables::sstable_set>(*ss1);
ss2->insert(sst2);
@@ -60,7 +59,6 @@ SEASTAR_TEST_CASE(test_time_series_sstable_set_read_modify_write) {
return test_env::do_with_async([] (test_env& env) {
simple_schema ss;
auto s = ss.schema();
fs::path tmp = env.tempdir().path();
auto pk = tests::generate_partition_key(s);
auto mut = mutation(s, pk);
@@ -69,7 +67,7 @@ SEASTAR_TEST_CASE(test_time_series_sstable_set_read_modify_write) {
sstable_writer_config cfg = env.manager().configure_writer("");
auto mr = make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), {mut});
auto sst1 = make_sstable_easy(env, tmp, std::move(mr), cfg, gen++);
auto sst1 = make_sstable_easy(env, std::move(mr), cfg, gen++);
auto ss1 = make_lw_shared<time_series_sstable_set>(ss.schema());
ss1->insert(sst1);
@@ -77,7 +75,7 @@ SEASTAR_TEST_CASE(test_time_series_sstable_set_read_modify_write) {
// Test that a random sstable_origin is stored and retrieved properly.
mr = make_flat_mutation_reader_from_mutations_v2(s, env.make_reader_permit(), {mut});
auto sst2 = make_sstable_easy(env, tmp, std::move(mr), cfg, gen++);
auto sst2 = make_sstable_easy(env, std::move(mr), cfg, gen++);
auto ss2 = make_lw_shared<time_series_sstable_set>(*ss1);
ss2->insert(sst2);

6
test/boost/sstable_test.cc
View File

@@ -242,9 +242,9 @@ SEASTAR_TEST_CASE(check_compressed_info_func) {
future<>
write_and_validate_sst(schema_ptr s, sstring dir, noncopyable_function<future<> (shared_sstable sst1, shared_sstable sst2)> func) {
return test_env::do_with(tmpdir(), [s = std::move(s), dir = std::move(dir), func = std::move(func)] (test_env& env, tmpdir& tmp) mutable {
return do_write_sst(env, s, dir, tmp.path().string(), 1).then([&env, &tmp, s = std::move(s), func = std::move(func)] (auto sst1) {
auto sst2 = env.make_sstable(s, tmp.path().string(), 2, sst1->get_version());
return test_env::do_with([s = std::move(s), dir = std::move(dir), func = std::move(func)] (test_env& env) mutable {
return do_write_sst(env, s, dir, env.tempdir().path().string(), 1).then([&env, s = std::move(s), func = std::move(func)] (auto sst1) {
auto sst2 = env.make_sstable(s, 2, sst1->get_version());
return func(std::move(sst1), std::move(sst2));
});
});

35
test/lib/sstable_test_env.hh
View File

@@ -55,6 +55,7 @@ class test_env {
db::nop_large_data_handler nop_ld_handler;
test_env_sstables_manager mgr;
reader_concurrency_semaphore semaphore;
unsigned generation = 1;
impl(test_env_config cfg);
impl(impl&&) = delete;
@@ -77,12 +78,28 @@ public:
return _impl->mgr.make_sstable(std::move(schema), dir, generation_from_value(generation), v, f, now, default_io_error_handler_gen(), buffer_size);
}
shared_sstable make_sstable(schema_ptr schema, unsigned long generation,
sstable::version_types v = sstables::get_highest_sstable_version(), sstable::format_types f = sstable::format_types::big,
size_t buffer_size = default_sstable_buffer_size, gc_clock::time_point now = gc_clock::now()) {
return make_sstable(std::move(schema), _impl->dir.path().native(), generation, std::move(v), std::move(f), buffer_size, now);
}
shared_sstable make_sstable(schema_ptr schema, sstable::version_types v = sstables::get_highest_sstable_version()) {
return make_sstable(std::move(schema), _impl->generation++, std::move(v));
}
struct sst_not_found : public std::runtime_error {
sst_not_found(const sstring& dir, unsigned long generation)
: std::runtime_error(format("no versions of sstable generation {} found in {}", generation, dir))
{}
};
// reusable_sst() opens the requested sstable for reading only (sstables are
// immutable, so an existing sstable cannot be opened for writing).
// It returns a future because opening requires reading from disk, and
// therefore may block. The future value is a shared sstable - a reference-
// counting pointer to an sstable - allowing for the returned handle to
// be passed around until no longer needed.
future<shared_sstable> reusable_sst(schema_ptr schema, sstring dir, unsigned long generation,
sstable::version_types version, sstable::format_types f = sstable::format_types::big) {
auto sst = make_sstable(std::move(schema), dir, generation, version, f);
@@ -92,9 +109,18 @@ public:
});
}
future<shared_sstable> reusable_sst(schema_ptr schema, unsigned long generation,
sstable::version_types version, sstable::format_types f = sstable::format_types::big) {
return reusable_sst(std::move(schema), _impl->dir.path().native(), std::move(generation), std::move(version), std::move(f));
}
// looks up the sstable in the given dir
future<shared_sstable> reusable_sst(schema_ptr schema, sstring dir, unsigned long generation);
future<shared_sstable> reusable_sst(schema_ptr schema, unsigned long generation) {
return reusable_sst(std::move(schema), _impl->dir.path().native(), generation);
}
test_env_sstables_manager& manager() { return _impl->mgr; }
reader_concurrency_semaphore& semaphore() { return _impl->semaphore; }
db::config& db_config() { return *_impl->db_config; }
@@ -120,15 +146,6 @@ public:
});
}
template <typename T, typename Func>
static inline auto do_with(T&& rval, Func&& func) {
return seastar::do_with(test_env(), std::forward<T>(rval), [func = std::move(func)] (test_env& env, T& val) mutable {
return futurize_invoke(func, env, val).finally([&env] {
return env.stop();
});
});
}
static inline future<> do_with_async(noncopyable_function<void (test_env&)> func, test_env_config cfg = {}) {
return seastar::async([func = std::move(func), cfg = std::move(cfg)] () mutable {
test_env env(std::move(cfg));

19
test/lib/sstable_utils.cc
View File

@@ -25,9 +25,12 @@ using namespace sstables;
using namespace std::chrono_literals;
sstables::shared_sstable make_sstable_containing(std::function<sstables::shared_sstable()> sst_factory, std::vector<mutation> muts) {
return make_sstable_containing(sst_factory(), std::move(muts));
}
sstables::shared_sstable make_sstable_containing(sstables::shared_sstable sst, std::vector<mutation> muts) {
tests::reader_concurrency_semaphore_wrapper semaphore;
auto sst = sst_factory();
schema_ptr s = muts[0].schema();
auto mt = make_lw_shared<replica::memtable>(s);
@@ -77,22 +80,26 @@ shared_sstable make_sstable(sstables::test_env& env, schema_ptr s, sstring dir,
mt->apply(m);
}
return make_sstable_easy(env, dir_path, mt, cfg, 1, version, mutations.size(), query_time);
auto sst = env.make_sstable(s, dir_path.string(), 1, version, sstable_format_types::big, default_sstable_buffer_size, query_time);
auto mr = mt->make_flat_reader(s, env.make_reader_permit());
sst->write_components(std::move(mr), mutations.size(), s, cfg, mt->get_encoding_stats()).get();
sst->load().get();
return sst;
}
shared_sstable make_sstable_easy(test_env& env, const fs::path& path, flat_mutation_reader_v2 rd, sstable_writer_config cfg,
shared_sstable make_sstable_easy(test_env& env, flat_mutation_reader_v2 rd, sstable_writer_config cfg,
int64_t generation, const sstables::sstable::version_types version, int expected_partition) {
auto s = rd.schema();
auto sst = env.make_sstable(s, path.string(), generation, version, sstable_format_types::big);
auto sst = env.make_sstable(s, generation, version, sstable_format_types::big);
sst->write_components(std::move(rd), expected_partition, s, cfg, encoding_stats{}).get();
sst->load().get();
return sst;
}
shared_sstable make_sstable_easy(test_env& env, const fs::path& path, lw_shared_ptr<replica::memtable> mt, sstable_writer_config cfg,
shared_sstable make_sstable_easy(test_env& env, lw_shared_ptr<replica::memtable> mt, sstable_writer_config cfg,
unsigned long gen, const sstable::version_types v, int estimated_partitions, gc_clock::time_point query_time) {
schema_ptr s = mt->schema();
auto sst = env.make_sstable(s, path.string(), gen, v, sstable_format_types::big, default_sstable_buffer_size, query_time);
auto sst = env.make_sstable(s, gen, v, sstable_format_types::big, default_sstable_buffer_size, query_time);
auto mr = mt->make_flat_reader(s, env.make_reader_permit());
sst->write_components(std::move(mr), estimated_partitions, s, cfg, mt->get_encoding_stats()).get();
sst->load().get();

5
test/lib/sstable_utils.hh
View File

@@ -26,6 +26,7 @@ using namespace sstables;
using namespace std::chrono_literals;
sstables::shared_sstable make_sstable_containing(std::function<sstables::shared_sstable()> sst_factory, std::vector<mutation> muts);
sstables::shared_sstable make_sstable_containing(sstables::shared_sstable sst, std::vector<mutation> muts);
inline future<> write_memtable_to_sstable_for_test(replica::memtable& mt, sstables::shared_sstable sst) {
return write_memtable_to_sstable(mt, sst, sst->manager().configure_writer("memtable"));
@@ -287,7 +288,7 @@ future<compaction_result> compact_sstables(compaction_manager& cm, sstables::com
std::function<shared_sstable()> creator, sstables::compaction_sstable_replacer_fn replacer = sstables::replacer_fn_no_op(),
can_purge_tombstones can_purge = can_purge_tombstones::yes);
shared_sstable make_sstable_easy(test_env& env, const fs::path& path, flat_mutation_reader_v2 rd, sstable_writer_config cfg,
shared_sstable make_sstable_easy(test_env& env, flat_mutation_reader_v2 rd, sstable_writer_config cfg,
int64_t generation = 1, const sstables::sstable::version_types version = sstables::get_highest_sstable_version(), int expected_partition = 1);
shared_sstable make_sstable_easy(test_env& env, const fs::path& path, lw_shared_ptr<replica::memtable> mt, sstable_writer_config cfg,
shared_sstable make_sstable_easy(test_env& env, lw_shared_ptr<replica::memtable> mt, sstable_writer_config cfg,
unsigned long gen = 1, const sstable::version_types v = sstables::get_highest_sstable_version(), int estimated_partitions = 1, gc_clock::time_point = gc_clock::now());

9
test/lib/tmpdir.cc
View File

@@ -21,6 +21,15 @@ void tmpdir::remove() noexcept {
}
}
tmpdir::sweeper::~sweeper() {
memory::scoped_critical_alloc_section dfg;
if (!_tmpd._path.empty()) {
for (const auto& ent : fs::directory_iterator(_tmpd._path)) {
fs::remove_all(ent.path());
}
}
}
tmpdir::tmpdir()
: _path(fs::temp_directory_path() / fs::path(fmt::format(FMT_STRING("scylla-{}"), utils::make_random_uuid()))) {
fs::create_directories(_path);

8
test/lib/tmpdir.hh
View File

@@ -24,6 +24,13 @@ class tmpdir {
private:
void remove() noexcept;
class sweeper {
const tmpdir& _tmpd;
public:
sweeper(const tmpdir& t) noexcept : _tmpd(t) {}
~sweeper();
};
public:
tmpdir();
@@ -35,4 +42,5 @@ public:
~tmpdir();
const fs::path& path() const noexcept { return _path; }
sweeper make_sweeper() const noexcept { return sweeper(*this); }
};