From 62ced6670231aa013acdfb2d1f4913a7b238e2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Jadwiszczak?= Date: Thu, 2 Feb 2023 11:01:39 +0100 Subject: [PATCH] schema: add scylla specific options to schema description Add `paxos_grace_seconds`, `tombstone_gc`, `cdc` and `synchronous_updates` options to schema description. Fixes: #12389 Fixes: scylladb/scylla-enterprise#2979 Closes #14275 --- schema/schema.cc | 27 +++++++++- test/boost/cql_query_test.cc | 53 ++++++++++++++++---- test/boost/multishard_mutation_query_test.cc | 35 +++++++++---- test/cql-pytest/test_describe.py | 3 ++ 4 files changed, 98 insertions(+), 20 deletions(-) diff --git a/schema/schema.cc b/schema/schema.cc index 116d30fe55..227127afc0 100644 --- a/schema/schema.cc +++ b/schema/schema.cc @@ -35,6 +35,8 @@ #include "utils/rjson.hh" #include "tombstone_gc_options.hh" #include "db/per_partition_rate_limit_extension.hh" +#include "db/tags/utils.hh" +#include "db/tags/extension.hh" constexpr int32_t schema::NAME_LENGTH; @@ -784,6 +786,15 @@ static bool is_index(replica::database& db, const table_id& id, const schema& s) return db.find_column_family(id).get_index_manager().is_index(s); } +static bool is_update_synchronously_view(const schema& s) { + auto tag_opt = db::find_tag(s, db::SYNCHRONOUS_VIEW_UPDATES_TAG_KEY); + if (!tag_opt.has_value()) { + return false; + } + + return *tag_opt == "true"; +} + sstring schema::element_type(replica::database& db) const { if (is_view()) { if (is_index(db, view_info()->base_id(), *this)) { @@ -927,8 +938,20 @@ std::ostream& schema::describe(replica::database& db, std::ostream& os, bool wit os << "\n AND memtable_flush_period_in_ms = " << memtable_flush_period(); os << "\n AND min_index_interval = " << min_index_interval(); os << "\n AND read_repair_chance = " << read_repair_chance(); - os << "\n AND speculative_retry = '" << speculative_retry().to_sstring() << "';"; - os << "\n"; + os << "\n AND speculative_retry = '" << speculative_retry().to_sstring() << "'"; + os << "\n AND paxos_grace_seconds = " << paxos_grace_seconds().count(); + + auto tombstone_gc_str = tombstone_gc_options().to_sstring(); + std::replace(tombstone_gc_str.begin(), tombstone_gc_str.end(), '"', '\''); + os << "\n AND tombstone_gc = " << tombstone_gc_str; + + if (cdc_options().enabled()) { + os << "\n AND cdc = " << cdc_options().to_sstring(); + } + if (is_view() && !is_index(db, view_info()->base_id(), *this)) { + os << "\n AND synchronous_updates = " << std::boolalpha << is_update_synchronously_view(*this); + } + os << ";\n"; if (with_internals) { for (auto& cdef : dropped_columns()) { diff --git a/test/boost/cql_query_test.cc b/test/boost/cql_query_test.cc index 14d7fb1490..ec41f8c1bc 100644 --- a/test/boost/cql_query_test.cc +++ b/test/boost/cql_query_test.cc @@ -44,6 +44,13 @@ #include "db/query_context.hh" #include "service/qos/qos_common.hh" #include "utils/UUID_gen.hh" +#include "tombstone_gc_extension.hh" +#include "db/tags/extension.hh" +#include "cdc/cdc_extension.hh" +#include "db/paxos_grace_seconds_extension.hh" +#include "db/per_partition_rate_limit_extension.hh" + + using namespace std::literals::chrono_literals; @@ -4127,6 +4134,18 @@ std::string normalize_white_space(const std::string& str) { return boost::regex_replace(boost::regex_replace(" " + str + " ", boost::regex("\\s+"), " "), boost::regex(", "), ","); } +cql_test_config describe_test_config() { + auto ext = std::make_shared(); + ext->add_schema_extension(db::tags_extension::NAME); + ext->add_schema_extension(cdc::cdc_extension::NAME); + ext->add_schema_extension(db::paxos_grace_seconds_extension::NAME); + ext->add_schema_extension(tombstone_gc_extension::NAME); + ext->add_schema_extension(db::per_partition_rate_limit_extension::NAME); + + auto cfg = seastar::make_shared(ext); + return cql_test_config(cfg); +} + SEASTAR_TEST_CASE(test_describe_simple_schema) { return do_with_cql_env_thread([] (cql_test_env& e) { std::unordered_map cql_create_tables { @@ -4148,7 +4167,10 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { " AND memtable_flush_period_in_ms = 0\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n"}, + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n" + }, {"cf1", "CREATE TABLE ks.cf1 (\n" " pk blob,\n" " ck blob,\n" @@ -4169,7 +4191,9 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { " AND memtable_flush_period_in_ms = 0\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n" + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n" }, {"CF2", "CREATE TABLE ks.\"CF2\" (\n" " pk blob,\n" @@ -4191,7 +4215,9 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { " AND memtable_flush_period_in_ms = 10\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n" + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n" }, {"Cf3", "CREATE TABLE ks.\"Cf3\" (\n" " pk blob,\n" @@ -4214,7 +4240,9 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { " AND memtable_flush_period_in_ms = 10\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n" + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n" }, {"cf4", "CREATE TABLE ks.cf4 (\n" " pk blob,\n" @@ -4236,7 +4264,9 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { " AND memtable_flush_period_in_ms = 10\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n" + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n" } }; @@ -4252,7 +4282,7 @@ SEASTAR_TEST_CASE(test_describe_simple_schema) { schema->describe(e.local_db(), ss, false); BOOST_CHECK_EQUAL(normalize_white_space(ss.str()), normalize_white_space(ct.second)); } - }); + }, describe_test_config()); } SEASTAR_TEST_CASE(test_describe_view_schema) { @@ -4279,7 +4309,9 @@ SEASTAR_TEST_CASE(test_describe_view_schema) { " AND memtable_flush_period_in_ms = 0\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n"; + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'};\n"; std::unordered_map cql_create_tables { {"cf_view", "CREATE MATERIALIZED VIEW \"KS\".cf_view AS\n" @@ -4301,7 +4333,10 @@ SEASTAR_TEST_CASE(test_describe_view_schema) { " AND memtable_flush_period_in_ms = 0\n" " AND min_index_interval = 128\n" " AND read_repair_chance = 0\n" - " AND speculative_retry = '99.0PERCENTILE';\n"}, + " AND speculative_retry = '99.0PERCENTILE'\n" + " AND paxos_grace_seconds = 43200\n" + " AND tombstone_gc = {'mode':'timeout','propagation_delay_in_seconds':'3600'}\n" + " AND synchronous_updates = false;\n"}, {"cf_index_index", "CREATE INDEX cf_index ON \"KS\".\"cF\"(col2);"}, {"cf_index1_index", "CREATE INDEX cf_index1 ON \"KS\".\"cF\"(pk);"}, {"cf_index2_index", "CREATE INDEX cf_index2 ON \"KS\".\"cF\"(pk1);"}, @@ -4325,7 +4360,7 @@ SEASTAR_TEST_CASE(test_describe_view_schema) { base_schema->describe(e.local_db(), base_ss, false); BOOST_CHECK_EQUAL(normalize_white_space(base_ss.str()), normalize_white_space(base_table)); } - }); + }, describe_test_config()); } shared_ptr cql_func_require_nofail( diff --git a/test/boost/multishard_mutation_query_test.cc b/test/boost/multishard_mutation_query_test.cc index 7f487ec786..7ed93757e7 100644 --- a/test/boost/multishard_mutation_query_test.cc +++ b/test/boost/multishard_mutation_query_test.cc @@ -21,6 +21,11 @@ #include "test/lib/test_utils.hh" #include "test/lib/random_utils.hh" #include "test/lib/random_schema.hh" +#include "tombstone_gc_extension.hh" +#include "db/tags/extension.hh" +#include "cdc/cdc_extension.hh" +#include "db/paxos_grace_seconds_extension.hh" +#include "db/per_partition_rate_limit_extension.hh" #include "test/lib/scylla_test_case.hh" @@ -33,6 +38,18 @@ const sstring KEYSPACE_NAME = "ks"; namespace { +static cql_test_config cql_config_with_extensions() { + auto ext = std::make_shared(); + ext->add_schema_extension(db::tags_extension::NAME); + ext->add_schema_extension(cdc::cdc_extension::NAME); + ext->add_schema_extension(db::paxos_grace_seconds_extension::NAME); + ext->add_schema_extension(tombstone_gc_extension::NAME); + ext->add_schema_extension(db::per_partition_rate_limit_extension::NAME); + + auto cfg = seastar::make_shared(ext); + return cql_test_config(cfg); +} + struct generated_table { schema_ptr schema; std::vector keys; @@ -201,7 +218,7 @@ SEASTAR_THREAD_TEST_CASE(test_abandoned_read) { require_eventually_empty_caches(env.db()); return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } static std::vector read_all_partitions_one_by_one(distributed& db, schema_ptr s, std::vector pkeys, @@ -557,7 +574,7 @@ SEASTAR_THREAD_TEST_CASE(test_read_all) { check_results_are_equal(results1, results3); return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } // Best run with SMP>=2 @@ -617,7 +634,7 @@ SEASTAR_THREAD_TEST_CASE(test_read_all_multi_range) { require_eventually_empty_caches(env.db()); return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } // Best run with SMP>=2 @@ -669,7 +686,7 @@ SEASTAR_THREAD_TEST_CASE(test_read_with_partition_row_limits) { } } } return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } // Best run with SMP>=2 @@ -720,7 +737,7 @@ SEASTAR_THREAD_TEST_CASE(test_evict_a_shard_reader_on_each_page) { require_eventually_empty_caches(env.db()); return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } // Best run with SMP>=2 @@ -775,7 +792,7 @@ SEASTAR_THREAD_TEST_CASE(test_read_reversed) { require_eventually_empty_caches(env.db()); return make_ready_future<>(); - }).get(); + }, cql_config_with_extensions()).get(); } namespace { @@ -1077,8 +1094,8 @@ run_fuzzy_test_workload(fuzzy_test_config cfg, distributed& d } // namespace SEASTAR_THREAD_TEST_CASE(fuzzy_test) { - auto db_cfg = make_shared(); - db_cfg->enable_commitlog(false); + auto cql_cfg = cql_config_with_extensions(); + cql_cfg.db_config->enable_commitlog(false); do_with_cql_env_thread([] (cql_test_env& env) -> future<> { // REPLACE RANDOM SEED HERE. @@ -1125,5 +1142,5 @@ SEASTAR_THREAD_TEST_CASE(fuzzy_test) { }).get(); return make_ready_future<>(); - }, db_cfg).get(); + }, cql_cfg).get(); } diff --git a/test/cql-pytest/test_describe.py b/test/cql-pytest/test_describe.py index f1c9894d29..e23729823a 100644 --- a/test/cql-pytest/test_describe.py +++ b/test/cql-pytest/test_describe.py @@ -829,6 +829,9 @@ def new_random_table(cql, keyspace, udts=[]): extras["min_index_interval"] = min_idx_interval extras["max_index_interval"] = random.randrange(min_idx_interval, 10000) extras["memtable_flush_period_in_ms"] = random.randrange(0, 10000) + extras["paxos_grace_seconds"] = random.randrange(1000, 100000) + + extras["tombstone_gc"] = f"{{'mode': 'timeout', 'propagation_delay_in_seconds': '{random.randrange(100, 100000)}'}}" extra_options = [f"{k} = {v}" for (k, v) in extras.items()] extra_str = " AND ".join(extra_options)