/* * Copyright (C) 2021-present ScyllaDB */ /* * SPDX-License-Identifier: AGPL-3.0-or-later */ #include #include #include #include #include #include #include "cql3/query_processor.hh" #include "cql3/statements/create_keyspace_statement.hh" #include "cql3/statements/create_table_statement.hh" #include "cql3/statements/create_type_statement.hh" #include "cql3/statements/update_statement.hh" #include "db/cql_type_parser.hh" #include "db/config.hh" #include "db/schema_tables.hh" #include "replica/database.hh" #include "gms/feature_service.hh" #include "locator/abstract_replication_strategy.hh" #include "locator/snitch_base.hh" #include "tools/schema_loader.hh" #include "utils/fb_utilities.hh" namespace { sstring read_file(std::filesystem::path path) { auto file = open_file_dma(path.native(), open_flags::ro).get(); auto fstream = make_file_input_stream(file); return util::read_entire_stream_contiguous(fstream).get(); } std::vector do_load_schemas(std::string_view schema_str) { cql3::cql_stats cql_stats; db::config cfg; cfg.enable_cache(false); cfg.volatile_system_keyspace_for_testing(true); replica::database_config dbcfg; dbcfg.available_memory = 1'000'000'000; // 1G service::migration_notifier migration_notifier; gms::feature_service feature_service(gms::feature_config_from_db_config(cfg)); feature_service.enable(feature_service.known_feature_set()); sharded token_metadata; sharded erm_factory; abort_source as; sharded sst_dir_sem; token_metadata.start([] () noexcept { return db::schema_tables::hold_merge_lock(); }).get(); auto stop_token_metadata = deferred_stop(token_metadata); erm_factory.start().get(); auto stop_erm_factory = deferred_stop(erm_factory); sst_dir_sem.start(1).get(); auto sst_dir_sem_stop = deferred_stop(sst_dir_sem); utils::fb_utilities::set_broadcast_address(gms::inet_address(0x7f000001)); // 127.0.0.1 if (!locator::i_endpoint_snitch::snitch_instance().local_is_initialized()) { locator::i_endpoint_snitch::create_snitch(cfg.endpoint_snitch()).get(); } replica::database db(cfg, dbcfg, migration_notifier, feature_service, token_metadata.local(), as, sst_dir_sem); auto stop_db = deferred_stop(db); // Mock system_schema keyspace to be able to parse modification statements // against system_schema.dropped_columns. db.create_keyspace(make_lw_shared( db::schema_tables::NAME, "org.apache.cassandra.locator.LocalStrategy", std::map{}, false), erm_factory.local()).get(); db.add_column_family(db.find_keyspace(db::schema_tables::NAME), db::schema_tables::dropped_columns(), {}); std::vector schemas; auto find_or_create_keyspace = [&] (const sstring& name) -> replica::keyspace& { try { return db.find_keyspace(name); } catch (replica::no_such_keyspace&) { // fall-though to below } auto raw_statement = cql3::query_processor::parse_statement( fmt::format("CREATE KEYSPACE {} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': '1'}}", name)); auto prepared_statement = raw_statement->prepare(db.as_data_dictionary(), cql_stats); auto* statement = prepared_statement->statement.get(); auto p = dynamic_cast(statement); assert(p); db.create_keyspace(p->get_keyspace_metadata(*token_metadata.local().get()), erm_factory.local()).get(); return db.find_keyspace(name); }; std::vector> raw_statements; try { raw_statements = cql3::query_processor::parse_statements(schema_str); } catch (...) { throw std::runtime_error(format("tools:do_load_schemas(): failed to parse CQL statements: {}", std::current_exception())); } for (auto& raw_statement : raw_statements) { auto cf_statement = dynamic_cast(raw_statement.get()); if (!cf_statement) { continue; // we don't support any non-cf statements here } auto& ks = find_or_create_keyspace(cf_statement->keyspace()); auto prepared_statement = cf_statement->prepare(db.as_data_dictionary(), cql_stats); auto* statement = prepared_statement->statement.get(); if (auto p = dynamic_cast(statement)) { db.create_keyspace(p->get_keyspace_metadata(*token_metadata.local().get()), erm_factory.local()).get(); } else if (auto p = dynamic_cast(statement)) { auto type = p->create_type(db.as_data_dictionary()); ks.add_user_type(std::move(type)); } else if (auto p = dynamic_cast(statement)) { schemas.push_back(p->get_cf_meta_data(db.as_data_dictionary())); } else if (auto p = dynamic_cast(statement)) { if (p->keyspace() != db::schema_tables::NAME && p->column_family() != db::schema_tables::DROPPED_COLUMNS) { throw std::runtime_error(fmt::format("tools::do_load_schemas(): expected modification statement to be against {}.{}, but it is against {}.{}", db::schema_tables::NAME, db::schema_tables::DROPPED_COLUMNS, p->keyspace(), p->column_family())); } auto schema = db::schema_tables::dropped_columns(); cql3::statements::modification_statement::json_cache_opt json_cache{}; cql3::update_parameters params(schema, cql3::query_options::DEFAULT, api::new_timestamp(), schema->default_time_to_live(), cql3::update_parameters::prefetch_data(schema)); auto pkeys = p->build_partition_keys(cql3::query_options::DEFAULT, json_cache); auto ckranges = p->create_clustering_ranges(cql3::query_options::DEFAULT, json_cache); auto updates = p->apply_updates(pkeys, ckranges, params, json_cache); if (updates.size() != 1) { throw std::runtime_error(fmt::format("tools::do_load_schemas(): expected one update per statement for {}.{}, got: {}", db::schema_tables::NAME, db::schema_tables::DROPPED_COLUMNS, updates.size())); } auto& mut = updates.front(); if (!mut.partition().row_tombstones().empty()) { throw std::runtime_error(fmt::format("tools::do_load_schemas(): expected only update against {}.{}, not deletes", db::schema_tables::NAME, db::schema_tables::DROPPED_COLUMNS)); } query::result_set rs(mut); for (auto& row : rs.rows()) { const auto keyspace_name = row.get_nonnull("keyspace_name"); const auto table_name = row.get_nonnull("table_name"); auto it = std::find_if(schemas.begin(), schemas.end(), [&] (schema_ptr s) { return s->ks_name() == keyspace_name && s->cf_name() == table_name; }); if (it == schemas.end()) { throw std::runtime_error(fmt::format("tools::do_load_schemas(): failed applying update to {}.{}, the table it applies to is not found: {}.{}", db::schema_tables::NAME, db::schema_tables::DROPPED_COLUMNS, keyspace_name, table_name)); } auto name = row.get_nonnull("column_name"); auto type = db::cql_type_parser::parse(keyspace_name, row.get_nonnull("type")); auto time = row.get_nonnull("dropped_time"); *it = schema_builder(*it).without_column(std::move(name), std::move(type), time.time_since_epoch().count()).build(); } } else { throw std::runtime_error(fmt::format("tools::do_load_schemas(): expected statement to be one of (create keyspace, create type, create table), got: {}", typeid(statement).name())); } } return schemas; } } // anonymous namespace namespace tools { future> load_schemas(std::string_view schema_str) { return async([schema_str] () mutable { return do_load_schemas(schema_str); }); } future load_one_schema(std::string_view schema_str) { return async([schema_str] () mutable { auto schemas = do_load_schemas(schema_str); if (schemas.size() != 1) { throw std::runtime_error(fmt::format("Schema string expected to contain exactly 1 schema, actually has {}", schemas.size())); } return std::move(schemas.front()); }); } future> load_schemas_from_file(std::filesystem::path path) { return async([path] () mutable { return do_load_schemas(read_file(path)); }); } future load_one_schema_from_file(std::filesystem::path path) { return async([path] () mutable { auto schemas = do_load_schemas(read_file(path)); if (schemas.size() != 1) { throw std::runtime_error(fmt::format("Schema file {} expected to contain exactly 1 schema, actually has {}", path.native(), schemas.size())); } return std::move(schemas.front()); }); } } // namespace tools