mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

service: raft: split off setup_group0_if_exist from setup_group0

Browse Source 
Currently setup_group0 is responsible to start existing group0 on restart
or create a new one and joining the cluster with it during bootstrap. We
want to create the server for existing group0 earlier, before we start
to accept messages because some messages may assume that the server
exists already. For that we split creation of exiting group0 server into
a separate function and call it on restart before the messaging service
starts accepting messages.

Fixes: #13887
This commit is contained in:
Gleb Natapov
2023-05-28 18:13:49 +03:00
parent acc035b504
commit f26179cd27
4 changed files with 57 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
main.cc
View File

@@ -1603,6 +1603,11 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl
group0_service.abort().get();
});
// Setup group0 early in case the node is bootsrapped already and the group exists
// Need to do it before allowing incomming messaging service connections since
// storage proxy's and migration manager's verbs may access group0
group0_service.setup_group0_if_exist(sys_ks.local(), ss.local(), qp.local(), mm.local(), cdc_generation_service.local()).get();
with_scheduling_group(maintenance_scheduling_group, [&] {
return messaging.invoke_on_all([&token_metadata] (auto& netw) {
return netw.start_listen(token_metadata.local());

60
service/raft/raft_group0.cc
View File

@@ -507,9 +507,7 @@ static future<bool> synchronize_schema(
const noncopyable_function<future<bool>()>& can_finish_early,
abort_source&);
future<> raft_group0::setup_group0(
db::system_keyspace& sys_ks, const std::unordered_set<gms::inet_address>& initial_contact_nodes,
std::optional<replace_info> replace_info, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, cdc::generation_service& cdc_gen_servic) {
future<bool> raft_group0::use_raft() {
assert(this_shard_id() == 0);
if (!_raft_gr.is_enabled()) {
@@ -517,31 +515,53 @@ future<> raft_group0::setup_group0(
// Note: if the local feature was enabled by every node earlier, that would enable the cluster
// SUPPORTS_RAFT feature, and the node should then refuse to start during feature check
// (because if the local feature is disabled, then the cluster feature - enabled in the cluster - is 'unknown' to us).
co_return;
co_return false;
}
if (((co_await _client.get_group0_upgrade_state()).second) == group0_upgrade_state::recovery) {
group0_log.warn("setup_group0: Raft RECOVERY mode, skipping group 0 setup.");
co_return false;
}
co_return true;
}
future<> raft_group0::setup_group0_if_exist(db::system_keyspace& sys_ks, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, cdc::generation_service& cdc_gen_service) {
if (!co_await use_raft()) {
co_return;
}
if (!sys_ks.bootstrap_complete()) {
// If bootsrap did not complete yet there is no group0 to setup
co_return;
}
auto group0_id = raft::group_id{co_await db::system_keyspace::get_raft_group0_id()};
if (group0_id) {
// Group 0 ID is present => we've already joined group 0 earlier.
group0_log.info("setup_group0: group 0 ID present. Starting existing Raft server.");
co_await start_server_for_group0(group0_id, ss, qp, mm, cdc_gen_service);
} else {
// Scylla has bootstrapped earlier but group 0 ID not present. This means we're upgrading.
// Upgrade will start through a feature listener created after we enter NORMAL state.
//
// See `raft_group0::finish_setup_after_join`.
upgrade_log.info(
"setup_group0: Scylla bootstrap completed before but group 0 ID not present."
" Internal upgrade-to-raft procedure will automatically start after every node finishes"
" upgrading to the new Scylla version.");
}
}
future<> raft_group0::setup_group0(
db::system_keyspace& sys_ks, const std::unordered_set<gms::inet_address>& initial_contact_nodes,
std::optional<replace_info> replace_info, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, cdc::generation_service& cdc_gen_servic) {
if (!co_await use_raft()) {
co_return;
}
if (sys_ks.bootstrap_complete()) {
auto group0_id = raft::group_id{co_await db::system_keyspace::get_raft_group0_id()};
if (group0_id) {
// Group 0 ID is present => we've already joined group 0 earlier.
group0_log.info("setup_group0: group 0 ID present. Starting existing Raft server.");
co_await start_server_for_group0(group0_id, ss, qp, mm, cdc_gen_servic);
} else {
// Scylla has bootstrapped earlier but group 0 ID not present. This means we're upgrading.
// Upgrade will start through a feature listener created after we enter NORMAL state.
//
// See `raft_group0::finish_setup_after_join`.
upgrade_log.info(
"setup_group0: Scylla bootstrap completed before but group 0 ID not present."
" Internal upgrade-to-raft procedure will automatically start after every node finishes"
" upgrading to the new Scylla version.");
}
// If the node is bootsraped the group0 server should be setup already
co_return;
}

12
service/raft/raft_group0.hh
View File

@@ -131,7 +131,6 @@ public:
//
// If the local RAFT feature is enabled, does one of the following:
// - join group 0 (if we're bootstrapping),
// - start existing group 0 server (if we bootstrapped before),
// - prepare us for the upgrade procedure, which will create group 0 later (if we're upgrading).
//
// Cannot be called twice.
@@ -140,6 +139,14 @@ public:
future<> setup_group0(db::system_keyspace&, const std::unordered_set<gms::inet_address>& initial_contact_nodes,
std::optional<replace_info>, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, cdc::generation_service& cdc_gen_service);
// Call during the startup procedure before networking is enabled during restart
//
// If the local RAFT feature is enabled start existing group 0 server.
//
// Cannot be called twice.
//
future<> setup_group0_if_exist(db::system_keyspace&, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, cdc::generation_service& cdc_gen_service);
// Call at the end of the startup procedure, after the node entered NORMAL state.
// `setup_group0()` must have finished earlier.
//
@@ -306,6 +313,9 @@ private:
// Load the initial Raft <-> IP address map as seen by
// the gossiper.
void load_initial_raft_address_map();
// Returns true if raft is enabled
future<bool> use_raft();
};
} // end of namespace service

1
service/storage_service.cc
View File

@@ -1713,6 +1713,7 @@ future<> storage_service::join_token_ring(cdc::generation_service& cdc_gen_servi
co_await _gossiper.start_gossiping(generation_number, app_states, advertise);
assert(_group0);
// if the node is bootstrapped the functin will do nothing since we already created group0 in main.cc
co_await _group0->setup_group0(_sys_ks.local(), initial_contact_nodes, raft_replace_info, *this, qp, _migration_manager.local(), cdc_gen_service);
raft::server* raft_server = co_await [this] () -> future<raft::server*> {