From ff17a29b54afb3ca82173a13ee7629daee35f3a7 Mon Sep 17 00:00:00 2001 From: Marcin Maliszkiewicz Date: Tue, 26 Mar 2024 17:26:19 +0100 Subject: [PATCH 1/2] service: qos: create separate function for reloading data accessor Scylla's main is already too long, it's better to contain this logic inside qos service. --- main.cc | 9 ++------- service/qos/service_level_controller.cc | 9 +++++++++ service/qos/service_level_controller.hh | 6 ++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/main.cc b/main.cc index f085e31729..4187f19024 100644 --- a/main.cc +++ b/main.cc @@ -1793,13 +1793,8 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl db.local().enable_autocompaction_toggle(); sl_controller.invoke_on_all([&qp, &group0_client] (qos::service_level_controller& controller) -> future<> { - return qos::get_service_level_distributed_data_accessor_for_current_version( - sys_ks.local(), - sys_dist_ks.local(), - qp.local(), group0_client - ).then([&controller] (auto data_accessor) { - controller.set_distributed_data_accessor(std::move(data_accessor)); - }); + return controller.reload_distributed_data_accessor( + qp.local(), group0_client, sys_ks.local(), sys_dist_ks.local()); }).get(); group0_service.start().get(); diff --git a/service/qos/service_level_controller.cc b/service/qos/service_level_controller.cc index 9d9a0520e7..a5c06dc341 100644 --- a/service/qos/service_level_controller.cc +++ b/service/qos/service_level_controller.cc @@ -88,6 +88,15 @@ void service_level_controller::set_distributed_data_accessor(service_level_distr } } +future<> service_level_controller::reload_distributed_data_accessor(cql3::query_processor& qp, service::raft_group0_client& g0, db::system_keyspace& sys_ks, db::system_distributed_keyspace& sys_dist_ks) { + auto accesor = co_await qos::get_service_level_distributed_data_accessor_for_current_version( + sys_ks, + sys_dist_ks, + qp, + g0); + set_distributed_data_accessor(std::move(accesor)); +} + future<> service_level_controller::drain() { if (this_shard_id() != global_controller) { co_return; diff --git a/service/qos/service_level_controller.hh b/service/qos/service_level_controller.hh index 36707b9cdc..259ae43630 100644 --- a/service/qos/service_level_controller.hh +++ b/service/qos/service_level_controller.hh @@ -105,6 +105,12 @@ public: void set_distributed_data_accessor(service_level_distributed_data_accessor_ptr sl_data_accessor); + /** + * Reloads data accessor, this is used to align it with service level version + * stored in scylla_local table. + */ + future<> reload_distributed_data_accessor(cql3::query_processor&, service::raft_group0_client&, db::system_keyspace&, db::system_distributed_keyspace&); + /** * Adds a service level configuration if it doesn't exists, and updates * an the existing one if it does exist. From e1fea3af6b0016732c53fa3e2eb4a6d5678ef53c Mon Sep 17 00:00:00 2001 From: Marcin Maliszkiewicz Date: Tue, 26 Mar 2024 17:34:28 +0100 Subject: [PATCH 2/2] main: reload service levels data accessor after join_cluster Setting data accessor implicitly depends on node joining the cluster with raft leader elected as only then service level mutation is put into scylla_local table. Calling it after join_cluster avoids starting new cluster with older version only to immediately migrate it to the latest one in the background. --- main.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/main.cc b/main.cc index 4187f19024..0c52b1fae4 100644 --- a/main.cc +++ b/main.cc @@ -1792,11 +1792,6 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl */ db.local().enable_autocompaction_toggle(); - sl_controller.invoke_on_all([&qp, &group0_client] (qos::service_level_controller& controller) -> future<> { - return controller.reload_distributed_data_accessor( - qp.local(), group0_client, sys_ks.local(), sys_dist_ks.local()); - }).get(); - group0_service.start().get(); auto stop_group0_service = defer_verbose_shutdown("group 0 service", [&group0_service] { sl_controller.local().abort_group0_operations(); @@ -1842,6 +1837,14 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl startlog.info("SSTable data integrity checker is {}.", cfg->enable_sstable_data_integrity_check() ? "enabled" : "disabled"); + // This implicitly depends on node joining the cluster (join_cluster()) + // with raft leader elected as only then service level mutation is put + // into scylla_local table. Calling it here avoids starting new cluster with + // older version only to immediately migrate it to the latest in the background. + sl_controller.invoke_on_all([&qp, &group0_client] (qos::service_level_controller& controller) -> future<> { + return controller.reload_distributed_data_accessor( + qp.local(), group0_client, sys_ks.local(), sys_dist_ks.local()); + }).get(); const qualified_name qualified_authorizer_name(auth::meta::AUTH_PACKAGE_NAME, cfg->authorizer()); const qualified_name qualified_authenticator_name(auth::meta::AUTH_PACKAGE_NAME, cfg->authenticator());