From 45dbe38658ea9d0a587ea2cd3389aecc454feb2a Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Wed, 31 Jan 2024 12:47:57 +0300 Subject: [PATCH] tablets: Make sure topology has enough endpoints for RF When creating a keyspace, scylla allows setting RF value smaller than there are nodes in the DC. With vnodes, when new nodes are bootstrapped, new tokens are inserted thus catching up with RF. With tablets, it's not the case as replica set remains unchanged. With tablets it's good chance not to mimic the vnodes behavior and require as many nodes to be up and running as the requested RF is. This patch implementes this in a lazy manned -- when creating a keyspace RF can be any, but when a new table is created the topology should meet RF requirements. If not met, user can bootstrap new nodes or ALTER KEYSPACE. Signed-off-by: Pavel Emelyanov --- locator/network_topology_strategy.cc | 15 ++++++++++++++ test/topology_custom/test_tablets.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 test/topology_custom/test_tablets.py diff --git a/locator/network_topology_strategy.cc b/locator/network_topology_strategy.cc index f13025c664..f2c94b4775 100644 --- a/locator/network_topology_strategy.cc +++ b/locator/network_topology_strategy.cc @@ -230,6 +230,19 @@ public: host_id_set& replicas() noexcept { return _replicas; } + + static void check_enough_endpoints(const token_metadata& tm, const std::unordered_map& dc_rf) { + const auto& dc_endpoints = tm.get_topology().get_datacenter_endpoints(); + auto endpoints_in = [&dc_endpoints](sstring dc) { + auto i = dc_endpoints.find(dc); + return i != dc_endpoints.end() ? i->second.size() : size_t(0); + }; + for (const auto& p : dc_rf) { + if (p.second > endpoints_in(p.first)) { + throw exceptions::configuration_exception(fmt::format("Datacenter {} doesn't have enough nodes for replication_factor={}", p.first, p.second)); + } + } + } }; future @@ -315,6 +328,8 @@ static unsigned calculate_initial_tablets_from_topology(const schema& s, const t } future network_topology_strategy::allocate_tablets_for_new_table(schema_ptr s, token_metadata_ptr tm, unsigned initial_scale) const { + natural_endpoints_tracker::check_enough_endpoints(*tm, _dc_rep_factor); + auto tablet_count = get_initial_tablets(); if (tablet_count == 0) { tablet_count = calculate_initial_tablets_from_topology(*s, tm->get_topology(), _dc_rep_factor) * initial_scale; diff --git a/test/topology_custom/test_tablets.py b/test/topology_custom/test_tablets.py new file mode 100644 index 0000000000..080fa17ebe --- /dev/null +++ b/test/topology_custom/test_tablets.py @@ -0,0 +1,30 @@ +# +# Copyright (C) 2024-present ScyllaDB +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +from cassandra.protocol import ConfigurationException +from test.pylib.manager_client import ManagerClient +import pytest +import logging +import asyncio + +logger = logging.getLogger(__name__) + + +@pytest.mark.asyncio +async def test_tablet_replication_factor_enough_nodes(manager: ManagerClient): + cfg = {'enable_user_defined_functions': False, + 'experimental_features': ['tablets', 'consistent-topology-changes']} + servers = await manager.servers_add(2, config=cfg) + + cql = manager.get_cql() + res = await cql.run_async("SELECT data_center FROM system.local") + this_dc = res[0].data_center + + await cql.run_async(f"CREATE KEYSPACE test WITH replication = {{'class': 'NetworkTopologyStrategy', '{this_dc}': 3}}") + with pytest.raises(ConfigurationException, match=f"Datacenter {this_dc} doesn't have enough nodes"): + await cql.run_async("CREATE TABLE test.test (pk int PRIMARY KEY, c int);") + + await cql.run_async(f"ALTER KEYSPACE test WITH replication = {{'class': 'NetworkTopologyStrategy', '{this_dc}': 2}}") + await cql.run_async("CREATE TABLE test.test (pk int PRIMARY KEY, c int);")