e639dcda0b
The test was flaky. The scenario looked like this: 1. Stop server 1. 2. Set its rf_rack_valid_keyspaces configuration option to true. 3. Create an RF-rack-invalid keyspace. 4. Start server 1 and expect a failure during start-up. It was wrong. We cannot predict when the Raft mutation corresponding to the newly created keyspace will arrive at the node or when it will be processed. If the check of the RF-rack-valid keyspaces we perform at start-up was done before that, it won't include the keyspace. This will lead to a test failure. Unfortunately, it's not feasible to perform a read barrier during start-up. What's more, although it would help the test, it wouldn't be useful otherwise. Because of that, we simply fix the test, at least for now. The new scenario looks like this: 1. Disable the rf_rack_valid_keyspaces configuration option on server 1. 2. Start the server. 3. Create an RF-rack-invalid keyspace. 4. Perform a read barrier on server 1. This will ensure that it has observed all Raft mutations, and we won't run into the same problem. 5. Stop the node. 6. Set its rf_rack_valid_keyspaces configuration option to true. 7. Try to start the node and observe a failure. This will make the test perform consistently. --- I ran the test (in dev mode, on my local machine) three times before these changes, and three times with them. I include the time results below. Before: ``` real 0m47.570s user 0m41.631s sys 0m8.634s real 0m50.495s user 0m42.499s sys 0m8.607s real 0m50.375s user 0m41.832s sys 0m8.789s ``` After: ``` real 0m50.509s user 0m43.535s sys 0m9.715s real 0m50.857s user 0m44.185s sys 0m9.811s real 0m50.873s user 0m44.289s sys 0m9.737s ``` Fixes SCYLLADB-1137
692 lines
28 KiB
Python
692 lines
28 KiB
Python
#
|
|
# Copyright (C) 2022-present ScyllaDB
|
|
#
|
|
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
|
|
#
|
|
import asyncio
|
|
import logging
|
|
import sys
|
|
|
|
from typing import List, Union
|
|
|
|
import pytest
|
|
from cassandra.policies import WhiteListRoundRobinPolicy
|
|
|
|
from test.cqlpy import nodetool
|
|
from cassandra import ConsistencyLevel, Unavailable
|
|
from cassandra.cluster import NoHostAvailable
|
|
from cassandra.protocol import InvalidRequest, ConfigurationException
|
|
from cassandra.query import SimpleStatement
|
|
from test.pylib.async_cql import _wrap_future
|
|
from test.pylib.manager_client import ManagerClient
|
|
from test.pylib.random_tables import RandomTables, TextType, Column
|
|
from test.pylib.rest_client import read_barrier
|
|
from test.pylib.util import unique_name
|
|
from test.cluster.conftest import cluster_con
|
|
|
|
logger = logging.getLogger(__name__)
|
|
CONFIG = {"endpoint_snitch": "GossipingPropertyFileSnitch"}
|
|
|
|
|
|
# Checks a cluster boot/operations in multi-dc environment with 5 nodes each in a separate DC
|
|
@pytest.mark.asyncio
|
|
async def test_multidc(request: pytest.FixtureRequest, manager: ManagerClient) -> None:
|
|
logger.info("Creating a new cluster")
|
|
for i in range(5):
|
|
s_info = await manager.server_add(
|
|
config=CONFIG,
|
|
property_file={'dc': f'dc{i}', 'rack': 'myrack1'}
|
|
)
|
|
logger.info(s_info)
|
|
random_tables = RandomTables(request.node.name, manager, unique_name(), 1)
|
|
logger.info("Creating new tables")
|
|
await random_tables.add_tables(ntables=3, ncolumns=3)
|
|
await random_tables.verify_schema()
|
|
|
|
|
|
cluster_config = [
|
|
([1, 2], 1),
|
|
([1, 1, 2, 2], 2)
|
|
]
|
|
|
|
|
|
# Simple put-get test for 2 DC with a different amount of nodes and different replication factors
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("nodes_list, rf", cluster_config)
|
|
async def test_putget_2dc_with_rf(
|
|
request: pytest.FixtureRequest, manager: ManagerClient, nodes_list: list[int], rf: int
|
|
) -> None:
|
|
ks = "test_ks"
|
|
cf = "test_cf"
|
|
table_name = "test_table_name"
|
|
columns = [Column("name", TextType), Column("value", TextType)]
|
|
logger.info("Create two servers in different DC's")
|
|
for rack_idx, dc_idx in enumerate(nodes_list):
|
|
s_info = await manager.server_add(
|
|
config=CONFIG,
|
|
property_file={"dc": f"dc{dc_idx}", "rack": f"rack{rack_idx}"},
|
|
)
|
|
logger.info(s_info)
|
|
conn = manager.get_cql()
|
|
random_tables = RandomTables(request.node.name, manager, ks, rf)
|
|
logger.info("Add table")
|
|
await random_tables.add_table(ncolumns=2, columns=columns, pks=1, name=table_name)
|
|
conn.execute(f"USE {ks}")
|
|
conn.execute(
|
|
f"CREATE COLUMNFAMILY {cf} ( key varchar, c varchar, v varchar, PRIMARY KEY(key, c)) WITH comment='test cf'"
|
|
)
|
|
|
|
logger.info("Perform insert/overwrite")
|
|
# create 100 records with values from 0 to 99
|
|
update_query = f"UPDATE {cf} SET v='value%d' WHERE key='k0' AND c='c%02d'"
|
|
query_batch = "BEGIN BATCH %s APPLY BATCH"
|
|
kvs = [update_query % (i, i) for i in range(100)]
|
|
query = SimpleStatement(
|
|
query_batch % ";".join(kvs), consistency_level=ConsistencyLevel.QUORUM
|
|
)
|
|
conn.execute(query)
|
|
nodetool.flush_keyspace(conn, ks)
|
|
# overwrite each second value
|
|
kvs = [update_query % (i * 4, i * 2) for i in range(50)]
|
|
query = SimpleStatement(
|
|
query_batch % "; ".join(kvs), consistency_level=ConsistencyLevel.QUORUM
|
|
)
|
|
conn.execute(query)
|
|
nodetool.flush_keyspace(conn, ks)
|
|
# overwrite each fifth value
|
|
kvs = [update_query % (i * 20, i * 5) for i in range(20)]
|
|
query = SimpleStatement(
|
|
query_batch % "; ".join(kvs), consistency_level=ConsistencyLevel.QUORUM
|
|
)
|
|
conn.execute(query)
|
|
nodetool.flush_keyspace(conn, ks)
|
|
|
|
logger.info("Check written data is correct")
|
|
query = SimpleStatement(f"SELECT * FROM {cf} WHERE key='k0'", consistency_level=ConsistencyLevel.QUORUM)
|
|
rows = list(conn.execute(query))
|
|
assert len(rows) == 100
|
|
for i, row in enumerate(rows):
|
|
if i % 5 == 0:
|
|
assert row[2] == f"value{i * 4}"
|
|
elif i % 2 == 0:
|
|
assert row[2] == f"value{i * 2}"
|
|
else:
|
|
assert row[2] == f"value{i}"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_read_or_write_to_dc_with_rf_0_fails(request: pytest.FixtureRequest, manager: ManagerClient):
|
|
"""
|
|
Verifies that operations using local consistency levels (LOCAL_QUORUM, LOCAL_ONE) fail
|
|
with a clear, actionable error message when the datacenter has replication factor 0.
|
|
|
|
Covers:
|
|
- https://github.com/scylladb/scylla/issues/8354 - Operations should not crash the DB
|
|
- https://github.com/scylladb/scylladb/issues/27893 - Should give a clear error about RF=0
|
|
"""
|
|
servers = []
|
|
ks = "test_ks"
|
|
table_name = "test_table_name"
|
|
dc_replication = {'dc1': 1, 'dc2': 0}
|
|
columns = [Column("name", TextType), Column("value", TextType)]
|
|
|
|
for i in [1, 2]:
|
|
servers.append(await manager.server_add(
|
|
config=CONFIG,
|
|
property_file={"dc": f"dc{i}", "rack": "myrack"},
|
|
))
|
|
|
|
dc1_connection = cluster_con([servers[0].ip_addr],
|
|
load_balancing_policy=WhiteListRoundRobinPolicy([servers[0].ip_addr])).connect()
|
|
dc2_connection = cluster_con([servers[1].ip_addr],
|
|
load_balancing_policy=WhiteListRoundRobinPolicy([servers[1].ip_addr])).connect()
|
|
|
|
random_tables = RandomTables(request.node.name, manager, ks, 1, dc_replication)
|
|
await random_tables.add_table(ncolumns=2, columns=columns, pks=1, name=table_name)
|
|
|
|
# sanity check: operations from dc1 (RF > 0) should succeed with all local consistency levels
|
|
local_cls = [ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.LOCAL_ONE]
|
|
|
|
for i, cl in enumerate(local_cls):
|
|
dc1_connection.execute(SimpleStatement(
|
|
f"INSERT INTO {ks}.{table_name} ({columns[0].name}, {columns[1].name}) VALUES ('k{i}', 'value{i}')", consistency_level=cl))
|
|
|
|
nodetool.flush(dc1_connection, f"{ks}.{table_name}")
|
|
|
|
select_query = SimpleStatement(f"SELECT * FROM {ks}.{table_name} WHERE {columns[0].name} = 'k{i}'", consistency_level=cl)
|
|
# asserting SELECT's results may seem excessive, but it could happen that it'd fail and return nothing and not throw,
|
|
# in which case it'd silently return nothing, which we don't want, and hence are asserting it's *really* working
|
|
result = list(dc1_connection.execute(select_query).one())
|
|
expected_row = [f"k{i}", f"value{i}"]
|
|
assert result == expected_row, \
|
|
f"Expected {expected_row} with CL={cl} from dc1, but got {result}"
|
|
|
|
def assert_operation_fails_with_rf0_error(cl: ConsistencyLevel, operation: str) -> None:
|
|
with pytest.raises((Unavailable, NoHostAvailable)) as exc_info:
|
|
dc2_connection.execute(SimpleStatement(operation, consistency_level=cl))
|
|
|
|
error_msg = str(exc_info.value)
|
|
assert "Cannot achieve consistency level LOCAL_" in error_msg and "use a non-local consistency level" in error_msg, \
|
|
f"Expected error indicating RF=0 and datacenter with CL={cl}, but received: {exc_info.value}"
|
|
|
|
# SELECT & INSERT from dc2 (with RF=0) using local CLs should fail with a clear error message
|
|
# indicating the replication factor is 0 and suggesting to use a non-local CL
|
|
for i, cl in enumerate(local_cls):
|
|
assert_operation_fails_with_rf0_error(cl,
|
|
f"SELECT * FROM {ks}.{table_name}")
|
|
|
|
assert_operation_fails_with_rf0_error(cl,
|
|
f"INSERT INTO {ks}.{table_name} ({columns[0].name}, {columns[1].name}) VALUES ('k_fail_{i}', 'value_fail_{i}')")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_and_alter_keyspace_with_altering_rf_and_racks(manager: ManagerClient):
|
|
"""
|
|
This test verifies that creating and altering a keyspace keeps it RF-rack-valid.
|
|
If an operation would make it RF-rack-invalid, it should fail.
|
|
We can add a new rack or a data center and the existing keyspaces must still work fine.
|
|
|
|
For context, see: scylladb/scylladb#23276.
|
|
"""
|
|
|
|
cql = None
|
|
cfg = {"rf_rack_valid_keyspaces": "true"}
|
|
cfg_zero_token = {"rf_rack_valid_keyspaces": "true", "join_ring": "false"}
|
|
|
|
async def create_aux(ks: str, rfs: Union[List[int], int]):
|
|
if type(rfs) is int:
|
|
rep_opts = f"'replication_factor': {rfs}"
|
|
else:
|
|
rep_opts = ", ".join([f"'dc{i + 1}': {rf}" for i, rf in enumerate(rfs)])
|
|
opts = f"replication = {{'class': 'NetworkTopologyStrategy', {rep_opts}}} AND tablets = {{'enabled': true}}"
|
|
await cql.run_async(f"CREATE KEYSPACE {ks} WITH {opts}")
|
|
|
|
async def create_ok(rfs: Union[List[int], int]) -> str:
|
|
ks = unique_name()
|
|
await create_aux(ks, rfs)
|
|
return ks
|
|
|
|
async def create_fail(rfs: Union[List[int], int], failed_dc: int, rf: int, rack_count: int):
|
|
ks = unique_name()
|
|
err = rf"Replication factor {rf} exceeds the number of racks|The keyspace '{ks}' is required to be RF-rack-valid. " \
|
|
f"That condition is violated for DC 'dc{failed_dc}': RF={rf} vs. rack count={rack_count}."
|
|
with pytest.raises((ConfigurationException, InvalidRequest), match=err):
|
|
await create_aux(ks, rfs)
|
|
|
|
async def alter_ok(ks: str, rfs: List[int]) -> None:
|
|
dcs = ", ".join([f"'dc{i + 1}': {rf}" for i, rf in enumerate(rfs)])
|
|
await cql.run_async(f"ALTER KEYSPACE {ks} WITH REPLICATION = {{'class': 'NetworkTopologyStrategy', {dcs}}}")
|
|
|
|
async def alter_fail(ks: str, rfs: List[int], failed_dc: int, rack_count: int) -> None:
|
|
rf = rfs[failed_dc - 1]
|
|
with pytest.raises((ConfigurationException, InvalidRequest)):
|
|
await alter_ok(ks, rfs)
|
|
|
|
# Step 1.
|
|
# -------
|
|
# dc1: r1=1, rzerotoken=1 (Note: zero-token nodes have NO impact. In this case, this rack should behave as if it never existed).
|
|
_ = await manager.server_add(property_file={"dc": "dc1", "rack": "r1"}, config=cfg)
|
|
_ = await manager.server_add(property_file={"dc": "dc1", "rack": "rzerotoken"}, config=cfg_zero_token)
|
|
cql = manager.get_cql()
|
|
|
|
ks1 = await create_ok([1])
|
|
await create_ok([0])
|
|
await create_ok(0)
|
|
await create_ok(1)
|
|
await create_fail([2], 1, 2, 1)
|
|
await create_fail(2, 1, 2, 1)
|
|
|
|
await alter_ok(ks1, [0])
|
|
await alter_ok(ks1, [1])
|
|
# Check if it works if the RF doesn't change.
|
|
await alter_ok(ks1, [1])
|
|
|
|
await alter_fail(ks1, [2], 1, 1)
|
|
|
|
# Step 2.
|
|
# -------
|
|
# dc1: r1=1, rzerotoken=1.
|
|
# dc2: r1=1.
|
|
_ = await manager.server_add(property_file={"dc": "dc2", "rack": "r1"}, config=cfg)
|
|
|
|
ks2 = await create_ok([1, 1])
|
|
|
|
tasks = [
|
|
create_ok([0, 1]),
|
|
create_ok([0, 0]),
|
|
create_ok([1, 0]),
|
|
create_ok(0),
|
|
create_ok(1),
|
|
create_fail([2, 1], 1, 2, 1),
|
|
create_fail([1, 2], 2, 2, 1),
|
|
create_fail([2, 0], 1, 2, 1)
|
|
]
|
|
|
|
# To reduce the latency.
|
|
async with asyncio.TaskGroup() as tg:
|
|
for task in tasks:
|
|
_ = tg.create_task(task)
|
|
|
|
# Edge case: global RF = 0.
|
|
await alter_ok(ks1, [0, 0])
|
|
await alter_ok(ks1, [1, 0])
|
|
# Check if it works if the RF doesn't change.
|
|
await alter_ok(ks1, [1, 0])
|
|
|
|
# Edge case: global RF = 0.
|
|
await alter_ok(ks1, [0])
|
|
await alter_ok(ks1, [1])
|
|
# Check if it works if the RF doesn't change.
|
|
await alter_ok(ks1, [1])
|
|
|
|
await alter_fail(ks1, [2, 0], 1, 1)
|
|
await alter_fail(ks1, [2], 1, 1)
|
|
|
|
await alter_ok(ks1, [1, 1])
|
|
await alter_ok(ks2, [0, 1])
|
|
await alter_ok(ks2, [1, 1])
|
|
await alter_ok(ks2, [1, 0])
|
|
await alter_ok(ks2, [1, 1])
|
|
|
|
await alter_fail(ks2, [2, 1], 1, 1)
|
|
await alter_fail(ks2, [1, 2], 2, 1)
|
|
|
|
# Step 3.
|
|
# -------
|
|
# dc1: r1=1, r2=1, rzerotoken=1.
|
|
# dc2: r1=1.
|
|
_ = await manager.server_add(property_file={"dc": "dc1", "rack": "r2"}, config=cfg)
|
|
|
|
ks3 = await create_ok([2, 1])
|
|
# RF = 1 is always OK!
|
|
ks4 = await create_ok([1, 1])
|
|
|
|
tasks = [
|
|
create_ok([2, 0]),
|
|
create_ok([0, 1]),
|
|
create_ok(0),
|
|
create_ok(1),
|
|
create_fail([1, 2], 2, 2, 1),
|
|
create_fail([2, 2], 2, 2, 1),
|
|
create_fail(2, 2, 2, 1)
|
|
]
|
|
|
|
# To reduce the latency.
|
|
async with asyncio.TaskGroup() as tg:
|
|
for task in tasks:
|
|
_ = tg.create_task(task)
|
|
|
|
# Altering from rack list to numeric not supported.
|
|
await alter_fail(ks1, [2, 1], 1, 2)
|
|
# await alter_ok(ks1, [2, 1])
|
|
await alter_fail(ks1, [2, 2], 2, 1)
|
|
|
|
await alter_fail(ks2, [2, 1], 1, 2)
|
|
await alter_ok(ks3, [2, 1])
|
|
await alter_fail(ks4, [2, 1], 1, 2)
|
|
# RF = 1 is always OK!
|
|
await alter_fail(ks3, [1, 1], 1, 2)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_arbiter_dc_rf_rack_valid_keyspaces(manager: ManagerClient):
|
|
"""
|
|
This test verifies that Scylla rejects RF-rack-invalid keyspaces
|
|
in presence of an arbiter data center.
|
|
|
|
For context, see: scylladb/scylladb#23276 and scylladb/scylladb#20356.
|
|
"""
|
|
|
|
cfg = {"rf_rack_valid_keyspaces": "true"}
|
|
zero_token_cfg = {"rf_rack_valid_keyspaces": "true", "join_ring": "false"}
|
|
|
|
# Regular DC.
|
|
_ = await manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r1"})
|
|
_ = await manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r2"})
|
|
_ = await manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r3"})
|
|
# Zero-token node. This rack should behave as if it didn't exist.
|
|
_ = await manager.server_add(config=zero_token_cfg, property_file={"dc": "dc1", "rack": "r4"})
|
|
|
|
# Arbiter DC.
|
|
_ = await manager.server_add(config=zero_token_cfg, property_file={"dc": "dc2", "rack": "r1"})
|
|
_ = await manager.server_add(config=zero_token_cfg, property_file={"dc": "dc2", "rack": "r2"})
|
|
_ = await manager.server_add(config=zero_token_cfg, property_file={"dc": "dc2", "rack": "r3"})
|
|
|
|
cql = manager.get_cql()
|
|
|
|
async def create_aux(ks: str, rfs: Union[List[int], int]):
|
|
if type(rfs) is int:
|
|
rep_opts = f"'replication_factor': {rfs}"
|
|
else:
|
|
rep_opts = f"'dc1': {rfs[0]}, 'dc2': {rfs[1]}"
|
|
opts = f"replication = {{'class': 'NetworkTopologyStrategy', {rep_opts}}} AND tablets = {{'enabled': true}}"
|
|
try:
|
|
await cql.run_async(f"CREATE KEYSPACE {ks} WITH {opts}")
|
|
finally:
|
|
await cql.run_async(f"DROP KEYSPACE IF EXISTS {ks}")
|
|
|
|
async def create_ok(rfs: Union[List[int], int]):
|
|
ks = unique_name()
|
|
await create_aux(ks, rfs)
|
|
|
|
async def create_fail(rfs: Union[List[int], int], failed_dc: int, rf: int, rack_count: int):
|
|
ks = unique_name()
|
|
err = f"Replication factor {rf} exceeds the number of racks|The keyspace '{ks}' is required to be RF-rack-valid. " \
|
|
f"That condition is violated for DC 'dc{failed_dc}': RF={rf} vs. rack count={rack_count}."
|
|
with pytest.raises((ConfigurationException, InvalidRequest), match=err):
|
|
await create_aux(ks, rfs)
|
|
logger.error(f"create_aux({ks}, {rfs}) should have failed")
|
|
|
|
valid_keyspaces = [
|
|
create_ok([0, 0]),
|
|
create_ok([1, 0]),
|
|
create_ok([2, 0]),
|
|
create_ok([3, 0]),
|
|
create_ok(0)
|
|
]
|
|
|
|
# We don't consider cases where the RF for dc1 is 2 and the RF for dc2 is a positive number
|
|
# because then we can't predict what error will say.
|
|
invalid_keyspaces = [
|
|
create_fail([4, 0], 1, 4, 3),
|
|
create_fail([0, 1], 2, 1, 0),
|
|
create_fail([0, 2], 2, 2, 0),
|
|
create_fail([0, 3], 2, 3, 0),
|
|
create_fail([1, 1], 2, 1, 0),
|
|
create_fail([1, 2], 2, 2, 0),
|
|
create_fail([1, 3], 2, 3, 0),
|
|
create_fail([3, 1], 2, 1, 0),
|
|
create_fail([3, 2], 2, 2, 0),
|
|
create_fail([3, 3], 2, 3, 0),
|
|
create_fail(1, 2, 1, 0),
|
|
create_fail(3, 2, 3, 0)
|
|
]
|
|
|
|
async with asyncio.TaskGroup() as tg:
|
|
for task in [*valid_keyspaces, *invalid_keyspaces]:
|
|
_ = tg.create_task(task)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_startup_with_keyspaces_violating_rf_rack_valid_keyspaces(manager: ManagerClient):
|
|
"""
|
|
This test verifies that starting a Scylla node fails when there's an RF-rack-invalid keyspace.
|
|
We aim to simulate the behavior of a node when upgrading to 2025.*.
|
|
|
|
For more context, see: scylladb/scylladb#23300.
|
|
"""
|
|
|
|
cfg_false = {"rf_rack_valid_keyspaces": "false"}
|
|
|
|
s1 = await manager.server_add(config=cfg_false, property_file={"dc": "dc1", "rack": "r1"})
|
|
_ = await manager.server_add(config=cfg_false, property_file={"dc": "dc1", "rack": "r2"})
|
|
_ = await manager.server_add(config=cfg_false, property_file={"dc": "dc1", "rack": "r3"})
|
|
_ = await manager.server_add(config=cfg_false, property_file={"dc": "dc2", "rack": "r4"})
|
|
# Note: This rack should behave as if it never existed.
|
|
_ = await manager.server_add(config={"join_ring": "false", "rf_rack_valid_keyspaces": "false"}, property_file={"dc": "dc1", "rack": "rzerotoken"})
|
|
|
|
# Current situation:
|
|
# DC1: {r1, r2, r3}, DC2: {r4}
|
|
|
|
cql = manager.get_cql()
|
|
|
|
async def create_keyspace(rfs: List[int], tablets: bool) -> str:
|
|
dcs = ", ".join([f"'dc{i + 1}': {rf}" for i, rf in enumerate(rfs)])
|
|
name = unique_name()
|
|
tablets = str(tablets).lower()
|
|
await cql.run_async(f"CREATE KEYSPACE {name} WITH REPLICATION = {{'class': 'NetworkTopologyStrategy', {dcs}}} "
|
|
f"AND tablets = {{'enabled': {tablets}}}")
|
|
logger.info(f"Created keyspace {name} with {rfs} and tablets={tablets}")
|
|
return name
|
|
|
|
valid_keyspaces = [
|
|
# For each DC: RF \in {0, 1, #racks}.
|
|
([0, 0], True),
|
|
([1, 0], True),
|
|
([3, 0], True),
|
|
([1, 1], True),
|
|
([3, 1], True),
|
|
# Reminder: Keyspaces not using tablets are all valid.
|
|
([0, 0], False),
|
|
([1, 0], False),
|
|
([2, 0], False),
|
|
([3, 0], False),
|
|
([4, 0], False),
|
|
([0, 1], False),
|
|
([1, 1], False),
|
|
([2, 1], False),
|
|
([3, 1], False),
|
|
([4, 1], False),
|
|
([0, 2], False),
|
|
([1, 2], False),
|
|
([2, 2], False),
|
|
([3, 2], False),
|
|
([4, 2], False)
|
|
]
|
|
|
|
# Populate RF-rack-valid keyspaces.
|
|
async with asyncio.TaskGroup() as tg:
|
|
for rfs, tablets in valid_keyspaces:
|
|
_ = tg.create_task(create_keyspace(rfs, tablets))
|
|
|
|
# Precondition: s1 has rf_rack_valid_keyspaces set to false.
|
|
# Postcondition: s1 still has rf_rack_valid_keyspaces set to false.
|
|
async def try_fail(rfs: List[int], dc: str, rf: int, rack_count: int):
|
|
running_servers = await manager.running_servers()
|
|
should_start = s1.server_id not in [server.server_id for server in running_servers]
|
|
if should_start:
|
|
await manager.server_start(s1.server_id)
|
|
|
|
ks = await create_keyspace(rfs, True)
|
|
# We need to wait for the new schema to propagate.
|
|
# Otherwise, it's not clear when the mutation
|
|
# corresponding to the created keyspace will
|
|
# arrive at server 1.
|
|
# It could happen only after the node performs
|
|
# the check upon start-up, effectively leading
|
|
# to a successful start-up, which we don't want.
|
|
# For more context, see issue: SCYLLADB-1137.
|
|
await read_barrier(manager.api, s1.ip_addr)
|
|
|
|
await manager.server_stop_gracefully(s1.server_id)
|
|
await manager.server_update_config(s1.server_id, "rf_rack_valid_keyspaces", "true")
|
|
|
|
err = f"The keyspace '{ks}' is required to be RF-rack-valid. " \
|
|
f"That condition is violated for DC '{dc}': RF={rf} vs. rack count={rack_count}."
|
|
await manager.server_start(s1.server_id, expected_error=err)
|
|
await cql.run_async(f"DROP KEYSPACE {ks}")
|
|
|
|
await manager.server_update_config(s1.server_id, "rf_rack_valid_keyspaces", "false")
|
|
|
|
# Test RF-rack-invalid keyspaces.
|
|
await try_fail([2, 0], "dc1", 2, 3)
|
|
await try_fail([3, 2], "dc2", 2, 1)
|
|
await try_fail([4, 1], "dc1", 4, 3)
|
|
|
|
# We need to perform a read barrier on the node to make
|
|
# sure that it processes the last DROP KEYSPACE.
|
|
# Otherwise, the node could think the RF-rack-invalid
|
|
# keyspace still exists.
|
|
await manager.server_start(s1.server_id)
|
|
await read_barrier(manager.api, s1.ip_addr)
|
|
await manager.server_stop_gracefully(s1.server_id)
|
|
|
|
await manager.server_update_config(s1.server_id, "rf_rack_valid_keyspaces", "true")
|
|
await manager.server_start(s1.server_id)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_startup_with_keyspaces_violating_rf_rack_valid_keyspaces_but_not_enforced(manager: ManagerClient):
|
|
"""
|
|
When the configuration option `rf_rack_valid_keyspaces` is enabled and there is an RF-rack-invalid keyspace,
|
|
starting a node fails. However, when the configuration option is disabled, but there still is a keyspace
|
|
that violates the condition, Scylla should print a warning informing the user about the fact. This test
|
|
verifies that.
|
|
|
|
For more context, see issue: scylladb/scylladb#23330.
|
|
"""
|
|
|
|
cfg = {"rf_rack_valid_keyspaces": False}
|
|
|
|
# One DC, 4 racks.
|
|
dc = "dc1"
|
|
s1, _, _, _ = await manager.servers_add(4, config=cfg, auto_rack_dc=dc)
|
|
|
|
cql = manager.get_cql()
|
|
# We need to set `max_schema_agreement_wait` to 0 to speed up this test.
|
|
assert hasattr(cql.cluster, "max_schema_agreement_wait")
|
|
cql.cluster.max_schema_agreement_wait = 0
|
|
|
|
async def create_ks(name: str, rf: int):
|
|
await cql.run_async(f"CREATE KEYSPACE {name} WITH replication = {{'class': 'NetworkTopologyStrategy', '{dc}': {rf}}} AND tablets = {{'enabled': true}}")
|
|
|
|
await create_ks("ks1", 1)
|
|
await create_ks("ks2", 2)
|
|
await create_ks("ks3", 3)
|
|
await create_ks("ks4", 4)
|
|
|
|
await manager.server_stop_gracefully(s1.server_id)
|
|
await manager.server_start(s1.server_id)
|
|
|
|
log = await manager.server_open_log(s1.server_id)
|
|
|
|
expected_pattern = r"Some existing keyspaces are not RF-rack-valid, i\.e\. the replication factor " \
|
|
r"does not match the number of racks in one of the datacenters. That may reduce " \
|
|
r"availability in case of a failure \(cf\. " \
|
|
r"https://docs\.scylladb\.com/manual/stable/reference/glossary\.html#term-RF-rack-valid-keyspace\)\. " \
|
|
r"Those keyspaces are: (ks2, ks3)|(ks3, ks2)"
|
|
|
|
await log.wait_for(expected_pattern)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_with_prefer_local(request: pytest.FixtureRequest, manager: ManagerClient) -> None:
|
|
logger.info("Creating a new cluster")
|
|
for i in range(3):
|
|
s_info = await manager.server_add(
|
|
config=CONFIG,
|
|
property_file={'dc': f'dc{i}', 'rack': 'myrack1', 'prefer_local': 'true'}
|
|
)
|
|
logger.info(s_info)
|
|
|
|
await manager.server_stop_gracefully(s_info.server_id)
|
|
await manager.server_start(s_info.server_id)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warn_create_and_alter_rf_rack_invalid_ks(manager: ManagerClient):
|
|
"""
|
|
When the configuration option `rf_rack_valid_keyspaces` is enabled, the user is not
|
|
allowed to create an RF-rack-invalid keyspace. When the option is disabled, that limitation
|
|
disappears.
|
|
|
|
However, since (at some point) we're going to get rid of that option and start always
|
|
enforcing the restriction, we'd like to let the user know that the keyspace they're
|
|
creating may not be valid in the future. Verify that the warning really appears.
|
|
|
|
For more context, see issue: scylladb/scylladb#23330.
|
|
"""
|
|
|
|
cfg = {"rf_rack_valid_keyspaces": False}
|
|
|
|
# Setup: DC1: 3 racks, DC2: 2 racks.
|
|
start_node_tasks = [
|
|
manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r1"}),
|
|
manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r2"}),
|
|
manager.server_add(config=cfg, property_file={"dc": "dc1", "rack": "r3"}),
|
|
manager.server_add(config=cfg, property_file={"dc": "dc2", "rack": "r4"}),
|
|
manager.server_add(config=cfg, property_file={"dc": "dc2", "rack": "r5"}),
|
|
]
|
|
await asyncio.gather(*start_node_tasks)
|
|
|
|
cql = manager.get_cql()
|
|
# We need to set `max_schema_agreement_wait` to 0 to speed up this test.
|
|
# Without it, the test takes 50 seconds (or 16 seconds if cases run in parallel).
|
|
# With it, the test takes about 9 seconds. All results on my local machine of course.
|
|
assert hasattr(cql.cluster, "max_schema_agreement_wait")
|
|
cql.cluster.max_schema_agreement_wait = 0
|
|
|
|
# Scenario 1. Creating an RF-rack-invalid keyspace.
|
|
###################################################
|
|
|
|
async def do_create_test(rf1: int, rf2: int, tablets: str, ok: bool):
|
|
ks = unique_name()
|
|
warning = f"Keyspace '{ks}' is not RF-rack-valid: the replication factor doesn't match " \
|
|
"the rack count in at least one datacenter. A rack failure may reduce availability. " \
|
|
"For more context, see: " \
|
|
"https://docs.scylladb.com/manual/stable/reference/glossary.html#term-RF-rack-valid-keyspace."
|
|
|
|
stmt = f"CREATE KEYSPACE {ks} WITH replication = " \
|
|
f"{{'class': 'NetworkTopologyStrategy', 'dc1': {rf1}, 'dc2': {rf2}}} AND " \
|
|
f"tablets = {{'enabled': {tablets}}}"
|
|
|
|
# We have to use `Session::execute_async` here to be able to obtain `warnings`.
|
|
# It's pretty convoluted, but we have to live with it...
|
|
result = cql.execute_async(stmt)
|
|
await _wrap_future(result)
|
|
|
|
if ok:
|
|
assert not hasattr(result, "warnings") or result.warnings is None or warning not in result.warnings
|
|
else:
|
|
assert hasattr(result, "warnings")
|
|
assert warning in result.warnings
|
|
|
|
# All of the statements below are OK: they don't use tablets.
|
|
await do_create_test(2, 2, "false", True)
|
|
await do_create_test(3, 3, "false", True)
|
|
await do_create_test(3, 2, "false", True)
|
|
await do_create_test(3, 1, "false", True)
|
|
await do_create_test(1, 2, "false", True)
|
|
await do_create_test(1, 1, "false", True)
|
|
|
|
# BAD: the RF doesn't match the number of racks in DC1.
|
|
await do_create_test(2, 2, "true", False)
|
|
# BAD: the RF doesn't match the number of racks in DC2.
|
|
await do_create_test(3, 3, "true", False)
|
|
# OK: the RFs match the number of racks in the DCs.
|
|
await do_create_test(3, 2, "true", True)
|
|
# OK: RF=#racks for DC1, RF=1 is always accepted.
|
|
await do_create_test(3, 1, "true", True)
|
|
# OK: RF=#racks for DC2, RF=1 is always accepted.
|
|
await do_create_test(1, 2, "true", True)
|
|
# OK: RF=1 is always accepted.
|
|
await do_create_test(1, 1, "true", True)
|
|
|
|
# Scenario 2. Altering an RF-rack-valid keyspace so that it becomes RF-rack-invalid.
|
|
####################################################################################
|
|
|
|
async def do_alter_test(rf1: int, rf2: int, tablets: str, ok: bool):
|
|
ks = unique_name()
|
|
warning = f"Keyspace '{ks}' is not RF-rack-valid: the replication factor doesn't match " \
|
|
"the rack count in at least one datacenter. A rack failure may reduce availability. " \
|
|
"For more context, see: " \
|
|
"https://docs.scylladb.com/manual/stable/reference/glossary.html#term-RF-rack-valid-keyspace."
|
|
|
|
await cql.run_async(f"CREATE KEYSPACE {ks} WITH replication = " \
|
|
f"{{'class': 'NetworkTopologyStrategy', 'dc1': 3, 'dc2': 2}} AND " \
|
|
f"tablets = {{'enabled': {tablets}}}")
|
|
|
|
stmt = f"ALTER KEYSPACE {ks} WITH replication = " \
|
|
f"{{'class': 'NetworkTopologyStrategy', 'dc1': {rf1}, 'dc2': {rf2}}} AND " \
|
|
f"tablets = {{'enabled': {tablets}}}"
|
|
|
|
# We have to use `Session::execute_async` here to be able to obtain `warnings`.
|
|
# It's pretty convoluted, but we have to live with it...
|
|
result = cql.execute_async(stmt)
|
|
await _wrap_future(result)
|
|
|
|
if ok:
|
|
assert not hasattr(result, "warnings") or result.warnings is None or warning not in result.warnings
|
|
else:
|
|
assert hasattr(result, "warnings")
|
|
assert warning in result.warnings
|
|
|
|
# All of the statements below are OK: they don't use tablets.
|
|
await do_alter_test(2, 2, "false", True)
|
|
await do_alter_test(3, 3, "false", True)
|
|
await do_alter_test(3, 2, "false", True)
|
|
await do_alter_test(3, 1, "false", True)
|
|
|
|
# BAD: the RF doesn't match the number of racks in DC1.
|
|
await do_alter_test(2, 2, "true", False)
|
|
# BAD: the RF doesn't match the number of racks in DC2.
|
|
await do_alter_test(3, 3, "true", False)
|
|
# OK: the RFs match the number of racks in the DCs.
|
|
await do_alter_test(3, 2, "true", True)
|
|
# OK: RF=#racks for DC1, RF=1 is always accepted.
|
|
await do_alter_test(3, 1, "true", True)
|