Merge 'Sanitize topology suites' skiplists' from Pavel Emelyanov

There are skip_in_<mode> lists in suite yaml that tells test.py not to run the test from it. This PR sanitizes these lists in two ways.

First, to skip pytests the skip-decorators are much more convenient, e.g. because they show the reason why the test is skipped.

Also, if a test wants to be opt-in-ed for some mode only, it's opt-out-ed in all other lists instead. There's run_in_<mode> list in suite for that.

Closes scylladb/scylladb#17964

* github.com:scylladb/scylladb:
  test: Do not duplicate test name in several skip-lists
  test: Mark tests with skip_mode instead of suite skip-list

test/topology/suite.yaml

@@ -11,10 +11,7 @@ run_first:
     - test_mutation_schema_change
 skip_in_release:
     - test_cluster_features
-    - test_replace_alive_node
-    - test_topology_failure_recovery
-    - test_coordinator_queue_management
-skip_in_debug:
-    - test_replace_alive_node
 run_in_release:
     - test_gossiper
+run_in_dev:
+    - test_replace_alive_node

test/topology/test_coordinator_queue_management.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 from test.pylib.manager_client import ManagerClient
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -11,6 +12,7 @@ import asyncio
 logger = logging.getLogger(__name__)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_coordinator_queue_management(manager: ManagerClient):
     """This test creates a 5 node cluster with 2 down nodes (A and B). After that it
        creates a queue of 3 topology operation: bootstrap, removenode A and removenode B

test/topology/test_topology_failure_recovery.py

@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient
 from test.pylib.internal_types import ServerInfo
 from test.pylib.scylla_cluster import ReplaceConfig
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -17,6 +18,7 @@ async def inject_error_on(manager, error_name, servers):
     await asyncio.gather(*errs)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_topology_streaming_failure(request, manager: ManagerClient):
     """Fail streaming while doing a topology operation"""
     # decommission failure

test/topology_custom/suite.yaml

@@ -10,20 +10,12 @@ run_first:
   - test_raft_recovery_basic
   - test_group0_schema_versioning
   - test_tablets_migration
-skip_in_release:
-  - test_raft_recovery_stuck
-  - test_shutdown_hang
-  - test_replace_ignore_nodes
-  - test_old_ip_notification_repro
-  - test_different_group0_ids
-  - test_group0_schema_versioning
-  - test_topology_failure_recovery
-  - test_node_shutdown_waits_for_pending_requests
 skip_in_debug:
   - test_shutdown_hang
   - test_replace
-  - test_replace_ignore_nodes
   - test_old_ip_notification_repro
-  - test_different_group0_ids
-  - test_group0_schema_versioning
   - test_node_shutdown_waits_for_pending_requests
+run_in_dev:
+  - test_group0_schema_versioning
+  - test_different_group0_ids
+  - test_replace_ignore_nodes

test/topology_custom/test_node_shutdown_waits_for_pending_requests.py

@@ -9,12 +9,14 @@ from cassandra.query import SimpleStatement # type: ignore
 from cassandra.cluster import ConsistencyLevel # type: ignore
 from cassandra.protocol import ReadTimeout # type: ignore
 from test.pylib.util import wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_node_shutdown_waits_for_pending_requests(manager: ManagerClient) -> None:
     """Reproducer for #16382"""
 

test/topology_custom/test_old_ip_notification_repro.py

@@ -11,12 +11,14 @@ import pytest
 from test.pylib.manager_client import ManagerClient
 from test.pylib.rest_client import inject_error
 from test.pylib.util import read_barrier, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_old_ip_notification_repro(manager: ManagerClient) -> None:
     """
     Regression test for #14257.

test/topology_custom/test_raft_recovery_stuck.py

@@ -11,11 +11,13 @@ import time
 from test.pylib.manager_client import ManagerClient
 from test.pylib.random_tables import RandomTables
 from test.pylib.util import unique_name, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 from test.topology.util import reconnect_driver, restart, enter_recovery_state, wait_for_upgrade_state, \
         wait_until_upgrade_finishes, delete_raft_data_and_upgrade_state, log_run_time
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 @log_run_time
 async def test_recover_stuck_raft_recovery(request, manager: ManagerClient):
     """

test/topology_custom/test_shutdown_hang.py

@@ -15,12 +15,14 @@ from cassandra.protocol import WriteTimeout # type: ignore
 
 from test.pylib.manager_client import ManagerClient
 from test.topology.util import wait_for_token_ring_and_group0_consistency
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_hints_manager_shutdown_hang(manager: ManagerClient) -> None:
     """Reproducer for #8079"""
     s1 = await manager.server_add(config={

test/topology_custom/test_topology_failure_recovery.py

@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient
 from test.pylib.internal_types import ServerInfo
 from test.pylib.scylla_cluster import ReplaceConfig
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -17,6 +18,7 @@ async def inject_error_on(manager, error_name, servers):
     await asyncio.gather(*errs)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_tablet_drain_failure_during_decommission(manager: ManagerClient):
     cfg = {'enable_user_defined_functions': False,
            'experimental_features': ['tablets', 'consistent-topology-changes']}

test/topology_experimental_raft/suite.yaml

@@ -12,13 +12,9 @@ run_first:
   - test_raft_ignore_nodes
   - test_tablets
 skip_in_release:
-  - test_blocked_bootstrap
-  - test_cdc_generation_clearing
-  - test_cdc_generation_publishing
   - test_raft_cluster_features
-  - test_raft_ignore_nodes
-  - test_topology_upgrade
 skip_in_debug:
   - test_cdc_generation_clearing
   - test_cdc_generation_publishing
+run_in_dev:
   - test_raft_ignore_nodes

test/topology_experimental_raft/test_blocked_bootstrap.py

@@ -4,6 +4,7 @@
 #
 from test.pylib.scylla_cluster import ReplaceConfig
 from test.pylib.manager_client import ManagerClient
+from test.topology.conftest import skip_mode
 
 import pytest
 import logging
@@ -12,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.skip(reason = "can't make it work with the new join procedure, without error recovery")
+@skip_mode('release', 'error injections are not supported in release mode')
 @pytest.mark.asyncio
 async def test_blocked_bootstrap(manager: ManagerClient):
     """

test/topology_experimental_raft/test_cdc_generation_clearing.py

@@ -8,6 +8,7 @@ from test.pylib.manager_client import ManagerClient
 from test.pylib.util import wait_for, wait_for_cql_and_get_hosts
 from test.topology.util import wait_for_cdc_generations_publishing, \
         check_system_topology_and_cdc_generations_v3_consistency
+from test.topology.conftest import skip_mode
 
 from cassandra.cluster import ConsistencyLevel # type: ignore # pylint: disable=no-name-in-module
 from cassandra.pool import Host # type: ignore # pylint: disable=no-name-in-module
@@ -23,6 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_cdc_generation_clearing(manager: ManagerClient):
     """Test that obsolete CDC generations are removed from CDC_GENERATIONS_V3 and TOPOLOGY.committed_cdc_generations
        if their timestamp is old enough according to the topology coordinator's clock."""

test/topology_experimental_raft/test_cdc_generation_publishing.py

@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient, ServerInfo
 from test.pylib.rest_client import inject_error
 from test.pylib.util import wait_for, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 from cassandra.cluster import ConsistencyLevel # type: ignore # pylint: disable=no-name-in-module
 from cassandra.query import SimpleStatement # type: ignore # pylint: disable=no-name-in-module
@@ -22,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_cdc_generations_are_published(request, manager: ManagerClient):
     """Test that the CDC generation publisher eventually publishes committed CDC generations in the correct order."""
     query_gen_timestamps = SimpleStatement(

test/topology_experimental_raft/test_topology_upgrade.py

@@ -12,11 +12,13 @@ import time
 from test.pylib.rest_client import HTTPError
 from test.pylib.manager_client import ManagerClient
 from test.pylib.util import wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 from test.topology.util import log_run_time, wait_until_topology_upgrade_finishes, \
         wait_for_cdc_generations_publishing, check_system_topology_and_cdc_generations_v3_consistency
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 @log_run_time
 async def test_topology_upgrade_basic(request, manager: ManagerClient):
     # First, force the first node to start in legacy mode due to the error injection