From 90dfcec86b296a9e0e6de7c661b66f1e79eb3866 Mon Sep 17 00:00:00 2001
From: Pavel Emelyanov <xemul@scylladb.com>
Date: Thu, 21 Mar 2024 18:53:28 +0300
Subject: [PATCH 1/2] test: Mark tests with skip_mode instead of suite
 skip-list

There are many tests that are skipped in release mode becuase they rely
on error-injection machinery which doesn't work in release mode. Most of
those tests are listed in suite's skip_in_release, but it's not very
handy, mainly because it's not clear why the test is there. The
skip_mode decoration is much more convenient.

Signed-off-by: Pavel Emelyanov <xemul@scylladb.com>
---
 test/topology/suite.yaml                                     | 2 --
 test/topology/test_coordinator_queue_management.py           | 2 ++
 test/topology/test_topology_failure_recovery.py              | 2 ++
 test/topology_custom/suite.yaml                              | 5 -----
 .../test_node_shutdown_waits_for_pending_requests.py         | 2 ++
 test/topology_custom/test_old_ip_notification_repro.py       | 2 ++
 test/topology_custom/test_raft_recovery_stuck.py             | 2 ++
 test/topology_custom/test_shutdown_hang.py                   | 2 ++
 test/topology_custom/test_topology_failure_recovery.py       | 2 ++
 test/topology_experimental_raft/suite.yaml                   | 4 ----
 test/topology_experimental_raft/test_blocked_bootstrap.py    | 2 ++
 .../test_cdc_generation_clearing.py                          | 2 ++
 .../test_cdc_generation_publishing.py                        | 2 ++
 test/topology_experimental_raft/test_topology_upgrade.py     | 2 ++
 14 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/test/topology/suite.yaml b/test/topology/suite.yaml
index 0a21bac13c..1b518f9f2a 100644
--- a/test/topology/suite.yaml
+++ b/test/topology/suite.yaml
@@ -12,8 +12,6 @@ run_first:
 skip_in_release:
     - test_cluster_features
     - test_replace_alive_node
-    - test_topology_failure_recovery
-    - test_coordinator_queue_management
 skip_in_debug:
     - test_replace_alive_node
 run_in_release:
diff --git a/test/topology/test_coordinator_queue_management.py b/test/topology/test_coordinator_queue_management.py
index 1c451b2e42..ae1d554e43 100644
--- a/test/topology/test_coordinator_queue_management.py
+++ b/test/topology/test_coordinator_queue_management.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 from test.pylib.manager_client import ManagerClient
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -11,6 +12,7 @@ import asyncio
 logger = logging.getLogger(__name__)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_coordinator_queue_management(manager: ManagerClient):
     """This test creates a 5 node cluster with 2 down nodes (A and B). After that it
        creates a queue of 3 topology operation: bootstrap, removenode A and removenode B
diff --git a/test/topology/test_topology_failure_recovery.py b/test/topology/test_topology_failure_recovery.py
index 0cc36c857e..4b2eb23cc1 100644
--- a/test/topology/test_topology_failure_recovery.py
+++ b/test/topology/test_topology_failure_recovery.py
@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient
 from test.pylib.internal_types import ServerInfo
 from test.pylib.scylla_cluster import ReplaceConfig
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -17,6 +18,7 @@ async def inject_error_on(manager, error_name, servers):
     await asyncio.gather(*errs)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_topology_streaming_failure(request, manager: ManagerClient):
     """Fail streaming while doing a topology operation"""
     # decommission failure
diff --git a/test/topology_custom/suite.yaml b/test/topology_custom/suite.yaml
index 52548470e9..ce98e1517d 100644
--- a/test/topology_custom/suite.yaml
+++ b/test/topology_custom/suite.yaml
@@ -11,14 +11,9 @@ run_first:
   - test_group0_schema_versioning
   - test_tablets_migration
 skip_in_release:
-  - test_raft_recovery_stuck
-  - test_shutdown_hang
   - test_replace_ignore_nodes
-  - test_old_ip_notification_repro
   - test_different_group0_ids
   - test_group0_schema_versioning
-  - test_topology_failure_recovery
-  - test_node_shutdown_waits_for_pending_requests
 skip_in_debug:
   - test_shutdown_hang
   - test_replace
diff --git a/test/topology_custom/test_node_shutdown_waits_for_pending_requests.py b/test/topology_custom/test_node_shutdown_waits_for_pending_requests.py
index 8c73779c0c..e2831b2a8d 100644
--- a/test/topology_custom/test_node_shutdown_waits_for_pending_requests.py
+++ b/test/topology_custom/test_node_shutdown_waits_for_pending_requests.py
@@ -9,12 +9,14 @@ from cassandra.query import SimpleStatement # type: ignore
 from cassandra.cluster import ConsistencyLevel # type: ignore
 from cassandra.protocol import ReadTimeout # type: ignore
 from test.pylib.util import wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_node_shutdown_waits_for_pending_requests(manager: ManagerClient) -> None:
     """Reproducer for #16382"""
 
diff --git a/test/topology_custom/test_old_ip_notification_repro.py b/test/topology_custom/test_old_ip_notification_repro.py
index dc362f70d3..ad288b7650 100644
--- a/test/topology_custom/test_old_ip_notification_repro.py
+++ b/test/topology_custom/test_old_ip_notification_repro.py
@@ -11,12 +11,14 @@ import pytest
 from test.pylib.manager_client import ManagerClient
 from test.pylib.rest_client import inject_error
 from test.pylib.util import read_barrier, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_old_ip_notification_repro(manager: ManagerClient) -> None:
     """
     Regression test for #14257.
diff --git a/test/topology_custom/test_raft_recovery_stuck.py b/test/topology_custom/test_raft_recovery_stuck.py
index 2f6102bb22..9f65f9bb8e 100644
--- a/test/topology_custom/test_raft_recovery_stuck.py
+++ b/test/topology_custom/test_raft_recovery_stuck.py
@@ -11,11 +11,13 @@ import time
 from test.pylib.manager_client import ManagerClient
 from test.pylib.random_tables import RandomTables
 from test.pylib.util import unique_name, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 from test.topology.util import reconnect_driver, restart, enter_recovery_state, wait_for_upgrade_state, \
         wait_until_upgrade_finishes, delete_raft_data_and_upgrade_state, log_run_time
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 @log_run_time
 async def test_recover_stuck_raft_recovery(request, manager: ManagerClient):
     """
diff --git a/test/topology_custom/test_shutdown_hang.py b/test/topology_custom/test_shutdown_hang.py
index 9e3ca2f8e7..8cb8dc7f17 100644
--- a/test/topology_custom/test_shutdown_hang.py
+++ b/test/topology_custom/test_shutdown_hang.py
@@ -15,12 +15,14 @@ from cassandra.protocol import WriteTimeout # type: ignore
 
 from test.pylib.manager_client import ManagerClient
 from test.topology.util import wait_for_token_ring_and_group0_consistency
+from test.topology.conftest import skip_mode
 
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_hints_manager_shutdown_hang(manager: ManagerClient) -> None:
     """Reproducer for #8079"""
     s1 = await manager.server_add(config={
diff --git a/test/topology_custom/test_topology_failure_recovery.py b/test/topology_custom/test_topology_failure_recovery.py
index 91cc6206cc..945c4c531a 100644
--- a/test/topology_custom/test_topology_failure_recovery.py
+++ b/test/topology_custom/test_topology_failure_recovery.py
@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient
 from test.pylib.internal_types import ServerInfo
 from test.pylib.scylla_cluster import ReplaceConfig
+from test.topology.conftest import skip_mode
 import pytest
 import logging
 import asyncio
@@ -17,6 +18,7 @@ async def inject_error_on(manager, error_name, servers):
     await asyncio.gather(*errs)
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_tablet_drain_failure_during_decommission(manager: ManagerClient):
     cfg = {'enable_user_defined_functions': False,
            'experimental_features': ['tablets', 'consistent-topology-changes']}
diff --git a/test/topology_experimental_raft/suite.yaml b/test/topology_experimental_raft/suite.yaml
index abcf821b2a..755e3d114c 100644
--- a/test/topology_experimental_raft/suite.yaml
+++ b/test/topology_experimental_raft/suite.yaml
@@ -12,12 +12,8 @@ run_first:
   - test_raft_ignore_nodes
   - test_tablets
 skip_in_release:
-  - test_blocked_bootstrap
-  - test_cdc_generation_clearing
-  - test_cdc_generation_publishing
   - test_raft_cluster_features
   - test_raft_ignore_nodes
-  - test_topology_upgrade
 skip_in_debug:
   - test_cdc_generation_clearing
   - test_cdc_generation_publishing
diff --git a/test/topology_experimental_raft/test_blocked_bootstrap.py b/test/topology_experimental_raft/test_blocked_bootstrap.py
index ffb6997910..641ddfc357 100644
--- a/test/topology_experimental_raft/test_blocked_bootstrap.py
+++ b/test/topology_experimental_raft/test_blocked_bootstrap.py
@@ -4,6 +4,7 @@
 #
 from test.pylib.scylla_cluster import ReplaceConfig
 from test.pylib.manager_client import ManagerClient
+from test.topology.conftest import skip_mode
 
 import pytest
 import logging
@@ -12,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.skip(reason = "can't make it work with the new join procedure, without error recovery")
+@skip_mode('release', 'error injections are not supported in release mode')
 @pytest.mark.asyncio
 async def test_blocked_bootstrap(manager: ManagerClient):
     """
diff --git a/test/topology_experimental_raft/test_cdc_generation_clearing.py b/test/topology_experimental_raft/test_cdc_generation_clearing.py
index a5b3c5010d..51486f7524 100644
--- a/test/topology_experimental_raft/test_cdc_generation_clearing.py
+++ b/test/topology_experimental_raft/test_cdc_generation_clearing.py
@@ -8,6 +8,7 @@ from test.pylib.manager_client import ManagerClient
 from test.pylib.util import wait_for, wait_for_cql_and_get_hosts
 from test.topology.util import wait_for_cdc_generations_publishing, \
         check_system_topology_and_cdc_generations_v3_consistency
+from test.topology.conftest import skip_mode
 
 from cassandra.cluster import ConsistencyLevel # type: ignore # pylint: disable=no-name-in-module
 from cassandra.pool import Host # type: ignore # pylint: disable=no-name-in-module
@@ -23,6 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_cdc_generation_clearing(manager: ManagerClient):
     """Test that obsolete CDC generations are removed from CDC_GENERATIONS_V3 and TOPOLOGY.committed_cdc_generations
        if their timestamp is old enough according to the topology coordinator's clock."""
diff --git a/test/topology_experimental_raft/test_cdc_generation_publishing.py b/test/topology_experimental_raft/test_cdc_generation_publishing.py
index 7385dd4f50..96e724f098 100644
--- a/test/topology_experimental_raft/test_cdc_generation_publishing.py
+++ b/test/topology_experimental_raft/test_cdc_generation_publishing.py
@@ -6,6 +6,7 @@
 from test.pylib.manager_client import ManagerClient, ServerInfo
 from test.pylib.rest_client import inject_error
 from test.pylib.util import wait_for, wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 
 from cassandra.cluster import ConsistencyLevel # type: ignore # pylint: disable=no-name-in-module
 from cassandra.query import SimpleStatement # type: ignore # pylint: disable=no-name-in-module
@@ -22,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 async def test_cdc_generations_are_published(request, manager: ManagerClient):
     """Test that the CDC generation publisher eventually publishes committed CDC generations in the correct order."""
     query_gen_timestamps = SimpleStatement(
diff --git a/test/topology_experimental_raft/test_topology_upgrade.py b/test/topology_experimental_raft/test_topology_upgrade.py
index 19c8989a48..23c3e91a5c 100644
--- a/test/topology_experimental_raft/test_topology_upgrade.py
+++ b/test/topology_experimental_raft/test_topology_upgrade.py
@@ -12,11 +12,13 @@ import time
 from test.pylib.rest_client import HTTPError
 from test.pylib.manager_client import ManagerClient
 from test.pylib.util import wait_for_cql_and_get_hosts
+from test.topology.conftest import skip_mode
 from test.topology.util import log_run_time, wait_until_topology_upgrade_finishes, \
         wait_for_cdc_generations_publishing, check_system_topology_and_cdc_generations_v3_consistency
 
 
 @pytest.mark.asyncio
+@skip_mode('release', 'error injections are not supported in release mode')
 @log_run_time
 async def test_topology_upgrade_basic(request, manager: ManagerClient):
     # First, force the first node to start in legacy mode due to the error injection

From 16343b3edca95a20f1213ecee47a6d55ed91f200 Mon Sep 17 00:00:00 2001
From: Pavel Emelyanov <xemul@scylladb.com>
Date: Thu, 21 Mar 2024 16:36:26 +0300
Subject: [PATCH 2/2] test: Do not duplicate test name in several skip-lists

Some tests are only run in dev mode for some reason. For such tests
there's run_in_dev list, no need in putting it in all the non-dev
skip_in_... ones.

Signed-off-by: Pavel Emelyanov <xemul@scylladb.com>
---
 test/topology/suite.yaml                   |  5 ++---
 test/topology_custom/suite.yaml            | 11 ++++-------
 test/topology_experimental_raft/suite.yaml |  2 +-
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/test/topology/suite.yaml b/test/topology/suite.yaml
index 1b518f9f2a..fcb3bc9a62 100644
--- a/test/topology/suite.yaml
+++ b/test/topology/suite.yaml
@@ -11,8 +11,7 @@ run_first:
     - test_mutation_schema_change
 skip_in_release:
     - test_cluster_features
-    - test_replace_alive_node
-skip_in_debug:
-    - test_replace_alive_node
 run_in_release:
     - test_gossiper
+run_in_dev:
+    - test_replace_alive_node
diff --git a/test/topology_custom/suite.yaml b/test/topology_custom/suite.yaml
index ce98e1517d..5da3ca23ad 100644
--- a/test/topology_custom/suite.yaml
+++ b/test/topology_custom/suite.yaml
@@ -10,15 +10,12 @@ run_first:
   - test_raft_recovery_basic
   - test_group0_schema_versioning
   - test_tablets_migration
-skip_in_release:
-  - test_replace_ignore_nodes
-  - test_different_group0_ids
-  - test_group0_schema_versioning
 skip_in_debug:
   - test_shutdown_hang
   - test_replace
-  - test_replace_ignore_nodes
   - test_old_ip_notification_repro
-  - test_different_group0_ids
-  - test_group0_schema_versioning
   - test_node_shutdown_waits_for_pending_requests
+run_in_dev:
+  - test_group0_schema_versioning
+  - test_different_group0_ids
+  - test_replace_ignore_nodes
diff --git a/test/topology_experimental_raft/suite.yaml b/test/topology_experimental_raft/suite.yaml
index 755e3d114c..d600553771 100644
--- a/test/topology_experimental_raft/suite.yaml
+++ b/test/topology_experimental_raft/suite.yaml
@@ -13,8 +13,8 @@ run_first:
   - test_tablets
 skip_in_release:
   - test_raft_cluster_features
-  - test_raft_ignore_nodes
 skip_in_debug:
   - test_cdc_generation_clearing
   - test_cdc_generation_publishing
+run_in_dev:
   - test_raft_ignore_nodes