[test.py] Add uniqueness to the test name

In CI test always executed with option --repeat=3 that leads to generate 3 test results with the same name. Junit plugin in CI cannot distinguish correctly the difference between these results. In case when we have two passes and one fail, the link to test result will sometimes be redirected to the incorrect one because the test name is the same.
To fix this ReportPlugin added that will be responsible to modify the test case name during junit report generation adding to the test name mode and run id.

Fixes: https://github.com/scylladb/scylladb/issues/17851

Fixes: https://github.com/scylladb/scylladb/issues/15973

test.py

@@ -795,6 +795,8 @@ class CQLApprovalTest(Test):
             "test/pylib/cql_repl/cql_repl.py",
             "--input={}".format(self.cql),
             "--output={}".format(self.tmpfile),
+            "--run_id={}".format(self.id),
+            "--mode={}".format(self.mode),
         ]
 
     async def run(self, options: argparse.Namespace) -> Test:
@@ -947,7 +949,10 @@ class PythonTest(Test):
             "-o",
             "junit_suite_name={}".format(self.suite.name),
             "--junit-xml={}".format(self.xmlout),
-            "-rs"]
+            "-rs",
+            "--run_id={}".format(self.id),
+            "--mode={}".format(self.mode)
+        ]
         if options.markers:
             self.args.append(f"-m={options.markers}")
 
@@ -1074,7 +1079,9 @@ class ToolTest(Test):
             "-o",
             "junit_family=xunit2",
             "--junit-xml={}".format(self.xmlout),
-            f"--mode={self.mode}"]
+            "--mode={}".format(self.mode),
+            "--run_id={}".format(self.id)
+        ]
         if options.markers:
             self.args.append(f"-m={options.markers}")
 

test/alternator/conftest.py

@@ -14,6 +14,7 @@ import boto3
 import requests
 import re
 
+from test.pylib.report_plugin import ReportPlugin
 from test.alternator.util import create_test_table, is_aws, scylla_log
 from urllib.parse import urlparse
 from functools import cache
@@ -49,8 +50,13 @@ def pytest_addoption(parser):
         help='Omit scylla\'s output from the test output')
     parser.addoption('--host', action='store', default='localhost',
         help='Scylla server host to connect to')
+    parser.addoption('--mode', action='store', required=True,
+                     help='Scylla build mode. Tests can use it to adjust their behavior.')
+    parser.addoption('--run_id', action='store', default=1,
+                     help='Run id for the test run')
 def pytest_configure(config):
     config.addinivalue_line("markers", "veryslow: mark test as very slow to run")
+    config.pluginmanager.register(ReportPlugin())
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--runveryslow"):

test/cql-pytest/conftest.py

@@ -21,6 +21,9 @@ import tempfile
 import time
 import random
 
+import sys
+sys.path.insert(1, sys.path[0] + '/../..')
+from test.pylib.report_plugin import ReportPlugin
 from util import unique_name, new_test_keyspace, keyspace_has_tablets, cql_session, local_process_id, is_scylla
 
 
@@ -42,6 +45,10 @@ def pytest_addoption(parser):
     # presence.
     parser.addoption('--omit-scylla-output', action='store_true',
         help='Omit scylla\'s output from the test output')
+    parser.addoption('--mode', action='store', required=True,
+                     help='Scylla build mode. Tests can use it to adjust their behavior.')
+    parser.addoption('--run_id', action='store', default=1,
+                     help='Run id for the test run')
 
 # "cql" fixture: set up client object for communicating with the CQL API.
 # The host/port combination of the server are determined by the --host and
@@ -262,3 +269,7 @@ def has_tablets(cql):
 def skip_without_tablets(scylla_only, has_tablets):
     if not has_tablets:
         pytest.skip("Test needs tablets experimental feature on")
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(ReportPlugin())

test/nodetool/conftest.py

@@ -16,6 +16,7 @@ import requests.exceptions
 
 from test.nodetool.rest_api_mock import set_expected_requests, expected_request, get_expected_requests, \
     get_unexpected_requests, expected_requests_manager
+from test.pylib.report_plugin import ReportPlugin
 
 
 def pytest_addoption(parser):
@@ -31,6 +32,9 @@ def pytest_addoption(parser):
                      help="Path to the jmx binary, only used with --nodetool=cassandra")
     parser.addoption('--run-within-unshare', action='store_true',
                      help="Setup the 'lo' network if launched with unshare(1)")
+    parser.addoption('--run_id', action='store', default=1,
+                     help='Run id for the test run')
+
 
 class ServerAddress(NamedTuple):
     ip: str
@@ -221,3 +225,7 @@ def nodetool(request, jmx, nodetool_path, rest_api_mock_server):
             return res
 
     return invoker
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(ReportPlugin())

test/object_store/conftest.py

@@ -21,10 +21,6 @@ def pytest_addoption(parser):
 
     parser.addoption('--manager-api', action='store', required=True,
                      help='Manager unix socket path')
-    parser.addoption('--mode', action='store', required=True,
-                     help='Scylla build mode. Tests can use it to adjust their behavior.')
-    parser.addoption('--run_id', action='store', default=None,
-                     help='Run id for the test run')
     parser.addoption('--tmpdir', action='store', type=str, dest='tmpdir',
                      help='Temporary directory where logs are stored')
     parser.addoption("--artifacts_dir_url", action='store', type=str, default=None, dest="artifacts_dir_url",

test/pylib/cql_repl/conftest.py

@@ -21,6 +21,7 @@ from cassandra.policies import RoundRobinPolicy    # type: ignore # pylint: disa
 from cassandra.connection import DRIVER_NAME       # type: ignore # pylint: disable=no-name-in-module
 from cassandra.connection import DRIVER_VERSION    # type: ignore # pylint: disable=no-name-in-module
 
+from test.pylib.report_plugin import ReportPlugin
 
 logger = logging.getLogger(__name__)
 logger.warning("Driver name %s", DRIVER_NAME)
@@ -42,6 +43,10 @@ def pytest_addoption(parser) -> None:
         help='CQL server port to connect to')
     parser.addoption('--ssl', action='store_true',
         help='Connect to CQL via an encrypted TLSv1.2 connection')
+    parser.addoption('--mode', action='store', required=True,
+                     help='Scylla build mode. Tests can use it to adjust their behavior.')
+    parser.addoption('--run_id', action='store', default=1,
+                     help='Run id for the test run')
 
 
 # "cql" fixture: set up client object for communicating with the CQL API.
@@ -130,3 +135,7 @@ def keyspace(cql, this_dc):             # pylint: disable=redefined-outer-name
                 f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}")
     yield keyspace_name
     cql.execute(f"DROP KEYSPACE {keyspace_name}")
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(ReportPlugin())

test/pylib/report_plugin.py

@@ -0,0 +1,15 @@
+import pytest
+
+
+class ReportPlugin:
+    # Pytest hook to modify test name to include mode and run_id
+    def pytest_configure(self, config):
+        self.config = config
+        self.mode = config.getoption("mode")
+        self.run_id = config.getoption("run_id")
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self):
+        outcome = yield
+        report = outcome.get_result()
+        report.nodeid = f"{report.nodeid}.{self.mode}.{self.run_id}"

test/rest_api/conftest.py

@@ -18,6 +18,8 @@ from cassandra.auth import PlainTextAuthProvider
 from cassandra.cluster import Cluster, ConsistencyLevel, ExecutionProfile, EXEC_PROFILE_DEFAULT
 from cassandra.policies import RoundRobinPolicy
 
+from test.pylib.report_plugin import ReportPlugin
+
 # Use the util.py library from ../cql-pytest:
 sys.path.insert(1, sys.path[0] + '/test/cql-pytest')
 from util import unique_name, new_test_keyspace, keyspace_has_tablets, is_scylla
@@ -34,6 +36,10 @@ def pytest_addoption(parser):
         help='Connect to CQL via an encrypted TLSv1.2 connection')
     parser.addoption('--api-port', action='store', default='10000',
         help='server REST API port to connect to')
+    parser.addoption('--mode', action='store', required=True,
+                     help='Scylla build mode. Tests can use it to adjust their behavior.')
+    parser.addoption('--run_id', action='store', default=1,
+                     help='Run id for the test run')
 
 class RestApiSession:
     def __init__(self, host, port):
@@ -156,3 +162,7 @@ def test_keyspace(cql, this_dc):
     cql.execute("CREATE KEYSPACE " + name + " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 1 }")
     yield name
     cql.execute("DROP KEYSPACE " + name)
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(ReportPlugin())

test/topology/conftest.py

@@ -12,6 +12,7 @@ import platform
 from functools import partial
 from typing import List, Optional, Dict
 from test.pylib.random_tables import RandomTables
+from test.pylib.report_plugin import ReportPlugin
 from test.pylib.util import unique_name
 from test.pylib.manager_client import ManagerClient, IPAddress
 from test.pylib.async_cql import event_loop, run_async
@@ -270,3 +271,7 @@ def skip_mode_fixture(request, mode):
     for reason, platform_key in skipped_funcs.get((request.function, mode), []):
         if platform_key is None or platform_key in platform.platform():
             pytest.skip(f'{request.node.name} skipped, reason: {reason}')
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(ReportPlugin())