From 8d1d206aff1e68cfa4bc5cd6dd52af4e034aebce Mon Sep 17 00:00:00 2001 From: Andrei Chekun Date: Tue, 11 Jun 2024 14:58:03 +0200 Subject: [PATCH] [test.py] Add uniqueness to the test name In CI test always executed with option --repeat=3 that leads to generate 3 test results with the same name. Junit plugin in CI cannot distinguish correctly the difference between these results. In case when we have two passes and one fail, the link to test result will sometimes be redirected to the incorrect one because the test name is the same. To fix this ReportPlugin added that will be responsible to modify the test case name during junit report generation adding to the test name mode and run id. Fixes: https://github.com/scylladb/scylladb/issues/17851 Fixes: https://github.com/scylladb/scylladb/issues/15973 --- test.py | 11 +++++++++-- test/alternator/conftest.py | 6 ++++++ test/cql-pytest/conftest.py | 11 +++++++++++ test/nodetool/conftest.py | 8 ++++++++ test/object_store/conftest.py | 4 ---- test/pylib/cql_repl/conftest.py | 9 +++++++++ test/pylib/report_plugin.py | 15 +++++++++++++++ test/rest_api/conftest.py | 10 ++++++++++ test/topology/conftest.py | 5 +++++ 9 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 test/pylib/report_plugin.py diff --git a/test.py b/test.py index 05eb4bc7e9..669a78b67a 100755 --- a/test.py +++ b/test.py @@ -795,6 +795,8 @@ class CQLApprovalTest(Test): "test/pylib/cql_repl/cql_repl.py", "--input={}".format(self.cql), "--output={}".format(self.tmpfile), + "--run_id={}".format(self.id), + "--mode={}".format(self.mode), ] async def run(self, options: argparse.Namespace) -> Test: @@ -947,7 +949,10 @@ class PythonTest(Test): "-o", "junit_suite_name={}".format(self.suite.name), "--junit-xml={}".format(self.xmlout), - "-rs"] + "-rs", + "--run_id={}".format(self.id), + "--mode={}".format(self.mode) + ] if options.markers: self.args.append(f"-m={options.markers}") @@ -1074,7 +1079,9 @@ class ToolTest(Test): "-o", "junit_family=xunit2", "--junit-xml={}".format(self.xmlout), - f"--mode={self.mode}"] + "--mode={}".format(self.mode), + "--run_id={}".format(self.id) + ] if options.markers: self.args.append(f"-m={options.markers}") diff --git a/test/alternator/conftest.py b/test/alternator/conftest.py index 685141a3e4..bac5b6df84 100644 --- a/test/alternator/conftest.py +++ b/test/alternator/conftest.py @@ -14,6 +14,7 @@ import boto3 import requests import re +from test.pylib.report_plugin import ReportPlugin from test.alternator.util import create_test_table, is_aws, scylla_log from urllib.parse import urlparse from functools import cache @@ -49,8 +50,13 @@ def pytest_addoption(parser): help='Omit scylla\'s output from the test output') parser.addoption('--host', action='store', default='localhost', help='Scylla server host to connect to') + parser.addoption('--mode', action='store', required=True, + help='Scylla build mode. Tests can use it to adjust their behavior.') + parser.addoption('--run_id', action='store', default=1, + help='Run id for the test run') def pytest_configure(config): config.addinivalue_line("markers", "veryslow: mark test as very slow to run") + config.pluginmanager.register(ReportPlugin()) def pytest_collection_modifyitems(config, items): if config.getoption("--runveryslow"): diff --git a/test/cql-pytest/conftest.py b/test/cql-pytest/conftest.py index b719b18997..2c1b0d5352 100644 --- a/test/cql-pytest/conftest.py +++ b/test/cql-pytest/conftest.py @@ -21,6 +21,9 @@ import tempfile import time import random +import sys +sys.path.insert(1, sys.path[0] + '/../..') +from test.pylib.report_plugin import ReportPlugin from util import unique_name, new_test_keyspace, keyspace_has_tablets, cql_session, local_process_id, is_scylla @@ -42,6 +45,10 @@ def pytest_addoption(parser): # presence. parser.addoption('--omit-scylla-output', action='store_true', help='Omit scylla\'s output from the test output') + parser.addoption('--mode', action='store', required=True, + help='Scylla build mode. Tests can use it to adjust their behavior.') + parser.addoption('--run_id', action='store', default=1, + help='Run id for the test run') # "cql" fixture: set up client object for communicating with the CQL API. # The host/port combination of the server are determined by the --host and @@ -262,3 +269,7 @@ def has_tablets(cql): def skip_without_tablets(scylla_only, has_tablets): if not has_tablets: pytest.skip("Test needs tablets experimental feature on") + + +def pytest_configure(config): + config.pluginmanager.register(ReportPlugin()) diff --git a/test/nodetool/conftest.py b/test/nodetool/conftest.py index cc8e8df844..4a5d0505c7 100644 --- a/test/nodetool/conftest.py +++ b/test/nodetool/conftest.py @@ -16,6 +16,7 @@ import requests.exceptions from test.nodetool.rest_api_mock import set_expected_requests, expected_request, get_expected_requests, \ get_unexpected_requests, expected_requests_manager +from test.pylib.report_plugin import ReportPlugin def pytest_addoption(parser): @@ -31,6 +32,9 @@ def pytest_addoption(parser): help="Path to the jmx binary, only used with --nodetool=cassandra") parser.addoption('--run-within-unshare', action='store_true', help="Setup the 'lo' network if launched with unshare(1)") + parser.addoption('--run_id', action='store', default=1, + help='Run id for the test run') + class ServerAddress(NamedTuple): ip: str @@ -221,3 +225,7 @@ def nodetool(request, jmx, nodetool_path, rest_api_mock_server): return res return invoker + + +def pytest_configure(config): + config.pluginmanager.register(ReportPlugin()) diff --git a/test/object_store/conftest.py b/test/object_store/conftest.py index a2dd4ad077..11667e2a10 100644 --- a/test/object_store/conftest.py +++ b/test/object_store/conftest.py @@ -21,10 +21,6 @@ def pytest_addoption(parser): parser.addoption('--manager-api', action='store', required=True, help='Manager unix socket path') - parser.addoption('--mode', action='store', required=True, - help='Scylla build mode. Tests can use it to adjust their behavior.') - parser.addoption('--run_id', action='store', default=None, - help='Run id for the test run') parser.addoption('--tmpdir', action='store', type=str, dest='tmpdir', help='Temporary directory where logs are stored') parser.addoption("--artifacts_dir_url", action='store', type=str, default=None, dest="artifacts_dir_url", diff --git a/test/pylib/cql_repl/conftest.py b/test/pylib/cql_repl/conftest.py index f2c4898e31..e6b0011957 100644 --- a/test/pylib/cql_repl/conftest.py +++ b/test/pylib/cql_repl/conftest.py @@ -21,6 +21,7 @@ from cassandra.policies import RoundRobinPolicy # type: ignore # pylint: disa from cassandra.connection import DRIVER_NAME # type: ignore # pylint: disable=no-name-in-module from cassandra.connection import DRIVER_VERSION # type: ignore # pylint: disable=no-name-in-module +from test.pylib.report_plugin import ReportPlugin logger = logging.getLogger(__name__) logger.warning("Driver name %s", DRIVER_NAME) @@ -42,6 +43,10 @@ def pytest_addoption(parser) -> None: help='CQL server port to connect to') parser.addoption('--ssl', action='store_true', help='Connect to CQL via an encrypted TLSv1.2 connection') + parser.addoption('--mode', action='store', required=True, + help='Scylla build mode. Tests can use it to adjust their behavior.') + parser.addoption('--run_id', action='store', default=1, + help='Run id for the test run') # "cql" fixture: set up client object for communicating with the CQL API. @@ -130,3 +135,7 @@ def keyspace(cql, this_dc): # pylint: disable=redefined-outer-name f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}") yield keyspace_name cql.execute(f"DROP KEYSPACE {keyspace_name}") + + +def pytest_configure(config): + config.pluginmanager.register(ReportPlugin()) diff --git a/test/pylib/report_plugin.py b/test/pylib/report_plugin.py new file mode 100644 index 0000000000..b9d5d96c77 --- /dev/null +++ b/test/pylib/report_plugin.py @@ -0,0 +1,15 @@ +import pytest + + +class ReportPlugin: + # Pytest hook to modify test name to include mode and run_id + def pytest_configure(self, config): + self.config = config + self.mode = config.getoption("mode") + self.run_id = config.getoption("run_id") + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self): + outcome = yield + report = outcome.get_result() + report.nodeid = f"{report.nodeid}.{self.mode}.{self.run_id}" diff --git a/test/rest_api/conftest.py b/test/rest_api/conftest.py index 326c74504b..b3518efa4f 100644 --- a/test/rest_api/conftest.py +++ b/test/rest_api/conftest.py @@ -18,6 +18,8 @@ from cassandra.auth import PlainTextAuthProvider from cassandra.cluster import Cluster, ConsistencyLevel, ExecutionProfile, EXEC_PROFILE_DEFAULT from cassandra.policies import RoundRobinPolicy +from test.pylib.report_plugin import ReportPlugin + # Use the util.py library from ../cql-pytest: sys.path.insert(1, sys.path[0] + '/test/cql-pytest') from util import unique_name, new_test_keyspace, keyspace_has_tablets, is_scylla @@ -34,6 +36,10 @@ def pytest_addoption(parser): help='Connect to CQL via an encrypted TLSv1.2 connection') parser.addoption('--api-port', action='store', default='10000', help='server REST API port to connect to') + parser.addoption('--mode', action='store', required=True, + help='Scylla build mode. Tests can use it to adjust their behavior.') + parser.addoption('--run_id', action='store', default=1, + help='Run id for the test run') class RestApiSession: def __init__(self, host, port): @@ -156,3 +162,7 @@ def test_keyspace(cql, this_dc): cql.execute("CREATE KEYSPACE " + name + " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 1 }") yield name cql.execute("DROP KEYSPACE " + name) + + +def pytest_configure(config): + config.pluginmanager.register(ReportPlugin()) diff --git a/test/topology/conftest.py b/test/topology/conftest.py index 7b163d28e8..167e5597c2 100644 --- a/test/topology/conftest.py +++ b/test/topology/conftest.py @@ -12,6 +12,7 @@ import platform from functools import partial from typing import List, Optional, Dict from test.pylib.random_tables import RandomTables +from test.pylib.report_plugin import ReportPlugin from test.pylib.util import unique_name from test.pylib.manager_client import ManagerClient, IPAddress from test.pylib.async_cql import event_loop, run_async @@ -270,3 +271,7 @@ def skip_mode_fixture(request, mode): for reason, platform_key in skipped_funcs.get((request.function, mode), []): if platform_key is None or platform_key in platform.platform(): pytest.skip(f'{request.node.name} skipped, reason: {reason}') + + +def pytest_configure(config): + config.pluginmanager.register(ReportPlugin())