d0e4045103
Previous way of executin repeat was to launch pytest for each repeat. That was resource consuming, since each time pytest was doing discovery of the tests. Now all repeats are done inside one pytest process.
230 lines
10 KiB
Python
230 lines
10 KiB
Python
#
|
|
# Copyright (C) 2024-present ScyllaDB
|
|
#
|
|
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
|
|
#
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import itertools
|
|
import re
|
|
import logging
|
|
import sys
|
|
from argparse import BooleanOptionalAction
|
|
from pathlib import Path
|
|
from random import randint
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from test import ALL_MODES, TEST_RUNNER, TOP_SRC_DIR
|
|
from test.pylib.cpp.item import CppTestFunction
|
|
from test.pylib.report_plugin import ReportPlugin
|
|
from test.pylib.util import get_configured_modes, get_modes_to_run
|
|
from test.pylib.suite.base import (
|
|
TestSuite,
|
|
get_testpy_test,
|
|
init_testsuite_globals,
|
|
prepare_dirs,
|
|
start_3rd_party_services,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from asyncio import AbstractEventLoop
|
|
from collections.abc import Generator
|
|
|
|
from test.pylib.cpp.item import CppTestFunction
|
|
from test.pylib.suite.base import Test
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
|
parser.addoption('--mode', choices=ALL_MODES, action="append", dest="modes",
|
|
help="Run only tests for given build mode(s)")
|
|
parser.addoption('--tmpdir', action='store', default=str(TOP_SRC_DIR / 'testlog'),
|
|
help='Path to temporary test data and log files. The data is further segregated per build mode.')
|
|
parser.addoption('--run_id', action='store', default=None, help='Run id for the test run')
|
|
parser.addoption('--byte-limit', action="store", default=randint(0, 2000), type=int,
|
|
help="Specific byte limit for failure injection (random by default)")
|
|
parser.addoption("--gather-metrics", action=BooleanOptionalAction, default=False,
|
|
help='Switch on gathering cgroup metrics')
|
|
parser.addoption('--random-seed', action="store",
|
|
help="Random number generator seed to be used by boost tests")
|
|
|
|
# Following option is to use with bare pytest command.
|
|
#
|
|
# For compatibility with reasons need to run bare pytest with --test-py-init option
|
|
# to run a test.py-compatible pytest session.
|
|
#
|
|
# TODO: remove this when we'll completely switch to bare pytest runner.
|
|
parser.addoption('--test-py-init', action='store_true', default=False,
|
|
help='Run pytest session in test.py-compatible mode. I.e., start all required services, etc.')
|
|
|
|
# Options for compatibility with test.py
|
|
parser.addoption('--save-log-on-success', default=False,
|
|
dest="save_log_on_success", action="store_true",
|
|
help="Save test log output on success and skip cleanup before the run.")
|
|
parser.addoption('--coverage', action='store_true', default=False,
|
|
help="When running code instrumented with coverage support"
|
|
"Will route the profiles to `tmpdir`/mode/coverage/`suite` and post process them in order to generate "
|
|
"lcov file per suite, lcov file per mode, and an lcov file for the entire run, "
|
|
"The lcov files can eventually be used for generating coverage reports")
|
|
parser.addoption("--coverage-mode", action='append', type=str, dest="coverage_modes",
|
|
help="Collect and process coverage only for the modes specified. implies: --coverage, default: All built modes")
|
|
parser.addoption("--cluster-pool-size", type=int,
|
|
help="Set the pool_size for PythonTest and its descendants. Alternatively environment variable "
|
|
"CLUSTER_POOL_SIZE can be used to achieve the same")
|
|
parser.addoption("--extra-scylla-cmdline-options", default=[],
|
|
help="Passing extra scylla cmdline options for all tests. Options should be space separated:"
|
|
" '--logger-log-level raft=trace --default-log-level error'")
|
|
parser.addoption('--x-log2-compaction-groups', action="store", default="0", type=int,
|
|
help="Controls number of compaction groups to be used by Scylla tests. Value of 3 implies 8 groups.")
|
|
parser.addoption('--repeat', action="store", default="1", type=int,
|
|
help="number of times to repeat test execution")
|
|
|
|
# Pass information about Scylla node from test.py to pytest.
|
|
parser.addoption("--scylla-log-filename",
|
|
help="Path to a log file of a ScyllaDB node (for suites with type: Python)")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def build_mode(request: pytest.FixtureRequest) -> str:
|
|
"""
|
|
This fixture returns current build mode.
|
|
This is for running tests through the test.py script, where only one mode is passed to the test
|
|
"""
|
|
# to avoid issues when there's no provided mode parameter, do it in two steps: get the parameter and if it's not
|
|
# None, get the first value from the list
|
|
mode = request.config.getoption("modes")
|
|
if mode:
|
|
return mode[0]
|
|
return "unknown"
|
|
|
|
@pytest.fixture(scope="function")
|
|
def get_params(request: pytest.FixtureRequest) -> Generator[None]:
|
|
# this dummy fixture only needed to modify the test name with run id and mode. We don't want to parametrize with
|
|
# some parameters, so we are returning existing params that function is accepting. This method only needed for
|
|
# pytest_generate_tests method
|
|
return request.param
|
|
|
|
|
|
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
|
if TEST_RUNNER == "runpy":
|
|
return
|
|
repeat_count = metafunc.config.getoption("--repeat")
|
|
modes = get_modes_to_run(metafunc.config)
|
|
|
|
all_combinations = [*itertools.product(modes, range(repeat_count))]
|
|
|
|
metafunc.fixturenames.append('get_params')
|
|
metafunc.parametrize(
|
|
'get_params',
|
|
range(len(all_combinations)),
|
|
ids=[f"%{mode}.{run_id + 1}%" for mode, run_id in all_combinations],
|
|
indirect=True
|
|
)
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def print_scylla_log_filename(request: pytest.FixtureRequest) -> Generator[None]:
|
|
"""Print out a path to a ScyllaDB log.
|
|
|
|
This is a fixture for Python test suites, because they are using a single node clusters created inside test.py,
|
|
but it is handy to have this information printed to a pytest log.
|
|
"""
|
|
|
|
yield
|
|
|
|
if scylla_log_filename := request.config.getoption("--scylla-log-filename"):
|
|
logger.info("ScyllaDB log file: %s", scylla_log_filename)
|
|
|
|
|
|
def testpy_test_fixture_scope(fixture_name: str, config: pytest.Config) -> str:
|
|
"""Dynamic scope for fixtures which rely on a current test.py suite/test.
|
|
|
|
test.py runs tests file-by-file as separate pytest sessions, so, `session` scope is effectively close to be the
|
|
same as `module` (can be a difference in the order.) In case of running tests with bare pytest command, we
|
|
need to use `module` scope to maintain same behavior as test.py, since we run all tests in one pytest session.
|
|
"""
|
|
if config.getoption("--test-py-init"):
|
|
return "module"
|
|
return "session"
|
|
|
|
testpy_test_fixture_scope.__test__ = False
|
|
|
|
|
|
@pytest.fixture(scope=testpy_test_fixture_scope)
|
|
async def testpy_test(request: pytest.FixtureRequest, build_mode: str) -> Test | None:
|
|
"""Create an instance of Test class for the current test.py test."""
|
|
|
|
if request.scope == "module":
|
|
return await get_testpy_test(path=request.path, options=request.config.option, mode=build_mode)
|
|
return None
|
|
|
|
|
|
def pytest_configure(config: pytest.Config) -> None:
|
|
config.pluginmanager.register(ReportPlugin())
|
|
|
|
|
|
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item | CppTestFunction]) -> None:
|
|
"""
|
|
This is a standard pytest method.
|
|
This is needed to modify the test names with dev mode and run id to differ them one from another
|
|
"""
|
|
testpy_run_id = config.getoption('run_id', None)
|
|
|
|
def modify_test_name(s: str, testpy_run_id) -> str:
|
|
"""
|
|
Modify the test name to extract run id from parameterized test name to the end of the name.
|
|
Convert names like:
|
|
cluster/test_multidc.py::test_multidc[%dev.1%]
|
|
cluster/test_multidc.py::test_putget_2dc_with_rf[%release.1%-nodes_list0-1]
|
|
to:
|
|
cluster/test_multidc.py::test_multidc.dev.1
|
|
cluster/test_multidc.py::test_putget_2dc_with_rf[nodes_list0-1].release.1
|
|
"""
|
|
match = re.search(r'\[%([a-zA-Z_][\w]*)\.(\d+)%(-[^]]+)?]', s)
|
|
if not match:
|
|
return s
|
|
|
|
mode = match.group(1)
|
|
run_id = match.group(2)
|
|
suffix = match.group(3)
|
|
if suffix:
|
|
s = re.sub(r'\[%[a-zA-Z_][\w]*\.\d+%(-[^]]+)]', f'[{suffix[1:]}]', s)
|
|
else:
|
|
s = re.sub(r'\[%[a-zA-Z_][\w]*\.\d+%]', '', s)
|
|
return f"{s}.{mode}.{testpy_run_id}" if testpy_run_id else f"{s}.{mode}.{run_id}"
|
|
|
|
for item in items:
|
|
if not isinstance(item, CppTestFunction):
|
|
# pytest_generate_tests is not triggered for C++ tests, so they have their own logic for test name
|
|
# modification that handled in CppTestFunction class.
|
|
item._nodeid = modify_test_name(item._nodeid, testpy_run_id)
|
|
item.name = modify_test_name(item.name, testpy_run_id)
|
|
|
|
|
|
def pytest_sessionstart(session: pytest.Session) -> None:
|
|
# test.py starts S3 mock and create/cleanup testlog by itself. Also, if we run with --collect-only option,
|
|
# we don't need this stuff.
|
|
if TEST_RUNNER != "pytest" or session.config.getoption("--collect-only"):
|
|
return
|
|
|
|
if not session.config.getoption("--test-py-init"):
|
|
return
|
|
|
|
init_testsuite_globals()
|
|
TestSuite.artifacts.add_exit_artifact(None, TestSuite.hosts.cleanup)
|
|
|
|
# Run stuff just once for the pytest session even running under xdist.
|
|
if "xdist" not in sys.modules or not sys.modules["xdist"].is_xdist_worker(request_or_session=session):
|
|
temp_dir = Path(session.config.getoption("--tmpdir")).absolute()
|
|
prepare_dirs(tempdir_base=temp_dir, modes=session.config.getoption("--mode") or get_configured_modes(), gather_metrics=session.config.getoption("--gather-metrics"), save_log_on_success=session.config.getoption("save_log_on_success"),)
|
|
start_3rd_party_services(tempdir_base=temp_dir, toxiproxy_byte_limit=session.config.getoption('byte_limit'))
|
|
|
|
|
|
def pytest_sessionfinish() -> None:
|
|
if getattr(TestSuite, "artifacts", None) is not None:
|
|
asyncio.get_event_loop().run_until_complete(TestSuite.artifacts.cleanup_before_exit())
|