3cb5838619
This changes needed for later integration for pytest executing the C++ tests to be able to gather resource metric.
862 lines
41 KiB
Python
Executable File
862 lines
41 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2015-present ScyllaDB
|
|
#
|
|
#
|
|
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
|
|
#
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import collections
|
|
from random import randint
|
|
|
|
import colorama
|
|
import glob
|
|
import itertools
|
|
import logging
|
|
import multiprocessing
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from typing import TYPE_CHECKING
|
|
|
|
import humanfriendly
|
|
import treelib
|
|
|
|
from scripts import coverage
|
|
from test import ALL_MODES, TOP_SRC_DIR, path_to
|
|
from test.pylib import coverage_utils
|
|
from test.pylib.suite.base import (
|
|
Test,
|
|
TestSuite,
|
|
init_testsuite_globals,
|
|
output_is_a_tty,
|
|
palette,
|
|
prepare_dirs,
|
|
start_3rd_party_services,
|
|
)
|
|
from test.pylib.suite.boost import BoostTest
|
|
from test.pylib.resource_gather import setup_cgroup, run_resource_watcher
|
|
from test.pylib.util import LogPrefixAdapter, get_configured_modes, ninja
|
|
|
|
if TYPE_CHECKING:
|
|
from typing import List
|
|
|
|
|
|
launch_time = time.monotonic()
|
|
|
|
|
|
class TabularConsoleOutput:
|
|
"""Print test progress to the console"""
|
|
|
|
def __init__(self, verbose: bool, test_count: int) -> None:
|
|
self.verbose = verbose
|
|
self.test_count = test_count
|
|
self.last_test_no = 0
|
|
|
|
def print_start_blurb(self) -> None:
|
|
print("="*80)
|
|
print("{:10s} {:^8s} {:^7s} {:8s} {}".format("[N/TOTAL]", "SUITE", "MODE", "RESULT", "TEST"))
|
|
print("-"*78)
|
|
print("")
|
|
|
|
def print_end_blurb(self) -> None:
|
|
print("-"*78)
|
|
|
|
def print_progress(self, test: Test) -> None:
|
|
self.last_test_no += 1
|
|
status = ""
|
|
if test.success:
|
|
logging.debug("Test {} is flaky {}".format(test.uname,
|
|
test.is_flaky_failure))
|
|
if test.is_flaky_failure:
|
|
status = palette.warn("[ FLKY ]")
|
|
else:
|
|
status = palette.ok("[ PASS ]")
|
|
else:
|
|
status = palette.fail("[ FAIL ]")
|
|
msg = "{:10s} {:^8s} {:^7s} {:8s} {}".format(
|
|
f"[{self.last_test_no}/{self.test_count}]",
|
|
test.suite.name, test.mode[:7],
|
|
status,
|
|
test.uname
|
|
)
|
|
if not self.verbose:
|
|
if test.success:
|
|
print("\033[A", end="\r")
|
|
print("\033[K", end="\r")
|
|
print(msg)
|
|
else:
|
|
print("\033[A", end="\r")
|
|
print("\033[K", end="\r")
|
|
print(f"{msg}\n")
|
|
else:
|
|
msg += " {:.2f}s".format(test.time_end - test.time_start)
|
|
print(msg)
|
|
|
|
|
|
def setup_signal_handlers(loop, signaled) -> None:
|
|
|
|
async def shutdown(loop, signo, signaled):
|
|
print("\nShutdown requested... Aborting tests:"),
|
|
signaled.signo = signo
|
|
signaled.set()
|
|
|
|
# Use a lambda to avoid creating a coroutine until
|
|
# the signal is delivered to the loop - otherwise
|
|
# the coroutine will be dangling when the loop is over,
|
|
# since it's never going to be invoked
|
|
for signo in [signal.SIGINT, signal.SIGTERM]:
|
|
loop.add_signal_handler(signo, lambda: asyncio.create_task(shutdown(loop, signo, signaled)))
|
|
|
|
|
|
def parse_cmd_line() -> argparse.Namespace:
|
|
""" Print usage and process command line options. """
|
|
|
|
parser = argparse.ArgumentParser(description="Scylla test runner")
|
|
parser.add_argument(
|
|
"name",
|
|
nargs="*",
|
|
action="store",
|
|
help="""Can be empty. List of test names, to look for in
|
|
suites. Each name is used as a substring to look for in the
|
|
path to test file, e.g. "mem" will run all tests that have
|
|
"mem" in their name in all suites, "boost/mem" will only enable
|
|
tests starting with "mem" in "boost" suite, and
|
|
"boost/memtable_test::test_hash_is_cached" to narrow down to
|
|
a certain test case. Default: run all tests in all suites.""",
|
|
)
|
|
parser.add_argument("--tmpdir", action="store", default=str(TOP_SRC_DIR / "testlog"),
|
|
help="Path to temporary test data and log files. The data is further segregated per build mode.")
|
|
parser.add_argument("--gather-metrics", action=argparse.BooleanOptionalAction, default=True)
|
|
parser.add_argument("--max-failures", type=int, default=0,
|
|
help="Maximum number of failures to tolerate before cancelling rest of tests.")
|
|
parser.add_argument('--mode', choices=ALL_MODES, action="append", dest="modes",
|
|
help="Run only tests for given build mode(s)")
|
|
parser.add_argument('--repeat', action="store", default="1", type=int,
|
|
help="number of times to repeat test execution")
|
|
parser.add_argument('--timeout', action="store", default="24000", type=int,
|
|
help="timeout value for test execution")
|
|
parser.add_argument('--verbose', '-v', action='store_true', default=False,
|
|
help='Verbose reporting')
|
|
parser.add_argument('--jobs', '-j', action="store", type=int,
|
|
help="Number of jobs to use for running the tests")
|
|
parser.add_argument('--save-log-on-success', "-s", default=False,
|
|
dest="save_log_on_success", action="store_true",
|
|
help="Save test log output on success.")
|
|
parser.add_argument('--list', dest="list_tests", action="store_true", default=False,
|
|
help="Print list of tests instead of executing them")
|
|
parser.add_argument('--skip',
|
|
dest="skip_patterns", action="append",
|
|
help="Skip tests which match the provided pattern")
|
|
parser.add_argument('--no-parallel-cases', dest="parallel_cases", action="store_false", default=True,
|
|
help="Do not run individual test cases in parallel")
|
|
parser.add_argument('--cpus', action="store",
|
|
help="Run the tests on those CPUs only (in taskset"
|
|
" acceptable format). Consider using --jobs too")
|
|
parser.add_argument('--log-level', action="store",
|
|
help="Log level for Python logging module. The log "
|
|
"is in {tmpdir}/test.py.log. Default: INFO",
|
|
default="INFO",
|
|
choices=["CRITICAL", "ERROR", "WARNING", "INFO",
|
|
"DEBUG"],
|
|
dest="log_level")
|
|
parser.add_argument('--markers', action='store', metavar='MARKEXPR',
|
|
help="Only run tests that match the given mark expression. The syntax is the same "
|
|
"as in pytest, for example: --markers 'mark1 and not mark2'. The parameter "
|
|
"is only supported by python tests for now, other tests ignore it. "
|
|
"By default, the marker filter is not applied and all tests will be run without exception."
|
|
"To exclude e.g. slow tests you can write --markers 'not slow'.")
|
|
parser.add_argument('--coverage', action = 'store_true', default = False,
|
|
help="When running code instrumented with coverage support"
|
|
"Will route the profiles to `tmpdir`/mode/coverage/`suite` and post process them in order to generate "
|
|
"lcov file per suite, lcov file per mode, and an lcov file for the entire run, "
|
|
"The lcov files can eventually be used for generating coverage reports")
|
|
parser.add_argument("--coverage-mode",action = 'append', type = str, dest = "coverage_modes",
|
|
help = "Collect and process coverage only for the modes specified. implies: --coverage, default: All built modes")
|
|
parser.add_argument("--coverage-keep-raw",action = 'store_true',
|
|
help = "Do not delete llvm raw profiles when processing coverage reports.")
|
|
parser.add_argument("--coverage-keep-indexed",action = 'store_true',
|
|
help = "Do not delete llvm indexed profiles when processing coverage reports.")
|
|
parser.add_argument("--coverage-keep-lcovs",action = 'store_true',
|
|
help = "Do not delete intermediate lcov traces when processing coverage reports.")
|
|
parser.add_argument("--artifacts_dir_url", action='store', type=str, default=None, dest="artifacts_dir_url",
|
|
help="Provide the URL to artifacts directory to generate the link to failed tests directory "
|
|
"with logs")
|
|
parser.add_argument("--cluster-pool-size", action="store", default=None, type=int,
|
|
help="Set the pool_size for PythonTest and its descendants. Alternatively environment variable "
|
|
"CLUSTER_POOL_SIZE can be used to achieve the same")
|
|
parser.add_argument('--manual-execution', action='store_true', default=False,
|
|
help='Let me manually run the test executable at the moment this script would run it')
|
|
parser.add_argument('--byte-limit', action="store", default=randint(0, 2000), type=int,
|
|
help="Specific byte limit for failure injection (random by default)")
|
|
scylla_additional_options = parser.add_argument_group('Additional options for Scylla tests')
|
|
scylla_additional_options.add_argument('--x-log2-compaction-groups', action="store", default="0", type=int,
|
|
help="Controls number of compaction groups to be used by Scylla tests. Value of 3 implies 8 groups.")
|
|
scylla_additional_options.add_argument('--extra-scylla-cmdline-options', action="store", default=[], type=str,
|
|
help="Passing extra scylla cmdline options for all tests. Options should be space separated:"
|
|
"'--logger-log-level raft=trace --default-log-level error'")
|
|
|
|
boost_group = parser.add_argument_group('boost suite options')
|
|
boost_group.add_argument('--random-seed', action="store",
|
|
help="Random number generator seed to be used by boost tests")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not args.jobs:
|
|
if not args.cpus:
|
|
nr_cpus = multiprocessing.cpu_count()
|
|
else:
|
|
nr_cpus = int(subprocess.check_output(
|
|
['taskset', '-c', args.cpus, 'python3', '-c',
|
|
'import os; print(len(os.sched_getaffinity(0)))']))
|
|
|
|
cpus_per_test_job = 1
|
|
sysmem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
|
testmem = 6e9 if os.sysconf('SC_PAGE_SIZE') > 4096 else 2e9
|
|
default_num_jobs_mem = ((sysmem - 4e9) // testmem)
|
|
args.jobs = min(default_num_jobs_mem, nr_cpus // cpus_per_test_job)
|
|
|
|
if not output_is_a_tty:
|
|
args.verbose = True
|
|
|
|
if not args.modes:
|
|
try:
|
|
args.modes = get_configured_modes()
|
|
except Exception:
|
|
print(palette.fail("Failed to read output of `ninja mode_list`: please run ./configure.py first"))
|
|
raise
|
|
|
|
if not args.coverage_modes and args.coverage:
|
|
args.coverage_modes = list(args.modes)
|
|
if "coverage" in args.coverage_modes:
|
|
args.coverage_modes.remove("coverage")
|
|
if not args.coverage_modes:
|
|
args.coverage = False
|
|
elif args.coverage_modes:
|
|
if "coverage" in args.coverage_modes:
|
|
raise RuntimeError("'coverage' mode is not allowed in --coverage-mode")
|
|
missing_coverage_modes = set(args.coverage_modes).difference(set(args.modes))
|
|
if len(missing_coverage_modes) > 0:
|
|
raise RuntimeError(f"The following modes weren't built or ran (using the '--mode' option): {missing_coverage_modes}")
|
|
args.coverage = True
|
|
|
|
args.tmpdir = os.path.abspath(args.tmpdir)
|
|
prepare_dirs(tempdir_base=pathlib.Path(args.tmpdir), modes=args.modes, gather_metrics=args.gather_metrics)
|
|
|
|
# Get the list of tests configured by configure.py
|
|
try:
|
|
out = ninja('unit_test_list')
|
|
# [1/1] List configured unit tests
|
|
args.tests = set(re.sub(r'.* List configured unit tests\n(.*)\n', r'\1', out, count=1, flags=re.DOTALL).split("\n"))
|
|
except Exception:
|
|
print(palette.fail("Failed to read output of `ninja unit_test_list`: please run ./configure.py first"))
|
|
raise
|
|
|
|
if args.extra_scylla_cmdline_options:
|
|
args.extra_scylla_cmdline_options = args.extra_scylla_cmdline_options.split()
|
|
|
|
return args
|
|
|
|
|
|
async def find_tests(options: argparse.Namespace) -> None:
|
|
|
|
for f in glob.glob(os.path.join("test", "*")):
|
|
if os.path.isdir(f) and os.path.isfile(os.path.join(f, "suite.yaml")):
|
|
for mode in options.modes:
|
|
suite = TestSuite.opt_create(f, options, mode)
|
|
await suite.add_test_list()
|
|
|
|
if not TestSuite.test_count():
|
|
if len(options.name):
|
|
print("Test {} not found".format(palette.path(options.name[0])))
|
|
sys.exit(1)
|
|
else:
|
|
print(palette.warn("No tests found. Please enable tests in ./configure.py first."))
|
|
sys.exit(0)
|
|
|
|
logging.info("Found %d tests, repeat count is %d, starting %d concurrent jobs",
|
|
TestSuite.test_count(), options.repeat, options.jobs)
|
|
print("Found {} tests.".format(TestSuite.test_count()))
|
|
|
|
|
|
async def run_all_tests(signaled: asyncio.Event, options: argparse.Namespace) -> None:
|
|
console = TabularConsoleOutput(options.verbose, TestSuite.test_count())
|
|
signaled_task = asyncio.create_task(signaled.wait())
|
|
pending = {signaled_task}
|
|
|
|
async def cancel(pending, msg):
|
|
for task in pending:
|
|
task.cancel(msg)
|
|
await asyncio.wait(pending, return_when=asyncio.ALL_COMPLETED)
|
|
print("...done")
|
|
|
|
async def reap(done, pending, signaled):
|
|
nonlocal console
|
|
if signaled.is_set():
|
|
await cancel(pending, "Signal received")
|
|
failed = 0
|
|
for coro in done:
|
|
result = coro.result()
|
|
if isinstance(result, bool):
|
|
continue # skip signaled task result
|
|
if not result.success:
|
|
failed += 1
|
|
if not result.did_not_run:
|
|
console.print_progress(result)
|
|
return failed
|
|
|
|
await start_3rd_party_services(tempdir_base=pathlib.Path(options.tmpdir), toxiproxy_byte_limit=options.byte_limit)
|
|
|
|
console.print_start_blurb()
|
|
max_failures = options.max_failures
|
|
failed = 0
|
|
try:
|
|
TestSuite.artifacts.add_exit_artifact(None, TestSuite.hosts.cleanup)
|
|
for test in TestSuite.all_tests():
|
|
# +1 for 'signaled' event
|
|
if len(pending) > options.jobs:
|
|
# Wait for some task to finish
|
|
done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
|
|
failed += await reap(done, pending, signaled)
|
|
if max_failures != 0 and max_failures <= failed:
|
|
print("Too much failures, stopping")
|
|
await cancel(pending, "Too much failures, stopping")
|
|
pending.add(asyncio.create_task(test.suite.run(test, options)))
|
|
# Wait & reap ALL tasks but signaled_task
|
|
# Do not use asyncio.ALL_COMPLETED to print a nice progress report
|
|
while len(pending) > 1:
|
|
done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
|
|
failed += await reap(done, pending, signaled)
|
|
if max_failures != 0 and max_failures <= failed:
|
|
print("Too much failures, stopping")
|
|
await cancel(pending, "Too much failures, stopping")
|
|
except asyncio.CancelledError:
|
|
return
|
|
finally:
|
|
await TestSuite.artifacts.cleanup_before_exit()
|
|
|
|
console.print_end_blurb()
|
|
|
|
|
|
def print_summary(failed_tests: List["Test"], cancelled_tests: int, options: argparse.Namespace) -> None:
|
|
rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
cpu_used = rusage.ru_stime + rusage.ru_utime
|
|
cpu_available = (time.monotonic() - launch_time) * multiprocessing.cpu_count()
|
|
utilization = cpu_used / cpu_available
|
|
print(f"CPU utilization: {utilization*100:.1f}%")
|
|
if failed_tests:
|
|
print("The following test(s) have failed: {}".format(
|
|
palette.path(" ".join([t.name for t in failed_tests]))))
|
|
if options.verbose:
|
|
for test in failed_tests:
|
|
test.print_summary()
|
|
print("-"*78)
|
|
if cancelled_tests > 0:
|
|
print(f"Summary: {len(failed_tests)} of the total {TestSuite.test_count()} tests failed, {cancelled_tests} cancelled")
|
|
else:
|
|
print(f"Summary: {len(failed_tests)} of the total {TestSuite.test_count()} tests failed")
|
|
|
|
|
|
def summarize_boost_tests(tests):
|
|
# in case we run a certain test multiple times
|
|
# - if any of the runs failed, the test is considered failed, and
|
|
# the last failed run is returned.
|
|
# - otherwise, the last successful run is returned
|
|
failed_test = None
|
|
passed_test = None
|
|
num_failed_tests = collections.defaultdict(int)
|
|
num_passed_tests = collections.defaultdict(int)
|
|
for test in tests:
|
|
error = None
|
|
for tag in ['Error', 'FatalError', 'Exception']:
|
|
error = test.find(tag)
|
|
if error is not None:
|
|
break
|
|
mode = test.attrib['mode']
|
|
if error is None:
|
|
passed_test = test
|
|
num_passed_tests[mode] += 1
|
|
else:
|
|
failed_test = test
|
|
num_failed_tests[mode] += 1
|
|
|
|
if failed_test is not None:
|
|
test = failed_test
|
|
else:
|
|
test = passed_test
|
|
|
|
num_failed = sum(num_failed_tests.values())
|
|
num_passed = sum(num_passed_tests.values())
|
|
num_total = num_failed + num_passed
|
|
if num_total == 1:
|
|
return test
|
|
if num_failed == 0:
|
|
return test
|
|
# we repeated this test for multiple times.
|
|
#
|
|
# Boost::test's XML logger schema does not allow us to put text directly in a
|
|
# TestCase tag, so create a dummy Message tag in the TestCase for carrying the
|
|
# summary. and the schema requires that the tags should be listed in following order:
|
|
# 1. TestSuite
|
|
# 2. Info
|
|
# 3. Error
|
|
# 3. FatalError
|
|
# 4. Message
|
|
# 5. Exception
|
|
# 6. Warning
|
|
# and both "file" and "line" are required in an "Info" tag, so appease it. assuming
|
|
# there is no TestSuite under tag TestCase, we always add Info as the first subelements
|
|
if num_passed == 0:
|
|
message = ET.Element('Info', file=test.attrib['file'], line=test.attrib['line'])
|
|
message.text = f'The test failed {num_failed}/{num_total} times'
|
|
test.insert(0, message)
|
|
else:
|
|
message = ET.Element('Info', file=test.attrib['file'], line=test.attrib['line'])
|
|
modes = ', '.join(f'{mode}={n}' for mode, n in num_failed_tests.items())
|
|
message.text = f'failed: {modes}'
|
|
test.insert(0, message)
|
|
|
|
message = ET.Element('Info', file=test.attrib['file'], line=test.attrib['line'])
|
|
modes = ', '.join(f'{mode}={n}' for mode, n in num_passed_tests.items())
|
|
message.text = f'passed: {modes}'
|
|
test.insert(0, message)
|
|
|
|
message = ET.Element('Info', file=test.attrib['file'], line=test.attrib['line'])
|
|
message.text = f'{num_failed} out of {num_total} times failed.'
|
|
test.insert(0, message)
|
|
return test
|
|
|
|
|
|
def write_consolidated_boost_junit_xml(tmpdir: str, mode: str) -> str:
|
|
# collects all boost tests sorted by their full names
|
|
boost_tests = itertools.chain.from_iterable(suite.boost_tests()
|
|
for suite in TestSuite.suites.values())
|
|
test_cases = itertools.chain.from_iterable(test.get_test_cases()
|
|
for test in boost_tests)
|
|
test_cases = sorted(test_cases, key=BoostTest.test_path_of_element)
|
|
|
|
xml = ET.Element("TestLog")
|
|
for full_path, tests in itertools.groupby(
|
|
test_cases,
|
|
key=BoostTest.test_path_of_element):
|
|
# dedup the tests with the same name, so only the representative one is
|
|
# preserved
|
|
test_case = summarize_boost_tests(tests)
|
|
test_case.attrib.pop('path')
|
|
test_case.attrib.pop('mode')
|
|
|
|
suite_name, test_name, _ = full_path
|
|
suite = xml.find(f"./TestSuite[@name='{suite_name}']")
|
|
if suite is None:
|
|
suite = ET.SubElement(xml, 'TestSuite', name=suite_name)
|
|
test = suite.find(f"./TestSuite[@name='{test_name}']")
|
|
if test is None:
|
|
test = ET.SubElement(suite, 'TestSuite', name=test_name)
|
|
test.append(test_case)
|
|
et = ET.ElementTree(xml)
|
|
xunit_file = f'{tmpdir}/{mode}/xml/boost.xunit.xml'
|
|
et.write(xunit_file, encoding='unicode')
|
|
return xunit_file
|
|
|
|
|
|
def boost_to_junit(boost_xml, junit_xml):
|
|
boost_root = ET.parse(boost_xml).getroot()
|
|
junit_root = ET.Element('testsuites')
|
|
|
|
def parse_tag_output(test_case_element: ET.Element, tag_output: str) -> str:
|
|
text_template = '''
|
|
[{level}] - {level_message}
|
|
[FILE] - {file_name}
|
|
[LINE] - {line_number}
|
|
'''
|
|
text = ''
|
|
tag_outputs = test_case_element.findall(tag_output)
|
|
if tag_outputs:
|
|
for tag_output in tag_outputs:
|
|
text += text_template.format(level='Info', level_message=tag_output.text,
|
|
file_name=tag_output.get('file'), line_number=tag_output.get('line'))
|
|
return text
|
|
|
|
# report produced {write_consolidated_boost_junit_xml} have the nested structure suite_boost -> [suite1, suite2, ...]
|
|
# so we are excluding the upper suite with name boost
|
|
for test_suite in boost_root.findall('./TestSuite/TestSuite'):
|
|
suite_time = 0.0
|
|
suite_test_total = 0
|
|
suite_test_fails_number = 0
|
|
|
|
junit_test_suite = ET.SubElement(junit_root, 'testsuite')
|
|
junit_test_suite.attrib['name'] = test_suite.attrib['name']
|
|
|
|
test_cases = test_suite.findall('TestCase')
|
|
for test_case in test_cases:
|
|
# convert the testing time: boost uses microseconds and Junit uses seconds
|
|
test_case_time = int(test_case.find('TestingTime').text) / 1_000_000
|
|
suite_time += test_case_time
|
|
suite_test_total += 1
|
|
|
|
junit_test_case = ET.SubElement(junit_test_suite, 'testcase')
|
|
junit_test_case.set('name', test_case.get('name'))
|
|
junit_test_case.set('time', str(test_case_time))
|
|
junit_test_case.set('file', test_case.get('file'))
|
|
junit_test_case.set('line', test_case.get('line'))
|
|
|
|
system_out = ET.SubElement(junit_test_case, 'system-out')
|
|
system_out.text = ''
|
|
for tag in ['Info', 'Message', 'Exception']:
|
|
output = parse_tag_output(test_case, tag)
|
|
if output:
|
|
system_out.text += output
|
|
|
|
junit_test_suite.set('tests', str(suite_test_total))
|
|
junit_test_suite.set('time', str(suite_time))
|
|
junit_test_suite.set('failures', str(suite_test_fails_number))
|
|
ET.ElementTree(junit_root).write(junit_xml, encoding='UTF-8')
|
|
|
|
|
|
def open_log(tmpdir: str, log_file_name: str, log_level: str) -> None:
|
|
pathlib.Path(tmpdir).mkdir(parents=True, exist_ok=True)
|
|
logging.basicConfig(
|
|
filename=os.path.join(tmpdir, log_file_name),
|
|
filemode="w",
|
|
level=log_level,
|
|
format="%(asctime)s.%(msecs)03d %(levelname)s> %(message)s",
|
|
datefmt="%H:%M:%S",
|
|
)
|
|
logging.critical("Started %s", " ".join(sys.argv))
|
|
|
|
|
|
async def main() -> int:
|
|
|
|
options = parse_cmd_line()
|
|
|
|
open_log(options.tmpdir, f"test.py.{'-'.join(options.modes)}.log", options.log_level)
|
|
|
|
init_testsuite_globals()
|
|
|
|
await find_tests(options)
|
|
if options.list_tests:
|
|
print('\n'.join([f"{t.suite.mode:<8} {type(t.suite).__name__[:-9]:<11} {t.name}"
|
|
for t in TestSuite.all_tests()]))
|
|
return 0
|
|
|
|
if options.manual_execution and TestSuite.test_count() > 1:
|
|
print('--manual-execution only supports running a single test, but multiple selected: {}'.format(
|
|
[t.path for t in TestSuite.tests()][:3])) # Print whole t.path; same shortname may be in different dirs.
|
|
return 1
|
|
|
|
signaled = asyncio.Event()
|
|
stop_event = asyncio.Event()
|
|
resource_watcher = run_resource_watcher(options.gather_metrics, signaled, stop_event, options.tmpdir)
|
|
|
|
setup_signal_handlers(asyncio.get_running_loop(), signaled)
|
|
|
|
try:
|
|
logging.info('running all tests')
|
|
await run_all_tests(signaled, options)
|
|
logging.info('after running all tests')
|
|
stop_event.set()
|
|
async with asyncio.timeout(5):
|
|
await resource_watcher
|
|
except Exception as e:
|
|
print(palette.fail(e))
|
|
raise
|
|
|
|
if signaled.is_set():
|
|
return -signaled.signo # type: ignore
|
|
|
|
failed_tests = [test for test in TestSuite.all_tests() if test.failed]
|
|
cancelled_tests = sum(1 for test in TestSuite.all_tests() if test.did_not_run)
|
|
|
|
print_summary(failed_tests, cancelled_tests, options)
|
|
|
|
for mode in options.modes:
|
|
junit_file = f"{options.tmpdir}/{mode}/allure/boost.junit.xml"
|
|
xunit_file = write_consolidated_boost_junit_xml(options.tmpdir, mode)
|
|
boost_to_junit(xunit_file, junit_file)
|
|
|
|
if 'coverage' in options.modes:
|
|
coverage.generate_coverage_report(path_to("coverage", "tests"))
|
|
|
|
if options.coverage:
|
|
await process_coverage(options)
|
|
|
|
# Note: failure codes must be in the ranges 0-124, 126-127,
|
|
# to cooperate with git bisect's expectations
|
|
return 0 if not failed_tests else 1
|
|
|
|
|
|
async def process_coverage(options):
|
|
total_processing_time = time.time()
|
|
logger = LogPrefixAdapter(logging.getLogger("coverage"), {'prefix' : 'coverage'})
|
|
modes_for_coverage = options.coverage_modes
|
|
# use about 75% of the machine's processing power.
|
|
concurrency = max(int(multiprocessing.cpu_count() * 0.75), 1)
|
|
logger.info(f"Processing coverage information for modes: {modes_for_coverage}, using {concurrency} cpus")
|
|
semaphore = asyncio.Semaphore(concurrency)
|
|
build_paths = [pathlib.Path(f"build/{mode}") for mode in modes_for_coverage]
|
|
paths_for_id_search = [bp / p for bp, p in itertools.product(build_paths, ["scylla", "test", "seastar"])]
|
|
logger.info("Getting binary ids for coverage conversion...")
|
|
files_to_ids_map = await coverage_utils.get_binary_ids_map(paths = paths_for_id_search,
|
|
filter = coverage_utils.PROFILED_ELF_TYPES,
|
|
semaphore = semaphore,
|
|
logger = logger)
|
|
logger.debug(f"Binary ids map is: {files_to_ids_map}")
|
|
logger.info("Done getting binary ids for coverage conversion")
|
|
# get the suits that have actually been ran
|
|
suits_to_exclude = ["pylib_test", "nodetool"]
|
|
sources_to_exclude = [line for line in open("coverage_excludes.txt", 'r').read().split('\n') if line and not line.startswith('#')]
|
|
ran_suites = list({test.suite for test in TestSuite.all_tests() if test.suite.need_coverage()})
|
|
|
|
def suite_coverage_path(suite) -> pathlib.Path:
|
|
return pathlib.Path(suite.options.tmpdir) / suite.mode / 'coverage' / suite.name
|
|
|
|
def pathsize(path : pathlib.Path):
|
|
if path.is_file():
|
|
return os.path.getsize(path)
|
|
elif path.is_dir():
|
|
return sum([os.path.getsize(f) for f in path.glob("**/*") if f.is_file()])
|
|
else:
|
|
return 0
|
|
class Stats:
|
|
def __init__(self, name = "", size = 0, time = 0) -> None:
|
|
self.name = name
|
|
self.size = size
|
|
self.time = time
|
|
def __add__(self, other):
|
|
return Stats(self.name,
|
|
size = self.size + other.size,
|
|
time = self.time + other.time)
|
|
def __str__(self):
|
|
name = f"{self.name} - " if self.name else ""
|
|
fields = []
|
|
if self.size:
|
|
fields.append(f"size: {humanfriendly.format_size(self.size)}")
|
|
if self.time:
|
|
fields.append(f"time: {humanfriendly.format_timespan(self.time)}")
|
|
fields = ', '.join(fields)
|
|
return f"{name}{fields}"
|
|
@property
|
|
def asstring(self):
|
|
return str(self)
|
|
|
|
# a nested map of: mode -> suite -> unified_coverage_file
|
|
suits_trace_files = {}
|
|
stats = treelib.Tree()
|
|
|
|
RAW_PROFILE_STATS = "raw profiles"
|
|
INDEXED_PROFILE_STATS = "indexed profiles"
|
|
LCOV_CONVERSION_STATS = "lcov conversion"
|
|
LCOV_SUITES_MEREGE_STATS = "lcov per suite merge"
|
|
LCOV_MODES_MERGE_STATS = "lcov merge for mode"
|
|
LCOV_MERGE_ALL_STATS = "lcov merge all stats"
|
|
ROOT_NODE = stats.create_node(tag = time.time(),
|
|
identifier = "root",
|
|
data = Stats("Coverage Processing Stats", 0, 0))
|
|
|
|
for suite in list(ran_suites):
|
|
coverage_path = suite_coverage_path(suite)
|
|
if not coverage_path.exists():
|
|
logger.warning(f"Coverage dir for suite '{suite.name}' in mode '{suite.mode}' wasn't found, common reasons:\n\t"
|
|
"1. The suite doesn't use any instrumented binaries.\n\t"
|
|
"2. The binaries weren't compiled with coverage instrumentation.")
|
|
continue
|
|
|
|
# 1. Transform every suite raw profiles into indexed profiles
|
|
raw_profiles = list(coverage_path.glob("*.profraw"))
|
|
if len(raw_profiles) == 0:
|
|
logger.warning(f"Couldn't find any raw profiles for suite '{suite.name}' in mode '{suite.mode}' ({coverage_path}):\n\t"
|
|
"1. The binaries are killed instead of terminating which bypasses profile dump.\n\t"
|
|
"2. The suite tempres with the LLVM_PROFILE_FILE which causes the profile to be dumped\n\t"
|
|
" to somewhere else.")
|
|
continue
|
|
mode_stats = stats.get_node(suite.mode)
|
|
if not mode_stats:
|
|
mode_stats = stats.create_node(tag = time.time(),
|
|
identifier = suite.mode,
|
|
parent = ROOT_NODE,
|
|
data = Stats(f"{suite.mode} mode processing stats", 0, 0))
|
|
|
|
raw_stats_node = stats.get_node(mode_stats.identifier + RAW_PROFILE_STATS)
|
|
if not raw_stats_node:
|
|
raw_stats_node = stats.create_node(tag = time.time(),
|
|
identifier = mode_stats.identifier + RAW_PROFILE_STATS,
|
|
parent = mode_stats,
|
|
data = Stats(RAW_PROFILE_STATS, 0, 0))
|
|
stat = stats.create_node(tag = time.time(),
|
|
identifier = raw_stats_node.identifier + suite.name,
|
|
parent = raw_stats_node,
|
|
data = Stats(suite.name, pathsize(coverage_path), 0))
|
|
raw_stats_node.data += stat.data
|
|
mode_stats.data.time += stat.data.time
|
|
mode_stats.data.size = max(mode_stats.data.size, raw_stats_node.data.size)
|
|
|
|
|
|
logger.info(f"{suite.name}: Converting raw profiles into indexed profiles - {stat.data}.")
|
|
start_time = time.time()
|
|
merge_result = await coverage_utils.merge_profiles(profiles = raw_profiles,
|
|
path_for_merged = coverage_path,
|
|
clear_on_success = (not options.coverage_keep_raw),
|
|
semaphore = semaphore,
|
|
logger = logger)
|
|
indexed_stats_node = stats.get_node(mode_stats.identifier +INDEXED_PROFILE_STATS)
|
|
if not indexed_stats_node:
|
|
indexed_stats_node = stats.create_node(tag = time.time(),
|
|
identifier = mode_stats.identifier +INDEXED_PROFILE_STATS,
|
|
parent = mode_stats,
|
|
data = Stats(INDEXED_PROFILE_STATS, 0, 0))
|
|
stat = stats.create_node(tag = time.time(),
|
|
identifier = indexed_stats_node.identifier + suite.name,
|
|
parent = indexed_stats_node,
|
|
data = Stats(suite.name, pathsize(coverage_path), time.time() - start_time))
|
|
indexed_stats_node.data += stat.data
|
|
mode_stats.data.time += stat.data.time
|
|
mode_stats.data.size = max(mode_stats.data.size, indexed_stats_node.data.size)
|
|
|
|
logger.info(f"{suite.name}: Done converting raw profiles into indexed profiles - {humanfriendly.format_timespan(stat.data.time)}.")
|
|
|
|
# 2. Transform every indexed profile into an lcov trace file,
|
|
# after this step, the dependency upon the build artifacts
|
|
# ends and processing of the files can be done using the source
|
|
# code only.
|
|
|
|
logger.info(f"{suite.name}: Converting indexed profiles into lcov trace files.")
|
|
start_time = time.time()
|
|
if len(merge_result.errors) > 0:
|
|
raise RuntimeError(merge_result.errors)
|
|
await coverage_utils.profdata_to_lcov(profiles = merge_result.generated_profiles,
|
|
excludes = sources_to_exclude,
|
|
known_file_ids = files_to_ids_map,
|
|
clear_on_success = (not options.coverage_keep_indexed),
|
|
semaphore = semaphore,
|
|
logger = logger
|
|
)
|
|
lcov_conversion_stats_node = stats.get_node(mode_stats.identifier + LCOV_CONVERSION_STATS)
|
|
if not lcov_conversion_stats_node:
|
|
lcov_conversion_stats_node = stats.create_node(tag = time.time(),
|
|
identifier = mode_stats.identifier + LCOV_CONVERSION_STATS,
|
|
parent = mode_stats,
|
|
data = Stats(LCOV_CONVERSION_STATS, 0, 0))
|
|
stat = stats.create_node(tag = time.time(),
|
|
identifier = lcov_conversion_stats_node.identifier + suite.name,
|
|
parent = lcov_conversion_stats_node,
|
|
data = Stats(suite.name, pathsize(coverage_path), time.time() - start_time))
|
|
lcov_conversion_stats_node.data += stat.data
|
|
mode_stats.data.time += stat.data.time
|
|
mode_stats.data.size = max(mode_stats.data.size, lcov_conversion_stats_node.data.size)
|
|
|
|
logger.info(f"{suite.name}: Done converting indexed profiles into lcov trace files - {humanfriendly.format_timespan(stat.data.time)}.")
|
|
|
|
# 3. combine all tracefiles
|
|
logger.info(f"{suite.name} in mode {suite.mode}: Combinig lcov trace files.")
|
|
start_time = time.time()
|
|
trace_files = list(coverage_path.glob("**/*.info"))
|
|
target_trace_file = coverage_path / (suite.name + ".info")
|
|
if len(trace_files) == 0: # No coverage data, can skip
|
|
logger.warning(f"{suite.name} in mode {suite.mode}: No coverage tracefiles found")
|
|
elif len(trace_files) == 1: # No need to merge, we can just rename the file
|
|
trace_files[0].rename(str(target_trace_file))
|
|
else:
|
|
await coverage_utils.lcov_combine_traces(lcovs = trace_files,
|
|
output_lcov = target_trace_file,
|
|
clear_on_success = (not options.coverage_keep_lcovs),
|
|
files_per_chunk = 10,
|
|
semaphore = semaphore,
|
|
logger = logger)
|
|
lcov_merge_stats_node = stats.get_node(mode_stats.identifier + LCOV_SUITES_MEREGE_STATS)
|
|
if not lcov_merge_stats_node:
|
|
lcov_merge_stats_node = stats.create_node(tag = time.time(),
|
|
identifier = mode_stats.identifier + LCOV_SUITES_MEREGE_STATS,
|
|
parent = mode_stats,
|
|
data = Stats(LCOV_SUITES_MEREGE_STATS, 0, 0))
|
|
stat = stats.create_node(tag = time.time(),
|
|
identifier = lcov_merge_stats_node.identifier + suite.name,
|
|
parent = lcov_merge_stats_node,
|
|
data = Stats(suite.name, pathsize(coverage_path), time.time() - start_time))
|
|
lcov_merge_stats_node.data += stat.data
|
|
mode_stats.data.time += stat.data.time
|
|
mode_stats.data.size = max(mode_stats.data.size, lcov_merge_stats_node.data.size)
|
|
|
|
suits_trace_files.setdefault(suite.mode, {})[suite.name] = target_trace_file
|
|
logger.info(f"{suite.name}: Done combinig lcov trace files - {humanfriendly.format_timespan(stat.data.time)}")
|
|
|
|
#4. combine the suite lcovs into per mode trace files
|
|
modes_trace_files = {}
|
|
for mode, suite_traces in suits_trace_files.items():
|
|
|
|
target_trace_file = pathlib.Path(options.tmpdir) / mode / "coverage" / f"{mode}_coverage.info"
|
|
start_time = time.time()
|
|
logger.info(f"Consolidating trace files for mode {mode}.")
|
|
await coverage_utils.lcov_combine_traces(lcovs = suite_traces.values(),
|
|
output_lcov = target_trace_file,
|
|
clear_on_success = False,
|
|
files_per_chunk = 10,
|
|
semaphore = semaphore,
|
|
logger = logger)
|
|
mode_stats = stats[mode]
|
|
stat = stats.create_node(tag = time.time(),
|
|
identifier = mode_stats.identifier + LCOV_MODES_MERGE_STATS,
|
|
parent = mode_stats,
|
|
data = Stats(LCOV_MODES_MERGE_STATS, None, time.time() - start_time))
|
|
mode_stats.data.time += stat.data.time
|
|
ROOT_NODE.data.size += mode_stats.data.size
|
|
modes_trace_files[mode] = target_trace_file
|
|
logger.info(f"Done consolidating trace files for mode {mode} - time: {humanfriendly.format_timespan(stat.data.time)}.")
|
|
#5. create one consolidated file with all trace information
|
|
logger.info(f"Consolidating all trace files for this run.")
|
|
start_time = time.time()
|
|
target_trace_file = pathlib.Path(options.tmpdir) / "test_coverage.info"
|
|
await coverage_utils.lcov_combine_traces(lcovs = modes_trace_files.values(),
|
|
output_lcov = target_trace_file,
|
|
clear_on_success = False,
|
|
files_per_chunk = 10,
|
|
semaphore = semaphore,
|
|
logger = logger)
|
|
stats.create_node(tag = time.time(),
|
|
identifier = LCOV_MERGE_ALL_STATS,
|
|
parent = ROOT_NODE,
|
|
data = Stats(LCOV_MERGE_ALL_STATS, None, time.time() - start_time))
|
|
logger.info(f"Done consolidating all trace files for this run - time: {humanfriendly.format_timespan(time.time() - start_time)}.")
|
|
|
|
logger.info(f"Creating textual report.")
|
|
proc = await asyncio.create_subprocess_shell(f"lcov --summary --rc lcov_branch_coverage=1 {options.tmpdir}/test_coverage.info 2>/dev/null > {options.tmpdir}/test_coverage_report.txt")
|
|
await proc.wait()
|
|
with open(pathlib.Path(options.tmpdir) /"test_coverage_report.txt") as f:
|
|
summary = f.readlines()
|
|
proc = await asyncio.create_subprocess_shell(f"lcov --list --rc lcov_branch_coverage=1 {options.tmpdir}/test_coverage.info 2>/dev/null >> {options.tmpdir}/test_coverage_report.txt")
|
|
await proc.wait()
|
|
logger.info(f"Done creating textual report. ({options.tmpdir}/test_coverage_report.txt)")
|
|
total_processing_time = time.time() - total_processing_time
|
|
ROOT_NODE.data.time = total_processing_time
|
|
|
|
stats_str ="\n" + stats.show(stdout=False,
|
|
data_property="asstring")
|
|
summary = ["\n" + l for l in summary]
|
|
logger.info(stats_str)
|
|
logger.info("".join(summary))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
colorama.init()
|
|
# gh-16583: ignore the inherited client host's ScyllaDB environment,
|
|
# since it may break the tests
|
|
if "SCYLLA_CONF" in os.environ:
|
|
del os.environ["SCYLLA_CONF"]
|
|
if "SCYLLA_HOME" in os.environ:
|
|
del os.environ["SCYLLA_HOME"]
|
|
|
|
if sys.version_info < (3, 11):
|
|
print("Python 3.11 or newer is required to run this program")
|
|
sys.exit(-1)
|
|
sys.exit(asyncio.run(main()))
|