diff --git a/Makefile b/Makefile index 83fb0152b..9616f4110 100644 --- a/Makefile +++ b/Makefile @@ -150,13 +150,13 @@ help: 'KUnit tests (see doc/KUNIT.md):' \ ' kunit-build build all SCST KUnit modules' \ ' kunit-clean clean test module builds; keep run reports' \ - ' kunit-list show all source labels and suites' \ + ' kunit-list show base/composite labels and source suites' \ ' kunit-run run already-built modules (root)' \ ' kunit-test build, then run tests (root)' \ ' KUNIT_TEST=y|n include all suites (default: n; kunit-build sets y)' \ ' KUNIT_LOG=y|n compile extra execution messages (default: n)' \ - ' KUNIT_LABEL space-separated labels to run (default: ALL)' \ - ' KUNIT_ARGS runner options and module paths; see --help' \ + ' KUNIT_LABEL base/composite labels to run (default: ALL)' \ + ' KUNIT_ARGS runner/list options; run also accepts module paths' \ '' \ 'Build variables:' \ ' WERROR=y|n treat warnings as errors (default: n)' \ diff --git a/doc/KUNIT.md b/doc/KUNIT.md index da5a16d3c..4e2811ef2 100644 --- a/doc/KUNIT.md +++ b/doc/KUNIT.md @@ -52,11 +52,13 @@ builds. Ordinary clean targets also remove KUnit build metadata. Reports and ## Labels and logging `KUNIT_LABEL` selects execution only; every test is compiled. The default `ALL` -runs every suite in the supplied module set. Select space-separated labels -or `ALL` alone. Empty, unknown or unavailable labels are errors. -`make kunit-list` always shows all source labels and suites, regardless of -`KUNIT_LABEL`, without a build, root privileges or loaded modules. The runner -validates the selection against the supplied `.ko` files. +runs every suite in the supplied module set. Select space-separated base or +composite labels, or `ALL` alone. Selected sets are combined and each suite +runs once. Empty, unknown or unavailable selections are errors. +`make kunit-list` always shows all source labels, composite descriptions and +their suites, regardless of `KUNIT_LABEL`, without a build, root privileges +or loaded modules. The runner validates selection against the supplied `.ko` +files. Base labels come directly from suite registrations: | Module | Labels | | --- | --- | @@ -75,6 +77,28 @@ sudo make kunit-run KUNIT_LABEL="UNMAP VDISK" sudo make -j kunit-test KDIR=/path/to/kernel-build KUNIT_LABEL=LIB ``` +Composite labels are defined in [labels.toml](../scripts/kunit/labels.toml). +For example, `KUNIT_LABEL=IO` combines `CDB`, `UNMAP` and `VDISK`. Add a table +with base labels, exact suite names, or both: + +```toml +[COMMANDS] +description = "Command decoding and virtual disk capacity" +labels = ["CDB"] +suites = ["scst-VDISK-capacity"] +``` + +Names must be uppercase and cannot replace a base label or `ALL`. References +to other composites and case filters are not supported. Every member of a +selected composite must exist in the supplied module set; missing members +are errors before module operations. Unselected composites do not require +their modules. Original suite names and base labels remain unchanged. + +To use a different file, pass `KUNIT_ARGS='--label-config "/path/to/labels.toml"'` +to either `kunit-list` or `kunit-run`. This replaces the default composite +configuration. The selected definitions and expanded suites are recorded in +`result.json`. + `KUNIT_LOG=y` includes optional `scst_kunit_log()` messages emitted during tests; it defaults to `n` and requires rebuilding. To display these messages, run with `KUNIT_ARGS=--verbose`. Assertion failure diagnostics are always enabled. @@ -146,8 +170,8 @@ entry; no build changes are needed. For a new suite: The registration supplies the native suite name (`scst-LIB-example` above) and label metadata. Use a literal uppercase label and an empty or hyphen-prefixed -suffix. Labels need no separate registration in Makefiles or the runner. Keep -registrations outside preprocessor conditionals so the source list is accurate. +suffix. Base labels need no separate registration in Makefiles or the runner. +Keep registrations outside preprocessor conditionals so the source list is accurate. Init-section suites cannot run through debugfs after initialization. A new test-owning module additionally needs its Kbuild integration and a path in diff --git a/scripts/kunit/inventory.py b/scripts/kunit/inventory.py index a24770ec5..e5ac004c3 100644 --- a/scripts/kunit/inventory.py +++ b/scripts/kunit/inventory.py @@ -5,6 +5,7 @@ import argparse from pathlib import Path import re +from labels import DEFAULT_LABEL_CONFIG, expand_labels, load_composites REGISTRATION = re.compile( r'^scst_kunit_suite\(\s*\w+\s*,\s*"([A-Z][A-Z0-9_]*)"\s*,\s*' @@ -27,19 +28,29 @@ def source_suites(paths): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('directories', nargs='+', type=Path) + parser.add_argument('--label-config', type=Path, default=DEFAULT_LABEL_CONFIG, + help='composite labels TOML (default: scripts/kunit/labels.toml)') args = parser.parse_args(argv) try: suites = source_suites(path for directory in args.directories for path in directory.rglob('*.c')) if not suites: raise ValueError('No source suites found') - available = sorted(set(suites.values())) + composites = load_composites(args.label_config.expanduser()) + expanded = {name: expand_labels(suites, [name], composites) for name in composites} + available = sorted(set(suites.values()) | composites.keys()) except (OSError, ValueError) as exc: parser.error(str(exc)) print('Source suites (the runner validates the built modules):') print('Available KUNIT_LABEL: ' + ' '.join(available) + ' ALL') for name, label in sorted(suites.items()): print(f' {label}: {name}') + if composites: + print('Composite labels:') + for name in sorted(composites): + print(f" {name}: {composites[name]['description']}".rstrip()) + for suite in sorted(expanded[name]): + print(f' {suite}') return 0 diff --git a/scripts/kunit/labels.py b/scripts/kunit/labels.py new file mode 100644 index 000000000..9185767f0 --- /dev/null +++ b/scripts/kunit/labels.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Expand source or module labels and configured sets into whole suites.""" + +from pathlib import Path +import re +import tomllib + + +DEFAULT_LABEL_CONFIG = Path(__file__).with_name('labels.toml') +LABEL = re.compile(r'[A-Z][A-Z0-9_]*') +SUITE = re.compile(r'scst-[A-Za-z0-9_-]+') + + +def load_composites(path=DEFAULT_LABEL_CONFIG): + with path.open('rb') as source: + definitions = tomllib.load(source) + result = {} + for name, entry in definitions.items(): + if not LABEL.fullmatch(name) or name == 'ALL' or not isinstance(entry, dict): + raise ValueError(f'{path}: invalid composite label {name!r}') + if entry.keys() - {'description', 'labels', 'suites'}: + raise ValueError(f'{path}: unknown fields in {name}') + description = entry.get('description', '') + if not isinstance(description, str) or '\n' in description or '\r' in description: + raise ValueError(f'{path}: {name}.description must be a single-line string') + for field, pattern in (('labels', LABEL), ('suites', SUITE)): + values = entry.get(field, []) + if not isinstance(values, list) or any( + not isinstance(value, str) or not pattern.fullmatch(value) for value in values): + raise ValueError(f'{path}: {name}.{field} must be an array of valid names') + labels, suites = entry.get('labels', []), entry.get('suites', []) + if not labels and not suites: + raise ValueError(f'{path}: composite label {name} is empty') + if invalid := set(labels) & (definitions.keys() | {'ALL'}): + raise ValueError(f'{path}: {name} must reference base labels only: {sorted(invalid)}') + result[name] = {'description': description, 'labels': labels, 'suites': suites} + return result + + +def expand_labels(suites, selected, composites): + """Validate requested sets against this inventory and keep its suite order. + + Unselected composites may refer to absent modules. A selected composite + must resolve completely, rather than silently running only part of it. + """ + base = set(suites.values()) + if conflicts := base & composites.keys(): + raise ValueError(f'Composite labels conflict with base labels: {sorted(conflicts)}') + if not selected or ('ALL' in selected and selected != ['ALL']): + raise ValueError('Select labels or ALL alone') + if selected == ['ALL']: + return list(suites) + if unknown := set(selected) - base - composites.keys(): + raise ValueError(f'Unknown or unavailable KUnit labels: {sorted(unknown)}') + chosen = set() + for name in selected: + if name in base: + chosen.update(suite for suite, label in suites.items() if label == name) + continue + entry = composites[name] + if missing := set(entry['labels']) - base: + raise ValueError(f'{name}: base labels absent from this inventory: {sorted(missing)}') + if missing := set(entry['suites']) - suites.keys(): + raise ValueError(f'{name}: suites absent from this inventory: {sorted(missing)}') + chosen.update(entry['suites']) + chosen.update(suite for suite, label in suites.items() if label in entry['labels']) + if not chosen: + raise ValueError('Selected labels contain no suites') + return [suite for suite in suites if suite in chosen] diff --git a/scripts/kunit/labels.toml b/scripts/kunit/labels.toml new file mode 100644 index 000000000..525db7906 --- /dev/null +++ b/scripts/kunit/labels.toml @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Composite labels select whole suites; base labels come from C registrations. + +[IO] +description = "Command decoding, UNMAP and virtual disk checks" +labels = ["CDB", "UNMAP", "VDISK"] diff --git a/scripts/kunit/run.py b/scripts/kunit/run.py index 64006245f..1cfb22cf6 100644 --- a/scripts/kunit/run.py +++ b/scripts/kunit/run.py @@ -23,6 +23,8 @@ import textwrap import time import tomllib +from labels import DEFAULT_LABEL_CONFIG, expand_labels, load_composites + sys.path.insert(0, str(Path(__file__).resolve().parent / "upstream")) import kunit_parser # noqa: E402 from kunit_printer import Printer # noqa: E402 @@ -323,16 +325,15 @@ def parse_labels(value): return sorted(set(labels)) -def select_modules(inventory, labels): +def select_modules(inventory, labels, composites): names = [item["name"] for item in inventory] if len(names) != len(set(names)): raise RunError("Duplicate module") suites = [suite for item in inventory for suite in item["suites"]] if not suites or len(suites) != len(set(suites)): raise RunError("Modules must have a nonempty, unique suite manifest") - available = {label for item in inventory for label in item["labels"].values()} - if labels != ["ALL"] and set(labels) - available: - raise RunError(f"Labels absent from supplied modules: {sorted(set(labels) - available)}") + selected = set(expand_labels({suite: label for item in inventory + for suite, label in item["labels"].items()}, labels, composites)) by_name = {item["name"]: item for item in inventory} ordered = [] visited = set() @@ -356,7 +357,7 @@ def select_modules(inventory, labels): for item in inventory: item["selected_suites"] = [suite for suite in item["suites"] - if labels == ["ALL"] or item["labels"][suite] in labels] + if suite in selected] for item in inventory: if item["selected_suites"]: visit(item["name"]) @@ -477,7 +478,8 @@ def unload_owned_kunit(capture, directory, timeout, report): raise RunError("; ".join(problems)) -def execute(modules, timeout, directory, report, labels, verbose=False): +def execute(modules, timeout, directory, report, labels, verbose=False, + label_config=DEFAULT_LABEL_CONFIG): if os.geteuid() != 0: raise RunError("Module execution requires root in the prepared test environment") for tool in ("modinfo", "insmod", "rmmod"): @@ -486,7 +488,10 @@ def execute(modules, timeout, directory, report, labels, verbose=False): inventory = [module_info(path) | {"path": str(path), "state": "not attempted"} for path in modules] report["modules"] = inventory - inventory = select_modules(inventory, labels) + composites = load_composites(label_config) + report["label_config"] = str(label_config) + report["composite_labels"] = {name: composites[name] for name in labels if name in composites} + inventory = select_modules(inventory, labels, composites) report["expected_suites"] = [suite for item in inventory for suite in item["selected_suites"]] report["started_suites"] = [] if len({item["vermagic"] for item in inventory}) != 1: @@ -587,7 +592,9 @@ def arguments(argv=None): parser.add_argument("modules", nargs="*", type=Path, help="test modules and their SCST dependencies; order is determined from modinfo") parser.add_argument("--labels", default="ALL", type=parse_labels, - help="space-separated labels to run, or ALL (default: ALL)") + help="space-separated base or composite labels, or ALL (default: ALL)") + parser.add_argument("--label-config", type=Path, default=DEFAULT_LABEL_CONFIG, + help="composite labels TOML (default: scripts/kunit/labels.toml)") parser.add_argument("--config", type=Path, help="TOML configuration file") parser.add_argument("--timeout", type=float, help="seconds per suite or module operation (default: 300)") parser.add_argument("--output", type=Path, help="parent for unique report directories") @@ -643,7 +650,7 @@ def arguments(argv=None): if "verbose" in settings and not isinstance(settings["verbose"], bool): parser.error("Config verbose must be a boolean") verbose = args.verbose if args.verbose is not None else settings.get("verbose", False) - return resolved, timeout, output.resolve(), verbose, args.labels + return resolved, timeout, output.resolve(), verbose, args.label_config.expanduser().resolve(), args.labels def show_suites(tests, report): @@ -870,7 +877,7 @@ def color_console(text): def main(argv=None): try: - modules, timeout, parent, verbose, labels = arguments(argv) + modules, timeout, parent, verbose, label_config, labels = arguments(argv) except (OSError, ValueError) as exc: print(f"Configuration error: {exc}", file=sys.stderr) return 2 @@ -890,7 +897,7 @@ def main(argv=None): fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: raise RunError("Another SCST KUnit runner holds the lock") from exc - execute(modules, timeout, directory, report, labels, verbose) + execute(modules, timeout, directory, report, labels, verbose, label_config) except KeyboardInterrupt: report["errors"].append("Interrupted; module operations may still be running; inspect their PIDs") except (RunError, OSError, ValueError, subprocess.SubprocessError) as exc: diff --git a/scripts/kunit/test_inventory.py b/scripts/kunit/test_inventory.py index 4eeefc2c8..a4b4ebc28 100644 --- a/scripts/kunit/test_inventory.py +++ b/scripts/kunit/test_inventory.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only """Source discovery must work before building and without kernel access.""" +import contextlib +import io import os from pathlib import Path import subprocess @@ -8,7 +10,7 @@ import sys import tempfile import unittest -from inventory import source_suites +from inventory import main, source_suites class SourceInventoryTests(unittest.TestCase): @@ -36,8 +38,10 @@ scst_kunit_suite(third, "OTHER", "", .test_cases = third_cases); self.assertEqual(source_suites(root.glob('*.c')), { 'scst-NEW_LABEL-boundary': 'NEW_LABEL', 'scst-NEW_LABEL': 'NEW_LABEL', 'scst-OTHER': 'OTHER'}) + config = root / 'labels.toml' + config.write_text('') command = [sys.executable, str(Path(__file__).with_name('inventory.py')), - str(root)] + str(root), '--label-config', str(config)] result = subprocess.run(command, cwd=tmp, env=dict(os.environ, PATH=''), capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) @@ -56,6 +60,28 @@ scst_kunit_suite(third, "OTHER", "", .test_cases = third_cases); with self.assertRaises(ValueError): source_suites([path]) + def test_composite_listing_includes_description_and_expanded_suites(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / 'suites.c').write_text(''' +scst_kunit_suite(cdb, "CDB", "", .test_cases = cases); +scst_kunit_suite(capacity, "VDISK", "-capacity", .test_cases = cases); +scst_kunit_suite(io, "VDISK", "-io", .test_cases = cases); +''') + config = root / 'custom labels.toml' + config.write_text('[IO]\ndescription = "Selected I/O suites"\n' + 'labels = ["CDB"]\nsuites = ["scst-VDISK-capacity"]\n') + with contextlib.redirect_stdout(io.StringIO()) as console: + self.assertEqual(main([str(root), '--label-config', str(config)]), 0) + self.assertIn('Available KUNIT_LABEL: CDB IO VDISK ALL', console.getvalue()) + self.assertEqual(console.getvalue().split('Composite labels:\n')[1], + ' IO: Selected I/O suites\n scst-CDB\n scst-VDISK-capacity\n') + config.write_text('[IO]\nlabels = ["UNKNOWN"]\n') + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()) as errors, self.assertRaises(SystemExit): + main([str(root), '--label-config', str(config)]) + self.assertIn('IO: base labels absent', errors.getvalue()) + if __name__ == '__main__': unittest.main() diff --git a/scripts/kunit/test_labels.py b/scripts/kunit/test_labels.py new file mode 100644 index 000000000..aa2d6d827 --- /dev/null +++ b/scripts/kunit/test_labels.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Composite selection checks using source/module inventories, without a kernel.""" + +from pathlib import Path +import tempfile +import unittest + +from labels import expand_labels, load_composites + + +class LabelTests(unittest.TestCase): + SUITES = {'scst-CDB': 'CDB', 'scst-LIB': 'LIB', 'scst-VDISK-capacity': 'VDISK', + 'scst-VDISK-io': 'VDISK', 'scst-VDISK-sync': 'VDISK'} + + def load(self, text): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'labels.toml' + path.write_text(text) + return load_composites(path) + + def test_union_preserves_inventory_order_and_deduplicates(self): + composites = self.load(''' +[IO] +description = "Command and virtual disk checks" +labels = ["VDISK", "CDB", "CDB"] +suites = ["scst-VDISK-io"] +[CAPACITY] +suites = ["scst-VDISK-capacity"] +''') + self.assertEqual(expand_labels(self.SUITES, ['IO', 'CAPACITY', 'VDISK'], composites), + ['scst-CDB', 'scst-VDISK-capacity', 'scst-VDISK-io', 'scst-VDISK-sync']) + self.assertEqual(expand_labels(self.SUITES, ['CAPACITY'], composites), ['scst-VDISK-capacity']) + self.assertEqual(expand_labels(self.SUITES, ['ALL'], composites), list(self.SUITES)) + self.assertEqual(expand_labels(self.SUITES, ['LIB'], composites), ['scst-LIB']) + + def test_base_labels_need_no_configuration(self): + self.assertEqual(self.load(''), {}) + self.assertEqual(expand_labels({'scst-NEW': 'NEW'}, ['NEW'], {}), ['scst-NEW']) + + def test_unknown_or_missing_members_never_select_a_partial_group(self): + for definition in ('labels = ["CDB", "MISSING"]', + 'suites = ["scst-CDB", "scst-missing"]'): + composites = self.load('[IO]\n' + definition) + for selected in (['IO'], ['LIB', 'IO']): + with self.subTest(definition=definition, selected=selected), self.assertRaisesRegex( + ValueError, 'IO: .*absent'): + expand_labels(self.SUITES, selected, composites) + # Unselected sets must not prevent a run using a subset of modules. + self.assertEqual(expand_labels(self.SUITES, ['LIB'], composites), ['scst-LIB']) + self.assertEqual(expand_labels(self.SUITES, ['ALL'], composites), list(self.SUITES)) + + def test_namespace_conflicts_are_errors_even_for_all(self): + composites = self.load('[VDISK]\nlabels = ["CDB"]') + for selected in (['VDISK'], ['CDB'], ['ALL']): + with self.subTest(selected=selected), self.assertRaisesRegex(ValueError, 'conflict'): + expand_labels(self.SUITES, selected, composites) + + def test_invalid_configurations(self): + for text in ('x = 1', '[ALL]\nlabels = ["CDB"]', '[lower]\nlabels = ["CDB"]', + '[IO]', '[IO]\nlabels = []\nsuites = []', '[IO]\nunknown = []', + '[IO]\nlabels = "CDB"', '[IO]\nlabels = [1]', + '[IO]\nsuites = [false]', '[IO]\nsuites = ["scst-*.case"]', + '[IO]\nsuites = ["../results"]', '[IO]\nlabels = ["ALL"]', + '[IO]\nlabels = ["IO"]', '[IO]\nlabels = ["OTHER"]\n[OTHER]\nlabels = ["CDB"]', + '[IO]\nlabels = ["CDB"]\ndescription = 1', + '[IO]\nlabels = ["CDB"]\ndescription = "line\\nline"', + '[IO]\nlabels = ["CDB"]\n[IO]\nlabels = ["LIB"]'): + with self.subTest(text=text), self.assertRaises(ValueError): + self.load(text) + + def test_invalid_selections(self): + for selected in ([], ['UNKNOWN'], ['ALL', 'LIB'], ['ALL', 'ALL']): + with self.subTest(selected=selected), self.assertRaises(ValueError): + expand_labels(self.SUITES, selected, {}) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/kunit/test_make.py b/scripts/kunit/test_make.py index 6731400a2..3dd81b0a4 100644 --- a/scripts/kunit/test_make.py +++ b/scripts/kunit/test_make.py @@ -12,13 +12,14 @@ import tempfile import unittest from inventory import source_suites +from labels import load_composites ROOT = Path(__file__).resolve().parents[2] MODULES = re.findall(r'[\w/-]+\.ko', (ROOT / 'scst/src/kunit/modules.mk').read_text()) SOURCE_FILES = sorted({path for module in MODULES for path in (ROOT / module).parent.joinpath('kunit').rglob('*.c')}) SUITES = source_suites(SOURCE_FILES) -LABELS = sorted(set(SUITES.values())) +LABELS = sorted(set(SUITES.values()) | load_composites().keys()) ENTRY_POINTS = [entry for entry in ('', 'scst', 'scst/src', 'scst/src/dev_handlers', 'iscsi-scst') if 'kunit/select.mk' in (ROOT / entry / 'Makefile').read_text()] OWNERS = { @@ -129,7 +130,9 @@ class SelectionTests(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn('Available KUNIT_LABEL: ' + ' '.join(LABELS) + ' ALL', result.stdout) listed = {line.strip() for line in result.stdout.splitlines() if line.startswith(' ')} - self.assertEqual(listed, {f'{label}: {suite}' for suite, label in SUITES.items()}) + self.assertTrue({f'{label}: {suite}' for suite, label in SUITES.items()} <= listed) + self.assertIn('Composite labels:', result.stdout) + self.assertIn(' IO: Command decoding, UNMAP and virtual disk checks', result.stdout) self.assertNotIn('Selected', result.stdout) self.assertNotIn('Flags:', result.stdout) @@ -147,6 +150,22 @@ class SelectionTests(unittest.TestCase): cwd=ROOT, env=env, capture_output=True, text=True) return result, json.loads(result.stdout) if result.returncode == 0 else [] + def test_list_and_run_accept_the_same_label_configuration(self): + with tempfile.TemporaryDirectory(prefix='kunit labels ') as tmp: + config = Path(tmp) / 'custom labels.toml' + config.write_text('[CAPACITY]\nsuites = ["scst-VDISK-capacity"]\n') + option = f'KUNIT_ARGS=--label-config "{config}"' + result = subprocess.run(['make', '-s', 'kunit-list', 'KUNIT_LABEL=UNKNOWN', option], + cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('CAPACITY', result.stdout.splitlines()[1]) + self.assertNotIn('Selected', result.stdout) + self.assertTrue(result.stdout.endswith(' CAPACITY:\n scst-VDISK-capacity\n')) + result, args = self.runner_arguments(option, selection='CAPACITY') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'CAPACITY', + '--label-config', str(config)]) + def test_runner_defaults_to_all_and_receives_explicit_labels(self): result, args = self.runner_arguments(selection=None) self.assertEqual(result.returncode, 0, result.stderr) @@ -188,7 +207,7 @@ class SelectionTests(unittest.TestCase): root = Path(tmp) paths = ['Makefile', 'scst/Makefile', 'scst/src/Makefile', 'scst/src/kunit/modules.mk', 'scst/src/kunit/select.mk', - 'scripts/kunit/inventory.py', + 'scripts/kunit/inventory.py', 'scripts/kunit/labels.py', 'scripts/kunit/labels.toml', 'scst/include/scst.h', 'scst/include/scst_const.h', 'scst/include/scst_user.h', 'iscsi-scst/Makefile', 'iscsi-scst/include/iscsi_scst.h', diff --git a/scripts/kunit/test_run.py b/scripts/kunit/test_run.py index f4661c2bb..ff6df0494 100644 --- a/scripts/kunit/test_run.py +++ b/scripts/kunit/test_run.py @@ -747,7 +747,7 @@ class LifecycleTests(unittest.TestCase): def execute(self, logs=None, labels=None, inventory=None, load_logs=None, loaded=(), missing=(), failure=None, unload_warning=False, preflight=None, kernel_mode='built-in', interface_error=None, remains=(), - fail_loaded=False, failure_code=1, cleanup_failure=False, observe=None, results=None): + fail_loaded=False, failure_code=1, cleanup_failure=False, observe=None, results=None, label_config=None): result = report() capture = Mock(messages=[]) inventory = inventory or [manifest()] @@ -814,8 +814,9 @@ class LifecycleTests(unittest.TestCase): patch.object(run, 'command', side_effect=operation) as command: try: run.execute([Path('/mock/' + i['name'] + '.ko') for i in inventory], - 10, Path(tmp), result, labels) - except run.RunError as exc: + 10, Path(tmp), result, labels, + label_config=label_config or run.DEFAULT_LABEL_CONFIG) + except (run.RunError, ValueError) as exc: result['errors'].append(str(exc)) operations = [] for call in command.call_args_list: @@ -1030,6 +1031,45 @@ class LifecycleTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): self.assertEqual(run.finish(result, Path(tmp)), 0) + def test_composite_selection_loads_dependencies_and_runs_each_suite_once(self): + with tempfile.TemporaryDirectory() as tmp: + config = Path(tmp) / 'custom labels.toml' + config.write_text('[IO]\nlabels = ["VDISK"]\nsuites = ["scst-LIB"]\n') + result, operations, _ = self.execute( + {'scst-LIB': PASS, 'scst-VDISK-capacity': VDISK_PASS}, + inventory=list(reversed(self.both_manifests())), labels=['IO', 'VDISK'], + label_config=config) + self.assertFalse(result['errors']) + self.assertEqual(operations, [('insmod', 'scst'), ('suite', 'scst-LIB'), + ('insmod', 'scst_vdisk'), ('suite', 'scst-VDISK-capacity'), + ('rmmod', 'scst_vdisk'), ('rmmod', 'scst')]) + self.assertEqual(result['label_config'], str(config)) + self.assertEqual(result['composite_labels']['IO']['suites'], ['scst-LIB']) + + def test_exact_suite_group_does_not_run_other_suites_in_its_module(self): + with tempfile.TemporaryDirectory() as tmp: + config = Path(tmp) / 'labels.toml' + config.write_text('[CAPACITY]\nsuites = ["scst-VDISK-capacity"]\n') + inventory = [manifest(), manifest('scst_vdisk', + {'scst-VDISK-capacity': 'VDISK', 'scst-VDISK-sync': 'VDISK'})] + result, operations, _ = self.execute({'scst-VDISK-capacity': VDISK_PASS}, + inventory=inventory, labels=['CAPACITY'], label_config=config) + self.assertFalse(result['errors']) + self.assertEqual([name for op, name in operations if op == 'suite'], ['scst-VDISK-capacity']) + self.assertEqual([name for op, name in operations if op == 'insmod'], ['scst', 'scst_vdisk']) + + def test_incomplete_composite_is_rejected_before_any_module_operation(self): + with tempfile.TemporaryDirectory() as tmp: + config = Path(tmp) / 'labels.toml' + for definition in ('labels = ["LIB", "VDISK"]', + 'suites = ["scst-LIB", "scst-VDISK-capacity"]'): + config.write_text('[IO]\n' + definition) + result, operations, _ = self.execute(labels=['IO'], label_config=config, + kernel_mode='load') + self.assertIn('absent', result['errors'][0]) + self.assertEqual(operations, []) + self.assertEqual(result['module_state'], 'not attempted') + def test_multiple_labels_and_nonmatching_suite_names(self): labels = {'scst-LIB': 'LIB', 'scst-CM': 'CM', 'scst-LIFECYCLE': 'LIFECYCLE', 'scst-LIFECYCLE-blocking': 'LIFECYCLE'} @@ -1336,7 +1376,7 @@ class ConfigurationTests(unittest.TestCase): config = root / 'settings.toml' config.write_text('timeout_seconds=12\nverbose=true\n') with patch.object(run, 'DEFAULT_MODULE_LIST', modules): - selected, timeout, _, verbose, labels = self.arguments(['--config', str(config)]) + selected, timeout, _, verbose, _, labels = self.arguments(['--config', str(config)]) self.assertEqual(selected, [core, handler]) self.assertEqual(timeout, 12) self.assertTrue(verbose) @@ -1375,7 +1415,7 @@ class ConfigurationTests(unittest.TestCase): (directory / 'scst.ko').touch() config = directory / 'settings.toml' config.write_text('module="scst.ko"\noutput_dir="reports"\ntimeout_seconds=9\n') - module, timeout, output, verbose, labels = self.arguments(['--config', str(config)]) + module, timeout, output, verbose, _, labels = self.arguments(['--config', str(config)]) self.assertFalse(verbose) self.assertEqual(module, [directory / 'scst.ko']) self.assertEqual(output, directory / 'reports') diff --git a/scst/src/kunit/select.mk b/scst/src/kunit/select.mk index c01619a24..e9c537967 100644 --- a/scst/src/kunit/select.mk +++ b/scst/src/kunit/select.mk @@ -47,7 +47,7 @@ scst-kunit-source-dirs = $(sort $(addsuffix kunit,$(dir $(SCST_KUNIT_MODULES)))) .PHONY: kunit-list kunit-list: @$(PYTHON) "$(scst-kunit-root)/scripts/kunit/inventory.py" \ - $(foreach dir,$(scst-kunit-source-dirs),"$(scst-kunit-root)/$(dir)") + $(foreach dir,$(scst-kunit-source-dirs),"$(scst-kunit-root)/$(dir)") $(KUNIT_ARGS) # A component clean invalidates the recorded build as well as a top-level clean. .PHONY: scst-kunit-clean-files