diff --git a/.github/workflows/kunit.yml b/.github/workflows/kunit.yml new file mode 100644 index 000000000..c3aff806c --- /dev/null +++ b/.github/workflows/kunit.yml @@ -0,0 +1,70 @@ +name: KUnit + +on: + push: + branches: [master] + pull_request: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + kunit: + name: Tooling and module build + runs-on: ubuntu-latest + container: fedora:44@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 + timeout-minutes: 20 + env: + # Update the image and kernel deliberately, independently of the regression matrix. + KERNEL_BUILD: 7.2.9-201.fc44 + KERNEL_DEVEL_SHA256: 17e14518fe28ea6897bc448bf3e50434ac5ed58d4ca10d3c1c20ef145a16e7cd + steps: + - name: Install build dependencies + run: | + dnf install -y gcc make elfutils-libelf-devel openssl-devel \ + python3 git tar gzip findutils diffutils gawk curl-minimal + - uses: actions/checkout@v4 + - name: Prepare pinned kernel headers + run: | + rpm_file="$RUNNER_TEMP/kernel-devel-$KERNEL_BUILD.x86_64.rpm" + curl --fail --location --retry 3 \ + "https://kojipkgs.fedoraproject.org/packages/kernel/${KERNEL_BUILD%-*}/${KERNEL_BUILD#*-}/x86_64/kernel-devel-$KERNEL_BUILD.x86_64.rpm" \ + --output "$rpm_file" + printf '%s %s\n' "$KERNEL_DEVEL_SHA256" "$rpm_file" | sha256sum --check + dnf install -y "$rpm_file" + kdir="/usr/src/kernels/$KERNEL_BUILD.x86_64" + test -s "$kdir/Module.symvers" + if ! grep -Eq '^CONFIG_KUNIT=[ym]$' "$kdir/.config"; then + echo "::error::The pinned kernel $KERNEL_BUILD does not enable KUnit" + exit 1 + fi + printf 'KDIR=%s\nREPORTS=%s/kunit\n' "$kdir" "$RUNNER_TEMP" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/kunit" + rpm -q kernel-devel gcc > "$RUNNER_TEMP/kunit/packages.txt" + cp "$kdir/.config" "$RUNNER_TEMP/kunit/kernel.config" + - name: Check tooling + run: | + python3 -m unittest discover -s scripts/kunit -p 'test_*.py' 2>&1 | tee "$REPORTS/tooling.log" + make kunit-list + - name: Build SCST KUnit modules + run: | + make -j"$(nproc)" kunit-build 2>&1 | tee "$REPORTS/scst-build.log" + tar -czf "$REPORTS/modules.tar.gz" -T .kunit-modules .kunit-modules + - name: Check cleanup + run: | + make kunit-clean 2>&1 | tee "$REPORTS/clean.log" + test ! -e .kunit-modules + test ! -e .kunit-modules.tmp + test -z "$(find scst/src iscsi-scst/kernel -type f \( -name '*.ko' -o -name '*.o' -o -name '*.cmd' \) -print -quit)" + - name: Save build evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: kunit + path: ${{ runner.temp }}/kunit/ diff --git a/.gitignore b/.gitignore index 018578d02..a5e0ae9cc 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,11 @@ .depend_d .depend_f .directory +.kunit-modules +.kunit-modules.tmp +.kunit-run.toml .tmp_versions +__pycache__/ built-in.a codegraph.db codegraph.db-* @@ -48,6 +52,7 @@ iscsi-scst/conftest/*/result-*.txt iscsi-scst/include/iscsi_scst_itf_ver.h iscsi-scst/usr/iscsi-scst-adm iscsi-scst/usr/iscsi-scstd +kunit-results/ Module.markers Module.symvers modules.order diff --git a/Makefile b/Makefile index 4f5cddae0..83fb0152b 100644 --- a/Makefile +++ b/Makefile @@ -137,103 +137,101 @@ SCST_SOURCE_FILES = $(shell if [ -e scripts/list-source-files ]; then \ fi) help: - @echo "Component tests (see tests/ginkgo/README.md):" - @echo " ginkgo-check / ginkgo-list: checks / dry run without SCST access" - @echo " ginkgo-config: validate GINKGO_CONFIG (JSON file)" - @echo " ginkgo-test: load built SCST modules, then run tests (root)" - @echo " ginkgo-test-build: build and load SCST, then run tests (root)" - @echo " GINKGO_LABEL selects test groups: local, iscsi, basic, stress, scsi, io" - @echo "" - @echo "Build variables:" - @echo " KUNIT_LABEL= : select KUnit labels to list" - @echo " WERROR=y|n : treat warnings as errors (default: n)" - @echo "" - @echo " tags : make tags" - @echo " cov-build : make coverity build" - @echo " shellcheck : check Bash scripts" - @echo " kunit-list : show KUnit labels and effective selection" - @echo "" - @echo " all : make all" - @echo " clean : clean files" - @echo " extraclean : clean + clean dependencies" - @echo " install : install" - @echo " uninstall : uninstall" - @echo "" - @echo " scst : make scst only" - @echo " scst_clean : scst: clean" - @echo " scst_extraclean : scst: clean + clean dependencies" - @echo " scst_install : scst: install" - @echo " scst_uninstall : scst: uninstall" - @echo "" - @echo " docs : make docs" - @echo " docs_clean : docs: clean" - @echo " docs_extraclean : docs: clean + clean dependencies" - @echo "" - @echo " scstadm : make scstadmin" - @echo " scstadm_clean : scstadmin: clean" - @echo " scstadm_extraclean : scstadmin: clean + clean dependencies" - @echo " scstadm_install : scstadmin: install" - @echo " scstadm_uninstall : scstadmin: uninstall" - @echo "" - @echo " qla : make QLA target driver" - @echo " qla_clean : qla target: clean" - @echo " qla_extraclean : qla target: clean + clean dependencies" - @echo " qla_install : qla target: install" - @echo " qla_uninstall : qla target: uninstall" - @echo "" - @echo " iscsi : make iSCSI target" - @echo " iscsi_clean : ISCSI target: clean" - @echo " iscsi_extraclean : ISCSI target: clean + clean dependencies" - @echo " iscsi_install : ISCSI target: install" - @echo " iscsi_uninstall : ISCSI target: uninstall" - @echo "" - @echo " emulex : make Emulex target" - @echo " emulex_clean : Emulex target: clean" - @echo " emulex_extraclean : Emulex target: clean + clean dependencies" - @echo " emulex_install : Emulex target: install" - @echo " emulex_uninstall : Emulex target: uninstall" - @echo "" - @echo " srpt : make SRP target" - @echo " srpt_clean : srp target: clean" - @echo " srpt_extraclean : srp target: clean + clean dependencies" - @echo " srpt_install : srp target: install" - @echo " srpt_uninstall : srp target: uninstall" - @echo "" - @echo " fcst : make FCoE target" - @echo " fcst_clean : FCoE target: clean" - @echo " fcst_extraclean : FCoE target: clean + clean dependencies" - @echo " fcst_install : FCoE target: install" - @echo " fcst_uninstall : FCoE target: uninstall" - @echo "" - @echo " scst_local : make scst_local target" - @echo " scst_local_clean : scst_local target: clean" - @echo " scst_local_extraclean : scst_local target: clean + clean dependencies" - @echo " scst_local_install : scst_local target: install" - @echo " scst_local_uninstall : scst_local target: uninstall" - @echo "" - @echo " usr : make user space targets" - @echo " usr_clean : usr target: clean" - @echo " usr_extraclean : usr target: clean + clean dependencies" - @echo " usr_install : usr target: install" - @echo " usr_uninstall : usr target: uninstall" - @echo "" - @echo " scst-rpm : make SCST RPM packages" - @echo " scst-dkms-rpm : make SCST DKMS RPM packages" - @echo " scstadm-rpm : make scstadmin RPM packages" - @echo " rpm : make both SCST and scstadmin RPM packages" - @echo " rpm-dkms : make both SCST DKMS and scstadmin RPM packages" - @echo " rpm-artifacts : build verified RPM artifacts" - @echo " docker-rpm-image : build the RPM builder Docker image" - @echo " docker-rpm : build RPM packages in Docker" - @echo "" - @echo " dpkg : make SCST dpkg packages" - @echo "" - @echo " 2perf : changes debug state to full performance" - @echo " 2release : changes debug state to release" - @echo " 2debug : changes debug state to full debug" - @echo "" - @echo " Note:" - @echo " - install and uninstall may need root privileges" + @printf '%s\n' \ + 'Component tests (see tests/ginkgo/README.md):' \ + ' ginkgo-check check the framework without SCST access' \ + ' ginkgo-list list tests without SCST access' \ + ' ginkgo-config validate GINKGO_CONFIG (JSON file)' \ + ' ginkgo-test load built modules, then run tests (root)' \ + ' ginkgo-test-build build and load modules, then run tests (root)' \ + ' GINKGO_LABEL select local, iscsi, basic, stress, scsi or io' \ + ' GINKGO_CONFIG component-test configuration file' \ + '' \ + 'KUnit tests (see doc/KUNIT.md):' \ + ' kunit-build build all SCST KUnit modules' \ + ' kunit-clean clean test module builds; keep run reports' \ + ' kunit-list show all source labels and suites' \ + ' kunit-run run already-built modules (root)' \ + ' kunit-test build, then run tests (root)' \ + ' KUNIT_TEST=y|n include all suites (default: n; kunit-build sets y)' \ + ' KUNIT_LOG=y|n compile extra execution messages (default: n)' \ + ' KUNIT_LABEL space-separated labels to run (default: ALL)' \ + ' KUNIT_ARGS runner options and module paths; see --help' \ + '' \ + 'Build variables:' \ + ' WERROR=y|n treat warnings as errors (default: n)' \ + '' \ + 'Build and maintenance targets:' \ + ' tags make tags' \ + ' cov-build make coverity build' \ + ' shellcheck check Bash scripts' \ + ' all make all' \ + ' clean clean files' \ + ' extraclean clean + clean dependencies' \ + ' install install' \ + ' uninstall uninstall' \ + ' scst make scst only' \ + ' scst_clean scst: clean' \ + ' scst_extraclean scst: clean + clean dependencies' \ + ' scst_install scst: install' \ + ' scst_uninstall scst: uninstall' \ + ' docs make docs' \ + ' docs_clean docs: clean' \ + ' docs_extraclean docs: clean + clean dependencies' \ + ' scstadm make scstadmin' \ + ' scstadm_clean scstadmin: clean' \ + ' scstadm_extraclean scstadmin: clean + clean dependencies' \ + ' scstadm_install scstadmin: install' \ + ' scstadm_uninstall scstadmin: uninstall' \ + ' qla make QLA target driver' \ + ' qla_clean qla target: clean' \ + ' qla_extraclean qla target: clean + clean dependencies' \ + ' qla_install qla target: install' \ + ' qla_uninstall qla target: uninstall' \ + ' iscsi make iSCSI target' \ + ' iscsi_clean ISCSI target: clean' \ + ' iscsi_extraclean ISCSI target: clean + clean dependencies' \ + ' iscsi_install ISCSI target: install' \ + ' iscsi_uninstall ISCSI target: uninstall' \ + ' emulex make Emulex target' \ + ' emulex_clean Emulex target: clean' \ + ' emulex_extraclean Emulex target: clean + clean dependencies' \ + ' emulex_install Emulex target: install' \ + ' emulex_uninstall Emulex target: uninstall' \ + ' srpt make SRP target' \ + ' srpt_clean srp target: clean' \ + ' srpt_extraclean srp target: clean + clean dependencies' \ + ' srpt_install srp target: install' \ + ' srpt_uninstall srp target: uninstall' \ + ' fcst make FCoE target' \ + ' fcst_clean FCoE target: clean' \ + ' fcst_extraclean FCoE target: clean + clean dependencies' \ + ' fcst_install FCoE target: install' \ + ' fcst_uninstall FCoE target: uninstall' \ + ' scst_local make scst_local target' \ + ' scst_local_clean scst_local target: clean' \ + ' scst_local_extraclean scst_local target: clean + clean dependencies' \ + ' scst_local_install scst_local target: install' \ + ' scst_local_uninstall scst_local target: uninstall' \ + ' usr make user space targets' \ + ' usr_clean usr target: clean' \ + ' usr_extraclean usr target: clean + clean dependencies' \ + ' usr_install usr target: install' \ + ' usr_uninstall usr target: uninstall' \ + ' scst-rpm make SCST RPM packages' \ + ' scst-dkms-rpm make SCST DKMS RPM packages' \ + ' scstadm-rpm make scstadmin RPM packages' \ + ' rpm make both SCST and scstadmin RPM packages' \ + ' rpm-dkms make both SCST DKMS and scstadmin RPM packages' \ + ' rpm-artifacts build verified RPM artifacts' \ + ' docker-rpm-image build the RPM builder Docker image' \ + ' docker-rpm build RPM packages in Docker' \ + ' dpkg make SCST dpkg packages' \ + ' 2perf changes debug state to full performance' \ + ' 2release changes debug state to release' \ + ' 2debug changes debug state to full debug' \ + '' \ + 'Installation and module execution may require root.' # Resolve the config relative to the caller, before entering tests/ginkgo. export GINKGO_CONFIG @@ -633,3 +631,32 @@ multiple-release-archives: 2perf 2release 2debug include scst/src/kunit/select.mk + +PYTHON ?= python3 + +.PHONY: kunit-build kunit-clean kunit-run kunit-test + +kunit-clean: scst-kunit-clean-files + @for target in $(SCST_KUNIT_COMPONENTS); do \ + $(MAKE) -C "$$target" clean || exit $$?; \ + done + +kunit-build: + @rm -f .kunit-modules .kunit-modules.tmp + @echo "Building all SCST KUnit tests" + @for target in $(SCST_KUNIT_COMPONENTS); do \ + $(MAKE) -C "$$target" kunit-build KUNIT_TEST=y || exit $$?; \ + done + @for module in $(SCST_KUNIT_MODULES); do \ + test -f "$$module" || { echo "Missing built module: $$module"; exit 2; }; \ + done + @printf '%s\n' $(SCST_KUNIT_MODULES) > .kunit-modules.tmp + @mv .kunit-modules.tmp .kunit-modules + +# Build and run sequentially even under make -j. +kunit-test: + @$(MAKE) kunit-build + @$(MAKE) kunit-run + +kunit-run: + @$(PYTHON) scripts/kunit/run.py --labels "$(KUNIT_LABEL)" $(KUNIT_ARGS) diff --git a/doc/KUNIT.md b/doc/KUNIT.md index f041a4705..fe153e8d1 100644 --- a/doc/KUNIT.md +++ b/doc/KUNIT.md @@ -1,45 +1,147 @@ # SCST KUnit tests KUnit checks SCST functions using private fixtures and simulated dependencies. -Use the latest upstream kernel with native KUnit; API backports are not supported. +Use a current kernel with native KUnit; API backports are not supported. Normal builds leave tests disabled and do not require KUnit headers. -## Build +## Requirements -Use a configured and prepared kernel build directory: +Build against a configured and prepared kernel tree with `CONFIG_KUNIT=y/m`. +Run tests in a disposable VM: ordinary SCST initialization discovers SCSI devices. +The runner requires Python 3.11+, kmod utilities, native KUnit autorun control, +`CONFIG_DEBUG_FS=y`, `CONFIG_KUNIT_DEBUGFS=y`, and debugfs mounted at +`/sys/kernel/debug`. The running kernel configuration must be readable through +`/proc/config.gz`, `/boot/config-$(uname -r)`, or +`/lib/modules/$(uname -r)/build/.config`. + +For built-in KUnit, boot with `kunit.enable=1 kunit.autorun=0`. For modular +KUnit, install `kunit.ko` matching the running kernel; the runner loads it with +`enable=1 autorun=0`. Leave KUnit's action and filters unset; `filter_glob=*.*` +is also valid. An explicitly empty glob selects no suites. + +An already-loaded KUnit must expose valid parameters through sysfs. The runner +rejects invalid or unreadable settings and leaves that module unchanged. If +the kernel hides `enable` or `autorun`, start with KUnit unloaded so the runner +can supply the required parameters itself. + +## Build and run + +| Target | Purpose | +| --- | --- | +| `kunit-build` | Build all test-owning modules and their dependencies. | +| `kunit-clean` | Clean those component builds and tooling caches; preserve run reports and settings. | +| `kunit-run` | Run selected suites from already-built modules. Requires root. | +| `kunit-test` | Build, then run after successful compilation, including under `make -j`. Requires root. | ```sh -make scst KDIR=/path/to/kernel-build CONFIG_SCST_KUNIT_TEST=y -make kunit-list +make kunit-build KDIR=/path/to/kernel-build +sudo make kunit-run ``` -The single `CONFIG_SCST_KUNIT_TEST` switch includes every suite in its owning -module. `KUNIT_LABEL` never limits compilation. KUnit supports out-of-tree -module builds with `CONFIG_KUNIT=y/m`. +`kunit-build` sets `KUNIT_TEST=y`. Ordinary builds default to `KUNIT_TEST=n`; +use `make scst KUNIT_TEST=y` to include tests in a core and handler build. +KUnit is supported only by out-of-tree module builds. + +Out-of-tree test builds keep the current SCST build mode (`2debug`, `2release` +or `2perf`); a clean tree defaults to debug. KUnit does not change this mode. + +Run `make kunit-clean KDIR=/path/to/kernel-build` to clean the test module +builds. Ordinary clean targets also remove KUnit build metadata. Reports and +`.kunit-run.toml` are preserved. + +## Labels and logging + +`KUNIT_LABEL` selects execution only; every test is compiled. The default `ALL` +runs every suite in the supplied module set. Select space-separated labels +or `ALL` alone. Empty, unknown or unavailable labels are errors. +`make kunit-list` always shows all source labels and suites, regardless of +`KUNIT_LABEL`, without a build, root privileges or loaded modules. The runner +validates the selection against the supplied `.ko` files. | Module | Labels | | --- | --- | | `scst.ko` | `LIB` | -Use a disposable VM for execution: ordinary SCST initialization still -discovers SCSI devices. Native KUnit -registers suites on module initialization and controls their execution. +The runner reads suite metadata from the modules and loads selected owners +with their dependencies. Unselected suites are not counted as skipped. + +```sh +sudo make kunit-run KUNIT_LABEL=LIB +sudo make -j kunit-test KDIR=/path/to/kernel-build KUNIT_LABEL=LIB +``` + +`KUNIT_LOG=y` includes optional `scst_kunit_log()` messages emitted during tests; +it defaults to `n` and requires rebuilding. To display these messages, run with +`KUNIT_ARGS=--verbose`. Assertion failure diagnostics are always enabled. + +## Runner options + +Pass runner options through `KUNIT_ARGS`; `--help` lists them. For example: + +```sh +sudo make kunit-run KUNIT_ARGS='--timeout 120 --verbose' +sudo make kunit-run KUNIT_ARGS='--config "/tmp/test settings.toml"' +sudo make kunit-run KUNIT_LABEL=LIB KUNIT_ARGS='"/tmp/test modules/scst.ko"' +``` + +Copy [config.example.toml](../scripts/kunit/config.example.toml) to +`.kunit-run.toml` for persistent settings. Command-line options override it. +Module paths come from positional arguments, then the configuration's `modules` +array, then `.kunit-modules` saved by the last successful build. When copying +a build to a VM, include this list and preserve its relative module paths. +Quote each explicit path containing spaces separately. + +## Results and cleanup + +Suites register without executing on module load; the runner starts selected +suites through debugfs. The console shows suite status, groups failure reasons +by suite, and identifies the SCST build and running kernel. `--verbose` adds +case logs and full diagnostics; `NO_COLOR` disables terminal colors. + +Each run saves logs, KTAP, module metadata and JSON/text reports under +`kunit-results/`; `--output` changes the parent directory. Read `report.txt` +for details, `messages.log` for kernel diagnostics and `commands.log` for +module-operation errors. Full reports are saved regardless of verbosity. +The runner exits with `0` for PASS/SKIP, `1` for FAIL and `2` for ERROR; +Make reports a failed recipe with its own nonzero exit status. + +Completed assertion failures allow subsequent suites to run. Kernel diagnostics +such as an Oops, missing results, lost logs or retained fixtures stop module +operations. Inspect the reports before retrying; the runner never forces +module removal or clears kernel logs. + +After completed tests, SCST modules are unloaded in reverse dependency order. +KUnit is removed only if the runner loaded it and SCST cleanup completed; +preloaded KUnit is left unchanged. External dependencies such as `dlm` are +loaded with `modprobe` and left loaded. A clean failure before SCST loads also +releases runner-owned KUnit; incomplete execution can retain modules. ## Add a suite -Keep suites and shared fixtures under their owning component's `kunit/` -directory. Compile suites as separate objects and add them to the owning -module's list in `scst/src/kunit/objects.mk`. Register suites with -`scst_kunit_suite()`, which supplies the native name and label metadata: +Keep suites and fixtures under their owning component's `kunit/` directory; +share fixtures only when multiple suites use them. Compile suites as separate +objects. To add a case to an existing suite, add its function and `KUNIT_CASE()` +entry; no build changes are needed. For a new suite: -```c -scst_kunit_suite(example_suite, "LIB", "-example", - .test_cases = example_cases, -); -``` +1. Create its source file and register it with `scst_kunit_suite()`: -`make kunit-list KUNIT_LABEL=LIB` discovers these registrations in source; -no build or loaded module is needed. Labels need no separate Makefile list. + ```c + scst_kunit_suite(example_suite, "LIB", "-example", + .test_cases = example_cases, + ); + ``` + +2. Add its object to the owning module's list in `scst/src/kunit/objects.mk`. + +The registration supplies the native suite name (`scst-LIB-example` above) and +label metadata. Use a literal uppercase label and an empty or hyphen-prefixed +suffix. Labels need no separate registration in Makefiles or the runner. Keep +registrations outside preprocessor conditionals so the source list is accurate. +Init-section suites cannot run through debugfs after initialization. + +A new test-owning module additionally needs its Kbuild integration and a path in +`scst/src/kunit/modules.mk`; new build components must provide `kunit-build`. +Dependency ordering comes from the built modules, without a Python owner list. Keep private implementation types in `*_priv.h` and test access declarations in `*_test.h` beside the owning `.c` file. Guard test declarations with @@ -52,8 +154,9 @@ redirects. Test interfaces are not installed headers or module exports. Fixtures must tolerate failed initialization and assertions. Bound worker waits; unfinished work must retain its state and module reference and report `scst_kunit_retain()`. Unsafe cases report `SKIP` pending separate production -fixes. Storage and transport integration belongs in component tests; upstream -QLogic initiator/HBA code is out of scope. +fixes. Storage and transport integration belongs in +[component tests](../tests/ginkgo/README.md); upstream QLogic initiator/HBA +code is out of scope. Tooling checks use temporary files and mocked operations, without SCST access: diff --git a/scripts/kunit/AGENTS.md b/scripts/kunit/AGENTS.md new file mode 100644 index 000000000..e0e429af8 --- /dev/null +++ b/scripts/kunit/AGENTS.md @@ -0,0 +1,12 @@ +# SCST KUnit tooling + +Read [the KUnit guide](../../doc/KUNIT.md). The runner owns module execution, +log capture and reporting; kernel fixtures belong beside their owning module. +`upstream/` is a pinned public Linux parser: preserve its provenance and keep +SCST-specific behavior outside it. + +Missing results, lost log records, retained fixtures and kernel diagnostics +must not become a pass. Preserve evidence on errors; never force module +removal or clear logs. Both console modes must retain full report artifacts. +Userspace regression tests must use temporary files and mocked module/log +operations. Running them does not authorize loading SCST on the host. diff --git a/scripts/kunit/config.example.toml b/scripts/kunit/config.example.toml new file mode 100644 index 000000000..18f1f2068 --- /dev/null +++ b/scripts/kunit/config.example.toml @@ -0,0 +1,8 @@ +# Copy to .kunit-run.toml for make kunit-run or python3 scripts/kunit/run.py. +# Select execution with KUNIT_LABEL= or --labels (default: ALL). +# Paths are relative to this file. CLI arguments override these settings. +# Omit modules to use the module list saved by make kunit-build. +# modules = ["scst/src/scst.ko"] +timeout_seconds = 300 +output_dir = "kunit-results" +verbose = false diff --git a/scripts/kunit/inventory.py b/scripts/kunit/inventory.py index 9f62a5380..a24770ec5 100644 --- a/scripts/kunit/inventory.py +++ b/scripts/kunit/inventory.py @@ -26,7 +26,6 @@ def source_suites(paths): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--labels', default='ALL') parser.add_argument('directories', nargs='+', type=Path) args = parser.parse_args(argv) try: @@ -35,20 +34,12 @@ def main(argv=None): if not suites: raise ValueError('No source suites found') available = sorted(set(suites.values())) - labels = args.labels.split() - if 'ALL' in labels and labels != ['ALL']: - raise ValueError('KUNIT_LABEL=ALL must be used alone') - if labels != ['ALL'] and (not labels or set(labels) - set(available)): - raise ValueError('Select KUNIT_LABEL from: ' + ' '.join(available) + ' ALL') - selected = available if labels == ['ALL'] else sorted(set(labels)) except (OSError, ValueError) as exc: parser.error(str(exc)) print('Source suites (the runner validates the built modules):') print('Available KUNIT_LABEL: ' + ' '.join(available) + ' ALL') - print('Selected: ' + ' '.join(selected)) for name, label in sorted(suites.items()): - if label in selected: - print(f' {label}: {name}') + print(f' {label}: {name}') return 0 diff --git a/scripts/kunit/run.py b/scripts/kunit/run.py new file mode 100644 index 000000000..e8354bd7d --- /dev/null +++ b/scripts/kunit/run.py @@ -0,0 +1,881 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Capture and report a local run of already-built SCST KUnit modules.""" + +import argparse +from collections import Counter +import errno +import fcntl +import gzip +import io +import json +import math +import os +from pathlib import Path +import platform +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import textwrap +import time +import tomllib + +sys.path.insert(0, str(Path(__file__).resolve().parent / "upstream")) +import kunit_parser # noqa: E402 +from kunit_printer import Printer # noqa: E402 + +DEFAULT_MODULE_LIST = Path(__file__).resolve().parents[2] / ".kunit-modules" +DEBUGFS = Path("/sys/kernel/debug/kunit") +MODULE_METADATA = ("version", "description", "author", "license", "git_commit", + "build_date", "build_number", "kver", "arch_type", "srcversion", "vermagic") + + +class RunError(Exception): + """Execution or evidence is incomplete.""" + + +def kernel_diagnostics(messages): + signatures = ( + "WARNING:", "BUG:", "Oops:", "Kernel panic", "KASAN:", "KCSAN:", + "UBSAN:", "kernel BUG at", "general protection fault", + "Unable to handle kernel", "Call Trace:", "Call trace:", "Backtrace:", + ) + groups = [] + active = False + oops_seen = False + fault = False + for line in messages: + signature = any(s in line for s in signatures) or re.search( + r"(?:soft|hard) LOCKUP|blocked for more than|rcu:.*stall", line, re.IGNORECASE) + trace = any(s in line for s in ("Call Trace:", "Call trace:", "Backtrace:")) + oops = line.startswith("Oops:") + # #PF and architecture-specific register lines are part of an Oops, + # not KTAP comments. Keep the block until test output resumes. + ktap = re.match(r"\s*(?:#(?:\s|$)|(?:not ok|ok) \d|\d+\.\.|KTAP version)", line) + continuation = active and (trace or (oops and fault and not oops_seen)) + if signature and not continuation: + groups.append(line) + active = True + oops_seen = oops + fault = any(s in line for s in ("BUG:", "kernel BUG at", "general protection fault", + "Unable to handle kernel")) + elif active and not ktap: + groups[-1] += "\n" + line + oops_seen |= oops + else: + active = False + return groups + + +def test_tree(test): + return {"name": test.name, "status": test.status.name, + "counts": vars(test.counts), "log": test.log, "skip_reason": test.skip_reason, + "children": [test_tree(child) for child in test.subtests]} + + +def associate_parameter_logs(test, block): + """Attach diagnostics before an unplanned first parameter to that case. + + The upstream parser keeps these on the parent. Move only an unambiguous + source span ending at the first leaf result; retain all other parent logs. + Plans, results and parser validation are unchanged. + """ + for child in test.subtests: + associate_parameter_logs(child, block) + if not test.subtests or test.expected_count is not None or test.subtests[0].subtests: + return + headers = [i for i, line in enumerate(block) + if line.strip() == f"# Subtest: {test.name}"] + if len(headers) != 1: + return + source = kunit_parser.LineStream(iter(enumerate(block[headers[0] + 1:], start=1))) + leading = kunit_parser.parse_diagnostic(source) + result = kunit_parser.TEST_RESULT.match(source.peek()) if source else None + if (not leading or not result or int(result[2]) != 1 or + result[4].strip() != test.subtests[0].name): + return + positions = [i for i in range(len(test.log) - len(leading) + 1) + if test.log[i:i + len(leading)] == leading] + if len(positions) == 1: + start = positions[0] + del test.log[start:start + len(leading)] + test.subtests[0].log[:0] = leading + + +def analyse(messages, expected): + """Parse each named suite, including module output without a root plan.""" + candidates = [] + for index, line in enumerate(messages): + match = re.fullmatch(r"([ \t]*)# Subtest: (\S.*?)\s*", line) + if match: + candidates.append((index, match[2], len(match[1].expandtabs()))) + # Parameterized cases can also have names beginning with scst-. Suite + # headers are the outermost subtests, not every matching name in the log. + depth = min((indent for _, _, indent in candidates), default=0) + headers = [(index, name) for index, name, indent in candidates if indent == depth] + retained = [line for line in messages if "SCST_KUNIT_RETAINED:" in line] + errors = ["Unfinished test work retained fixture and module; inspect the VM"] if retained else [] + names = [name for _, name in headers] + if not names and expected: + errors.append("No SCST test results were recorded") + elif missing := sorted(set(expected) - set(names)): + errors.append("Missing suite results: " + ", ".join(missing)) + if unexpected := sorted(set(names) - set(expected)): + errors.append("Unexpected suite results: " + ", ".join(unexpected)) + if repeated := sorted(name for name, count in Counter(names).items() if count > 1): + errors.append("Repeated suite results: " + ", ".join(repeated)) + results = [] + consumed = set() + details = io.StringIO() + printer = Printer(output=details) + for position, (start, name) in enumerate(headers): + limit = headers[position + 1][0] if position + 1 < len(headers) else len(messages) + end_pattern = re.compile(r"\s*(?:not ok|ok) (\d+) (?:- )?" + re.escape(name) + + r"(?:\s+#.*)?\s*$") + end = next((i for i in range(start + 1, limit) + if end_pattern.fullmatch(messages[i])), None) + if end is None: + errors.append(f"Missing final result for {name}") + block = messages[start:limit] + number = 1 + else: + block = messages[start:end + 1] + number = int(end_pattern.fullmatch(messages[end])[1]) + consumed.update(range(start, start + len(block))) + # Suite numbers can be global across previously loaded modules. + # Preserve all case numbers/plans and validate those upstream. + lines = kunit_parser.LineStream(iter(enumerate(block, start=1))) + parse_output = io.StringIO() + parsed = kunit_parser.parse_test(lines, number, [], True, Printer(output=parse_output)) + for case in parsed.subtests: + associate_parameter_logs(case, block) + if parsed.counts.errors or parsed.counts.crashed or not parsed.subtests: + errors.append(f"Incomplete or invalid KTAP for {name}") + details.write(parse_output.getvalue()) + else: + kunit_parser.print_test(parsed, False, printer) + results.append(test_tree(parsed)) + for index, line in enumerate(messages): + if index not in consumed and re.match(r"\s*(?:not ok|ok) \d+\b|\s+\d+\.\.\d+", line): + errors.append(f"Unexpected KTAP outside a selected suite: {line.strip()}") + counts = {key: sum(s["counts"][key] for s in results) + for key in ("passed", "failed", "skipped", "crashed", "errors")} + failed = any(s["status"] == "FAILURE" for s in results) or counts["failed"] > 0 + status = ("ERROR" if errors else "FAIL" if failed else + "PASS" if counts["passed"] else "SKIP") + return {"test_status": status, "suites": results, "counts": counts, + "parse_errors": errors, "retained_fixtures": retained, + "diagnostics": kernel_diagnostics(messages)}, details.getvalue() + + +class KernelLog: + """Read only new /dev/kmsg records, checking the sequence for overruns.""" + + def __init__(self, directory): + self.fd = os.open("/dev/kmsg", os.O_RDONLY | os.O_NONBLOCK) + try: + os.lseek(self.fd, 0, os.SEEK_END) + self.raw = (directory / "kernel.log").open("wb") + except BaseException: + os.close(self.fd) + raise + self.sequence = None + self.messages = [] + self.previous_kernel = False + + def accept(self, record): + self.raw.write(record) + self.raw.flush() + try: + header, body = record.split(b";", 1) + fields = header.split(b",") + priority, sequence, timestamp = map(int, fields[:3]) + flags = fields[3] + if min(priority, sequence, timestamp) < 0: + raise ValueError("negative kmsg metadata") + except (ValueError, IndexError) as exc: + raise RunError("Malformed /dev/kmsg record; see kernel.log") from exc + if self.sequence is not None and sequence != self.sequence + 1: + raise RunError(f"Lost kernel log records between {self.sequence} and {sequence}") + self.sequence = sequence + # Userspace cannot write LOG_KERN records. Keep injected messages in + # the raw evidence, but do not let them supply test results/diagnostics. + if priority >> 3: + self.previous_kernel = False + return + escaped = body.split(b"\n", 1)[0] + decoded = re.sub(rb"\\x([0-9a-fA-F]{2})", + lambda match: bytes([int(match[1], 16)]), escaped) + lines = decoded.decode("utf-8", errors="replace").split("\n") + if b"c" in flags and self.previous_kernel and self.messages: + self.messages[-1] += lines.pop(0) + self.messages.extend(lines) + self.previous_kernel = True + + def drain(self): + # Bound each batch so a noisy system cannot defeat the command timeout. + for _ in range(256): + try: + record = os.read(self.fd, 1024 * 1024) + except BlockingIOError: + return False + except OSError as exc: + if exc.errno == errno.EPIPE: + raise RunError("Kernel log overrun; results are incomplete") from exc + raise + if not record: + raise RunError("Kernel log closed unexpectedly") + self.accept(record) + return True + + def close(self): + os.close(self.fd) + self.raw.close() + + +def command(argv, capture, directory, timeout, report): + if argv[0] in ("modprobe", "insmod", "rmmod"): + print("Running: " + shlex.join(argv), flush=True) + with (directory / "commands.log").open("ab", buffering=0) as output: + output.write(("\n$ " + shlex.join(argv) + "\n").encode()) + output_start = output.tell() + process = subprocess.Popen(argv, stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + operation = {"argv": argv, "pid": process.pid, "returncode": None} + report["commands"].append(operation) + deadline = time.monotonic() + timeout + while True: + capture.drain() + operation["returncode"] = process.poll() + # The child can print and exit after the first drain. Read again + # after observing its exit before declaring the log complete. + if operation["returncode"] is not None and not capture.drain(): + if operation["returncode"]: + # Keep a bounded error excerpt; commands.log retains all output. + with (directory / "commands.log").open("rb") as source: + source.seek(max(output_start, os.fstat(source.fileno()).st_size - 4096)) + operation["output"] = source.read().decode("utf-8", errors="replace").strip() + return operation["returncode"] + if time.monotonic() >= deadline: + raise RunError(f"Timed out waiting for PID {process.pid}; it may still be running. " + "No automatic unload will be attempted") + time.sleep(0.02) + + +def command_error(report, operation, code): + output = report["commands"][-1].get("output", "") if report["commands"] else "" + message = f"{operation} exited with {code}" + return RunError(message + ("\n" + output if output else "; see commands.log")) + + +def module_info(module): + info = {} + for field in ("name", "scst_kunit_suite", "depends", *MODULE_METADATA): + result = subprocess.run(["modinfo", "-F", field, str(module)], text=True, + capture_output=True, timeout=10, check=True) + info[field] = result.stdout.strip() + if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_-]*", info["name"]): + raise RunError("Invalid module name") + info["name"] = info["name"].replace("-", "_") + labels = {} + for record in info["scst_kunit_suite"].splitlines(): + match = re.fullmatch(r"([A-Z][A-Z0-9_]*):(scst-[A-Za-z0-9_-]+)", record) + if not match or match[1] == "ALL" or match[2] in labels: + raise RunError("Invalid SCST label manifest; rebuild with kunit-build") + labels[match[2]] = match[1] + if not labels: + raise RunError("Module has no KUnit label manifest; rebuild with kunit-build") + info["suites"] = list(labels) + info["labels"] = labels + dependencies = info["depends"].split(",") if info["depends"] else [] + if any(not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_-]*", name) for name in dependencies): + raise RunError(f"Invalid dependencies for {info['name']}") + info["depends"] = [name.replace("-", "_") for name in dependencies] + return info + + +def parse_labels(value): + labels = value.split() + if not labels or any(re.fullmatch(r"[A-Z][A-Z0-9_]*", label) is None for label in labels): + raise ValueError("Select labels with --labels '' or --labels ALL") + if "ALL" in labels and len(labels) != 1: + raise ValueError("ALL must be used alone") + return sorted(set(labels)) + + +def select_modules(inventory, labels): + names = [item["name"] for item in inventory] + if len(names) != len(set(names)): + raise RunError("Duplicate module") + suites = [suite for item in inventory for suite in item["suites"]] + if not suites or len(suites) != len(set(suites)): + raise RunError("Modules must have a nonempty, unique suite manifest") + available = {label for item in inventory for label in item["labels"].values()} + if labels != ["ALL"] and set(labels) - available: + raise RunError(f"Labels absent from supplied modules: {sorted(set(labels) - available)}") + by_name = {item["name"]: item for item in inventory} + ordered = [] + visited = set() + visiting = set() + + def visit(name): + if name in visiting: + raise RunError(f"Cyclic module dependency at {name}") + if name in visited: + return + visiting.add(name) + item = by_name[name] + for dependency in item["depends"]: + if dependency in by_name: + visit(dependency) + elif dependency == "scst": + raise RunError(f"Supply scst.ko from the same build as {name}") + visiting.remove(name) + visited.add(name) + ordered.append(item) + + for item in inventory: + item["selected_suites"] = [suite for suite in item["suites"] + if labels == ["ALL"] or item["labels"][suite] in labels] + for item in inventory: + if item["selected_suites"]: + visit(item["name"]) + return ordered + + +def kernel_config(): + release = platform.release() + for path in (Path("/proc/config.gz"), Path(f"/boot/config-{release}"), + Path(f"/lib/modules/{release}/build/.config")): + if not path.is_file(): + continue + source = gzip.open(path, "rt") if path.suffix == ".gz" else path.open() + with source: + values = {} + for line in source: + if line.startswith("CONFIG_") and "=" in line: + key, value = line.strip().split("=", 1) + values[key] = value + elif re.fullmatch(r"# CONFIG_\w+ is not set\n?", line): + values[line.split()[1]] = "n" + return values + raise RunError("Cannot read running kernel configuration; provide /proc/config.gz " + "or /boot/config-") + + +def check_kunit_interfaces(): + if not DEBUGFS.is_dir(): + raise RunError("Mount debugfs at /sys/kernel/debug and enable CONFIG_KUNIT_DEBUGFS") + # NULL and an empty charp are different: an empty glob selects no suites, + # while an empty action is an unknown action. Attribute filters may be empty. + neutral = {"action": ("(null)",), "filter_glob": ("(null)", "*", "*.*"), + "filter": ("(null)", ""), "filter_action": ("(null)", "")} + for name, allowed in neutral.items(): + value = Path(f"/sys/module/kunit/parameters/{name}").read_text().removesuffix("\n") + if value not in allowed: + raise RunError(f"kunit.{name}={value!r} prevents unfiltered suite registration; " + "use KUnit without action or filters (filter_glob=*.* is also valid)") + + +def check_preloaded_kunit(): + for name, expected, allowed in (("enable", "1", ("y", "1")), ("autorun", "0", ("n", "0"))): + path = Path(f"/sys/module/kunit/parameters/{name}") + try: + value = path.read_text().strip() + except OSError as exc: + raise RunError(f"Cannot verify preloaded KUnit: {path.name} is unreadable.\n" + "Module left unchanged. If it is unused, run 'rmmod kunit' as root, " + "then retry; the runner will load it with enable=1 autorun=0.") from exc + if value.lower() not in allowed: + raise RunError(f"Preloaded kunit.{name}={value!r}; require {name}={expected}. " + "Module left unchanged; correct its parameters before retrying") + check_kunit_interfaces() + + +def check_kernel(): + config = kernel_config() + if config.get("CONFIG_KUNIT") not in ("y", "m"): + raise RunError("The running kernel requires CONFIG_KUNIT=y or m") + if "CONFIG_KUNIT_AUTORUN_ENABLED" not in config: + raise RunError("The kernel lacks native KUnit autorun control; use the latest upstream kernel") + if config.get("CONFIG_KUNIT_DEBUGFS") != "y": + raise RunError("Enable CONFIG_KUNIT_DEBUGFS in the test kernel") + if config["CONFIG_KUNIT"] == "m": + if Path("/sys/module/kunit").exists(): + check_preloaded_kunit() + return "preloaded" + if shutil.which("modprobe") is None: + raise RunError("Required command not found: modprobe") + return "load" + if Path("/sys/module/kunit/initstate").exists(): + raise RunError("Running KUnit does not match the built-in kernel configuration") + options = {} + for token in shlex.split(Path("/proc/cmdline").read_text()): + if token == "--": + break + key, _, value = token.partition("=") + options[key.replace("-", "_")] = value.lower() + if options.get("kunit.autorun") not in ("0", "n", "no", "false"): + raise RunError("Boot the test VM with kunit.autorun=0 before running SCST tests") + enabled = options.get("kunit.enable", config.get("CONFIG_KUNIT_DEFAULT_ENABLED", "n").lower()) + if enabled not in ("1", "y", "yes", "true"): + raise RunError("KUnit is disabled; boot with kunit.enable=1") + check_kunit_interfaces() + return "built-in" + + +def prepare_dependencies(inventory, capture, directory, timeout, report): + # Use installed kernel modules only for external prerequisites. SCST owners + # come from the supplied build, and KUnit has its own parameter/ownership rules. + names = sorted({name for item in inventory for name in item["depends"]} + - {item["name"] for item in inventory} - {"kunit"}) + report["dependencies"] = names + for name in names: + if Path(f"/sys/module/{name}").exists(): + continue + if shutil.which("modprobe") is None: + raise RunError(f"Required command not found: modprobe (dependency {name})") + code = command(["modprobe", "--ignore-install", name], capture, directory, timeout, report) + if code: + raise command_error(report, f"Loading dependency {name}", code) + evidence, _ = analyse(capture.messages, []) + if evidence["parse_errors"] or evidence["diagnostics"]: + raise RunError(f"Unexpected kernel output while loading dependency {name}; execution stopped") + + +def unload_owned_kunit(capture, directory, timeout, report): + report["kunit_state"] = "unloading" + code = command(["rmmod", "kunit"], capture, directory, timeout, report) + if code: + report["kunit_state"] = "retained" + raise command_error(report, "Could not unload runner-owned kunit", code) + if Path("/sys/module/kunit").exists(): + report["kunit_state"] = "retained" + raise RunError("Could not unload runner-owned kunit: module remains present after rmmod") + report["kunit_state"] = "unloaded" + + +def execute(modules, timeout, directory, report, labels, verbose=False): + if os.geteuid() != 0: + raise RunError("Module execution requires root in the prepared test environment") + for tool in ("modinfo", "insmod", "rmmod"): + if shutil.which(tool) is None: + raise RunError(f"Required command not found: {tool}") + inventory = [module_info(path) | {"path": str(path), "state": "not attempted"} + for path in modules] + report["modules"] = inventory + inventory = select_modules(inventory, labels) + report["expected_suites"] = [suite for item in inventory for suite in item["selected_suites"]] + report["started_suites"] = [] + if len({item["vermagic"] for item in inventory}) != 1: + raise RunError("Modules have different vermagic values; rebuild them for the same kernel") + for name in (item["name"] for item in inventory): + if Path(f"/sys/module/{name}").exists(): + raise RunError(f"{name} is already loaded; refusing to disturb it") + kernel_mode = check_kernel() + report["kunit_state"] = "not loaded" if kernel_mode == "load" else kernel_mode + capture = KernelLog(directory) + try: + prepare_dependencies(inventory, capture, directory, timeout, report) + if kernel_mode == "load": + report["kunit_state"] = "loading" + # Fail if another process loaded KUnit in the meantime: modprobe + # otherwise succeeds without applying the requested parameters. + code = command(["modprobe", "--first-time", "--ignore-install", "kunit", + "enable=1", "autorun=0"], capture, directory, timeout, report) + if code: + raise command_error(report, "modprobe kunit", code) + report["kunit_state"] = "retained" + check_kunit_interfaces() + startup, _ = analyse(capture.messages, []) + if startup["parse_errors"]: + raise RunError("Unexpected KUnit results during preparation; execution stopped") + if startup["diagnostics"]: + report["stop_reason"] = "kernel diagnostics while preparing KUnit" + return + completed = [] + for item in inventory: + report["module_state"] = item["state"] = "loading" + code = command(["insmod", item["path"]], capture, directory, timeout, report) + if code: + error = command_error(report, f"insmod {item['name']}", code) + if not Path(f"/sys/module/{item['name']}").exists(): + item["state"] = "not loaded" + present = any(Path(f"/sys/module/{entry['name']}").exists() for entry in inventory) + report["module_state"] = "retained" if present else "not loaded" + evidence, _ = analyse(capture.messages, []) + if (code > 0 and kernel_mode == "load" and not present and not report["started_suites"] + and not evidence["parse_errors"] and not evidence["diagnostics"]): + try: + unload_owned_kunit(capture, directory, timeout, report) + except (RunError, OSError) as exc: + raise RunError(f"{error}\nCleanup: {exc}") from exc + raise error + report["module_state"] = item["state"] = "retained" + tests, _ = analyse(capture.messages, completed) + if tests["parse_errors"]: + raise RunError("Incomplete or unexpected tests; modules retained for inspection") + if tests["diagnostics"]: + report["stop_reason"] = f"kernel diagnostics while loading {item['name']}" + return + for suite in item["selected_suites"]: + trigger = DEBUGFS / suite / "run" + if not trigger.is_file(): + raise RunError(f"Missing KUnit run interface for {suite}; modules retained") + # A debugfs write blocks in KUnit. Use a child so the parent can + # drain kmsg and enforce its deadline without unloading live work. + report["started_suites"].append(suite) + code = command([sys.executable, "-c", + "import pathlib,sys; pathlib.Path(sys.argv[1]).write_text('run\\n')", + str(trigger)], capture, directory, timeout, report) + if code: + raise RunError(f"Execution of {suite} exited with {code}; modules retained") + completed.append(suite) + tests, _ = analyse(capture.messages, completed) + show_suites(tests, report) + if tests["parse_errors"]: + raise RunError("Incomplete or unexpected tests; modules retained for inspection") + if tests["diagnostics"]: + report["stop_reason"] = f"kernel diagnostics during {suite}" + return + for item in reversed(inventory): + report["module_state"] = item["state"] = "unloading" + code = command(["rmmod", item["name"]], capture, directory, timeout, report) + if code: + raise RunError(f"rmmod {item['name']} exited with {code}; see commands.log") + if Path(f"/sys/module/{item['name']}").exists(): + raise RunError(f"{item['name']} remains loaded after rmmod") + item["state"] = "unloaded" + if kernel_diagnostics(capture.messages): + report["module_state"] = "retained" if any( + item["state"] != "unloaded" for item in inventory) else "unloaded" + return + report["module_state"] = "unloaded" + if kernel_mode == "load": + unload_owned_kunit(capture, directory, timeout, report) + finally: + report["messages"] = capture.messages + capture.close() + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("modules", nargs="*", type=Path, + help="test modules and their SCST dependencies; order is determined from modinfo") + parser.add_argument("--labels", default="ALL", type=parse_labels, + help="space-separated labels to run, or ALL (default: ALL)") + parser.add_argument("--config", type=Path, help="TOML configuration file") + parser.add_argument("--timeout", type=float, help="seconds per suite or module operation (default: 300)") + parser.add_argument("--output", type=Path, help="parent for unique report directories") + parser.add_argument("--verbose", action=argparse.BooleanOptionalAction, default=None, + help="print all cases; full details are always saved in report.txt") + args = parser.parse_args(argv) + config = args.config or Path(".kunit-run.toml") + settings = {} + if args.config or config.exists(): + with config.open("rb") as source: + settings = tomllib.load(source) + if settings.keys() - {"module", "modules", "timeout_seconds", "output_dir", "verbose"}: + parser.error("Config accepts only modules, module, timeout_seconds, output_dir and verbose") + if "module" in settings and "modules" in settings: + parser.error("Use either module or modules in config") + timeout = args.timeout if args.timeout is not None else settings.get("timeout_seconds", 300) + if (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) or not 0 < timeout < 86400): + parser.error("Timeout must be positive and less than 86400 seconds") + for field in ("module", "output_dir"): + if field in settings and (not isinstance(settings[field], str) or not settings[field]): + parser.error(f"Config {field} must be a nonempty path") + configured = settings.get("modules", [settings["module"]] if "module" in settings else None) + if configured is not None and (not isinstance(configured, list) or not configured or + any(not isinstance(path, str) or not path for path in configured)): + parser.error("Config modules must be a nonempty array of paths") + module_base = Path.cwd() + if args.modules: + modules = args.modules + elif configured is not None: + modules = [Path(path) for path in configured] + module_base = config.parent + else: + if not DEFAULT_MODULE_LIST.is_file(): + parser.error("No KUnit build recorded; run make kunit-build or specify module paths") + paths = DEFAULT_MODULE_LIST.read_text().splitlines() + if not paths or any(not path.strip() for path in paths): + parser.error("Invalid KUnit module list; rerun make kunit-build") + modules = [Path(path) for path in paths] + module_base = DEFAULT_MODULE_LIST.parent + output = args.output or Path(settings.get("output_dir", "kunit-results")) + output = output.expanduser() + resolved = [] + for module in modules: + module = module.expanduser() + if not module.is_absolute(): + module = module_base / module + if not module.is_file(): + parser.error(f"Module not found: {module}") + resolved.append(module.resolve()) + if args.output is None and "output_dir" in settings and not output.is_absolute(): + output = config.parent / output + if "verbose" in settings and not isinstance(settings["verbose"], bool): + parser.error("Config verbose must be a boolean") + verbose = args.verbose if args.verbose is not None else settings.get("verbose", False) + return resolved, timeout, output.resolve(), verbose, args.labels + + +def show_suites(tests, report): + """Print progress without interleaving case diagnostics with the suite list.""" + printed = report.setdefault("printed_suites", []) + for suite in tests["suites"]: + if suite["name"] not in printed: + print(color_console(f"[{test_status(suite)}] {suite['name']}"), flush=True) + printed.append(suite["name"]) + + +def test_status(test): + return {"SUCCESS": "PASS", "FAILURE": "FAIL", "SKIPPED": "SKIP", + "TEST_CRASHED": "CRASH"}.get(test["status"], "ERROR") + + +def case_log_lines(log): + """Normalize assertion blocks without flattening their value indentation.""" + block = [] + margin = None + for line in log: + assertion = re.match(r"([ \t]*)# .*: (?:EXPECTATION|ASSERTION) FAILED at ", line) + if assertion or (margin is not None and line.strip() and not line.startswith(margin)): + # printk and assertion messages can start at column zero even + # when the KUnit assertion itself has a nesting prefix. + yield from textwrap.dedent("\n".join(block)).splitlines() + block = [] + margin = assertion[1] if assertion else None + block.append(line) + yield from textwrap.dedent("\n".join(block)).splitlines() + + +def case_details(tests, verbose=False): + """Group failed cases after progress; keep diagnostics within their case.""" + diagnostic_lines = {line for block in tests["diagnostics"] for line in block.splitlines()} + bookkeeping = re.compile(r"\s*# (?:module: .*|(?:.*: )?pass:\d+ fail:\d+ skip:\d+ total:\d+)\s*$") + + def describe(test, path="", root=False): + name = f"{path}/{test['name']}" if path else test["name"] + status = test_status(test) + lines = [] + if verbose or status not in ("PASS", "SKIP"): + log = [line for line in test["log"] if line not in diagnostic_lines + and (verbose or not bookkeeping.fullmatch(line))] + if (not root and not test["children"]) or log or test["skip_reason"]: + # A parent's status aggregates its children; don't repeat FAIL + # for it merely because one of its parameters failed. + if not root: + heading = name if test["children"] else f"[{status}] {name}" + lines.append(" " + heading) + if test["skip_reason"]: + lines.append(" " + test["skip_reason"]) + lines.extend(" " + line if line else "" for line in case_log_lines(log)) + lines.append("") + for child in test["children"]: + lines.extend(describe(child, "" if root else name)) + return lines + + lines = [] + for suite in tests["suites"]: + body = describe(suite, root=True) + if body: + lines.extend([suite["name"], *body]) + return (["Cases:" if verbose else "Failures:", "", *lines] if lines else []) + + +def module_metadata(modules): + """Describe the supplied artifacts using their embedded build information.""" + lines = [] + for module in modules: + fields = [(field, module.get(field)) for field in MODULE_METADATA if module.get(field)] + if not fields: + continue + lines.append(f"Module: {module['name']}") + for field, value in fields: + lines.extend(textwrap.wrap(f" {field}: {' '.join(value.split())}", width=100, + subsequent_indent=" ", break_long_words=False, + break_on_hyphens=False)) + lines.append("") + return lines + + +def core_metadata(modules): + """Keep the normal summary focused on identifying the SCST build.""" + core = next((item for item in modules if item["name"] == "scst"), {}) + fields = (("version", "SCST"), ("git_commit", "Commit"), + ("build_number", "Build number"), ("build_date", "Build date"), + ("kver", "Built for kernel"), ("arch_type", "Architecture")) + return [f"{label}: {' '.join(core[field].split())}" for field, label in fields if core.get(field)] + + +def finish(report, directory, verbose=False): + messages = report.pop("messages", []) + expected = report.get("expected_suites", []) + started = report.get("started_suites", expected if report["module_state"] != "not attempted" else []) + tests, details = analyse(messages, started) + report["not_run_suites"] = [suite for suite in expected if suite not in started] + if not started and not tests["suites"]: + tests["test_status"] = "NOT_RUN" + report.update(tests) + report["errors"].extend(tests["parse_errors"]) + if report["not_run_suites"] and not report["errors"] and not tests["diagnostics"]: + report["errors"].append("Execution stopped before all selected suites were started") + if report["errors"]: + report["status"] = "ERROR" + elif tests["diagnostics"]: + report["status"] = "FAIL" + elif report["not_run_suites"] or not tests["suites"] or report["module_state"] != "unloaded": + report["status"] = "ERROR" + elif tests["test_status"] == "FAIL": + report["status"] = "FAIL" + else: + report["status"] = tests["test_status"] + counts = report["counts"] + events = len(tests["diagnostics"]) + lines = [f"Result: {report['status']}", + f"Tests: {report['test_status']} ({counts['passed']} passed, {counts['failed']} failed, " + f"{counts['skipped']} skipped, {counts['crashed']} crashed, {counts['errors']} parse errors)", + f"Suites: {len(started)}/{len(expected)} started, {len(report['not_run_suites'])} not run", + f"Kernel diagnostics: {events} " + ("event" if events == 1 else "events"), + f"Modules: {report['module_state']}"] + lines.extend(f"Module {item['name']}: {item.get('state', 'unknown')}" for item in report.get("modules", [])) + if "kunit_state" in report: + lines.append(f"KUnit: {report['kunit_state']}") + if "labels" in report: + lines.append("Labels: " + " ".join(report["labels"])) + if "kernel" in report: + lines.append(f"Kernel: {report['kernel']} ({report['architecture']})") + if "elapsed_seconds" in report: + lines.append(f"Elapsed: {report['elapsed_seconds']} seconds") + lines.extend("Error: " + error for error in report["errors"]) + for headline, count in Counter(block.splitlines()[0] for block in tests["diagnostics"]).items(): + lines.append(f"Kernel: {headline}" + (f" ({count} events)" if count > 1 else "")) + if tests["diagnostics"]: + lines.append("Full kernel diagnostics: messages.log and report.txt") + lines.extend("NOT_RUN: " + suite for suite in report["not_run_suites"]) + summary = "\n".join(lines) + "\n" + metadata = "\n".join(module_metadata(report.get("modules", []))) + diagnostics = "\n\n".join(tests["diagnostics"]) + def successful_logs(test, parent=""): + name = f"{parent}/{test['name']}" if parent else test["name"] + if test["status"] in ("SUCCESS", "SKIPPED"): + for line in test["log"]: + yield f"{name}: {line}" + for child in test["children"]: + yield from successful_logs(child, name) + logs = "\n".join(line for suite in tests["suites"] for line in successful_logs(suite)) + (directory / "messages.log").write_text("\n".join(messages) + "\n") + (directory / "report.txt").write_text(details + "\n" + logs + "\n" + summary + metadata + "\n\n" + diagnostics + "\n") + saved = {key: value for key, value in report.items() if key != "printed_suites"} + (directory / "result.json").write_text(json.dumps(saved, indent=2) + "\n") + show_suites(tests, report) + modules = report.get("modules", []) + summary_lines = case_details(tests, verbose) + metadata_lines = module_metadata(modules) if verbose else core_metadata(modules) + summary_lines.extend(metadata_lines) + if metadata_lines and summary_lines[-1]: + summary_lines.append("") + if "kernel" in report: + summary_lines.append(f"Runtime kernel: {report['kernel']} ({report['architecture']})") + if verbose: + summary_lines.extend(f"Module {item['name']}: {item.get('state', 'unknown')}" + for item in report.get("modules", [])) + if report.get("labels"): + summary_lines.append("Labels: " + " ".join(report["labels"])) + if summary_lines and summary_lines[-1]: + summary_lines.append("") + missing = len(report["not_run_suites"]) + not_run = f"{missing} " + ("suite" if missing == 1 else "suites") + " not run" + if reason := report.get("stop_reason"): + summary_lines.append(f"Stopped: {reason}; {not_run}") + elif events and report["not_run_suites"]: + summary_lines.append(f"Stopped: kernel diagnostics; {not_run}") + for error in report["errors"]: + wrapped = textwrap.wrap("Error: " + " ".join(error.split()), width=88, + subsequent_indent=" ") + summary_lines.extend(wrapped if verbose else wrapped[:6]) + if not verbose and len(wrapped) > 6: + summary_lines.append(" ... full error in report.txt") + if started or tests["suites"]: + extra = "" + if counts["crashed"] or counts["errors"]: + extra = f", {counts['crashed']} crashed, {counts['errors']} parse errors" + summary_lines.append(f"Tests: {tests['test_status']} - {counts['passed']} passed, " + f"{counts['failed']} failed, {counts['skipped']} skipped{extra}") + summary_lines.append(f"Suites: {len(started)}/{len(expected)} started, " + f"{len(report['not_run_suites'])} not run") + else: + summary_lines.append(f"Tests: NOT_RUN ({not_run})") + kernel_status = "FAIL" if events else "ERROR" if report["errors"] else "PASS" + if not events and not started and not report.get("commands"): + kernel_status = "NOT_RUN" + summary_lines.append(f"Kernel: {kernel_status} - {events} diagnostic " + + ("event" if events == 1 else "events")) + if verbose: + summary_lines.extend(tests["diagnostics"]) + summary_lines.extend("NOT_RUN: " + suite for suite in report["not_run_suites"]) + elif events: + headlines = Counter(block.splitlines()[0] for block in tests["diagnostics"]) + for headline, count in list(headlines.items())[:3]: + summary_lines.append(" " + headline + (f" ({count} events)" if count > 1 else "")) + summary_lines.append(" Full diagnostics: messages.log and report.txt") + state = f"Modules: {report['module_state']}" + if "kunit_state" in report: + state += f"; KUnit {report['kunit_state']}" + summary_lines.append(state) + elapsed = f" ({report['elapsed_seconds']:.2f}s)" if "elapsed_seconds" in report else "" + summary_lines.append(f"Result: {report['status']}{elapsed}") + print("\n" + color_console("\n".join(summary_lines)) + f"\nReports: {directory}") + + return {"PASS": 0, "SKIP": 0, "FAIL": 1, "ERROR": 2}[report["status"]] + + +def color_console(text): + if not sys.stdout.isatty() or "NO_COLOR" in os.environ or os.environ.get("TERM") == "dumb": + return text + colors = {"PASS": 32, "SUCCESS": 32, "FAIL": 31, "FAILURE": 31, "ERROR": 31, + "BUG": 31, "CRASH": 31, "WARNING": 33, "SKIP": 33, "SKIPPED": 33, "NOT_RUN": 33, + "retained": 33} + return re.sub(r"\b(" + "|".join(colors) + r")\b", + lambda match: f"\033[{colors[match[0]]}m{match[0]}\033[0m", text) + + +def main(argv=None): + try: + modules, timeout, parent, verbose, labels = arguments(argv) + except (OSError, ValueError) as exc: + print(f"Configuration error: {exc}", file=sys.stderr) + return 2 + parent.mkdir(parents=True, exist_ok=True) + directory = Path(tempfile.mkdtemp(prefix=time.strftime("%Y%m%d-%H%M%S-"), dir=parent)) + report = {"status": "ERROR", "module_paths": [str(path) for path in modules], + "labels": labels, "module_state": "not attempted", + "kernel": platform.release(), "architecture": platform.machine(), + "errors": [], "commands": []} + print(f"Reports: {directory}", flush=True) + started = time.monotonic() + try: + # Serialize only this runner. Other module administration must stay idle. + fd = os.open("/run/lock/scst-kunit.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise RunError("Another SCST KUnit runner holds the lock") from exc + execute(modules, timeout, directory, report, labels, verbose) + except KeyboardInterrupt: + report["errors"].append("Interrupted; module operations may still be running; inspect their PIDs") + except (RunError, OSError, ValueError, subprocess.SubprocessError) as exc: + report["errors"].append(str(exc)) + report["elapsed_seconds"] = round(time.monotonic() - started, 3) + return finish(report, directory, verbose) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/kunit/test_clean.py b/scripts/kunit/test_clean.py new file mode 100644 index 000000000..1861678ca --- /dev/null +++ b/scripts/kunit/test_clean.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise cleanup recipes in temporary trees without touching real modules.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +COMPONENTS = next(line.split(':=', 1)[1].split() + for line in (ROOT / 'scst/src/kunit/modules.mk').read_text().splitlines() + if line.startswith('SCST_KUNIT_COMPONENTS := ')) + + +class CleanTests(unittest.TestCase): + def check_clean(self, directory, goal, fail=False, tooling=True): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for path in ('Makefile', 'scst/Makefile', 'scst/src/Makefile', + 'scst/src/dev_handlers/Makefile', 'scst/src/kunit/select.mk', + 'scst/src/kunit/modules.mk', 'scst/include/scst_const.h', + 'iscsi-scst/Makefile', 'iscsi-scst/usr/Makefile'): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / path, target) + # Other components are deliberately inert; kunit-clean must not visit them. + for component in ('qla2x00t-32gbit/qla2x00-target', 'srpt', 'scst_local', 'fcst', 'usr', 'scstadmin'): + target = root / component + target.mkdir(parents=True) + (target / 'Makefile').write_text('clean extraclean:;@rm -f visited.o\n') + (target / 'visited.o').touch() + kernel = root / 'kernel' + kernel.mkdir() + (kernel / 'Makefile').write_text( + 'clean:\n' + '\t@test "$(FAIL_CLEAN)" != y\n' + '\t@find "$(M)" -type f \\( -name "*.o" -o -name "*.ko" \\) -delete\n') + artifacts = ['scst/src/scst.ko', 'scst/src/kunit/lib/suites/test.o', + 'scst/src/dev_handlers/scst_vdisk.ko', + 'scst/src/dev_handlers/kunit/vdisk/suites/test.o', + 'iscsi-scst/kernel/iscsi-scst.ko', + 'iscsi-scst/kernel/kunit/suites/test.o'] + metadata = ['.kunit-modules', '.kunit-modules.tmp'] + if tooling: + metadata += ['scripts/kunit/__pycache__/run.pyc', + 'scripts/kunit/upstream/__pycache__/kunit_parser.pyc'] + generated = ['scst/include/build_mode.h', 'scst/include/build_mode.h.tmp', + 'scst/include/scst_itf_ver.h', + 'iscsi-scst/include/iscsi_scst_itf_ver.h', + 'iscsi-scst/conftest/sock_recvmsg/result-test.txt'] + preserved = ['.kunit-run.toml', 'kunit-results/run/report.txt', + 'scst/build_mode', 'scst/src/kunit/lib/suites/test.c'] + for path in artifacts + metadata + generated + preserved: + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text('preserve me\n' if path in preserved else '') + env = {key: value for key, value in os.environ.items() + if key not in ('MAKEFLAGS', 'MFLAGS', 'KDIR', 'KVER') + and not key.startswith(('KUNIT_', 'CONFIG_SCST_KUNIT_'))} + result = subprocess.run(['make', '-s', '-j4', '-C', str(root / directory), goal, + 'KDIR=' + str(kernel), 'KVER=test', + 'FAIL_CLEAN=' + ('y' if fail else 'n'), + 'OFED_FLAVOR=in-tree', + 'OFED_KERNEL_IB_DEVEL_RPM_INSTALLED=false', + 'OFED_COMPAT_RDMA_DEVEL_RPM_INSTALLED=false'], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, not fail, result.stdout + result.stderr) + for path in metadata: + self.assertFalse((root / path).exists(), path) + if not tooling: + self.assertFalse((root / 'scripts').exists()) + for path in preserved: + self.assertEqual((root / path).read_text(), 'preserve me\n', path) + if not fail and directory == '' and goal in ('clean', 'extraclean', 'kunit-clean'): + for path in artifacts + generated: + if goal == 'kunit-clean' and path.split('/')[0] not in COMPONENTS: + continue + self.assertFalse((root / path).exists(), path) + if goal == 'kunit-clean': + self.assertTrue((root / 'qla2x00t-32gbit/qla2x00-target/visited.o').exists()) + + def test_top_level_cleanup_removes_build_state_and_preserves_reports(self): + for goal in ('kunit-clean', 'clean', 'extraclean'): + with self.subTest(goal=goal): + self.check_clean('', goal) + + def test_component_cleanup_invalidates_module_list(self): + for directory in ('scst', 'scst/src', 'scst/src/dev_handlers', 'iscsi-scst'): + if directory != 'scst' and 'kunit/select.mk' not in (ROOT / directory / 'Makefile').read_text(): + continue + for goal in ('clean', 'extraclean'): + with self.subTest(directory=directory, goal=goal): + self.check_clean(directory, goal) + for goal in ('scst_clean', 'scst_extraclean', 'iscsi_clean', 'iscsi_extraclean'): + if goal.startswith('iscsi_') and 'iscsi-scst' not in COMPONENTS: + continue + with self.subTest(goal=goal): + self.check_clean('', goal) + + def test_failed_cleanup_still_invalidates_module_list(self): + self.check_clean('', 'kunit-clean', fail=True) + + def test_cleanup_without_packaged_tooling(self): + # Debian DKMS ships module sources without the scripts/ directory. + for directory in ('', 'scst', 'scst/src', 'iscsi-scst'): + if directory and 'kunit/select.mk' not in (ROOT / directory / 'Makefile').read_text(): + continue + for goal in ('clean', 'extraclean'): + with self.subTest(directory=directory, goal=goal): + self.check_clean(directory, goal, tooling=False) + self.check_clean('', 'kunit-clean', tooling=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/kunit/test_inventory.py b/scripts/kunit/test_inventory.py index 9ffb66919..4eeefc2c8 100644 --- a/scripts/kunit/test_inventory.py +++ b/scripts/kunit/test_inventory.py @@ -31,16 +31,19 @@ scst_kunit_suite(example, "NEW_LABEL", "-boundary", .test_cases = cases, ); scst_kunit_suite(second, "NEW_LABEL", "", .test_cases = other_cases); +scst_kunit_suite(third, "OTHER", "", .test_cases = third_cases); ''') self.assertEqual(source_suites(root.glob('*.c')), { - 'scst-NEW_LABEL-boundary': 'NEW_LABEL', 'scst-NEW_LABEL': 'NEW_LABEL'}) + 'scst-NEW_LABEL-boundary': 'NEW_LABEL', 'scst-NEW_LABEL': 'NEW_LABEL', + 'scst-OTHER': 'OTHER'}) command = [sys.executable, str(Path(__file__).with_name('inventory.py')), - '--labels', 'NEW_LABEL', str(root)] + str(root)] result = subprocess.run(command, cwd=tmp, env=dict(os.environ, PATH=''), capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn('Selected: NEW_LABEL', result.stdout) + self.assertIn('Available KUNIT_LABEL: NEW_LABEL OTHER ALL', result.stdout) self.assertIn('scst-NEW_LABEL-boundary', result.stdout) + self.assertIn('scst-OTHER', result.stdout) def test_duplicate_names_and_reserved_labels_are_rejected(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/scripts/kunit/test_make.py b/scripts/kunit/test_make.py index 4d5c91fe7..5f1468f0a 100644 --- a/scripts/kunit/test_make.py +++ b/scripts/kunit/test_make.py @@ -1,9 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only """Build and execution selection checks using inert kernel and runner fixtures.""" +import json +import os from pathlib import Path import re +import shutil import subprocess +import sys import tempfile import unittest @@ -58,6 +62,19 @@ class SelectionTests(unittest.TestCase): self.assertNotIn('SCST_KUNIT_ENABLED', result.stdout) self.assertEqual(observed, set(SUITES)) + def test_logging_switch_preserves_suite_inventory(self): + for owner in sorted(OWNERS): + for tests in ('y', 'n'): + outputs = [] + for log in ('y', 'n'): + result = self.inspect_kbuild(owner, ['CONFIG_KUNIT=y', + 'CONFIG_SCST_KUNIT_TEST=' + tests, 'CONFIG_SCST_KUNIT_LOG=' + log]) + self.assertEqual(result.returncode, 0, result.stderr) + outputs.append(result.stdout.split(' flags=', 1)[0]) + self.assertEqual('-DSCST_KUNIT_LOG_ENABLED=1' in result.stdout, tests == log == 'y') + self.assertEqual('-DSCST_KUNIT_LOG_ENABLED=0' in result.stdout, tests == 'y' and log == 'n') + self.assertEqual(*outputs) + def test_kernel_support_and_normal_build(self): for owner in sorted(OWNERS): for kunit in ('y', 'm', 'n', ''): @@ -98,18 +115,256 @@ class SelectionTests(unittest.TestCase): 'KBUILD_EXTMOD=external'], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - def test_make_lists_runtime_selection_without_config_flags(self): - for selection, valid in [('', False), ('ALL', True), ('LIB LIB', True), - ('UNKNOWN', False), ('ALL LIB', False), ('ALL ALL', False)]: + def test_make_lists_all_suites_regardless_of_runtime_labels(self): + for selection in (None, '', 'ALL', 'LIB', 'LIB LIB', 'UNKNOWN', 'ALL LIB', 'ALL ALL'): for directory in ENTRY_POINTS: with self.subTest(selection=selection, directory=directory): - result = subprocess.run(['make', '-s', 'kunit-list', 'KUNIT_LABEL=' + selection], - cwd=ROOT / directory, capture_output=True, text=True) - self.assertEqual(result.returncode == 0, valid, result.stderr) - if valid: - expected = ' '.join(LABELS) if selection == 'ALL' else ' '.join(sorted(set(selection.split()))) - self.assertIn('Selected: ' + (expected or '(none)'), result.stdout) - self.assertNotIn('Flags:', result.stdout) + command = ['make', '-s', 'kunit-list'] + if selection is not None: + command.append('KUNIT_LABEL=' + selection) + result = subprocess.run(command, cwd=ROOT / directory, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('Available KUNIT_LABEL: ' + ' '.join(LABELS) + ' ALL', result.stdout) + listed = {line.strip() for line in result.stdout.splitlines() if line.startswith(' ')} + self.assertEqual(listed, {f'{label}: {suite}' for suite, label in SUITES.items()}) + self.assertNotIn('Selected', result.stdout) + self.assertNotIn('Flags:', result.stdout) + + def runner_arguments(self, *options, selection='LIB'): + env = {key: value for key, value in os.environ.items() + if key not in ('TEST', 'TEST_LOG', 'KUNIT_LABEL', 'MAKEFLAGS', 'MFLAGS') + and not key.startswith('KUNIT_') and not key.endswith('_KUNIT_TEST')} + if selection is not None: + env['KUNIT_LABEL'] = selection + with tempfile.TemporaryDirectory() as tmp: + capture = Path(tmp) / 'capture.py' + capture.write_text('import json, sys\nprint(json.dumps(sys.argv[1:]))\n') + result = subprocess.run(['make', '-s', 'kunit-run', + f'PYTHON={sys.executable} {capture}', *options], + cwd=ROOT, env=env, capture_output=True, text=True) + return result, json.loads(result.stdout) if result.returncode == 0 else [] + + def test_runner_defaults_to_all_and_receives_explicit_labels(self): + result, args = self.runner_arguments(selection=None) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'ALL']) + # Runtime validation uses the supplied modules, not this source tree. + for selection in ('LIB', 'ALL', 'LIB LIB', '', 'UNKNOWN', 'ALL LIB'): + result, args = self.runner_arguments(selection=selection) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', selection]) + for option in ('STRESS=HT', 'PERF=HT'): + result, args = self.runner_arguments(option, selection=None) + self.assertNotEqual(result.returncode, 0) + self.assertIn('not implemented', result.stderr) + self.assertEqual(args, []) + + def test_runner_preserves_paths_and_extra_arguments(self): + result, args = self.runner_arguments( + 'KUNIT_ARGS="/tmp/test modules/core.ko" "/tmp/test modules/handler.ko" ' + '--config "/tmp/test config.toml" --timeout 12 --verbose --output "run results"') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'LIB', + '/tmp/test modules/core.ko', '/tmp/test modules/handler.ko', + '--config', '/tmp/test config.toml', '--timeout', '12', '--verbose', + '--output', 'run results']) + + def test_obsolete_options_are_rejected_instead_of_silently_running_all(self): + for option in ('TEST=LIB', 'KUNIT_LABELS=LIB', 'TEST_LOG=y', 'KUNIT_MODULE=/tmp/core.ko', + 'KUNIT_MODULES=/tmp/core.ko', 'KUNIT_VERBOSE=y', + 'KUNIT_CONFIG=test.toml', 'KUNIT_TIMEOUT=1', 'KUNIT_OUTPUT=logs'): + result, _ = self.runner_arguments(option, selection=None) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Use KUNIT_LABEL and KUNIT_ARGS', result.stderr) + + def build_components(self, *options, fail='', goals=('kunit-build',), selection=None, + uid='1000', missing=False, run_status=0): + # Exercise real component recipes against an inert kernel Makefile. + # All generated headers, module stand-ins and runner records stay here. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + paths = ['Makefile', 'scst/Makefile', 'scst/src/Makefile', + 'scst/src/kunit/modules.mk', 'scst/src/kunit/select.mk', + 'scripts/kunit/inventory.py', + 'scst/include/scst.h', 'scst/include/scst_const.h', + 'scst/include/scst_user.h', 'iscsi-scst/Makefile', + 'iscsi-scst/include/iscsi_scst.h', + 'iscsi-scst/conftest/sock_recvmsg/sock_recvmsg.c', + 'iscsi-scst/conftest/sock_recvmsg/Kbuild'] + for path in paths: + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + if (ROOT / path).is_file(): + shutil.copyfile(ROOT / path, target) + (root / 'scst/src/dev_handlers').mkdir() + (root / 'scst/src/certs').mkdir() + (root / 'scst/src/certs/Makefile').write_text('all:;@:\n') + (root / 'iscsi-scst/kernel').mkdir() + for path in SOURCE_FILES: + target = root / path.relative_to(ROOT) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + (root / 'iscsi-scst/conftest/sock_recvmsg/result-kunit-build-test.txt').write_text( + '-DSOCK_RECVMSG_HAS_FOUR_ARGS=0\n') + kdir = root / 'kernel' + (kdir / 'include/config').mkdir(parents=True) + (kdir / 'include/config/kernel.release').write_text('kunit-build-test\n') + fields = ['$(patsubst $(KUNIT_TEST_ROOT)/%,%,$(M))', '$(KUNIT_LABEL)', '$(KDIR)', + '$(KVER)', '$(ARCH)', '$(CROSS_COMPILE)', '$(CC)', + '$(PASS_CC_TO_MAKE)', '$(WERROR)'] + fields += ['$(CONFIG_SCST_KUNIT_TEST)', '$(CONFIG_SCST_KUNIT_LOG)'] + fields += ['$@'] + (kdir / 'Makefile').write_text( + '.PHONY: all scst.ko scst_vdisk.ko scst_user.ko iscsi-scst.ko\n' + 'all scst.ko scst_vdisk.ko scst_user.ko iscsi-scst.ko:\n' + '\t@test "$(M)" = "$(KUNIT_TEST_ROOT)/scst/src" || ' + 'test -f "$(KUNIT_TEST_ROOT)/scst/src/Module.symvers"\n' + "\t@printf '%s\\n' '" + '|'.join(fields) + "' >> \"$${KUNIT_TEST_LOG}\"\n" + + '\t@echo build >> "$${KUNIT_TEST_EVENTS}"\n' + '\t@test "$(FAIL_COMPONENT)" != "$(M:$(KUNIT_TEST_ROOT)/%=%)"\n' + '\t@touch "$(M)/Module.symvers"\n' + '\t@if [ "$(MISSING_MODULE)" != y ]; then touch "$(M)/$@"; fi\n') + binary = root / 'bin' + binary.mkdir() + scripts = { + 'id': '#!/bin/sh\nprintf "%s\\n" "$KUNIT_TEST_UID"\n', + 'sudo': '#!/bin/sh\necho sudo >> "$KUNIT_TEST_EVENTS"\nexec "$@"\n', + 'runner': f'#!{sys.executable}\nimport json, os, pathlib, sys\n' + 'if sys.argv[1].endswith("inventory.py"):\n' + ' os.execv(sys.executable, [sys.executable, *sys.argv[1:]])\n' + 'with open(os.environ["KUNIT_TEST_EVENTS"], "a") as log:\n' + ' log.write("runner\\n")\n' + 'pathlib.Path(os.environ["KUNIT_TEST_ARGS"]).write_text(' + 'json.dumps(sys.argv[1:]))\n' + 'sys.exit(int(os.environ["KUNIT_TEST_RUN_STATUS"]))\n', + } + for name, source in scripts.items(): + target = binary / name + target.write_text(source) + target.chmod(0o755) + records, events, args = root / 'build.log', root / 'events.log', root / 'args.json' + module_list = root / '.kunit-modules' + module_list.write_text('old.ko\n') + env = {key: value for key, value in os.environ.items() + if key not in ('TEST', 'TEST_LOG', 'KUNIT_LABEL', 'MAKEFLAGS', 'MFLAGS', 'KDIR', 'KVER', 'STRESS', 'PERF') + and not key.startswith('KUNIT_') and not key.endswith('_KUNIT_TEST')} + env.update(PATH=str(binary) + os.pathsep + env['PATH'], + KUNIT_TEST_ROOT=str(root), KUNIT_TEST_LOG=str(records), + KUNIT_TEST_EVENTS=str(events), KUNIT_TEST_ARGS=str(args), + KUNIT_TEST_UID=uid, KUNIT_TEST_RUN_STATUS=str(run_status)) + if selection is not None: + env['KUNIT_LABEL'] = selection + result = subprocess.run(['make', '-s', '-j4', *goals, 'KDIR=' + str(kdir), + 'ARCH=arm64', 'CROSS_COMPILE=aarch64-linux-gnu-', + 'CC=clang', 'PASS_CC_TO_MAKE=y', 'WERROR=y', + 'OFED_FLAVOR=in-tree', 'OFED_KERNEL_IB_DEVEL_RPM_INSTALLED=false', + 'OFED_COMPAT_RDMA_DEVEL_RPM_INSTALLED=false', + 'PYTHON=' + str(binary / 'runner'), + 'FAIL_COMPONENT=' + fail, + 'MISSING_MODULE=' + ('y' if missing else 'n'), *options], + cwd=root, env=env, capture_output=True, text=True) + rows = ([line.split('|') for line in records.read_text().splitlines()] + if records.exists() else []) + for row in rows: + self.assertEqual(row[2:9], [str(kdir), 'kunit-build-test', 'arm64', + 'aarch64-linux-gnu-', 'clang', 'y', 'y']) + modules = module_list.read_text().splitlines() if module_list.exists() else None + operations = events.read_text().splitlines() if events.exists() else [] + arguments = json.loads(args.read_text()) if args.exists() else [] + return result, rows, modules, operations, arguments + + def assert_full_build(self, result, rows, modules): + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(modules, MODULES) + self.assertEqual(rows[0][0], 'scst/src') + self.assertEqual(rows[0][-1], 'scst.ko') + self.assertEqual(sorted(row[0] + '/' + row[-1] for row in rows), sorted(MODULES)) + self.assertTrue(all(row[9] == 'y' for row in rows), rows) + + def test_full_build_for_root_and_regular_user(self): + for uid in ('0', '1000'): + result, rows, modules, events, args = self.build_components(uid=uid) + self.assert_full_build(result, rows, modules) + self.assertNotIn('sudo', events) + self.assertFalse(args) + + def test_labels_do_not_limit_build(self): + for selection in ('LIB', 'ALL', ''): + result, rows, modules, _, _ = self.build_components('KUNIT_LABEL=' + selection) + self.assert_full_build(result, rows, modules) + + def test_normal_build_keeps_tests_disabled(self): + for selection in (None, 'LIB'): + result, rows, _, events, _ = self.build_components(goals=('scst',), selection=selection) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(rows) + self.assertTrue(all(row[9] == 'n' for row in rows)) + self.assertNotIn('sudo', events) + + def test_public_build_and_logging_options_reach_all_owners(self): + for enabled in ('y', 'n'): + result, rows, modules, _, _ = self.build_components('KUNIT_LOG=' + enabled) + self.assert_full_build(result, rows, modules) + self.assertTrue(all(row[10] == enabled for row in rows), rows) + result, rows, _, _, _ = self.build_components('KUNIT_TEST=y', 'KUNIT_LOG=y', goals=('scst',)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(rows) + self.assertTrue(all(row[9:11] == ['y', 'y'] for row in rows), rows) + + def test_invalid_build_switch_and_categories(self): + for option in ('KUNIT_TEST=n', 'KUNIT_TEST=bad', 'KUNIT_TEST=', 'KUNIT_LOG=bad', 'KUNIT_LOG=', 'STRESS=HT', 'PERF=HT'): + result, rows, _, events, _ = self.build_components(option) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows, []) + self.assertEqual(events, []) + + def test_run_does_not_build_or_elevate(self): + for uid in ('1000', '0'): + result, rows, _, events, args = self.build_components('KUNIT_LABEL=LIB', goals=('kunit-run',), uid=uid) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(rows, []) + self.assertEqual(events, ['runner']) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'LIB']) + + def test_combined_target_orders_full_build_and_selected_execution(self): + for uid in ('0', '1000'): + result, rows, modules, events, args = self.build_components('KUNIT_LABEL=LIB', + 'KUNIT_ARGS="/tmp/test modules/core.ko" --output "run results"', + goals=('kunit-test',), uid=uid) + self.assert_full_build(result, rows, modules) + self.assertEqual(events, ['build'] * len(rows) + ['runner']) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'LIB', '/tmp/test modules/core.ko', + '--output', 'run results']) + + def test_combined_target_leaves_label_validation_to_runner(self): + for selection in ('', 'UNKNOWN'): + result, rows, modules, events, args = self.build_components( + goals=('kunit-test',), selection=selection, run_status=2) + self.assertNotEqual(result.returncode, 0) + self.assertTrue(modules) + self.assertTrue(rows) + self.assertEqual(events, ['build'] * len(rows) + ['runner']) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', selection]) + + def test_combined_target_defaults_to_all_after_successful_build(self): + result, rows, modules, events, args = self.build_components(goals=('kunit-test',)) + self.assert_full_build(result, rows, modules) + self.assertEqual(events, ['build'] * len(rows) + ['runner']) + self.assertEqual(args, ['scripts/kunit/run.py', '--labels', 'ALL']) + + def test_failed_or_incomplete_build_prevents_execution(self): + for options in ({'fail': 'scst/src'}, {'missing': True}): + result, _, modules, events, args = self.build_components('KUNIT_LABEL=LIB', goals=('kunit-test',), **options) + self.assertNotEqual(result.returncode, 0) + self.assertIsNone(modules) + self.assertNotIn('sudo', events) + self.assertFalse(args) + + def test_runner_failure_propagates(self): + result, _, _, events, _ = self.build_components('KUNIT_LABEL=LIB', goals=('kunit-test',), run_status=1) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(events[-1], 'runner') + self.assertNotIn('sudo', events) if __name__ == '__main__': diff --git a/scripts/kunit/test_run.py b/scripts/kunit/test_run.py new file mode 100644 index 000000000..e299bff40 --- /dev/null +++ b/scripts/kunit/test_run.py @@ -0,0 +1,1277 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Userspace runner checks. No test loads a module or opens /dev/kmsg.""" + +import contextlib +import errno +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +import run + +PASS = [' # Subtest: scst-LIB', ' 1..1', + ' ok 1 scst_lun_flat_encoding', 'ok 7 - scst-LIB'] +FAIL = [line.replace('ok ', 'not ok ') if 'ok ' in line else line for line in PASS] +SKIP = [line + ' # SKIP unavailable' if 'ok ' in line else line for line in PASS] +VDISK_PASS = [line.replace('scst-LIB', 'scst-VDISK-capacity') for line in PASS] +OOPS = ['BUG: kernel NULL pointer dereference, address: 0000000000000000', + '#PF: supervisor write access in kernel mode', 'PGD 0 P4D 0', + 'Oops: Oops: 0002 [#1] SMP NOPTI', 'CPU: 0 PID: 1 Comm: kunit_try_catch', + 'Tainted: [D]=DIE, [N]=TEST', 'RIP: 0010:test_case+0x10/0x20 [scst]', + 'Call Trace:', ' ', ' kunit_try_run_case+0x10/0x20 [kunit]', ' ', + '---[ end trace 00000000 ]---', 'RIP: 0010:test_case+0x10/0x20 [scst]'] + + +def report(): + return {'modules': [{'name': 'scst', 'suites': ['scst-LIB']}], 'module_state': 'not attempted', + 'commands': [], 'errors': [], 'expected_suites': ['scst-LIB']} + + +class ParseTests(unittest.TestCase): + def test_module_output_and_statuses(self): + for lines, status in ((PASS, 'PASS'), (FAIL, 'FAIL'), (SKIP, 'SKIP')): + with self.subTest(status=status): + parsed, _ = run.analyse(lines, ['scst-LIB']) + self.assertEqual(parsed['test_status'], status) + self.assertFalse(parsed['parse_errors']) + + def test_envelope_and_unrelated_output(self): + parsed, _ = run.analyse(['SCST initializing', 'KTAP version 1', '1..1'] + + PASS + ['SCST unloaded'], ['scst-LIB']) + self.assertEqual(parsed['counts']['passed'], 1) + + def test_parameterized_case(self): + lines = [PASS[0], ' 1..1', ' KTAP version 1', + ' # Subtest: boundary', ' 1..2', + ' ok 1 zero', ' not ok 2 maximum', + ' not ok 1 boundary', 'not ok 2 - scst-LIB'] + parsed, _ = run.analyse(lines, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'FAIL') + self.assertEqual(parsed['counts']['failed'], 1) + + def test_skip_reasons_are_preserved(self): + parsed, details = run.analyse(SKIP, ['scst-LIB']) + suite = parsed['suites'][0] + self.assertEqual(suite['skip_reason'], 'unavailable') + self.assertEqual(suite['children'][0]['skip_reason'], 'unavailable') + self.assertIn('unavailable', details) + + def test_parser_output_is_local_to_each_run(self): + console = io.StringIO() + with contextlib.redirect_stdout(console): + _, first = run.analyse(PASS, ['scst-LIB']) + _, second = run.analyse(VDISK_PASS, ['scst-VDISK-capacity']) + self.assertEqual(console.getvalue(), '') + self.assertIn('scst_lun_flat_encoding', first) + self.assertNotIn('scst-VDISK-capacity', first) + self.assertIn('scst-VDISK-capacity', second) + + def test_bad_evidence_cannot_pass(self): + cases = [[], PASS[:-1], PASS[:2] + PASS[3:], PASS + PASS, + [line.replace('1..1', '1..2') for line in PASS], + [line.replace('ok 1 scst_', 'ok 2 scst_') for line in PASS], + [PASS[0], ' 1..0', PASS[-1]], + [line.replace('scst-LIB', 'scst-unexpected') for line in PASS]] + for lines in cases: + with self.subTest(lines=lines): + parsed, _ = run.analyse(lines, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'ERROR') + + def test_missing_compiled_suite(self): + parsed, _ = run.analyse(PASS, ['scst-LIB', 'scst-extra']) + self.assertEqual(parsed['test_status'], 'ERROR') + + def test_trailing_results_and_foreign_suite_cannot_pass(self): + for extra in ([PASS[-1]], [' not ok 2 stray'], [' 1..99'], + [' # Subtest: foreign-suite'], + [line.replace('scst-LIB', 'foreign-suite') for line in PASS]): + with self.subTest(extra=extra): + parsed, _ = run.analyse(PASS + extra, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'ERROR') + + def test_parameter_name_does_not_become_a_suite(self): + logs = [PASS[0], ' 1..1', ' # Subtest: scst-parameter', + ' 1..1', ' ok 1 value', ' ok 1 scst-parameter', PASS[-1]] + parsed, _ = run.analyse(logs, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'PASS') + self.assertEqual(len(parsed['suites']), 1) + + def test_module_envelopes_between_suites(self): + logs = ['KTAP version 1', '1..1'] + PASS + ['KTAP version 1', '1..1'] + VDISK_PASS + parsed, _ = run.analyse(logs, ['scst-LIB', 'scst-VDISK-capacity']) + self.assertEqual(parsed['test_status'], 'PASS') + + def test_diagnostic_groups_include_trace_without_consuming_results(self): + warning = ['WARNING: CPU: 0 PID: 1 at example', 'CPU: 0 PID: 1 Comm: insmod', + 'Call Trace:', ' ', ' example+0x1/0x2', ' ', + '---[ end trace 00000000 ]---'] + parsed, _ = run.analyse(warning + PASS + ['unrelated', 'BUG: another failure'], ['scst-LIB']) + self.assertEqual(parsed['diagnostics'], ['\n'.join(warning), 'BUG: another failure']) + self.assertEqual(parsed['counts']['passed'], 1) + + def test_oops_header_and_trace_form_one_event(self): + logs = FAIL[:2] + OOPS + [' # assertion detail'] + FAIL[2:] + parsed, _ = run.analyse(logs, ['scst-LIB']) + self.assertEqual(parsed['diagnostics'], ['\n'.join(OOPS)]) + self.assertEqual(parsed['counts']['failed'], 1) + self.assertEqual(run.kernel_diagnostics(OOPS + OOPS), ['\n'.join(OOPS)] * 2) + + def test_independent_oops_warning_and_bare_trace_are_not_merged(self): + first = ['WARNING: unrelated', 'Call Trace:', ' warning_frame'] + oops = OOPS[3:] + logs = first + oops + PASS + ['Call Trace:', ' bare_frame'] + self.assertEqual(run.kernel_diagnostics(logs), + ['\n'.join(first), '\n'.join(oops), 'Call Trace:\n bare_frame']) + + def test_crash_console_is_brief_and_artifacts_keep_full_evidence(self): + logs = FAIL[:2] + OOPS + [' # assertion detail'] + FAIL[2:] + for verbose in (False, True): + with self.subTest(verbose=verbose), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': logs, 'module_state': 'retained', + 'started_suites': ['scst-LIB'], + 'expected_suites': ['scst-LIB', 'scst-VDISK-capacity']} + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp), verbose), 1) + output = console.getvalue() + self.assertIn('Kernel: FAIL - 1 diagnostic event', output) + self.assertIn('1/2 started, 1 not run', output) + self.assertEqual('NOT_RUN: scst-VDISK-capacity' in output, verbose) + self.assertIn(' [FAIL] scst_lun_flat_encoding', output) + self.assertIn('assertion detail', output) + self.assertEqual('Call Trace:' in output, verbose) + self.assertFalse(result['parse_errors']) + for artifact in ('messages.log', 'report.txt', 'result.json'): + content = (Path(tmp) / artifact).read_text() + self.assertIn('test_case+0x10/0x20', content) + self.assertIn('assertion detail', content) + self.assertEqual(result['not_run_suites'], ['scst-VDISK-capacity']) + + def test_repeated_diagnostic_headlines_are_summarized(self): + with tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': OOPS + OOPS + PASS, 'module_state': 'retained'} + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp)), 1) + self.assertIn('Kernel: FAIL - 2 diagnostic events', console.getvalue()) + self.assertIn(' ' + OOPS[0] + ' (2 events)', console.getvalue()) + self.assertNotIn('Call Trace:', console.getvalue()) + + def test_unstarted_selection_cannot_pass_without_a_diagnostic(self): + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + result = report() | {'messages': PASS, 'module_state': 'unloaded', + 'started_suites': ['scst-LIB'], + 'expected_suites': ['scst-LIB', 'scst-VDISK-capacity']} + self.assertEqual(run.finish(result, Path(tmp)), 2) + self.assertIn('Execution stopped', result['errors'][0]) + self.assertEqual(result['not_run_suites'], ['scst-VDISK-capacity']) + + def test_preflight_console_is_compact_and_full_report_keeps_selection(self): + with tempfile.TemporaryDirectory() as tmp: + names = [f'scst-suite-{i}' for i in range(15)] + result = report() | {'expected_suites': names, + 'errors': ['Cannot verify preloaded KUnit: enable is unreadable.\n' + 'Module left unchanged. Leave KUnit unloaded before retrying.']} + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp)), 2) + output = console.getvalue() + self.assertIn('Tests: NOT_RUN (15 suites not run)', output) + self.assertNotIn('NOT_RUN: scst-suite-', output) + self.assertNotIn('0 passed', output) + self.assertLess(len(output.splitlines()), 10) + saved = (Path(tmp) / 'report.txt').read_text() + self.assertTrue(all('NOT_RUN: ' + name in saved for name in names)) + + def test_color_is_terminal_only_and_reports_remain_plain(self): + for tty, disabled in ((True, False), (False, False), (True, True)): + with self.subTest(tty=tty, disabled=disabled), tempfile.TemporaryDirectory() as tmp: + console = io.StringIO() + environment = {'TERM': 'xterm'} | ({'NO_COLOR': ''} if disabled else {}) + with contextlib.redirect_stdout(console), patch.object(console, 'isatty', return_value=tty), \ + patch.dict(os.environ, environment, clear=True): + sample = run.color_console('PASS FAIL NOT_RUN') + run.finish(report() | {'messages': PASS, 'module_state': 'unloaded'}, Path(tmp)) + colored = tty and not disabled + for color in (31, 32, 33): + self.assertEqual(f'\033[{color}m' in sample, colored) + self.assertEqual('\033[' in console.getvalue(), colored) + for name in ('messages.log', 'report.txt', 'result.json'): + self.assertNotIn('\033[', (Path(tmp) / name).read_text()) + + def test_console_modes_preserve_full_report(self): + for verbose in (False, True): + with self.subTest(verbose=verbose), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': PASS, 'module_state': 'unloaded'} + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp), verbose), 0) + self.assertEqual('scst_lun_flat_encoding' in console.getvalue(), verbose) + self.assertIn('scst_lun_flat_encoding', (Path(tmp) / 'report.txt').read_text()) + self.assertIn('scst-LIB', console.getvalue()) + + def test_failed_case_is_visible_in_default_console_and_json(self): + logs = FAIL[:2] + [' # assertion detail'] + FAIL[2:] + with tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': logs, 'module_state': 'unloaded'} + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp)), 1) + self.assertIn(' [FAIL] scst_lun_flat_encoding', console.getvalue()) + self.assertIn('assertion detail', console.getvalue()) + data = json.loads((Path(tmp) / 'result.json').read_text()) + self.assertIn('assertion detail', '\n'.join(data['suites'][0]['children'][0]['log'])) + + def test_parameter_failure_details_are_visible_with_and_without_a_plan(self): + for planned in (False, True): + for verbose in (False, True): + with self.subTest(planned=planned, verbose=verbose), tempfile.TemporaryDirectory() as tmp: + logs = [PASS[0], ' 1..1', ' KTAP version 1', + ' # Subtest: boundary'] + if planned: + logs.append(' 1..3') + logs += [' # first parameter assertion detail', ' not ok 1 first', + ' # second parameter assertion detail', ' not ok 2 second', + ' ok 3 third', ' not ok 1 boundary', 'not ok 1 scst-LIB'] + result = report() | {'messages': logs, 'module_state': 'unloaded'} + with contextlib.redirect_stdout(io.StringIO()) as console: + self.assertEqual(run.finish(result, Path(tmp), verbose), 1) + # Incremental output and final reporting must not duplicate diagnostics. + run.show_suites(result, result) + output = console.getvalue() + for parameter in ('first', 'second'): + detail = f'{parameter} parameter assertion detail' + self.assertEqual(output.count(detail), 1) + self.assertIn(f'[FAIL] boundary/{parameter}', output) + self.assertIn(detail, (Path(tmp) / 'report.txt').read_text()) + self.assertIn(detail, (Path(tmp) / 'result.json').read_text()) + case = result['suites'][0]['children'][0]['children'][ + 0 if parameter == 'first' else 1] + self.assertIn(detail, '\n'.join(case['log'])) + heading = f' [FAIL] boundary/{parameter}\n' + section = output.split(heading, 1)[1].split('\n\n', 1)[0] + self.assertIn(detail, section) + parent = result['suites'][0]['children'][0] + self.assertNotIn('first parameter assertion detail', '\n'.join(parent['log'])) + self.assertEqual('[PASS] boundary/third' in output, verbose) + self.assertEqual(result['counts']['failed'], 2) + self.assertEqual(result['counts']['passed'], 1) + self.assertFalse(result['errors']) + + def test_progress_precedes_grouped_failures_and_keeps_assertion_indentation(self): + logs = [PASS[0], ' 1..1', ' # Subtest: boundary', + ' # first assertion', ' Expected value == 1, but', + ' value == 0', ' 1 == 1', ' not ok 1 first', + ' # second assertion', ' not ok 2 second', + ' # boundary: pass:0 fail:2 skip:0 total:2', ' not ok 1 boundary', + ' # module: scst', ' # Totals: pass:0 fail:2 skip:0 total:2', + 'not ok 1 scst-LIB'] + with tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': logs + VDISK_PASS, 'module_state': 'unloaded', + 'expected_suites': ['scst-LIB', 'scst-VDISK-capacity']} + parsed, _ = run.analyse(logs, ['scst-LIB']) + with contextlib.redirect_stdout(io.StringIO()) as console: + run.show_suites(parsed, result) + self.assertEqual(console.getvalue(), '[FAIL] scst-LIB\n') + self.assertEqual(run.finish(result, Path(tmp)), 1) + output = console.getvalue() + self.assertEqual(output.count('[FAIL] scst-LIB'), 1) + self.assertLess(output.index('[PASS] scst-VDISK-capacity'), output.index('Failures:')) + self.assertIn('Failures:\n\nscst-LIB\n [FAIL] boundary/first\n', output) + self.assertIn(' Expected value == 1, but\n value == 0\n 1 == 1', output) + self.assertIn('\n\n [FAIL] boundary/second\n', output) + self.assertNotIn('[FAIL] boundary\n', output) + self.assertNotIn('# Totals:', output) + self.assertNotIn('# module:', output) + self.assertNotIn('pass:0 fail:2', output) + self.assertIn('# Totals:', (Path(tmp) / 'report.txt').read_text()) + + def test_mixed_printk_and_assertions_keep_consistent_console_indentation(self): + for verbose in (False, True): + for indent in ('', ' ', ' '): + with self.subTest(verbose=verbose, indent=indent), tempfile.TemporaryDirectory() as tmp: + diagnostic = ['[42]: scst: ***ERROR***: injected backend error', + indent + '# boundary: EXPECTATION FAILED at suite.c:42', + indent + 'Expected actual == wanted, but', + indent + ' actual == 0 (0x0)', + indent + ' wanted == 1 (0x1)', + 'custom assertion message', ' continuation of the message', + indent + '# boundary: ASSERTION FAILED at suite.c:43', + indent + 'Expected rc == 0, but', + indent + ' rc == -1 (0xffffffffffffffff)'] + logs = [PASS[0], ' 1..1', ' # Subtest: boundary', + *diagnostic, ' not ok 1 first', + *diagnostic, ' not ok 2 second', + ' not ok 1 boundary', 'not ok 1 scst-LIB'] + result = report() | {'messages': logs, 'module_state': 'unloaded'} + with contextlib.redirect_stdout(io.StringIO()) as console: + self.assertEqual(run.finish(result, Path(tmp), verbose), 1) + output = console.getvalue() + expected = (' # boundary: EXPECTATION FAILED at suite.c:42\n' + ' Expected actual == wanted, but\n' + ' actual == 0 (0x0)\n' + ' wanted == 1 (0x1)\n' + ' custom assertion message\n' + ' continuation of the message\n' + ' # boundary: ASSERTION FAILED at suite.c:43\n' + ' Expected rc == 0, but\n' + ' rc == -1 (0xffffffffffffffff)') + for parameter in ('first', 'second'): + section = output.split(f' [FAIL] boundary/{parameter}\n', 1)[1] + self.assertIn(expected, section.split('\n\n', 1)[0]) + self.assertEqual(result['counts']['failed'], 2) + saved = json.loads((Path(tmp) / 'result.json').read_text()) + cases = saved['suites'][0]['children'][0]['children'] + self.assertEqual([case['log'] for case in cases], [diagnostic, diagnostic]) + self.assertEqual((Path(tmp) / 'messages.log').read_text(), '\n'.join(logs) + '\n') + + def test_passing_first_parameter_logs_do_not_leak_into_failed_parent(self): + logs = [PASS[0], ' 1..1', ' # Subtest: boundary', + ' # expected negative-path message', ' ok 1 first', + ' # actual failure', ' not ok 2 second', + ' # parent teardown detail', ' not ok 1 boundary', 'not ok 1 scst-LIB'] + for verbose in (False, True): + with self.subTest(verbose=verbose), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': logs, 'module_state': 'unloaded'} + with contextlib.redirect_stdout(io.StringIO()) as console: + self.assertEqual(run.finish(result, Path(tmp), verbose), 1) + output = console.getvalue() + self.assertEqual('expected negative-path message' in output, verbose) + self.assertIn('parent teardown detail', output) + parent = result['suites'][0]['children'][0] + self.assertEqual(parent['log'], [' # parent teardown detail']) + self.assertEqual(parent['children'][0]['log'], [' # expected negative-path message']) + self.assertEqual(parent['children'][1]['log'], [' # actual failure']) + self.assertIn('expected negative-path message', (Path(tmp) / 'report.txt').read_text()) + + def test_parameter_plan_validation_is_not_changed_by_log_association(self): + for plan in (' 1..3', ' 1..1'): + logs = [PASS[0], ' 1..1', ' # Subtest: boundary', plan, + ' # first failure', ' not ok 1 first', + ' ok 2 second', ' not ok 1 boundary', 'not ok 1 scst-LIB'] + parsed, _ = run.analyse(logs, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'ERROR') + self.assertTrue(parsed['parse_errors']) + + def test_module_metadata_defaults_and_full_reports(self): + metadata = {'version': '3.11.0-pre', 'description': 'SCSI target core', + 'author': 'SCST contributors', 'license': 'GPL', 'git_commit': 'a' * 40, + 'build_date': '2026-10-13 12:00:00 UTC', 'build_number': '42', + 'kver': '7.2.9', 'arch_type': 'x86_64', 'srcversion': 'ABC123', + 'vermagic': '7.2.9 SMP preempt mod_unload'} + for verbose in (False, True): + for failed in (False, True): + with self.subTest(verbose=verbose, failed=failed), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': FAIL if failed else PASS, 'module_state': 'unloaded', + 'kernel': '7.2.9-test', 'architecture': 'x86_64'} + result['modules'][0].update(metadata) + console = io.StringIO() + with contextlib.redirect_stdout(console): + self.assertEqual(run.finish(result, Path(tmp), verbose), int(failed)) + output = console.getvalue() + saved = (Path(tmp) / 'report.txt').read_text() + data = json.loads((Path(tmp) / 'result.json').read_text()) + self.assertEqual(output.count('Module: scst\n'), int(verbose)) + self.assertIn('\n\n' + ('Module: scst' if verbose else 'SCST: 3.11.0-pre'), output) + last_field = 'vermagic' if verbose else 'arch_type' + self.assertIn(metadata[last_field] + '\n\nRuntime kernel:', output) + self.assertIn('Runtime kernel: 7.2.9-test (x86_64)', output) + self.assertIn(('Module scst: unknown' if verbose else + 'Runtime kernel: 7.2.9-test (x86_64)') + '\n\nTests:', output) + for key, value in metadata.items(): + labels = {'version': 'SCST', 'git_commit': 'Commit', + 'build_number': 'Build number', 'build_date': 'Build date', + 'kver': 'Built for kernel', 'arch_type': 'Architecture'} + if verbose: + self.assertIn(f' {key}: {value}', output) + elif key in labels: + self.assertIn(f'{labels[key]}: {value}', output) + else: + self.assertNotIn(f'{key}:', output) + self.assertIn(f' {key}: {value}', saved) + self.assertEqual(data['modules'][0][key], value) + + def test_each_module_keeps_its_own_metadata_and_missing_fields_are_optional(self): + for verbose in (False, True): + with self.subTest(verbose=verbose), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': PASS, 'module_state': 'unloaded', + 'modules': [{'name': 'scst', 'version': '3.11.0-pre', + 'git_commit': 'CORE'}, + {'name': 'scst_vdisk', 'version': '3.11.0-pre', + 'srcversion': 'HANDLER', 'git_commit': ''}]} + with contextlib.redirect_stdout(io.StringIO()) as console: + self.assertEqual(run.finish(result, Path(tmp), verbose), 0) + output = console.getvalue() + self.assertEqual(output.count('version: 3.11.0-pre'), 2 if verbose else 0) + self.assertEqual('SCST: 3.11.0-pre' in output, not verbose) + self.assertEqual('Module: scst_vdisk' in output, verbose) + self.assertEqual('srcversion: HANDLER' in output, verbose) + self.assertIn('git_commit: CORE' if verbose else 'Commit: CORE', output) + self.assertNotIn('build_date:', output) + if verbose: + self.assertIn('git_commit: CORE\n\nModule: scst_vdisk', output) + else: + self.assertIn('Commit: CORE\n\nTests:', output) + saved = (Path(tmp) / 'report.txt').read_text() + core, handler = saved.split('Module: scst_vdisk', 1) + self.assertIn('git_commit: CORE', core) + self.assertIn('srcversion: HANDLER', handler) + self.assertNotIn('git_commit:', handler) + data = json.loads((Path(tmp) / 'result.json').read_text()) + self.assertEqual(data['modules'][1]['srcversion'], 'HANDLER') + + def test_verbose_shows_optional_case_logs_without_changing_results(self): + logs = PASS[:2] + [' # scst_lun_flat_encoding: encoded LUN = 0x5234'] + PASS[2:] + for verbose in (False, True): + with self.subTest(verbose=verbose), tempfile.TemporaryDirectory() as tmp: + console = io.StringIO() + with contextlib.redirect_stdout(console): + run.finish(report() | {'messages': logs, 'module_state': 'unloaded'}, Path(tmp), verbose) + self.assertEqual('encoded LUN' in console.getvalue(), verbose) + self.assertIn('encoded LUN', (Path(tmp) / 'report.txt').read_text()) + self.assertIn('Tests: PASS', console.getvalue()) + + def test_kernel_diagnostics(self): + for message in ('WARNING: at scst', 'BUG: KASAN: use-after-free', + 'watchdog: BUG: soft lockup', 'rcu: detected stalls', + 'INFO: task insmod blocked for more than 120 seconds'): + with self.subTest(message=message): + parsed, _ = run.analyse(PASS + [message], ['scst-LIB']) + self.assertEqual(parsed['diagnostics'], [message]) + + def test_reports_and_exit_codes(self): + for lines, state, errors, expected in ((PASS, 'unloaded', [], 0), + (FAIL, 'unloaded', [], 1), (SKIP, 'unloaded', [], 0), + (PASS[:-1], 'retained', [], 2), + (PASS, 'loading', ['timeout'], 2), + (PASS + ['# SCST_KUNIT_RETAINED: worker'], 'retained', [], 2), + (PASS + ['WARNING: test'], 'retained', [], 1), + (PASS + ['BUG: unload'], 'unloaded', [], 1)): + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as tmp: + result = report() | {'messages': lines, 'module_state': state, 'errors': errors} + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), expected) + data = json.loads((Path(tmp) / 'result.json').read_text()) + self.assertNotIn('messages', data) + self.assertTrue((Path(tmp) / 'report.txt').is_file()) + + def test_preflight_failure_reports_not_run_without_missing_results(self): + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + result = report() | {'errors': ['KUnit preparation failed']} + self.assertEqual(run.finish(result, Path(tmp)), 2) + self.assertEqual(result['test_status'], 'NOT_RUN') + self.assertEqual(result['errors'], ['KUnit preparation failed']) + self.assertFalse(result['parse_errors']) + self.assertEqual(result['expected_suites'], ['scst-LIB']) + + +class CaptureTests(unittest.TestCase): + def test_failed_command_keeps_bounded_stderr_without_prior_commands(self): + child = Mock(pid=42) + child.poll.return_value = 1 + capture = Mock() + capture.drain.return_value = False + def start(argv, **kwargs): + kwargs['stdout'].write(b'x' * 5000 + b'\ninsmod: Unknown symbol dlm_new_lockspace\n') + return child + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + (directory / 'commands.log').write_text('old command output\n') + result = report() + with patch.object(run.subprocess, 'Popen', side_effect=start): + self.assertEqual(run.command(['insmod', 'test.ko'], capture, directory, 10, result), 1) + error = str(run.command_error(result, 'insmod scst', 1)) + self.assertIn('Unknown symbol dlm_new_lockspace', error) + self.assertNotIn('old command output', error) + self.assertLessEqual(len(result['commands'][0]['output']), 4096) + self.assertGreater((directory / 'commands.log').stat().st_size, 5000) + + def capture(self): + capture = run.KernelLog.__new__(run.KernelLog) + capture.raw = io.BytesIO() + capture.sequence = None + capture.messages = [] + capture.previous_kernel = False + capture.fd = 987 + return capture + + def test_sequence_and_raw_evidence(self): + capture = self.capture() + capture.accept(b'6,100,10,-;first\n') + capture.accept(b'6,101,11,-;second\n SUBSYSTEM=test\n') + self.assertEqual(capture.messages, ['first', 'second']) + with self.assertRaisesRegex(run.RunError, 'Lost kernel log records'): + capture.accept(b'6,103,12,-;after gap\n') + self.assertIn(b'after gap', capture.raw.getvalue()) + + def test_read_overrun(self): + with patch.object(run.os, 'read', side_effect=OSError(errno.EPIPE, 'overrun')): + with self.assertRaisesRegex(run.RunError, 'overrun'): + self.capture().drain() + + def test_injected_results_are_raw_evidence_only(self): + capture = self.capture() + for number, line in enumerate(PASS + ['BUG: injected', 'SCST_KUNIT_RETAINED: injected']): + capture.accept(f'14,{number},10,-;{line}\n'.encode()) + parsed, _ = run.analyse(capture.messages, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'ERROR') + self.assertFalse(parsed['diagnostics']) + self.assertFalse(parsed['retained_fixtures']) + self.assertIn(b'injected', capture.raw.getvalue()) + with self.assertRaisesRegex(run.RunError, 'Lost kernel log records'): + capture.accept(b'6,7,11,-;gap after ignored userspace\n') + + def test_fragments_escapes_and_extended_headers(self): + capture = self.capture() + capture.accept(b'6,1,10,-,extra; # Subtest: scst-\n') + capture.accept(b'6,2,11,c;LIB\n') + capture.accept(b'6,3,12,-; 1..1\n') + capture.accept(b'6,4,13,-; ok 1 escaped_\\x5c_name\n') + capture.accept(b'6,5,14,-;ok 7 - scst-LIB\n') + parsed, _ = run.analyse(capture.messages, ['scst-LIB']) + self.assertEqual(parsed['test_status'], 'PASS') + self.assertIn('escaped_\\_name', capture.messages[2]) + self.assertIn(b'\\x5c', capture.raw.getvalue()) + + def test_userspace_record_breaks_fragment_chain(self): + capture = self.capture() + capture.accept(b'6,1,10,-;first\n') + capture.accept(b'14,2,11,-;ignored\n') + capture.accept(b'6,3,12,c;fragment\n') + self.assertEqual(capture.messages, ['first', 'fragment']) + + def test_malformed_metadata_preserves_raw_record(self): + for record in (b'6,1,10;missing flags\n', b'-1,1,10,-;priority\n', + b'6,-1,10,-;sequence\n', b'6,1,bad,-;timestamp\n'): + with self.subTest(record=record): + capture = self.capture() + with self.assertRaises(run.RunError): + capture.accept(record) + self.assertEqual(capture.raw.getvalue(), record) + + def test_malformed_record(self): + with self.assertRaisesRegex(run.RunError, 'Malformed'): + self.capture().accept(b'not a kmsg record') + + def test_timeout_does_not_kill_or_unload(self): + child = Mock(pid=42) + child.poll.return_value = None + capture = Mock() + capture.drain.return_value = False + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(run.subprocess, 'Popen', return_value=child), \ + patch.object(run.time, 'monotonic', side_effect=[0, 11]): + with self.assertRaisesRegex(run.RunError, 'PID 42'): + run.command(['insmod', 'test.ko'], capture, Path(tmp), 10, report()) + child.kill.assert_not_called() + child.terminate.assert_not_called() + child.wait.assert_not_called() + + def test_command_reads_messages_printed_between_drain_and_exit(self): + pending, messages = [], [] + def drain(): + messages.extend(pending) + pending.clear() + return False + def exited(): + pending.append('WARNING: final module message') + return 0 + child = Mock(pid=42) + child.poll.side_effect = exited + capture = Mock() + capture.drain.side_effect = drain + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(run.subprocess, 'Popen', return_value=child): + self.assertEqual(run.command(['rmmod', 'test'], capture, Path(tmp), 10, report()), 0) + self.assertEqual(messages, ['WARNING: final module message']) + self.assertFalse(pending) + + +def manifest(name='scst', labels=None, vermagic='test-kernel', depends=None): + if labels is None: + labels = {'scst-LIB': 'LIB'} + if depends is None: + depends = () if name == 'scst' else ('scst',) + return {'name': name, 'suites': list(labels), 'labels': labels, 'vermagic': vermagic, 'depends': list(depends)} + + +def suite_log(suite): + return [line.replace('scst-LIB', suite) for line in PASS] + + +class LifecycleTests(unittest.TestCase): + def test_label_manifest_validation(self): + for records, valid in [('LIB:scst-LIB', True), ('LIB:scst-LIB\nCM:scst-CM', True), + ('', False), ('scst-LIB', False), ('LIB:../bad', False), + ('LIB:scst-LIB\nCM:scst-LIB', False), ('ALL:scst-LIB', False)]: + fields = {'name': 'scst', 'vermagic': 'test-kernel', 'scst_kunit_suite': records, + 'version': '3.11', 'srcversion': 'ABC123', 'depends': 'dlm,kunit'} + with self.subTest(records=records), patch.object(run.subprocess, 'run', + side_effect=lambda argv, **kw: Mock(stdout=fields.get(argv[2], ""))): + if valid: + info = run.module_info(Path('/mock/scst.ko')) + self.assertEqual(info['labels'], {r.split(':')[1]: r.split(':')[0] for r in records.splitlines()}) + self.assertEqual(info['version'], '3.11') + else: + with self.assertRaises(run.RunError): + run.module_info(Path('/mock/scst.ko')) + + def test_metadata_comes_from_the_requested_module_file(self): + fields = {'name': 'scst', 'scst_kunit_suite': 'LIB:scst-LIB', 'depends': '', + 'version': '3.11.0-pre', 'git_commit': 'embedded-commit', + 'build_number': '0', 'vermagic': 'test-kernel'} + path = Path('/mock/test modules/scst.ko') + with patch.object(run.subprocess, 'run', + side_effect=lambda argv, **kw: Mock(stdout=fields.get(argv[2], ''))) as query: + info = run.module_info(path) + self.assertEqual(info['git_commit'], 'embedded-commit') + self.assertEqual(info['build_number'], '0') + self.assertEqual(info['build_date'], '') + self.assertEqual(info['labels'], {'scst-LIB': 'LIB'}) + for call in query.call_args_list: + self.assertEqual(call.args[0][:2], ['modinfo', '-F']) + self.assertEqual(call.args[0][-1], str(path)) + self.assertEqual({call.args[0][2] for call in query.call_args_list}, + set(run.MODULE_METADATA) | {'name', 'scst_kunit_suite', 'depends'}) + + def execute(self, logs=None, labels=None, inventory=None, load_logs=None, + loaded=(), missing=(), failure=None, unload_warning=False, preflight=None, + kernel_mode='built-in', interface_error=None, remains=(), + fail_loaded=False, failure_code=1, cleanup_failure=False, observe=None): + result = report() + capture = Mock(messages=[]) + inventory = inventory or [manifest()] + logs = {'scst-LIB': PASS} if logs is None else logs + load_logs = load_logs or {} + present = set(loaded) + labels = ['ALL'] if labels is None else labels + def operation(argv, *args): + if observe: + observe(argv) + if argv[0] == 'modprobe': + name = 'kunit' if '--first-time' in argv else argv[-1] + if name == 'kunit': + self.assertEqual(argv, ['modprobe', '--first-time', '--ignore-install', 'kunit', + 'enable=1', 'autorun=0']) + else: + self.assertEqual(argv, ['modprobe', '--ignore-install', name]) + capture.messages.extend(load_logs.get(name, [])) + elif argv[0] == 'insmod': + name = Path(argv[1]).stem + capture.messages.extend(load_logs.get(name, [])) + elif argv[0] == 'rmmod': + name = argv[1] + self.assertEqual(argv, ['rmmod', name]) + if unload_warning: + capture.messages.append('WARNING: at module exit') + else: + name = Path(argv[-1]).parent.name + capture.messages.extend(logs.get(name, [])) + if failure == (argv[0], name) or failure == ('suite', name): + if fail_loaded: + present.add(name) + if isinstance(failure_code, Exception): + raise failure_code + return failure_code + if cleanup_failure and argv == ['rmmod', 'kunit']: + return 1 + if argv[0] in ('modprobe', 'insmod'): + self.assertNotIn(name, present) + present.add(name) + elif argv[0] == 'rmmod' and name not in remains: + present.discard(name) + return 0 + with tempfile.TemporaryDirectory() as tmp: + debugfs = Path(tmp) / 'kunit' + debugfs.mkdir() + for item in inventory: + for suite in item['suites']: + path = debugfs / suite + path.mkdir(exist_ok=True) + if suite not in missing: + (path / 'run').touch() + with patch.object(run.os, 'geteuid', return_value=0), \ + patch.object(run.shutil, 'which', return_value='/mock/tool'), \ + patch.object(run, 'module_info', side_effect=inventory), \ + patch.object(run, 'check_kernel', side_effect=preflight, return_value=kernel_mode), \ + patch.object(run, 'check_kunit_interfaces', side_effect=interface_error), \ + patch.object(run, 'DEBUGFS', debugfs), \ + patch.object(Path, 'exists', lambda p: str(p) in ['/sys/module/' + n for n in present]), \ + patch.object(run, 'KernelLog', return_value=capture), \ + patch.object(run, 'command', side_effect=operation) as command: + try: + run.execute([Path('/mock/' + i['name'] + '.ko') for i in inventory], + 10, Path(tmp), result, labels) + except run.RunError as exc: + result['errors'].append(str(exc)) + operations = [] + for call in command.call_args_list: + argv = call.args[0] + operations.append(('modprobe', 'kunit' if '--first-time' in argv else argv[-1]) if argv[0] == 'modprobe' else + (argv[0], Path(argv[1]).stem) if argv[0] == 'insmod' else + (argv[0], argv[1]) if argv[0] == 'rmmod' else + ('suite', Path(argv[-1]).parent.name)) + return result, operations, capture + + @staticmethod + def both_manifests(): + return [manifest(), manifest('scst_vdisk', {'scst-VDISK-capacity': 'VDISK'})] + + def test_complete_pass_fail_and_skip_allow_cleanup(self): + for messages in (PASS, FAIL, SKIP): + result, operations, capture = self.execute({'scst-LIB': messages}) + self.assertEqual(result['module_state'], 'unloaded') + self.assertEqual(operations, [('insmod', 'scst'), ('suite', 'scst-LIB'), ('rmmod', 'scst')]) + capture.close.assert_called_once() + + def test_results_stream_before_next_owner_without_summary_duplicates(self): + console = io.StringIO() + def observe(argv): + if argv[0] == 'insmod' and argv[1].endswith('scst_vdisk.ko'): + self.assertIn('[FAIL] scst-LIB', console.getvalue()) + with contextlib.redirect_stdout(console): + result, operations, _ = self.execute( + {'scst-LIB': FAIL, 'scst-VDISK-capacity': VDISK_PASS}, + inventory=self.both_manifests(), observe=observe) + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run.finish(result, Path(tmp)), 1) + saved = json.loads((Path(tmp) / 'result.json').read_text()) + self.assertNotIn('printed_suites', saved) + self.assertEqual(console.getvalue().count('[FAIL] scst-LIB'), 1) + self.assertEqual(console.getvalue().count('[PASS] scst-VDISK-capacity'), 1) + self.assertEqual(operations[-2:], [('rmmod', 'scst_vdisk'), ('rmmod', 'scst')]) + + def test_oops_stops_before_next_suite_and_explains_why(self): + inventory = [manifest('scst', {'scst-LIB': 'LIB', 'scst-CDB': 'CDB'})] + console = io.StringIO() + with contextlib.redirect_stdout(console): + result, operations, _ = self.execute({'scst-LIB': FAIL[:2] + OOPS + FAIL[2:]}, + inventory=inventory, kernel_mode='load') + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(run.finish(result, Path(tmp)), 1) + self.assertEqual(operations, [('modprobe', 'kunit'), ('insmod', 'scst'), ('suite', 'scst-LIB')]) + self.assertEqual(result['not_run_suites'], ['scst-CDB']) + self.assertIn('Stopped: kernel diagnostics during scst-LIB; 1 suite not run', console.getvalue()) + self.assertEqual(result['kunit_state'], 'retained') + + def test_modular_kunit_is_owned_until_scst_cleanup_finishes(self): + for messages in (PASS, FAIL, SKIP): + result, operations, _ = self.execute(logs={'scst-LIB': messages}, kernel_mode='load') + self.assertFalse(result['errors']) + self.assertEqual(operations, [('modprobe', 'kunit'), ('insmod', 'scst'), + ('suite', 'scst-LIB'), ('rmmod', 'scst'), ('rmmod', 'kunit')]) + self.assertEqual(result['kunit_state'], 'unloaded') + + def test_external_dependencies_precede_kunit_and_preloaded_ones_are_reused(self): + inventory = [manifest(depends=('dlm', 'kunit'))] + result, operations, _ = self.execute(inventory=inventory, kernel_mode='load') + self.assertFalse(result['errors']) + self.assertEqual(operations[:3], [('modprobe', 'dlm'), ('modprobe', 'kunit'), ('insmod', 'scst')]) + self.assertNotIn(('rmmod', 'dlm'), operations) + result, operations, _ = self.execute(inventory=inventory, kernel_mode='load', loaded=('dlm',)) + self.assertFalse(result['errors']) + self.assertNotIn(('modprobe', 'dlm'), operations) + + def test_only_selected_owners_prepare_external_dependencies(self): + inventory = [manifest(depends=('dlm', 'kunit')), + manifest('scst_vdisk', {'scst-VDISK-capacity': 'VDISK'}, depends=('scst', 'dlm', 'crc32c')), + manifest('scst_user', {'scst-USER-cleanup': 'USER'}, depends=('scst', 'unselected_dep'))] + result, operations, _ = self.execute(inventory=inventory, labels=['VDISK'], + logs={'scst-VDISK-capacity': VDISK_PASS}) + self.assertFalse(result['errors']) + self.assertEqual([name for op, name in operations if op == 'modprobe'], ['crc32c', 'dlm']) + self.assertEqual([name for op, name in operations if op == 'insmod'], ['scst', 'scst_vdisk']) + + def test_dependency_failure_or_diagnostics_stop_before_kunit_loading(self): + for options in ({'failure': ('modprobe', 'dlm')}, + {'load_logs': {'dlm': ['WARNING: dependency initialization']}}, + {'load_logs': {'dlm': PASS}}): + result, operations, _ = self.execute(inventory=[manifest(depends=('dlm',))], + kernel_mode='load', **options) + self.assertTrue(result['errors']) + self.assertEqual(operations, [('modprobe', 'dlm')]) + self.assertEqual(result['kunit_state'], 'not loaded') + + def test_failed_initial_load_releases_only_runner_owned_kunit(self): + for mode in ('load', 'preloaded', 'built-in'): + result, operations, _ = self.execute(kernel_mode=mode, failure=('insmod', 'scst'), + loaded=('kunit',) if mode == 'preloaded' else ()) + self.assertIn('insmod scst', result['errors'][0]) + self.assertEqual(result['module_state'], 'not loaded') + self.assertEqual(('rmmod', 'kunit') in operations, mode == 'load') + self.assertEqual(result['kunit_state'], 'unloaded' if mode == 'load' else mode) + + def test_unsafe_failed_load_does_not_unload_kunit(self): + for options in ({'load_logs': {'scst': ['WARNING: init failed']}}, + {'load_logs': {'scst': PASS}}, {'fail_loaded': True}, + {'failure_code': -9}, {'failure_code': run.RunError('lost records')}, + {'failure_code': run.RunError('timeout; child still running')}): + result, operations, _ = self.execute(kernel_mode='load', failure=('insmod', 'scst'), **options) + self.assertTrue(result['errors']) + self.assertNotIn(('rmmod', 'kunit'), operations) + self.assertEqual(result['kunit_state'], 'retained') + + def test_failed_cleanup_keeps_original_load_error(self): + result, operations, _ = self.execute(kernel_mode='load', failure=('insmod', 'scst'), cleanup_failure=True) + self.assertEqual(operations[-1], ('rmmod', 'kunit')) + self.assertEqual(result['kunit_state'], 'retained') + self.assertTrue(any('insmod scst' in error for error in result['errors'])) + self.assertTrue(any('Could not unload' in error for error in result['errors'])) + + def test_preloaded_kunit_is_preserved_after_pass_fail_and_skip(self): + for messages in (PASS, FAIL, SKIP): + with self.subTest(messages=messages): + result, operations, _ = self.execute(kernel_mode='preloaded', loaded=('kunit',), + logs={'scst-LIB': messages}) + self.assertFalse(result['errors']) + self.assertEqual(result['kunit_state'], 'preloaded') + self.assertEqual(operations, [('insmod', 'scst'), ('suite', 'scst-LIB'), ('rmmod', 'scst')]) + + def test_preloaded_kunit_is_preserved_after_operation_errors(self): + for failure in (('insmod', 'scst'), ('suite', 'scst-LIB'), ('rmmod', 'scst')): + with self.subTest(failure=failure): + result, operations, _ = self.execute(kernel_mode='preloaded', loaded=('kunit',), + failure=failure) + self.assertTrue(result['errors']) + self.assertEqual(result['kunit_state'], 'preloaded') + self.assertEqual(operations[-1], failure) + self.assertFalse(any(name == 'kunit' for op, name in operations)) + + def test_kunit_preparation_failure_stops_before_loading_scst(self): + for options in ({'failure': ('modprobe', 'kunit')}, + {'interface_error': run.RunError('no run interface')}, + {'load_logs': {'kunit': PASS}}, + {'load_logs': {'kunit': ['WARNING: KUnit startup']}}): + with self.subTest(options=options): + result, operations, _ = self.execute(kernel_mode='load', **options) + self.assertEqual(operations, [('modprobe', 'kunit')]) + self.assertEqual(result['module_state'], 'not attempted') + + def test_unsafe_scst_results_preserve_owned_and_preloaded_kunit(self): + for messages in (PASS[:-1], PASS + ['WARNING: in test'], + PASS + ['# SCST_KUNIT_RETAINED: worker']): + for mode in ('load', 'preloaded'): + with self.subTest(messages=messages, mode=mode): + result, operations, _ = self.execute(kernel_mode=mode, logs={'scst-LIB': messages}, + loaded=('kunit',) if mode == 'preloaded' else ()) + self.assertEqual(result['kunit_state'], 'preloaded' if mode == 'preloaded' else 'retained') + self.assertFalse(any(op == 'rmmod' for op, name in operations)) + + def test_kunit_cleanup_failure_cannot_pass(self): + for options in ({'failure': ('rmmod', 'kunit')}, {'remains': ('kunit',)}): + with self.subTest(options=options): + result, operations, _ = self.execute(kernel_mode='load', **options) + self.assertEqual(operations[-1], ('rmmod', 'kunit')) + self.assertEqual(result['module_state'], 'unloaded') + self.assertIn('Could not unload', result['errors'][0]) + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), 2) + self.assertEqual(result['test_status'], 'PASS') + + def test_selected_handler_does_not_execute_dependency_tests(self): + inventory = self.both_manifests() + [manifest('scst_user', {'scst-USER-cleanup': 'USER'})] + result, operations, _ = self.execute({'scst-VDISK-capacity': VDISK_PASS}, labels=['VDISK'], + inventory=inventory, loaded=('scst_user',)) + self.assertFalse(result['errors']) + self.assertEqual(operations, [('insmod', 'scst'), ('insmod', 'scst_vdisk'), + ('suite', 'scst-VDISK-capacity'), ('rmmod', 'scst_vdisk'), ('rmmod', 'scst')]) + self.assertEqual(result['expected_suites'], ['scst-VDISK-capacity']) + self.assertEqual(result['modules'][2]['state'], 'not attempted') + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), 0) + + def test_multiple_labels_and_nonmatching_suite_names(self): + labels = {'scst-LIB': 'LIB', 'scst-CM': 'CM', 'scst-LIFECYCLE': 'LIFECYCLE', + 'scst-LIFECYCLE-blocking': 'LIFECYCLE'} + selected = ['scst-CM', 'scst-LIFECYCLE', 'scst-LIFECYCLE-blocking'] + result, operations, _ = self.execute({s: suite_log(s) for s in selected}, + labels=['CM', 'LIFECYCLE'], inventory=[manifest(labels=labels)]) + self.assertEqual([name for op, name in operations if op == 'suite'], selected) + self.assertEqual(result['module_state'], 'unloaded') + + def test_incomplete_unexpected_and_retained_results_stop_execution(self): + for messages in (PASS[:-1], [], PASS + PASS, PASS + VDISK_PASS, + PASS + ['# SCST_KUNIT_RETAINED: worker']): + result, operations, _ = self.execute({'scst-LIB': messages}, inventory=self.both_manifests()) + self.assertTrue(result['errors']) + self.assertEqual(operations, [('insmod', 'scst'), ('suite', 'scst-LIB')]) + self.assertEqual(result['modules'][1]['state'], 'not attempted') + + def test_automatic_execution_is_an_error(self): + result, operations, _ = self.execute(load_logs={'scst': PASS}) + self.assertTrue(result['errors']) + self.assertEqual(operations, [('insmod', 'scst')]) + + def test_diagnostics_stop_after_load_or_suite(self): + for options in ({'load_logs': {'scst': ['BUG: core init']}}, + {'logs': {'scst-LIB': PASS + ['WARNING: in test']}}): + result, operations, _ = self.execute(inventory=self.both_manifests(), **options) + self.assertEqual(result['module_state'], 'retained') + self.assertFalse(any(op == 'rmmod' or name == 'scst_vdisk' for op, name in operations)) + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), 1) + self.assertFalse(result['parse_errors']) + expected = ['scst-LIB', 'scst-VDISK-capacity'] if 'load_logs' in options else ['scst-VDISK-capacity'] + self.assertEqual(result['not_run_suites'], expected) + + def test_missing_run_interface_retains_module(self): + result, operations, _ = self.execute(missing=('scst-LIB',)) + self.assertIn('Missing KUnit run interface', result['errors'][0]) + self.assertEqual(operations, [('insmod', 'scst')]) + + def test_preflight_and_loaded_module_errors_prevent_operations(self): + for options in ({'loaded': ('scst',)}, {'preflight': run.RunError('bad VM')}): + result, operations, _ = self.execute(**options) + self.assertTrue(result['errors']) + self.assertEqual(operations, []) + + def test_unknown_label_and_bad_module_sets_prevent_operations(self): + core, handler = self.both_manifests() + for inventory, labels in [([core], ['UNKNOWN']), ([core, core], ['ALL']), + ([handler], ['VDISK']), + ([manifest(labels={})], ['ALL']), + ([core, manifest('qla2xxx_scst')], ['ALL']), + ([core, manifest('scst_vdisk')], ['ALL'])]: + result, operations, _ = self.execute(inventory=inventory, labels=labels) + self.assertTrue(result['errors']) + self.assertEqual(operations, []) + + def test_two_modules_execute_and_cleanup_in_dependency_order(self): + result, operations, _ = self.execute({'scst-LIB': PASS, 'scst-VDISK-capacity': VDISK_PASS}, + inventory=self.both_manifests()) + self.assertEqual(operations, [('insmod', 'scst'), ('suite', 'scst-LIB'), + ('insmod', 'scst_vdisk'), ('suite', 'scst-VDISK-capacity'), ('rmmod', 'scst_vdisk'), ('rmmod', 'scst')]) + self.assertEqual(result['module_state'], 'unloaded') + + def test_dependency_metadata_orders_new_owners_and_transitive_dependencies(self): + core = manifest() + middle = manifest('scst_extra', {'scst-EXTRA': 'EXTRA'}) + leaf = manifest('scst_new', {'scst-NEW': 'NEW'}, depends=('scst_extra',)) + result, operations, _ = self.execute({'scst-NEW': suite_log('scst-NEW')}, + inventory=[leaf, middle, core], labels=['NEW']) + self.assertFalse(result['errors']) + self.assertEqual(operations, [('insmod', 'scst'), ('insmod', 'scst_extra'), + ('insmod', 'scst_new'), ('suite', 'scst-NEW'), ('rmmod', 'scst_new'), + ('rmmod', 'scst_extra'), ('rmmod', 'scst')]) + + def test_dependency_cycles_prevent_module_operations(self): + core = manifest(depends=('scst_vdisk',)) + handler = manifest('scst_vdisk', {'scst-VDISK': 'VDISK'}) + result, operations, _ = self.execute(inventory=[handler, core]) + self.assertIn('Cyclic module dependency', result['errors'][0]) + self.assertEqual(operations, []) + + def test_new_module_owners_need_only_a_valid_manifest(self): + fields = {'name': 'scst-new', 'scst_kunit_suite': 'NEW_LABEL:scst-NEW_LABEL', + 'depends': 'scst-extra,kunit', 'vermagic': 'test-kernel'} + with patch.object(run.subprocess, 'run', + side_effect=lambda argv, **kw: Mock(stdout=fields.get(argv[2], ''))): + info = run.module_info(Path('/mock/scst-new.ko')) + self.assertEqual(info['name'], 'scst_new') + self.assertEqual(info['depends'], ['scst_extra', 'kunit']) + self.assertEqual(info['labels'], {'scst-NEW_LABEL': 'NEW_LABEL'}) + + def test_all_module_owners(self): + names = ['scst', 'scst_vdisk', 'scst_user', 'iscsi_scst'] + inventory = [manifest(name, {f'scst-test-{i}': f'TEST{i}'}) for i, name in enumerate(names)] + for failed in (False, True): + with self.subTest(failed=failed): + logs = {f'scst-test-{i}': suite_log(f'scst-test-{i}') for i in range(len(names))} + if failed: + logs['scst-test-0'] = [line.replace('scst-LIB', 'scst-test-0') for line in FAIL] + result, operations, _ = self.execute(logs, inventory=inventory) + self.assertFalse(result['errors']) + self.assertEqual([name for op, name in operations if op == 'insmod'], names) + self.assertEqual([name for op, name in operations if op == 'suite'], list(logs)) + self.assertEqual([name for op, name in operations if op == 'rmmod'], list(reversed(names))) + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), int(failed)) + self.assertEqual(result['not_run_suites'], []) + self.assertEqual(result['counts']['failed'], int(failed)) + self.assertEqual(result['counts']['passed'], len(names) - int(failed)) + + def test_each_selected_suite_must_finish(self): + labels = {s: 'VDISK' for s in ('scst-VDISK-capacity', 'scst-VDISK-io', 'scst-VDISK-sync')} + for completed in range(4): + logs = {s: suite_log(s) for s in list(labels)[:completed]} + result, operations, _ = self.execute(logs, labels=['VDISK'], + inventory=[manifest(), manifest('scst_vdisk', labels)]) + self.assertEqual(result['module_state'], 'unloaded' if completed == 3 else 'retained') + self.assertEqual(len([op for op in operations if op[0] == 'suite']), min(3, completed + 1)) + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.finish(result, Path(tmp)), 0 if completed == 3 else 2) + self.assertEqual(result['not_run_suites'], list(labels)[completed + 1:]) + if completed < 3: + self.assertTrue(result['parse_errors']) + + def test_kernel_mismatch_rejected_before_loading(self): + inventory = self.both_manifests() + inventory[1]['vermagic'] = 'different-kernel' + result, operations, _ = self.execute(inventory=inventory) + self.assertIn('vermagic', result['errors'][0]) + self.assertEqual(operations, []) + + def test_operation_failures_retain_dependencies(self): + for failure in [('insmod', 'scst'), ('suite', 'scst-LIB'), ('insmod', 'scst_vdisk'), + ('suite', 'scst-VDISK-capacity'), ('rmmod', 'scst_vdisk')]: + result, operations, _ = self.execute({'scst-LIB': PASS, 'scst-VDISK-capacity': VDISK_PASS}, + inventory=self.both_manifests(), failure=failure) + self.assertTrue(result['errors']) + self.assertEqual(operations[-1], failure) + self.assertNotEqual(result['module_state'], 'unloaded') + + def test_unload_warning_stops_before_core_cleanup(self): + result, operations, _ = self.execute({'scst-LIB': PASS, 'scst-VDISK-capacity': VDISK_PASS}, + inventory=self.both_manifests(), unload_warning=True) + self.assertEqual(operations[-1], ('rmmod', 'scst_vdisk')) + self.assertEqual(result['module_state'], 'retained') + + def test_interruption_preserves_log(self): + capture = Mock(messages=PASS[:2]) + result = report() + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(run.os, 'geteuid', return_value=0), \ + patch.object(run.shutil, 'which', return_value='/mock/tool'), \ + patch.object(run, 'module_info', return_value=manifest()), \ + patch.object(Path, 'exists', return_value=False), \ + patch.object(run, 'check_kernel'), \ + patch.object(run, 'KernelLog', return_value=capture), \ + patch.object(run, 'command', side_effect=KeyboardInterrupt): + with self.assertRaises(KeyboardInterrupt): + run.execute([Path('/mock/scst.ko')], 10, Path(tmp), result, ['LIB']) + self.assertEqual(result['messages'], PASS[:2]) + capture.close.assert_called_once() + + def test_main_retains_reports_after_error(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + module = directory / 'scst.ko' + module.touch() + real_open = os.open + def local_lock(path, flags, mode=0o777): + self.assertEqual(path, '/run/lock/scst-kunit.lock') + return real_open(directory / 'lock', flags, mode) + with patch.object(run.os, 'open', side_effect=local_lock), \ + patch.object(run, 'execute', side_effect=run.RunError('operation failed')), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(run.main([str(module), '--labels', 'LIB', '--output', str(directory)]), 2) + data = json.loads(next(directory.glob('*/result.json')).read_text()) + self.assertIn('operation failed', data['errors']) + self.assertEqual(data['labels'], ['LIB']) + self.assertEqual(data['status'], 'ERROR') + + +class KernelSetupTests(unittest.TestCase): + def check(self, config=None, cmdline='kunit.autorun=0 kunit.enable=1', filters=None, + module=False, debugfs=True, modprobe=True, parameters=None): + settings = {'CONFIG_KUNIT': 'y', 'CONFIG_KUNIT_DEBUGFS': 'y', 'CONFIG_KUNIT_AUTORUN_ENABLED': 'y', + 'CONFIG_KUNIT_DEFAULT_ENABLED': 'y'} + settings.update(config or {}) + settings = {key: value for key, value in settings.items() if value is not None} + filters = filters or {} + parameters = parameters or {} + def read(path, *args, **kwargs): + if str(path) == '/proc/cmdline': + return cmdline + self.assertTrue(str(path).startswith('/sys/module/kunit/parameters/')) + if path.name in ('enable', 'autorun'): + value = parameters.get(path.name, FileNotFoundError(str(path))) + if isinstance(value, OSError): + raise value + return value + return filters.get(path.name, '(null)\n') + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(run, 'DEBUGFS', Path(tmp) if debugfs else Path(tmp) / 'missing'), \ + patch.object(run, 'kernel_config', return_value=settings), \ + patch.object(run.shutil, 'which', return_value='/mock/modprobe' if modprobe else None), \ + patch.object(Path, 'exists', return_value=module), \ + patch.object(Path, 'read_text', read): + return run.check_kernel() + + def test_prepared_kernel_and_neutral_parameters(self): + self.check() + self.check(cmdline='kunit.autorun=1 kunit.autorun=0') + self.check(filters={'filter_glob': '(null)'}) + self.check(config={'CONFIG_KUNIT_AUTORUN_ENABLED': 'n'}) + for glob in ('*', '*.*'): + self.check(filters={'filter_glob': glob, 'filter': '', 'filter_action': ''}) + + def test_modular_kunit_preparation(self): + self.assertEqual(self.check(config={'CONFIG_KUNIT': 'm'}, cmdline=''), 'load') + with self.assertRaisesRegex(run.RunError, 'modprobe'): + self.check(config={'CONFIG_KUNIT': 'm'}, modprobe=False) + + def test_preloaded_kunit_uses_effective_parameters(self): + for values in ({'enable': 'Y\n', 'autorun': 'N\n'}, {'enable': '1\n', 'autorun': '0\n'}): + with self.subTest(values=values): + self.assertEqual(self.check(config={'CONFIG_KUNIT': 'm', 'CONFIG_KUNIT_DEFAULT_ENABLED': 'n'}, + module=True, parameters=values, cmdline='kunit.enable=0 kunit.autorun=1', + modprobe=False), 'preloaded') + + def test_preloaded_kunit_rejects_incompatible_parameters(self): + for name, invalid in (('enable', 'N'), ('autorun', 'Y'), ('enable', ''), ('autorun', 'unknown')): + with self.subTest(name=name, invalid=invalid), self.assertRaisesRegex(run.RunError, + 'Preloaded kunit.' + name + '.*Module left unchanged'): + self.check(config={'CONFIG_KUNIT': 'm'}, module=True, + parameters={'enable': 'Y', 'autorun': 'N', name: invalid}) + + def test_preloaded_kunit_rejects_hidden_or_unreadable_parameters(self): + for name in ('enable', 'autorun'): + for problem in (FileNotFoundError('hidden'), PermissionError('denied')): + with self.subTest(name=name, problem=problem), self.assertRaisesRegex(run.RunError, + '(?s)Cannot verify preloaded KUnit.*' + name + '.*rmmod kunit'): + self.check(config={'CONFIG_KUNIT': 'm'}, module=True, + parameters={'enable': 'Y', 'autorun': 'N', name: problem}) + + def test_preloaded_kunit_still_requires_debugfs_and_neutral_filters(self): + for options in ({'debugfs': False}, {'filters': {'action': 'list'}}, + {'filters': {'filter_glob': 'scst-LIB*'}}, {'filters': {'filter_glob': ''}}, + {'filters': {'filter': 'speed=slow'}}, {'filters': {'filter_action': 'skip'}}): + with self.subTest(options=options), self.assertRaises(run.RunError): + self.check(config={'CONFIG_KUNIT': 'm'}, module=True, + parameters={'enable': 'Y', 'autorun': 'N'}, **options) + + def test_unsafe_or_unverifiable_setup_is_rejected(self): + for options in ({'config': {'CONFIG_KUNIT': 'n'}}, {'module': True}, {'debugfs': False}, + {'config': {'CONFIG_KUNIT_AUTORUN_ENABLED': None}}, + {'config': {'CONFIG_KUNIT_DEBUGFS': 'n'}}, + {'cmdline': ''}, {'cmdline': 'kunit.autorun=0 kunit.autorun=1'}, + {'cmdline': '-- kunit.autorun=0'}, {'cmdline': 'kunit.autorun=0 kunit.enable=0'}): + with self.subTest(options=options), self.assertRaises(run.RunError): + self.check(**options) + for name in ('action', 'filter_glob', 'filter', 'filter_action'): + with self.subTest(name=name), self.assertRaises(run.RunError): + self.check(filters={name: 'custom'}) + for name in ('action', 'filter_glob'): + with self.subTest(name=name), self.assertRaises(run.RunError): + self.check(filters={name: ''}) + + def test_config_reader_preserves_unset_options(self): + source = 'CONFIG_KUNIT=y\n# CONFIG_KUNIT_AUTORUN_ENABLED is not set\n' + with patch.object(Path, 'is_file', lambda p: str(p) == '/proc/config.gz'), \ + patch.object(run.gzip, 'open', return_value=io.StringIO(source)): + values = run.kernel_config() + self.assertEqual(values['CONFIG_KUNIT'], 'y') + self.assertEqual(values['CONFIG_KUNIT_AUTORUN_ENABLED'], 'n') + with patch.object(Path, 'is_file', return_value=False), self.assertRaises(run.RunError): + run.kernel_config() + + +class ConfigurationTests(unittest.TestCase): + @staticmethod + def arguments(args): + return run.arguments(['--labels', 'LIB', *args]) + + def test_labels_default_to_all_and_explicit_values_are_normalized(self): + with tempfile.TemporaryDirectory() as tmp: + module = Path(tmp) / 'scst.ko' + module.touch() + self.assertEqual(run.arguments([str(module)])[-1], ['ALL']) + for args in (['--labels', ''], ['--labels', 'ALL LIB'], ['--labels', 'lib']): + with self.subTest(args=args), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + run.arguments(args) + self.assertEqual(run.parse_labels('LIB LIB VDISK'), ['LIB', 'VDISK']) + self.assertEqual(run.parse_labels('ALL'), ['ALL']) + + def test_default_uses_recorded_build_in_dependency_order(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + core, handler = root / 'core.ko', root / 'handler with spaces.ko' + core.touch() + handler.touch() + (root / 'stale.ko').touch() + modules = root / '.kunit-modules' + modules.write_text('core.ko\nhandler with spaces.ko\n') + config = root / 'settings.toml' + config.write_text('timeout_seconds=12\nverbose=true\n') + with patch.object(run, 'DEFAULT_MODULE_LIST', modules): + selected, timeout, _, verbose, labels = self.arguments(['--config', str(config)]) + self.assertEqual(selected, [core, handler]) + self.assertEqual(timeout, 12) + self.assertTrue(verbose) + + def test_explicit_paths_then_config_override_recorded_build(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + configured, explicit = root / 'configured.ko', root / 'explicit.ko' + configured.touch() + explicit.touch() + config = root / 'settings.toml' + config.write_text('modules=["configured.ko"]\n') + modules = root / '.kunit-modules' + modules.write_text('missing-stale-module.ko\n') + with patch.object(run, 'DEFAULT_MODULE_LIST', modules): + self.assertEqual(self.arguments(['--config', str(config)])[0], [configured]) + self.assertEqual(self.arguments([str(explicit), '--config', str(config)])[0], + [explicit]) + + def test_missing_invalid_or_stale_build_list_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = root / 'settings.toml' + config.touch() + modules = root / '.kunit-modules' + for content in (None, '', '\n', 'missing.ko\n'): + if content is not None: + modules.write_text(content) + with self.subTest(content=content), patch.object(run, 'DEFAULT_MODULE_LIST', modules), \ + contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + self.arguments(['--config', str(config)]) + + def test_config_paths_and_cli_override(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + (directory / 'scst.ko').touch() + config = directory / 'settings.toml' + config.write_text('module="scst.ko"\noutput_dir="reports"\ntimeout_seconds=9\n') + module, timeout, output, verbose, labels = self.arguments(['--config', str(config)]) + self.assertFalse(verbose) + self.assertEqual(module, [directory / 'scst.ko']) + self.assertEqual(output, directory / 'reports') + self.assertEqual(timeout, 9) + self.assertEqual(self.arguments(['--config', str(config), '--timeout', '2'])[1], 2) + for value in ('nan', 'inf', '0', '-1'): + with self.subTest(value=value), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + self.arguments(['--config', str(config), '--timeout', value]) + + def test_verbose_cli_and_config(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + (directory / 'scst.ko').touch() + config = directory / 'settings.toml' + config.write_text('module="scst.ko"\nverbose=true\n') + args = ['--config', str(config)] + self.assertTrue(self.arguments(args)[3]) + self.assertFalse(self.arguments(args + ['--no-verbose'])[3]) + config.write_text('module="scst.ko"\nverbose=false\n') + self.assertTrue(self.arguments(args + ['--verbose'])[3]) + config.write_text('module="scst.ko"\nverbose="yes"\n') + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + self.arguments(args) + + def test_multiple_module_config_and_cli_override(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + core, handler = directory / 'scst.ko', directory / 'scst_vdisk.ko' + core.touch() + handler.touch() + config = directory / 'settings.toml' + config.write_text('modules=["scst.ko", "scst_vdisk.ko"]\n') + self.assertEqual(self.arguments(['--config', str(config)])[0], [core, handler]) + self.assertEqual(self.arguments([str(core), '--config', str(config)])[0], [core]) + self.assertEqual(self.arguments([str(core), str(handler)])[0], [core, handler]) + for settings in ('modules=[]', 'modules="scst.ko"', 'modules=[1]', + 'module="scst.ko"\nmodules=["scst.ko"]'): + config.write_text(settings) + with self.subTest(settings=settings), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + self.arguments(['--config', str(config)]) + + +if __name__ == '__main__': + unittest.main() diff --git a/scst/Makefile b/scst/Makefile index 72205be76..2c7d58ffb 100644 --- a/scst/Makefile +++ b/scst/Makefile @@ -36,17 +36,18 @@ endef all: include/build_mode.h cd $(SCST_DIR) && $(MAKE) $@ +.PHONY: kunit-build +kunit-build: include/build_mode.h + $(MAKE) -C $(SCST_DIR) $@ + install: include/build_mode.h cd $(SCST_DIR) && $(MAKE) $@ uninstall: cd $(SCST_DIR) && $(MAKE) $@ -clean: - rm -f include/build_mode.h - cd $(SCST_DIR) && $(MAKE) $@ - -extraclean: +clean extraclean: + rm -f include/build_mode.h include/build_mode.h.tmp cd $(SCST_DIR) && $(MAKE) $@ 2debug: diff --git a/scst/src/Makefile b/scst/src/Makefile index 8dc0998e6..a2836e851 100644 --- a/scst/src/Makefile +++ b/scst/src/Makefile @@ -88,6 +88,16 @@ all: $(SCST_INTF_VER_FILE) $(SCST_KUNIT_FLAGS) \ $(shell [ -n "$(PASS_CC_TO_MAKE)" ] && echo CC="$(CC)") +scst-kunit-handlers = $(notdir $(filter scst/src/dev_handlers/%,$(SCST_KUNIT_MODULES_scst))) + +.PHONY: kunit-build +kunit-build: $(SCST_INTF_VER_FILE) + $(MAKE) -C $(KDIR) M=$(SCST_MOD_DIR) $(SCST_KUNIT_FLAGS) \ + $(shell [ -n "$(PASS_CC_TO_MAKE)" ] && echo CC="$(CC)") scst.ko + $(if $(scst-kunit-handlers),$(MAKE) -C $(KDIR) M=$(SCST_DH_MOD_DIR) \ + $(SCST_KUNIT_FLAGS) \ + $(shell [ -n "$(PASS_CC_TO_MAKE)" ] && echo CC="$(CC)") $(scst-kunit-handlers),@:) + scst: $(MAKE) -C $(KDIR) M=$(SCST_MOD_DIR) \ $(SCST_KUNIT_FLAGS) \ diff --git a/scst/src/kunit/select.mk b/scst/src/kunit/select.mk index 7dc82896f..c01619a24 100644 --- a/scst/src/kunit/select.mk +++ b/scst/src/kunit/select.mk @@ -4,21 +4,57 @@ include $(scst-kunit-root)/scst/src/kunit/modules.mk PYTHON ?= python3 -ifneq ($(origin CONFIG_SCST_KUNIT_TEST),undefined) -ifneq ($(filter-out y n,$(CONFIG_SCST_KUNIT_TEST)),) -$(error CONFIG_SCST_KUNIT_TEST must be y or n) +ifneq ($(filter kunit-build kunit-list kunit-run kunit-test,$(MAKECMDGOALS)),) +scst-kunit-obsolete := $(foreach var,TEST TEST_LOG KUNIT_LABELS KUNIT_MODULE KUNIT_MODULES \ + KUNIT_CONFIG KUNIT_TIMEOUT KUNIT_OUTPUT KUNIT_VERBOSE,\ + $(if $(filter undefined,$(origin $(var))),,$(var))) +ifneq ($(strip $(scst-kunit-obsolete)),) +$(error Use KUNIT_LABEL and KUNIT_ARGS; unsupported variables: $(scst-kunit-obsolete)) endif -ifneq ($(words $(CONFIG_SCST_KUNIT_TEST)),1) -$(error CONFIG_SCST_KUNIT_TEST must be y or n) +ifneq ($(strip $(STRESS) $(PERF)),) +$(error SCST KUnit supports KUNIT_LABEL labels only; STRESS and PERF are not implemented) endif -SCST_KUNIT_FLAGS := CONFIG_SCST_KUNIT_TEST=$(CONFIG_SCST_KUNIT_TEST) endif +# Public Makefile options map to the private switches passed to Kbuild. +ifneq ($(filter kunit-build kunit-test,$(MAKECMDGOALS)),) +ifneq ($(filter-out y,$(KUNIT_TEST) $(CONFIG_SCST_KUNIT_TEST)),) +$(error kunit-build requires KUNIT_TEST=y) +endif +KUNIT_TEST := y +endif +KUNIT_TEST ?= $(if $(CONFIG_SCST_KUNIT_TEST),$(CONFIG_SCST_KUNIT_TEST),n) +KUNIT_LOG ?= $(if $(CONFIG_SCST_KUNIT_LOG),$(CONFIG_SCST_KUNIT_LOG),n) + +ifneq ($(filter-out y n,$(KUNIT_TEST)),) +$(error KUNIT_TEST must be y or n) +endif +ifneq ($(words $(KUNIT_TEST)),1) +$(error KUNIT_TEST must be y or n) +endif +ifneq ($(filter-out y n,$(KUNIT_LOG)),) +$(error KUNIT_LOG must be y or n) +endif +ifneq ($(words $(KUNIT_LOG)),1) +$(error KUNIT_LOG must be y or n) +endif +SCST_KUNIT_FLAGS := CONFIG_SCST_KUNIT_TEST=$(KUNIT_TEST) CONFIG_SCST_KUNIT_LOG=$(KUNIT_LOG) + # Labels belong to suites, not the build inventory. KUNIT_LABEL ?= ALL scst-kunit-source-dirs = $(sort $(addsuffix kunit,$(dir $(SCST_KUNIT_MODULES)))) .PHONY: kunit-list kunit-list: - @$(PYTHON) "$(scst-kunit-root)/scripts/kunit/inventory.py" --labels "$(KUNIT_LABEL)" \ + @$(PYTHON) "$(scst-kunit-root)/scripts/kunit/inventory.py" \ $(foreach dir,$(scst-kunit-source-dirs),"$(scst-kunit-root)/$(dir)") + +# A component clean invalidates the recorded build as well as a top-level clean. +.PHONY: scst-kunit-clean-files +clean extraclean: scst-kunit-clean-files +scst-kunit-clean-files: + @rm -f "$(scst-kunit-root)/.kunit-modules" "$(scst-kunit-root)/.kunit-modules.tmp" + @if [ -d "$(scst-kunit-root)/scripts/kunit" ]; then \ + find "$(scst-kunit-root)/scripts/kunit" -type d -name __pycache__ -prune \ + -exec rm -rf {} +; \ + fi