diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9941f6a8..592e657b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: pull_request: branches: - master + workflow_dispatch: + +permissions: + contents: read env: WERROR: 'y' diff --git a/.github/workflows/ginkgo.yml b/.github/workflows/ginkgo.yml new file mode 100644 index 000000000..594e679ce --- /dev/null +++ b/.github/workflows/ginkgo.yml @@ -0,0 +1,49 @@ +name: Ginkgo + +on: + push: + branches: [master] + pull_request: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + ginkgo: + name: Build and run in VMs + runs-on: ubuntu-24.04 + timeout-minutes: 220 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Install build and VM tools + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-x86 qemu-utils openssh-client e2fsprogs fakeroot + - name: Build SCST image and check Ginkgo framework + timeout-minutes: 40 + run: bash scripts/ci/build-image ginkgo "$RUNNER_TEMP/ginkgo-image" + - name: Run local and iSCSI tests against the compiled SCST + env: + GINKGO_LABEL: ${{ github.event_name == 'pull_request' && 'basic || smoke' || '' }} + run: | + python3 scripts/ci/vm.py ginkgo "$RUNNER_TEMP/ginkgo-image" \ + "$RUNNER_TEMP/ginkgo-results" --timeout 10000 --ginkgo-label "$GINKGO_LABEL" + - name: Save build logs, reports and serial consoles + if: always() + uses: actions/upload-artifact@v4 + with: + name: ginkgo-results + retention-days: 7 + path: | + ${{ runner.temp }}/ginkgo-image/*.log + ${{ runner.temp }}/ginkgo-image/*.json + ${{ runner.temp }}/ginkgo-image/*.txt + ${{ runner.temp }}/ginkgo-results/ diff --git a/.github/workflows/kunit.yml b/.github/workflows/kunit.yml index c3aff806c..c27a0e70a 100644 --- a/.github/workflows/kunit.yml +++ b/.github/workflows/kunit.yml @@ -16,55 +16,40 @@ defaults: jobs: kunit: - name: Tooling and module build - runs-on: ubuntu-latest - container: fedora:44@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 - timeout-minutes: 20 - env: - # Update the image and kernel deliberately, independently of the regression matrix. - KERNEL_BUILD: 7.2.9-201.fc44 - KERNEL_DEVEL_SHA256: 17e14518fe28ea6897bc448bf3e50434ac5ed58d4ca10d3c1c20ef145a16e7cd + name: Build and run in VM + runs-on: ubuntu-24.04 + timeout-minutes: 60 steps: - - name: Install build dependencies - run: | - dnf install -y gcc make elfutils-libelf-devel openssl-devel \ - python3 git tar gzip findutils diffutils gawk curl-minimal - uses: actions/checkout@v4 - - name: Prepare pinned kernel headers + with: + persist-credentials: false + - name: Install VM tools run: | - rpm_file="$RUNNER_TEMP/kernel-devel-$KERNEL_BUILD.x86_64.rpm" - curl --fail --location --retry 3 \ - "https://kojipkgs.fedoraproject.org/packages/kernel/${KERNEL_BUILD%-*}/${KERNEL_BUILD#*-}/x86_64/kernel-devel-$KERNEL_BUILD.x86_64.rpm" \ - --output "$rpm_file" - printf '%s %s\n' "$KERNEL_DEVEL_SHA256" "$rpm_file" | sha256sum --check - dnf install -y "$rpm_file" - kdir="/usr/src/kernels/$KERNEL_BUILD.x86_64" - test -s "$kdir/Module.symvers" - if ! grep -Eq '^CONFIG_KUNIT=[ym]$' "$kdir/.config"; then - echo "::error::The pinned kernel $KERNEL_BUILD does not enable KUnit" - exit 1 - fi - printf 'KDIR=%s\nREPORTS=%s/kunit\n' "$kdir" "$RUNNER_TEMP" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/kunit" - rpm -q kernel-devel gcc > "$RUNNER_TEMP/kunit/packages.txt" - cp "$kdir/.config" "$RUNNER_TEMP/kunit/kernel.config" - - name: Check tooling + sudo apt-get update + sudo apt-get install -y qemu-system-x86 qemu-utils e2fsprogs fakeroot + - name: Check VM supervisor + run: python3 -m unittest discover -s scripts/ci -p 'test_*.py' + - name: Build kernel-matched image and KUnit modules + run: bash scripts/ci/build-image kunit "$RUNNER_TEMP/kunit-image" + - name: Exercise VM success, panic and timeout handling run: | - python3 -m unittest discover -s scripts/kunit -p 'test_*.py' 2>&1 | tee "$REPORTS/tooling.log" - make kunit-list - - name: Build SCST KUnit modules + python3 scripts/ci/vm.py self-test "$RUNNER_TEMP/kunit-image" \ + "$RUNNER_TEMP/kunit-supervisor" --timeout 180 --accel tcg + - name: Run KUnit run: | - make -j"$(nproc)" kunit-build 2>&1 | tee "$REPORTS/scst-build.log" - tar -czf "$REPORTS/modules.tar.gz" -T .kunit-modules .kunit-modules - - name: Check cleanup - run: | - make kunit-clean 2>&1 | tee "$REPORTS/clean.log" - test ! -e .kunit-modules - test ! -e .kunit-modules.tmp - test -z "$(find scst/src iscsi-scst/kernel -type f \( -name '*.ko' -o -name '*.o' -o -name '*.cmd' \) -print -quit)" - - name: Save build evidence + python3 scripts/ci/vm.py kunit "$RUNNER_TEMP/kunit-image" \ + "$RUNNER_TEMP/kunit-results" --timeout 1200 + - name: Save build and runtime evidence if: always() uses: actions/upload-artifact@v4 with: - name: kunit - path: ${{ runner.temp }}/kunit/ + name: kunit-results + retention-days: 7 + path: | + ${{ runner.temp }}/kunit-image/*.log + ${{ runner.temp }}/kunit-image/*.json + ${{ runner.temp }}/kunit-image/*.txt + ${{ runner.temp }}/kunit-image/kernel.config + ${{ runner.temp }}/kunit-image/modules.tar.gz + ${{ runner.temp }}/kunit-supervisor/ + ${{ runner.temp }}/kunit-results/ diff --git a/doc/CI.md b/doc/CI.md new file mode 100644 index 000000000..1a531257d --- /dev/null +++ b/doc/CI.md @@ -0,0 +1,67 @@ +# Tests in GitHub Actions + +KUnit and Ginkgo have independent workflows. Each builds its own SCST modules +and runs tests in disposable QEMU guests within a single job: + +- `.github/workflows/kunit.yml`: KUnit modules, supervisor checks for guest + success, panic and timeout, then `make kunit-run`. +- `.github/workflows/ginkgo.yml`: ordinary SCST modules, `make ginkgo-check`, + then local and TCP iSCSI tests through `make ginkgo-test`. Pull requests use + `basic || smoke`; pushes to `master` and manual runs select all tests. + +Both run on pushes and pull requests to `master`, or via `workflow_dispatch`. +The Debian package build in `.github/workflows/ci.yml` remains independent. +Ginkgo uses its locally built image directly; no image artifact is uploaded. + +Docker builds a committed Git snapshot with the pinned Fedora kernel and +matching headers. QEMU loads SCST only into the guest kernel. It tries KVM +first and falls back to TCG; use `--accel tcg` or `--accel kvm` to select one. +The kernel pins are in `scripts/ci/kernel.env` and `kernel.SHA256SUMS`; update +them together. The Fedora base image is pinned in `scripts/ci/Dockerfile`. + +Ginkgo uses separate target and initiator VMs on a private QEMU network, +a disposable 1 GiB block disk and guest ext4 backing files. No host block +devices or external initiators are used. + +## Reports and failures + +The supervisor keeps serial consoles outside the guests and enforces a hard +deadline. Kernel diagnostics, missing or incomplete test reports, a lost +initiator and timeouts fail the job, even if QEMU exits with status zero. +The KUnit workflow checks panic and timeout handling explicitly under TCG. + +After each VM run, the step prints the native test output, including summaries +and failures. Failed runs also print kernel diagnostics and the tail of each +serial console; full logs remain in the artifacts. + +Build logs, test reports and serial consoles are uploaded with `if: always()` +and retained for seven days. Ginkgo also saves build metadata and package +versions. VM disks and private SSH keys are not uploaded. A guest crash may +prevent its final report, but the external serial log survives. Upload still +requires a working GitHub runner. + +## Local reproduction + +Image construction requires Docker with Buildx, fakeroot, tar and mkfs.ext4. +Execution requires Python 3.11+, QEMU with the stream network backend, +qemu-img, ssh-keygen and sha256sum. Use new output directories for each run. +These commands boot disposable guests; self-test intentionally crashes one. + +```sh +bash scripts/ci/build-image kunit /tmp/scst-kunit-image +python3 scripts/ci/vm.py self-test /tmp/scst-kunit-image /tmp/scst-vm-selftest --timeout 180 --accel tcg +python3 scripts/ci/vm.py kunit /tmp/scst-kunit-image /tmp/scst-kunit-results --timeout 1200 + +bash scripts/ci/build-image ginkgo /tmp/scst-ginkgo-image +python3 scripts/ci/vm.py ginkgo /tmp/scst-ginkgo-image /tmp/scst-ginkgo-results --timeout 10000 +``` + +Add `--ginkgo-label 'basic || smoke'` to reproduce the pull-request selection. +The selected filter is printed in stdout and recorded in `result.json`; +both transport reports must confirm that filter. + +Supervisor checks without VM or SCST access: + +```sh +python3 -m unittest discover -s scripts/ci -p 'test_*.py' +``` diff --git a/doc/KUNIT.md b/doc/KUNIT.md index 4e2811ef2..358479de2 100644 --- a/doc/KUNIT.md +++ b/doc/KUNIT.md @@ -4,6 +4,8 @@ KUnit checks SCST functions using private fixtures and simulated dependencies. Use a current kernel with native KUnit; API backports are not supported. Normal builds leave tests disabled and do not require KUnit headers. +For GitHub Actions, see [CI](CI.md). + ## Requirements Build against a configured and prepared kernel tree with `CONFIG_KUNIT=y/m`. diff --git a/scripts/ci/Dockerfile b/scripts/ci/Dockerfile new file mode 100644 index 000000000..28c977e9d --- /dev/null +++ b/scripts/ci/Dockerfile @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0-only +FROM fedora:44@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 AS runtime +ARG SUITE +ARG KERNEL_BUILD +ENV KVER=${KERNEL_BUILD}.x86_64 +ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOCACHE=/opt/go-cache GOMODCACHE=/opt/go-mod +COPY scripts/ci/kernel.SHA256SUMS /tmp/kernel.SHA256SUMS +RUN dnf install -y --setopt=install_weak_deps=False \ + make kmod python3 perl perl-Config-General perl-Data-Dumper \ + systemd systemd-udev dracut util-linux procps-ng iproute \ + openssh-server openssh-clients iscsi-initiator-utils sg3_utils fio \ + findutils diffutils tar gzip curl-minimal && \ + if [ "$SUITE" = ginkgo ]; then dnf install -y golang; fi && \ + mkdir /tmp/kernel && cd /tmp/kernel && \ + for package in kernel-core kernel-modules-core kernel-modules \ + kernel-modules-extra kernel-modules-internal kernel-devel; do \ + curl --fail --location --retry 3 \ + "https://kojipkgs.fedoraproject.org/packages/kernel/${KERNEL_BUILD%-*}/${KERNEL_BUILD#*-}/x86_64/$package-$KVER.rpm" \ + --output "$package-$KVER.rpm"; \ + done && \ + sha256sum --check /tmp/kernel.SHA256SUMS && \ + dnf install -y --setopt=install_weak_deps=False \ + "./kernel-core-$KVER.rpm" "./kernel-modules-core-$KVER.rpm" \ + "./kernel-modules-$KVER.rpm" "./kernel-modules-extra-$KVER.rpm" \ + "./kernel-modules-internal-$KVER.rpm" && \ + depmod "$KVER" && \ + dracut --force --no-hostonly --no-hostonly-cmdline --no-early-microcode \ + --add-drivers 'virtio_pci virtio_blk virtio_net 9p 9pnet_virtio' \ + --filesystems ext4 /boot/ci-initramfs "$KVER" && \ + cp "/lib/modules/$KVER/vmlinuz" /boot/ci-vmlinuz && \ + cp "/lib/modules/$KVER/config" "/boot/config-$KVER" && \ + dnf clean all + +FROM runtime AS build +ARG SUITE +ARG SOURCE_COMMIT +RUN dnf install -y --setopt=install_weak_deps=False \ + gcc elfutils-libelf-devel openssl openssl-devel perl-ExtUtils-MakeMaker git \ + /tmp/kernel/kernel-devel-*.rpm && dnf clean all +COPY . /scst +WORKDIR /scst +RUN bash scripts/ci/build-guest "$SUITE" "$SOURCE_COMMIT" + +FROM runtime +COPY --from=build /scst /scst +COPY --from=build /opt/go-cache /opt/go-cache +COPY --from=build /opt/go-mod /opt/go-mod +COPY --from=build /opt/ci-build /opt/ci-build +COPY scripts/ci/scst-ci.service /etc/systemd/system/scst-ci.service +COPY scripts/ci/scst-ci.target /etc/systemd/system/scst-ci.target +RUN rm -rf /tmp/kernel /tmp/kernel.SHA256SUMS /var/cache/* && \ + mkdir -p /ci/input /ci/reports /ci/backing /root/.ssh /run/sshd && \ + chmod 700 /root/.ssh && \ + sed -i 's/^root:[^:]*:/root::/' /etc/shadow +ENV GOPROXY=off diff --git a/scripts/ci/build-guest b/scripts/ci/build-guest new file mode 100644 index 000000000..8d6a463f1 --- /dev/null +++ b/scripts/ci/build-guest @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0-only +# Runs only in the image build container; never loads modules. +set -euo pipefail +suite=$1 +commit=$2 +export KDIR="/usr/src/kernels/$KVER" +mkdir -p /opt/ci-build /opt/go-cache /opt/go-mod +exec > >(tee /opt/ci-build/build.log) 2>&1 +test -s "$KDIR/Module.symvers" +cp "$KDIR/.config" /opt/ci-build/kernel.config +rpm -qa | sort > /opt/ci-build/packages.txt +python3 - "$suite" "$commit" "$KVER" <<'PY' +import json, pathlib, sys +pathlib.Path('/opt/ci-build/image.json').write_text(json.dumps(dict( + suite=sys.argv[1], commit=sys.argv[2], kernel=sys.argv[3]), indent=2) + '\n') +PY +opts=("-j$(nproc)" "GIT_COMMIT=${commit:0:12}" "KDIR=$KDIR" "KVER=$KVER") +case "$suite" in +kunit) + grep -Eq '^CONFIG_KUNIT=[ym]$' "$KDIR/.config" + grep -qx 'CONFIG_KUNIT_DEBUGFS=y' "$KDIR/.config" + modinfo -k "$KVER" -n kunit + modinfo -k "$KVER" -n dlm + python3 -m unittest discover -s scripts/kunit -p 'test_*.py' + make kunit-list + make "${opts[@]}" kunit-build + # Exercise cleanup without losing the exact modules to be executed. + tar -czf /opt/ci-build/modules.tar.gz -T .kunit-modules .kunit-modules + make kunit-clean + test ! -e .kunit-modules + test ! -e .kunit-modules.tmp + test -z "$(find scst/src iscsi-scst/kernel -type f \( -name '*.ko' -o -name '*.o' -o -name '*.cmd' \) -print -quit)" + tar -xzf /opt/ci-build/modules.tar.gz + ;; +ginkgo) + modinfo -k "$KVER" -n iscsi_tcp + modinfo -k "$KVER" -n dlm + # Drivers consume the core's Module.symvers. Keep these builds ordered. + make "${opts[@]}" scst + make "${opts[@]}" scst_local + make "${opts[@]}" iscsi INFINIBAND_ENABLED=false + make "${opts[@]}" scstadm + make ginkgo-check + # Populate Go's build cache at the same /scst path used by the guest. + # No package downloads or kernel compilation are needed in a test VM. + make ginkgo-list GINKGO_LABEL= + ;; +*) exit 2 ;; +esac +# Keep the sources, modules and Go caches, but omit intermediate kernel objects. +find /scst -type f \( -name '*.o' -o -name '*.cmd' -o -name '*.mod.c' \) -delete +# Module signing keys must never enter a VM image or CI artifact. +# The ordinary scst target generates these, even without modules_install. +rm -f /scst/scst/src/certs/scst_module_key.priv /scst/scst/src/certs/scst_module_key.der +test -z "$(find /scst -type f -name '*.priv' -print -quit)" diff --git a/scripts/ci/build-image b/scripts/ci/build-image new file mode 100644 index 000000000..d9631adf9 --- /dev/null +++ b/scripts/ci/build-image @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0-only +# Build a bootable, version-matched test image without booting or loading SCST. +set -euo pipefail +if [[ $# != 2 || ! $1 =~ ^(kunit|ginkgo)$ ]]; then + echo "Usage: $0 kunit|ginkgo OUTPUT_DIRECTORY" >&2 + exit 2 +fi +suite=$1 +out=$(realpath -m "$2") +mkdir "$out" +root=$(git rev-parse --show-toplevel) +commit=$(git rev-parse HEAD) +# shellcheck source=kernel.env +source "$root/scripts/ci/kernel.env" +work=$(mktemp -d) +cleanup() { + rm -rf -- "$work" +} +trap cleanup EXIT +mkdir "$work/source" "$work/root" +# A committed snapshot excludes local profiles, generated files and Git credentials. +git -C "$root" archive "$commit" | tar -x -C "$work/source" +docker build --progress=plain \ + --build-arg "KERNEL_BUILD=$KERNEL_BUILD" --build-arg "SUITE=$suite" \ + --build-arg "SOURCE_COMMIT=$commit" --output "type=tar,dest=$work/rootfs.tar" \ + -f "$work/source/scripts/ci/Dockerfile" "$work/source" 2>&1 | tee "$out/image-build.log" +# Preserve guest owners without host root, mounts or loop devices. Extraction +# and mkfs must share one fakeroot session so ownership metadata survives. +fakeroot -- bash -euo pipefail /dev/stdin "$work" "$out" <<'SH' + tar -xpf "$1/rootfs.tar" -C "$1/root" + rm "$1/rootfs.tar" + chown 0:0 "$1/root" + # Fedora ships these with mode 000. fakeroot preserves metadata but cannot + # bypass host DAC when mkfs reads file contents; keep them root-only. + for file in "$1/root/etc"/{shadow,gshadow}{,-}; do + if [[ -f $file ]]; then chmod 600 "$file"; fi + done + cp -a "$1/root/opt/ci-build/." "$2/" + cp "$1/root/boot/ci-vmlinuz" "$2/vmlinuz" + cp "$1/root/boot/ci-initramfs" "$2/initramfs" + # Otherwise systemd can mistake the VM for a container and skip udev. + rm -f "$1/root/.dockerenv" "$1/root/run/.containerenv" + size=$(du -sm "$1/root" | cut -f1) + truncate -s "$((size + 2048))M" "$2/rootfs.img" + mkfs.ext4 -q -F -d "$1/root" "$2/rootfs.img" +SH +(cd "$out" && sha256sum vmlinuz initramfs rootfs.img image.json > SHA256SUMS) diff --git a/scripts/ci/guest b/scripts/ci/guest new file mode 100644 index 000000000..6796cc678 --- /dev/null +++ b/scripts/ci/guest @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0-only +# Invoked only by the dedicated guest target. The host owns the hard deadline. +set -euo pipefail +mount -t 9p -o trans=virtio,version=9p2000.L,ro ci-input /ci/input +mount -t 9p -o trans=virtio,version=9p2000.L ci-reports /ci/reports +mode=$(cat /ci/input/mode) +token=$(cat /ci/input/token) +finish() { + status=$? + trap - EXIT + dmesg --facility=kern > /ci/reports/dmesg.log || status=1 + python3 - "$mode" "$token" "$status" <<'PY' +import json, os, pathlib, platform, sys +report = json.loads(pathlib.Path('/opt/ci-build/image.json').read_text()) +report.update(mode=sys.argv[1], token=sys.argv[2], exit_code=int(sys.argv[3]), + running_kernel=platform.release()) +path = pathlib.Path('/ci/reports/completion.tmp') +with path.open('w') as output: + json.dump(report, output) + output.flush() + os.fsync(output.fileno()) +path.rename('/ci/reports/completion.json') +PY + sync + systemctl --no-block poweroff + exit "$status" +} +trap finish EXIT +test "$(uname -r)" = "$(python3 -c 'import json; print(json.load(open("/opt/ci-build/image.json"))["kernel"])')" +mountpoint -q /sys/kernel/debug || mount -t debugfs none /sys/kernel/debug +cd /scst +printf '%s\n' "$mode" > /ci/reports/started +export PATH=/usr/sbin:/usr/bin:/sbin:/bin +export CGO_ENABLED=0 GOTOOLCHAIN=local GOPROXY=off +export GOCACHE=/opt/go-cache GOMODCACHE=/opt/go-mod +case "$mode" in +smoke) exit 0 ;; +panic) echo c > /proc/sysrq-trigger; exit 1 ;; +hang) sleep infinity ;; +kunit) + make kunit-run KUNIT_ARGS='--timeout 120 --output /ci/reports/kunit' \ + 2>&1 | tee /ci/reports/tests.log + ;; +ginkgo|initiator) + while [[ ! -e /ci/input/ssh-ready ]]; do sleep 1; done + ip link set eth0 up + if [[ $mode == initiator ]]; then address=192.0.2.2; else address=192.0.2.1; fi + ip address add "$address/24" dev eth0 + if [[ $mode == initiator ]]; then + install -m 600 /ci/input/authorized_keys /root/.ssh/authorized_keys + install -m 600 /ci/input/ssh_host_key /run/ci-host-key + mkdir -p /run/sshd /etc/iscsi + printf '%s\n' 'InitiatorName=iqn.2026-10.org.scst:ci' > /etc/iscsi/initiatorname.iscsi + /usr/sbin/sshd -D -e -o HostKey=/run/ci-host-key -o UsePAM=yes \ + -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no \ + -o PermitRootLogin=prohibit-password & + modprobe iscsi_tcp + systemctl start iscsid.service + # Publish readiness only once SSH accepts connections and iscsid is alive. + python3 - <<'PY' +import socket, time +for attempt in range(100): + try: + with socket.create_connection(('127.0.0.1', 22), timeout=1): + break + except OSError: + time.sleep(.1) +else: + raise SystemExit('sshd did not start') +PY + pgrep -x iscsid + touch /ci/reports/ready + sleep infinity + else + install -m 600 /ci/input/id_ed25519 /root/.ssh/id_ed25519 + install -m 600 /ci/input/known_hosts /root/.ssh/known_hosts + while [[ ! -e /ci/input/peer-ready ]]; do sleep 1; done + # This disk is created by the host supervisor exclusively for this run. + test "$(lsblk -dn -o SERIAL /dev/disk/by-id/virtio-scst-ci-data)" = scst-ci-data + python3 - <<'PY' +import json, pathlib +config = dict(disposable_host=True, + admin_command=['/usr/bin/perl', '-I/scst/scstadmin/scstadmin.sysfs/scst-1.0.0/lib', + '/scst/scstadmin/scstadmin.sysfs/scstadmin'], + report_dir='/ci/reports/ginkgo', backing_dir='/ci/backing', + command_timeout='90s', cleanup_timeout='5m', settle_timeout='30s', suite_timeout='75m', + io=dict(block_device='/dev/disk/by-id/virtio-scst-ci-data', region_mb=16, timeout='5m'), + iscsi=dict(target_address='192.0.2.1', port=3260, login_timeout='30s', + initiator=dict(host='192.0.2.2', user='root', port=22, + identity_file='/root/.ssh/id_ed25519', lock_file='/tmp/scst-ginkgo.lock', + disposable_host=True))) +pathlib.Path('/ci/config.json').write_text(json.dumps(config)) +PY + label=$(cat /ci/input/ginkgo-label) + make ginkgo-test GINKGO_CONFIG=/ci/config.json GINKGO_LABEL="$label" \ + 2>&1 | tee /ci/reports/tests.log + fi + ;; +*) echo "Unknown guest mode: $mode" >&2; exit 2 ;; +esac diff --git a/scripts/ci/kernel.SHA256SUMS b/scripts/ci/kernel.SHA256SUMS new file mode 100644 index 000000000..7e6510e2c --- /dev/null +++ b/scripts/ci/kernel.SHA256SUMS @@ -0,0 +1,6 @@ +6bcacdf74e749f6e6a9ae33658a837063d74db92d0b30093dae041cfda194c50 kernel-core-7.2.9-201.fc44.x86_64.rpm +0f472f1786781b14497a829782ec33cf6d0a140f7d4f7bf2c91fcc50ae3c2a93 kernel-modules-core-7.2.9-201.fc44.x86_64.rpm +259c4c853a4a93d8ca98b940ab2cbc13753876892b4583890a5ada9b6839177a kernel-modules-7.2.9-201.fc44.x86_64.rpm +17e14518fe28ea6897bc448bf3e50434ac5ed58d4ca10d3c1c20ef145a16e7cd kernel-devel-7.2.9-201.fc44.x86_64.rpm +b9ed178585dd4a6ef2f9f7817f847178d37aa1c98ff2b4f22d775d7abc254dc1 kernel-modules-extra-7.2.9-201.fc44.x86_64.rpm +809af41fcd0f6f79dea710615969f5f45b3203cf76ff0b00035cf510099d910d kernel-modules-internal-7.2.9-201.fc44.x86_64.rpm diff --git a/scripts/ci/kernel.env b/scripts/ci/kernel.env new file mode 100644 index 000000000..e62dbfecb --- /dev/null +++ b/scripts/ci/kernel.env @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Keep the runtime kernel and build headers together; update SHA256SUMS as well. +KERNEL_BUILD=7.2.9-201.fc44 diff --git a/scripts/ci/scst-ci.service b/scripts/ci/scst-ci.service new file mode 100644 index 000000000..2fe3d7605 --- /dev/null +++ b/scripts/ci/scst-ci.service @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Unit] +Description=Run SCST tests in the disposable guest +After=systemd-udev-settle.service +Wants=systemd-udev-settle.service + +[Service] +Type=oneshot +ExecStart=/bin/bash /scst/scripts/ci/guest +StandardOutput=journal+console +StandardError=journal+console +TimeoutStartSec=infinity diff --git a/scripts/ci/scst-ci.target b/scripts/ci/scst-ci.target new file mode 100644 index 000000000..42c4d66d8 --- /dev/null +++ b/scripts/ci/scst-ci.target @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Unit] +Description=Disposable SCST CI guest +Requires=basic.target scst-ci.service +After=basic.target +AllowIsolate=yes diff --git a/scripts/ci/test_vm.py b/scripts/ci/test_vm.py new file mode 100644 index 000000000..ed6443bf5 --- /dev/null +++ b/scripts/ci/test_vm.py @@ -0,0 +1,289 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Supervisor failure contracts; no SCST, module operations or real VM access.""" + +from contextlib import ExitStack, redirect_stdout +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import Mock, patch + +import vm + + +class SupervisorTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.output = Path(self.temporary.name) + self.manifest = dict(commit='a' * 40, suite='kunit', kernel='test-kernel') + + def completion(self, **changes): + result = dict(self.manifest, mode='smoke', token='this-run', exit_code=0, + running_kernel='test-kernel') + result.update(changes) + (self.output / 'completion.json').write_text(json.dumps(result)) + + def test_clean_qemu_exit_without_completion_is_not_success(self): + with self.assertRaisesRegex(vm.VMError, 'completion'): + vm.validate_completion(self.output, 'smoke', 'this-run', self.manifest) + + def test_report_must_match_this_boot_build_and_kernel(self): + for changes in ({'token': 'old-run'}, {'commit': 'b' * 40}, + {'running_kernel': 'wrong-kernel'}, {'exit_code': 1}): + with self.subTest(changes=changes): + self.completion(**changes) + with self.assertRaises(vm.VMError): + vm.validate_completion(self.output, 'smoke', 'this-run', self.manifest) + + def test_complete_smoke_boot(self): + self.completion() + vm.validate_completion(self.output, 'smoke', 'this-run', self.manifest) + + def test_kernel_panic_survives_missing_guest_report(self): + (self.output / 'target.serial.log').write_text( + '[1.0] Kernel panic - not syncing: test crash\n[1.1] Call Trace:\n') + self.assertTrue(vm.diagnostics(self.output)) + + def test_initiator_crash_is_not_hidden_by_target(self): + (self.output / 'initiator.serial.log').write_text('[1.0] BUG: unable to handle page fault\n') + self.assertTrue(vm.diagnostics(self.output)) + + def test_daemon_warning_is_not_a_kernel_fault(self): + (self.output / 'initiator.serial.log').write_text( + '[120.175797] bash[481]: WARNING: daemon configuration\n') + self.assertFalse(vm.diagnostics(self.output)) + + def test_boot_failure_has_a_shorter_deadline(self): + target = Mock() + target.poll.return_value = None + with self.assertRaisesRegex(vm.VMError, 'boot deadline'): + vm.wait_guest(target, None, float('inf'), self.output / 'ready', + self.output / 'release', self.output / 'started', 0) + + def test_deadline_does_not_depend_on_guest_response(self): + target = Mock() + target.poll.return_value = None + with self.assertRaisesRegex(vm.VMError, 'deadline'): + vm.wait_guest(target, None, 0, self.output / 'ready', self.output / 'release') + + def test_peer_loss_aborts_target_wait(self): + target, peer = Mock(), Mock() + target.poll.return_value = None + peer.poll.return_value = 0 + with self.assertRaisesRegex(vm.VMError, 'initiator exited'): + vm.wait_guest(target, peer, float('inf'), self.output / 'ready', self.output / 'release') + + def test_stuck_qemu_is_killed_after_term(self): + process = Mock(pid=123) + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired('qemu', 5), 0] + with patch('vm.os.killpg') as kill: + vm.stop(process) + self.assertEqual(kill.call_count, 2) + + def test_all_skipped_kunit_is_not_a_pass(self): + self.completion(mode='kunit') + results = self.output / 'kunit/run' + results.mkdir(parents=True) + (results / 'result.json').write_text(json.dumps(dict( + status='SKIP', counts=dict(passed=0), not_run_suites=[], module_state='unloaded'))) + with self.assertRaisesRegex(vm.VMError, 'KUnit did not complete'): + vm.validate_completion(self.output, 'kunit', 'this-run', self.manifest) + + def test_ginkgo_requires_both_transport_reports(self): + self.manifest['suite'] = 'ginkgo' + self.completion(mode='ginkgo') + with self.assertRaisesRegex(vm.VMError, 'both local and iSCSI'): + vm.validate_completion(self.output, 'ginkgo', 'this-run', self.manifest) + + def test_ginkgo_cannot_pass_with_empty_or_failed_transport(self): + self.manifest['suite'] = 'ginkgo' + self.completion(mode='ginkgo') + for group in ('local', 'iscsi'): + (self.output / 'ginkgo' / group).mkdir(parents=True) + for succeeded, count, state in ((True, 0, 'passed'), (False, 8, 'failed'), + (True, 8, 'skipped'), (True, 8, 'passed')): + with self.subTest(succeeded=succeeded, count=count, state=state): + for group in ('local', 'iscsi'): + (self.output / 'ginkgo' / group / 'report.json').write_text(json.dumps([ + dict(SuiteSucceeded=succeeded if group == 'iscsi' else True, + SuiteConfig=dict(LabelFilter=''), + PreRunStats=dict(SpecsThatWillRun=count if group == 'iscsi' else 31), + SpecReports=[dict(LeafNodeType='BeforeSuite', State='passed'), + dict(LeafNodeType='It', State=state)])])) + if succeeded and count and state == 'passed': + vm.validate_completion(self.output, 'ginkgo', 'this-run', self.manifest) + else: + with self.assertRaisesRegex(vm.VMError, 'incomplete or failed Ginkgo'): + vm.validate_completion(self.output, 'ginkgo', 'this-run', self.manifest) + + def test_ginkgo_rejects_unexpected_filter_in_either_transport(self): + self.manifest['suite'] = 'ginkgo' + self.completion(mode='ginkgo') + for group in ('local', 'iscsi'): + (self.output / 'ginkgo' / group).mkdir(parents=True) + for expected, actual in (('basic || smoke', ''), ('', 'basic || smoke')): + for mismatched in ('local', 'iscsi'): + with self.subTest(expected=expected, mismatched=mismatched): + for group in ('local', 'iscsi'): + (self.output / 'ginkgo' / group / 'report.json').write_text(json.dumps([ + dict(SuiteSucceeded=True, PreRunStats=dict(SpecsThatWillRun=2), + SuiteConfig=dict(LabelFilter=actual if group == mismatched else expected), + SpecReports=[dict(LeafNodeType='It', State='passed')])])) + with self.assertRaisesRegex(vm.VMError, 'label filter differs'): + vm.validate_completion(self.output, 'ginkgo', 'this-run', self.manifest, + expected) + + def test_ginkgo_filter_is_written_to_guest_input_before_boot(self): + for index, label in enumerate(('', 'basic || smoke')): + with self.subTest(label=label): + work = self.output / f'work-{index}' + output = self.output / f'output-{index}' + work.mkdir() + output.mkdir() + process = Mock() + process.poll.return_value = 0 + + def boot(*args, **kwargs): + self.assertEqual((work / 'target/ginkgo-label').read_text(), label) + return process + + with patch('vm.subprocess.run'), patch('vm.subprocess.Popen', side_effect=boot), \ + ExitStack() as stack: + vm.launch(stack, self.output, work, output, 'target', 'ginkgo', + 'this-run', 'tcg', ginkgo_label=label) + + +class ConsoleTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.image = self.root / 'image' + self.image.mkdir() + self.output = self.root / 'results' + + def run_guest(self, mode, console=None, exit_code=0, completion='valid', + error=None, serial=None, ginkgo_label=''): + manifest = dict(commit='a' * 40, suite=mode, kernel='test-kernel') + (self.image / 'image.json').write_text(json.dumps(manifest)) + + def launch(stack, image, work, output, role, guest_mode, token, accel, network, + ginkgo_label=''): + directory = output / role + directory.mkdir() + if network: + network.touch() + (output / f'{role}.serial.log').write_text((serial or {}).get(role, 'boot log\n')) + if role == 'target': + if console is not None: + (directory / 'tests.log').write_bytes(console) + if completion == 'valid': + (directory / 'completion.json').write_text(json.dumps(dict( + manifest, mode=mode, token=token, exit_code=exit_code, + running_kernel=manifest['kernel']))) + elif completion == 'malformed': + (directory / 'completion.json').write_text('{') + if mode == 'kunit': + report = directory / 'kunit/run/result.json' + report.parent.mkdir(parents=True) + report.write_text(json.dumps(dict(status='PASS', counts=dict(passed=2), + not_run_suites=[], module_state='unloaded'))) + else: + for transport in ('local', 'iscsi'): + report = directory / 'ginkgo' / transport / 'report.json' + report.parent.mkdir(parents=True) + report.write_text(json.dumps([dict( + SuiteSucceeded=True, PreRunStats=dict(SpecsThatWillRun=2), + SuiteConfig=dict(LabelFilter=ginkgo_label), + SpecReports=[dict(LeafNodeType='It', State='passed')])])) + return Mock() + + stdout = io.StringIO() + with patch('vm.launch', side_effect=launch), patch('vm.configure_ssh'), \ + patch('vm.wait_guest', side_effect=error), redirect_stdout(stdout): + result = vm.run(self.image, self.output, mode, 10, 'tcg', ginkgo_label) + self.assertEqual(json.loads((self.output / 'result.json').read_text()), result) + if console is not None: + self.assertEqual((self.output / 'target/tests.log').read_bytes(), console) + return result, stdout.getvalue() + + def test_kunit_success_prints_suites_and_counts(self): + result, output = self.run_guest('kunit', console=( + b'[PASS] scst-LIB\nTests: PASS - 2 passed, 0 failed, 1 skipped\nResult: PASS\n')) + self.assertEqual(result['status'], 'PASS') + self.assertIn('[PASS] scst-LIB', output) + self.assertIn('2 passed, 0 failed, 1 skipped', output) + self.assertIn('kunit: PASS', output) + self.assertNotIn('boot log', output) + + def test_ginkgo_success_prints_both_transport_summaries(self): + result, output = self.run_guest('ginkgo', console=( + b'Running Suite: local\n2 Passed | 0 Failed\n' + b'Running Suite: iSCSI\n2 Passed | 0 Failed\n')) + self.assertEqual(result['status'], 'PASS') + self.assertIn('Running Suite: local', output) + self.assertIn('Running Suite: iSCSI', output) + self.assertEqual(output.count('2 Passed | 0 Failed'), 2) + self.assertIn('ginkgo: PASS', output) + self.assertIn('Label filter: ', output) + self.assertEqual(result['ginkgo_label'], '') + + def test_ginkgo_pr_filter_reaches_both_suites_and_is_reported(self): + result, output = self.run_guest('ginkgo', ginkgo_label='basic || smoke') + self.assertEqual(result['status'], 'PASS') + self.assertEqual(result['ginkgo_label'], 'basic || smoke') + self.assertIn('Label filter: basic || smoke', output) + + def test_failed_tests_print_assertions_and_preserve_failed_verdict(self): + for mode in ('kunit', 'ginkgo'): + with self.subTest(mode=mode): + self.output = self.root / mode + result, output = self.run_guest(mode, exit_code=1, console=( + b'[FAIL] failing_test\nExpected 2 to equal 3\n')) + self.assertEqual(result['status'], 'ERROR') + self.assertIn('failing_test', output) + self.assertIn('Expected 2 to equal 3', output) + self.assertIn('Error: guest failed', output) + self.assertIn(f'{mode}: ERROR', output) + + def test_crash_before_tests_prints_diagnostics_without_a_guest_report(self): + serial = '[1.0] Kernel panic - not syncing: test crash\n' + 'noise\n' * 100 + result, output = self.run_guest('kunit', completion='missing', serial={'target': serial}) + self.assertEqual(result['status'], 'ERROR') + self.assertIn('No guest test output was saved', output) + self.assertIn('missing or invalid guest completion report', output) + self.assertIn('Kernel panic - not syncing: test crash', output) + self.assertEqual((self.output / 'target.serial.log').read_text(), serial) + + def test_malformed_completion_does_not_hide_partial_test_output(self): + result, output = self.run_guest('kunit', completion='malformed', + console=b'[PASS] scst-LIB\npartial \xff') + self.assertEqual(result['status'], 'ERROR') + self.assertIn('[PASS] scst-LIB', output) + self.assertIn('partial \ufffd', output) + self.assertIn('missing or invalid guest completion report', output) + + def test_timeout_prints_bounded_tails_from_both_guests(self): + serial = {'target': 'early boot\n' + 'x' * 70000 + '\n' + 'progress\n' * 100, + 'initiator': 'initiator connection lost\n'} + result, output = self.run_guest('ginkgo', console=b'Running Suite: local\n', + error=vm.VMError('guest deadline expired'), serial=serial) + self.assertEqual(result['status'], 'ERROR') + self.assertIn('Running Suite: local', output) + self.assertIn('guest deadline expired', output) + self.assertIn('target.serial.log: last 80 lines', output) + self.assertIn('initiator.serial.log: last 80 lines', output) + self.assertIn('initiator connection lost', output) + self.assertNotIn('early boot', output) + self.assertEqual(output.count(' progress\n'), 80) + self.assertLess(len(output), 2000) + for role, log in serial.items(): + self.assertEqual((self.output / f'{role}.serial.log').read_text(), log) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ci/vm.py b/scripts/ci/vm.py new file mode 100644 index 000000000..52973b057 --- /dev/null +++ b/scripts/ci/vm.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Supervise disposable guests; all deadlines and serial logs live outside them.""" + +import argparse +from contextlib import ExitStack +import json +import os +from pathlib import Path +import re +import secrets +import signal +import subprocess +import sys +import tempfile +import time + +# Use the same kernel diagnostic policy for both kinds of test. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'kunit')) +from run import kernel_diagnostics # noqa: E402 + + +class VMError(Exception): + pass + + +def stop(process): + """Kill only our own QEMU process group, including on interruption.""" + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + process.wait(timeout=5) + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + + +def wait_guest(target, peer, deadline, ready, release, started=None, boot_deadline=None): + while target.poll() is None: + if peer is not None: + if peer.poll() is not None: + raise VMError('initiator exited before the target completed') + if ready.exists() and not release.exists(): + release.touch() + if time.monotonic() >= deadline: + raise VMError('guest deadline expired') + if (boot_deadline is not None and time.monotonic() >= boot_deadline + and (not started.exists() or peer is not None and not ready.exists())): + raise VMError('guest did not become ready before the boot deadline') + time.sleep(.1) + if target.returncode != 0: + raise VMError(f'QEMU exited with {target.returncode}') + + +def validate_completion(directory, mode, token, manifest, ginkgo_label=''): + try: + result = json.loads((directory / 'completion.json').read_text()) + except (OSError, ValueError) as exc: + raise VMError('missing or invalid guest completion report') from exc + expected = dict(manifest, mode=mode, token=token, exit_code=0, + running_kernel=manifest['kernel']) + if result != expected: + raise VMError(f'guest failed or completion identity differs: {result}') + if mode == 'kunit': + paths = list((directory / 'kunit').glob('*/result.json')) + if len(paths) != 1: + raise VMError('expected exactly one KUnit result') + report = json.loads(paths[0].read_text()) + if (report['status'] != 'PASS' or report['counts']['passed'] == 0 + or report['not_run_suites'] or report['module_state'] != 'unloaded'): + raise VMError('KUnit did not complete successfully') + elif mode == 'ginkgo': + paths = list((directory / 'ginkgo').glob('*/report.json')) + if len(paths) != 2: + raise VMError('expected reports from both local and iSCSI suites') + for path in paths: + reports = json.loads(path.read_text()) + if (len(reports) != 1 or not reports[0]['SuiteSucceeded'] + or reports[0]['PreRunStats']['SpecsThatWillRun'] <= 0 + or not any(spec['LeafNodeType'] == 'It' and spec['State'] == 'passed' + for spec in reports[0]['SpecReports'])): + raise VMError(f'incomplete or failed Ginkgo suite: {path}') + if reports[0]['SuiteConfig']['LabelFilter'] != ginkgo_label: + raise VMError(f'Ginkgo label filter differs from the requested selection: {path}') + + +def diagnostics(output): + # Serial logs cannot be overwritten by a guest. dmesg covers console loss + # on normal completion, but is intentionally not required after a crash. + paths = list(output.glob('*.serial.log')) + list(output.glob('*/dmesg.log')) + found = [] + for path in paths: + lines = path.read_text(errors='replace').splitlines() + if path.name.endswith('.serial.log'): + # journal+console prefixes userspace output with process[pid]. A + # daemon's WARNING is not a kernel diagnostic. Completed guests + # additionally supply a LOG_KERN-only dmesg capture. + lines = [line for line in lines if not re.match( + r'^\[\s*\d+\.\d+\]\s+[^\s\[]+\[\d+\]:', line)] + events = kernel_diagnostics(lines) + if events: + found.append({'file': path.name, 'events': events}) + return found + + +def print_log(path, tail=False): + try: + with path.open('rb') as source: + if tail: + source.seek(0, os.SEEK_END) + source.seek(max(0, source.tell() - 65536)) + text = source.read().decode('utf-8', errors='replace') + except OSError as exc: + print(f'Unable to read {path.name}: {exc}') + return + lines = text.splitlines() + if tail: + lines = lines[-80:] + # Distinguish guest output from the supervisor's messages. + for line in lines: + print(' ' + line) + + +def print_results(output, result): + """Replay native test output without changing the supervisor's verdict.""" + mode = result['mode'] + if mode in ('kunit', 'ginkgo'): + print(f'\n=== {mode} test output ===') + if mode == 'ginkgo': + print(f"Label filter: {result['ginkgo_label'] or ''}") + path = output / 'target/tests.log' + if path.is_file(): + print_log(path) + else: + print('No guest test output was saved.') + if result['error']: + print(f"\nError: {result['error']}") + for item in result['diagnostics']: + print(f"\n=== Kernel diagnostics: {item['file']} ===") + for event in item['events']: + for line in event.splitlines(): + print(' ' + line) + if result['status'] != 'PASS': + for path in sorted(output.glob('*.serial.log')): + print(f'\n=== {path.name}: last 80 lines (up to 64 KiB) ===') + print_log(path, tail=True) + print(f"\n{mode}: {result['status']} ({result['elapsed_seconds']}s); {output}", flush=True) + + +def launch(stack, image, work, output, role, mode, token, accel, network=None, ginkgo_label=''): + config, reports = work / role, output / role + config.mkdir() + reports.mkdir() + (config / 'mode').write_text(mode) + (config / 'token').write_text(token) + if mode == 'ginkgo': + (config / 'ginkgo-label').write_text(ginkgo_label) + overlay = work / f'{role}.qcow2' + subprocess.run(['qemu-img', 'create', '-q', '-f', 'qcow2', '-F', 'raw', + '-b', str(image / 'rootfs.img'), str(overlay)], check=True) + serial = stack.enter_context((output / f'{role}.serial.log').open('wb')) + command = [ + 'qemu-system-x86_64', '-machine', 'accel=' + ('kvm:tcg' if accel == 'auto' else accel), + '-m', '1024' if role == 'initiator' else '2048', + '-cpu', 'host' if accel == 'kvm' else 'max', + '-smp', '1' if role == 'initiator' else '2', '-display', 'none', '-monitor', 'none', + '-serial', 'stdio', '-no-reboot', '-nic', 'none', + '-kernel', str(image / 'vmlinuz'), '-initrd', str(image / 'initramfs'), + '-append', 'root=/dev/disk/by-id/virtio-scst-ci-root rw console=ttyS0,115200 earlyprintk=serial ' + 'systemd.unit=scst-ci.target net.ifnames=0 biosdevname=0 selinux=0 ' + 'panic=1 oops=panic kunit.enable=1 kunit.autorun=0', + '-drive', f'file={overlay},format=qcow2,if=none,id=root', + '-device', 'virtio-blk-pci,drive=root,serial=scst-ci-root', + '-fsdev', f'local,id=input,path={config},security_model=none,readonly=on', + '-device', 'virtio-9p-pci,fsdev=input,mount_tag=ci-input', + # Preserve guest ownership for scstadmin's secure config-file checks + # while QEMU itself runs as an unprivileged host user. + '-fsdev', f'local,id=reports,path={reports},security_model=mapped-xattr', + '-device', 'virtio-9p-pci,fsdev=reports,mount_tag=ci-reports', + ] + if mode == 'ginkgo': + disk = work / 'data.raw' + with disk.open('wb') as file: + file.truncate(1024 ** 3) + command += ['-drive', f'file={disk},format=raw,if=none,id=data', + '-device', 'virtio-blk-pci,drive=data,serial=scst-ci-data,' + 'logical_block_size=4096,physical_block_size=4096'] + if network: + server = 'on' if role == 'target' else 'off' + command += ['-netdev', f'stream,id=peer,server={server},addr.type=unix,addr.path={network}', + '-device', 'virtio-net-pci,netdev=peer,mac=52:54:00:12:34:' + + ('01' if role == 'target' else '02')] + (output / f'{role}.command.json').write_text(json.dumps(command, indent=2) + '\n') + process = subprocess.Popen(command, stdout=serial, stderr=subprocess.STDOUT, + start_new_session=True) + stack.callback(stop, process) + return process + + +def configure_ssh(work): + # Generated per run, used only on the private guest-to-guest network and + # removed with work. Private keys are never uploaded as artifacts. + for name in ('id_ed25519', 'ssh_host_key'): + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', + '-f', str(work / name)], check=True) + target, peer = work / 'target', work / 'initiator' + (target / 'id_ed25519').write_bytes((work / 'id_ed25519').read_bytes()) + (target / 'known_hosts').write_text('192.0.2.2 ' + (work / 'ssh_host_key.pub').read_text()) + (peer / 'ssh_host_key').write_bytes((work / 'ssh_host_key').read_bytes()) + (peer / 'authorized_keys').write_bytes((work / 'id_ed25519.pub').read_bytes()) + (target / 'ssh-ready').touch() + (peer / 'ssh-ready').touch() + + +def run(image, output, mode, timeout, accel, ginkgo_label=''): + manifest = json.loads((image / 'image.json').read_text()) + if mode in ('kunit', 'ginkgo') and manifest['suite'] != mode: + raise VMError('image was built for a different test suite') + output.mkdir(parents=True, exist_ok=False) + token = secrets.token_hex(16) + result = dict(mode=mode, image=manifest, status='ERROR', error=None) + if mode == 'ginkgo': + result['ginkgo_label'] = ginkgo_label + started = time.monotonic() + try: + with tempfile.TemporaryDirectory(prefix='scst-ci-') as temporary, ExitStack() as stack: + work = Path(temporary) + network = work / 'network.sock' if mode == 'ginkgo' else None + target = launch(stack, image, work, output, 'target', mode, token, accel, network, + ginkgo_label) + peer = None + if network: + # The listening QEMU must create its socket before its peer connects. + while not network.exists(): + if target.poll() is not None or time.monotonic() - started > 10: + raise VMError('target did not create the private network') + time.sleep(.05) + peer = launch(stack, image, work, output, 'initiator', 'initiator', token, accel, network) + configure_ssh(work) + wait_guest(target, peer, started + timeout, output / 'initiator/ready', + work / 'target/peer-ready', output / 'target/started', started + 240) + validate_completion(output / 'target', mode, token, manifest, ginkgo_label) + result['status'] = 'PASS' + except (VMError, OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as exc: + result['error'] = str(exc) + finally: + result['diagnostics'] = diagnostics(output) + if result['diagnostics']: + result['status'] = 'ERROR' + result['error'] = result['error'] or 'kernel diagnostics in guest logs' + result['elapsed_seconds'] = round(time.monotonic() - started, 2) + (output / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print_results(output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=('kunit', 'ginkgo', 'self-test')) + parser.add_argument('image', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('--timeout', type=int, default=10000) + parser.add_argument('--accel', choices=('auto', 'tcg', 'kvm'), default='auto') + parser.add_argument('--ginkgo-label', default='', help='Ginkgo label expression (default: all tests)') + args = parser.parse_args() + if args.timeout <= 0: + parser.error('--timeout must be positive') + if args.ginkgo_label and args.mode != 'ginkgo': + parser.error('--ginkgo-label requires ginkgo mode') + image, output = args.image.resolve(), args.output.resolve() + # QEMU's key-value arguments cannot represent arbitrary comma-containing paths. + if any(',' in str(path) for path in (image, output, Path(tempfile.gettempdir()))): + parser.error('image, output and temporary paths must not contain commas') + subprocess.run(['sha256sum', '--check', 'SHA256SUMS'], cwd=image, check=True) + if args.mode == 'self-test': + smoke = run(image, output / 'smoke', 'smoke', args.timeout, args.accel) + if smoke['status'] != 'PASS': + return 1 + panic = run(image, output / 'panic', 'panic', args.timeout, args.accel) + hang = run(image, output / 'hang', 'hang', args.timeout, args.accel) + # Expected failure must be the injected fault, never an arbitrary boot error. + return int(not (panic['status'] == 'ERROR' + and any('Kernel panic' in event for item in panic['diagnostics'] for event in item['events']) + and hang['error'] == 'guest deadline expired' + and (output / 'hang/target/started').is_file() + and (output / 'hang/target/started').read_text().strip() == 'hang')) + return int(run(image, output, args.mode, args.timeout, args.accel, + args.ginkgo_label)['status'] != 'PASS') + + +if __name__ == '__main__': + def interrupted(signum, _frame): + raise KeyboardInterrupt(f'signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + sys.exit(main()) diff --git a/tests/ginkgo/README.md b/tests/ginkgo/README.md index ecc37e288..766ddd210 100644 --- a/tests/ginkgo/README.md +++ b/tests/ginkgo/README.md @@ -4,6 +4,8 @@ Ginkgo/Gomega tests exercise management through `scstadmin`, SCSI commands through sg3_utils, and data verification through fio. Transports are `scst_local` and TCP iSCSI with a separate initiator controlled over SSH. +For GitHub Actions, see [CI](../../doc/CI.md). + ## Setup and run Use a disposable Linux target with root access, an empty SCST configuration, @@ -49,7 +51,14 @@ with `basic` (management), `scsi` or `io`; add `fileio` or `blockio` to narrow the backend. For example, `local && io && fileio` needs no physical disk. `GINKGO_FOCUS` additionally filters spec names by regexp. +GitHub pull requests use `basic || smoke` on both transports. `smoke` covers +SCSI identity, capacity and LUN discovery, rejected writes and reads, an UNMAP +transfer-boundary case, asynchronous FUA/cache commands and both fio backends. +Pushes to `master` run all specs, including stress and the full parameter tables. + ```sh +# List the pull-request selection: +make ginkgo-list GINKGO_LABEL='basic || smoke' # List every spec and its labels without accessing either host: make ginkgo-list GINKGO_LABEL='' # All specs; prepare both hosts and a block device first: diff --git a/tests/ginkgo/internal/dataio/specs.go b/tests/ginkgo/internal/dataio/specs.go index 240c6d198..af89c9f33 100644 --- a/tests/ginkgo/internal/dataio/specs.go +++ b/tests/ginkgo/internal/dataio/specs.go @@ -10,19 +10,19 @@ import ( // each suite owns its target/session lifecycle and supplies the exported disk. func Register(create func(SpecContext, Profile) *Endpoint) bool { return Describe("Exported SCSI devices", func() { - It("returns INQUIRY and VPD identity", Label("scsi"), func(ctx SpecContext) { + It("returns INQUIRY and VPD identity", Label("scsi", "smoke"), func(ctx SpecContext) { Expect(create(ctx, Profile{Handler: "vdisk_fileio"}).Identity(ctx)).To(Succeed()) }) - It("reports matching READ CAPACITY(10) and (16) geometry", Label("scsi"), func(ctx SpecContext) { + It("reports matching READ CAPACITY(10) and (16) geometry", Label("scsi", "smoke"), func(ctx SpecContext) { Expect(create(ctx, Profile{Handler: "vdisk_fileio"}).Capacity(ctx)).To(Succeed()) }) - It("reports only the initiator's accessible LUNs", Label("scsi"), func(ctx SpecContext) { + It("reports only the initiator's accessible LUNs", Label("scsi", "smoke"), func(ctx SpecContext) { Expect(create(ctx, Profile{Handler: "vdisk_fileio", ExtraLUNs: true}).LUNs(ctx)).To(Succeed()) }) - It("rejects a read-only WRITE with DATA PROTECT and preserves data", Label("scsi", "negative"), func(ctx SpecContext) { + It("rejects a read-only WRITE with DATA PROTECT and preserves data", Label("scsi", "negative", "smoke"), func(ctx SpecContext) { Expect(create(ctx, Profile{Handler: "vdisk_fileio", ReadOnly: true}).ReadOnlyWrite(ctx)).To(Succeed()) }) - It("rejects an out-of-range READ with ILLEGAL REQUEST and remains usable", Label("scsi", "negative"), func(ctx SpecContext) { + It("rejects an out-of-range READ with ILLEGAL REQUEST and remains usable", Label("scsi", "negative", "smoke"), func(ctx SpecContext) { Expect(create(ctx, Profile{Handler: "vdisk_fileio"}).OutOfRange(ctx)).To(Succeed()) }) DescribeTable("bounds UNMAP descriptors and preserves surrounding data", Label("scsi", "fileio", "unmap"), @@ -39,7 +39,7 @@ func Register(create func(SpecContext, Profile) *Endpoint) bool { Entry("one complete descriptor unmaps its range", unmapCase{parameterLength: 24, descriptorLength: 16, unmapped: 1}), Entry("unaligned descriptor length keeps the complete descriptor", unmapCase{parameterLength: 25, descriptorLength: 17, unmapped: 1}), Entry("CDB length excludes a partial second descriptor", unmapCase{parameterLength: 39, descriptorLength: 32, unmapped: 1}), - Entry("CDB length bounds an oversized descriptor list", unmapCase{parameterLength: 24, descriptorLength: 32, unmapped: 1}), + Entry("CDB length bounds an oversized descriptor list", Label("smoke"), unmapCase{parameterLength: 24, descriptorLength: 32, unmapped: 1}), Entry("descriptor length excludes a complete trailing descriptor", unmapCase{parameterLength: 40, descriptorLength: 16, unmapped: 1}), Entry("two complete descriptors unmap both ranges", unmapCase{parameterLength: 40, descriptorLength: 32, unmapped: 2}), Entry("maximum descriptor length is bounded by the transfer", unmapCase{parameterLength: 24, descriptorLength: 0xffff, unmapped: 1})) @@ -50,12 +50,12 @@ func Register(create func(SpecContext, Profile) *Endpoint) bool { Expect(e.CacheCommands(ctx, !options.WriteThrough && !options.NVCache)).To(Succeed()) }, Entry("buffered synchronous 512", FileioOptions{BlockSize: 512}), - Entry("buffered asynchronous 512", FileioOptions{Async: true, BlockSize: 512}), + Entry("buffered asynchronous 512", Label("smoke"), FileioOptions{Async: true, BlockSize: 512}), Entry("buffered synchronous 4096", FileioOptions{BlockSize: 4096}), Entry("direct asynchronous 4096", FileioOptions{Async: true, Direct: true, BlockSize: 4096}), Entry("asynchronous write-through FUA control", FileioOptions{Async: true, WriteThrough: true}), Entry("asynchronous nonvolatile-cache FUA control", FileioOptions{Async: true, NVCache: true})) - DescribeTable("verifies concurrent 4K, 64K and 128K fio jobs", Label("io", "fio"), func(ctx SpecContext, handler string) { + DescribeTable("verifies concurrent 4K, 64K and 128K fio jobs", Label("io", "fio", "smoke"), func(ctx SpecContext, handler string) { e := create(ctx, Profile{Handler: handler, FIO: true}) Expect(e.Capacity(ctx)).To(Succeed()) Expect(e.FIO(ctx)).To(Succeed())