diff --git a/.github/workflows/ginkgo.yml b/.github/workflows/ginkgo.yml index acf6b5223..4f81d77e3 100644 --- a/.github/workflows/ginkgo.yml +++ b/.github/workflows/ginkgo.yml @@ -36,7 +36,7 @@ jobs: run: bash scripts/ci/build-image ginkgo "$RUNNER_TEMP/ginkgo-image" - name: Run local and iSCSI tests against the compiled SCST env: - GINKGO_LABEL: ${{ github.event_name == 'pull_request' && 'basic || smoke' || '' }} + GINKGO_LABEL: ${{ github.event_name == 'pull_request' && 'ci-pr' || github.event_name == 'push' && 'ci-push' || '' }} run: | python3 scripts/ci/vm.py ginkgo "$RUNNER_TEMP/ginkgo-image" \ "$RUNNER_TEMP/ginkgo-results" --timeout 10000 --ginkgo-label "$GINKGO_LABEL" diff --git a/doc/CI.md b/doc/CI.md index 1a531257d..6588bf359 100644 --- a/doc/CI.md +++ b/doc/CI.md @@ -7,7 +7,10 @@ and runs tests in disposable QEMU guests within a single job: success, panic and timeout, then `make kunit-run`. - `.github/workflows/ginkgo.yml`: ordinary SCST modules, `make ginkgo-check`, then local and TCP iSCSI tests through `make ginkgo-test`. Pull requests use - `basic || smoke`; pushes to `master` and manual runs select all tests. + the `ci-pr` label set; pushes to `master` use `ci-push`. Define these sets + in [labels.json](../tests/ginkgo/internal/testrun/labels.json) by combining + ordinary Ginkgo labels. Manual runs select all tests, including + timing-dependent race scenarios. Both run on pushes and pull requests to `master`, or via `workflow_dispatch`. The Debian package build in `.github/workflows/ci.yml` remains independent. @@ -56,9 +59,12 @@ bash scripts/ci/build-image ginkgo /tmp/scst-ginkgo-image python3 scripts/ci/vm.py ginkgo /tmp/scst-ginkgo-image /tmp/scst-ginkgo-results --timeout 10000 ``` -Add `--ginkgo-label 'basic || smoke'` to reproduce the pull-request selection. -The selected filter is printed in stdout and recorded in `result.json`; -both transport reports must confirm that filter. +Add `--ginkgo-label ci-pr` to reproduce the pull-request selection. +Use `--ginkgo-label ci-push` for the master push selection; omit the option +for the full manual selection. +Ordinary Ginkgo expressions are also accepted. The resolved filter is printed +in stdout and recorded in `result.json`; both transport reports must confirm +that filter. Supervisor checks without VM or SCST access: diff --git a/scripts/ci/test_vm.py b/scripts/ci/test_vm.py index ed6443bf5..c0a0981dd 100644 --- a/scripts/ci/test_vm.py +++ b/scripts/ci/test_vm.py @@ -232,11 +232,15 @@ class ConsoleTests(unittest.TestCase): self.assertIn('Label filter: ', output) self.assertEqual(result['ginkgo_label'], '') - def test_ginkgo_pr_filter_reaches_both_suites_and_is_reported(self): - result, output = self.run_guest('ginkgo', ginkgo_label='basic || smoke') - self.assertEqual(result['status'], 'PASS') - self.assertEqual(result['ginkgo_label'], 'basic || smoke') - self.assertIn('Label filter: basic || smoke', output) + def test_ginkgo_label_sets_reach_both_suites_and_are_reported(self): + for label, expression in (('basic || smoke', 'basic || smoke'), + ('ci-pr', 'basic || smoke'), ('ci-push', '!race')): + with self.subTest(label=label): + self.output = self.root / label + result, output = self.run_guest('ginkgo', ginkgo_label=label) + self.assertEqual(result['status'], 'PASS') + self.assertEqual(result['ginkgo_label'], expression) + self.assertIn(f'Label filter: {expression}', output) def test_failed_tests_print_assertions_and_preserve_failed_verdict(self): for mode in ('kunit', 'ginkgo'): diff --git a/scripts/ci/vm.py b/scripts/ci/vm.py index 52973b057..b54b0410a 100644 --- a/scripts/ci/vm.py +++ b/scripts/ci/vm.py @@ -24,6 +24,15 @@ class VMError(Exception): pass +def resolve_ginkgo_label(label): + path = Path(__file__).resolve().parents[2] / 'tests/ginkgo/internal/testrun/labels.json' + sets = json.loads(path.read_text()) + expression = sets.get(label.strip(), label) + if label.strip() in sets and (not isinstance(expression, str) or not expression.strip()): + raise VMError(f'Ginkgo label set {label!r} must be a nonempty expression') + return expression + + def stop(process): """Kill only our own QEMU process group, including on interruption.""" if process.poll() is None: @@ -220,6 +229,8 @@ def run(image, output, mode, timeout, accel, ginkgo_label=''): manifest = json.loads((image / 'image.json').read_text()) if mode in ('kunit', 'ginkgo') and manifest['suite'] != mode: raise VMError('image was built for a different test suite') + if mode == 'ginkgo': + ginkgo_label = resolve_ginkgo_label(ginkgo_label) output.mkdir(parents=True, exist_ok=False) token = secrets.token_hex(16) result = dict(mode=mode, image=manifest, status='ERROR', error=None) @@ -265,7 +276,8 @@ def main(): parser.add_argument('output', type=Path) parser.add_argument('--timeout', type=int, default=10000) parser.add_argument('--accel', choices=('auto', 'tcg', 'kvm'), default='auto') - parser.add_argument('--ginkgo-label', default='', help='Ginkgo label expression (default: all tests)') + parser.add_argument('--ginkgo-label', default='', + help='Ginkgo label set or expression (default: all tests)') args = parser.parse_args() if args.timeout <= 0: parser.error('--timeout must be positive') diff --git a/tests/ginkgo/README.md b/tests/ginkgo/README.md index 78aa1e311..94fec9af5 100644 --- a/tests/ginkgo/README.md +++ b/tests/ginkgo/README.md @@ -51,12 +51,25 @@ with `basic` (management), `scsi` or `io`; add `fileio` or `blockio` to narrow the backend. For example, `local && io && fileio` needs no physical disk. `GINKGO_FOCUS` additionally filters spec names by regexp. -GitHub pull requests use `basic || smoke` on both transports. -Pushes to `master` run all specs, including stress and the full parameter tables. +GitHub pull requests use the `ci-pr` label set; pushes to `master` use `ci-push`. +Their definitions live in [labels.json](internal/testrun/labels.json), shared +by the local runner and VM supervisor: + +| Label set | Ginkgo expression | +| --- | --- | +| `ci-pr` | `basic || smoke` | +| `ci-push` | `!race` | + +Each set aliases a whole filter; its definition combines ordinary Ginkgo labels. +Pass either a set name or a Ginkgo expression as `GINKGO_LABEL`. Manual workflow +runs use an empty filter and select all specs, including timing-dependent +`race` scenarios. Other stress tests remain in `ci-push`. ```sh # List the pull-request selection: -make ginkgo-list GINKGO_LABEL='basic || smoke' +make ginkgo-list GINKGO_LABEL=ci-pr +# List the master push selection: +make ginkgo-list GINKGO_LABEL=ci-push # List every spec and its labels without accessing either host: make ginkgo-list GINKGO_LABEL='' # All specs; prepare both hosts and a block device first: diff --git a/tests/ginkgo/internal/testrun/labels.go b/tests/ginkgo/internal/testrun/labels.go new file mode 100644 index 000000000..4c43e9b6a --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +package testrun + +import ( + _ "embed" + "encoding/json" + "fmt" + "strings" +) + +// Shared with scripts/ci/vm.py so guest reports use the expected filter. +// +//go:embed labels.json +var labelSetsJSON []byte + +// Named sets alias a whole filter; ordinary Ginkgo expressions pass through. +func resolveLabelFilter(filter string) (string, error) { + var sets map[string]string + if err := json.Unmarshal(labelSetsJSON, &sets); err != nil { + return "", fmt.Errorf("Ginkgo label sets: %w", err) + } + if expression, ok := sets[strings.TrimSpace(filter)]; ok { + if strings.TrimSpace(expression) == "" { + return "", fmt.Errorf("Ginkgo label set %q is empty", filter) + } + return expression, nil + } + return filter, nil +} diff --git a/tests/ginkgo/internal/testrun/labels.json b/tests/ginkgo/internal/testrun/labels.json new file mode 100644 index 000000000..810906318 --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels.json @@ -0,0 +1,4 @@ +{ + "ci-pr": "basic || smoke", + "ci-push": "!race" +} diff --git a/tests/ginkgo/internal/testrun/labels_test.go b/tests/ginkgo/internal/testrun/labels_test.go new file mode 100644 index 000000000..29721989e --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only +package testrun + +import ( + "testing" + + "github.com/onsi/ginkgo/v2/types" +) + +func TestLabelSetSelection(t *testing.T) { + for _, tc := range []struct { + filter string + labels []string + want bool + }{ + {"ci-pr", []string{"local", "basic"}, true}, + {"ci-pr", []string{"iscsi", "scsi", "smoke"}, true}, + {"ci-pr", []string{"iscsi", "stress", "race", "issue-367"}, false}, + {"ci-pr", []string{"local", "stress"}, false}, + {"ci-push", []string{"local", "stress"}, true}, + {" ci-push ", []string{"iscsi", "scsi"}, true}, + {"ci-push", []string{"iscsi", "stress", "race", "issue-367"}, false}, + {"", []string{"iscsi", "race"}, true}, + {"iscsi && race", []string{"iscsi", "race"}, true}, + {"iscsi && race", []string{"local", "stress"}, false}, + } { + t.Run(tc.filter, func(t *testing.T) { + expression, err := resolveLabelFilter(tc.filter) + if err != nil { + t.Fatal(err) + } + matches, err := types.ParseLabelFilter(expression) + if err != nil { + t.Fatal(err) + } + if got := matches(tc.labels); got != tc.want { + t.Fatalf("filter %q (%q) matches %v: got %t, want %t", + tc.filter, expression, tc.labels, got, tc.want) + } + }) + } +} diff --git a/tests/ginkgo/internal/testrun/run.go b/tests/ginkgo/internal/testrun/run.go index 81d224bf0..ba28abb13 100644 --- a/tests/ginkgo/internal/testrun/run.go +++ b/tests/ginkgo/internal/testrun/run.go @@ -29,6 +29,11 @@ func Run(t *testing.T, name string, iscsi bool, assign func(Environment)) { var cli *admin.Client RegisterFailHandler(Fail) suiteConfig, reporterConfig := GinkgoConfiguration() + filter, err := resolveLabelFilter(suiteConfig.LabelFilter) + if err != nil { + t.Fatal(err) + } + suiteConfig.LabelFilter = filter if suiteConfig.ParallelTotal != 1 { t.Fatal("SCST component tests share host state; parallel execution is unsupported") } diff --git a/tests/ginkgo/iscsi/acg_reassign_test.go b/tests/ginkgo/iscsi/acg_reassign_test.go index 2d0d18d77..68b073c51 100644 --- a/tests/ginkgo/iscsi/acg_reassign_test.go +++ b/tests/ginkgo/iscsi/acg_reassign_test.go @@ -90,7 +90,7 @@ func (f *fixture) raceACG(ctx context.Context, kernel *testhost.KernelLog, from, "ACG move completed before teardown started; this round is inconclusive") } -var _ = DescribeTable("ACG reassignment during iSCSI teardown", Label("iscsi", "stress", "sessions", "issue-367"), +var _ = DescribeTable("ACG reassignment during iSCSI teardown", Label("iscsi", "stress", "race", "sessions", "issue-367"), func(ctx SpecContext, recoverTCP bool) { path := filepath.Join(artifactDir, fmt.Sprintf("issue-367-%t-kernel.log", recoverTCP)) kernel := must(testhost.StartKernelLog(path))