From 3d4f88273b2e7de75edc52fa2a51b9ab2362e44d Mon Sep 17 00:00:00 2001 From: Gleb Chesnokov Date: Fri, 16 Oct 2026 23:24:37 +0300 Subject: [PATCH] tests: Add named CI selections for Ginkgo Keep CI selection policy in shared ci-pr and ci-push label sets. Resolve them to ordinary Ginkgo expressions for local discovery and execution, and before booting CI guests so report validation still checks the exact filter. Preserve ordinary expressions and the full manual selection. Mark both ACG teardown scenarios as race and exclude that category from master push runs. Keep their strict overlap, recovery and cleanup checks unchanged. PR runs retain basic/smoke coverage through ci-pr. Document the named selections and check their filtering behavior and propagation through the VM supervisor. --- .github/workflows/ginkgo.yml | 2 +- doc/CI.md | 14 +++++-- scripts/ci/test_vm.py | 14 ++++--- scripts/ci/vm.py | 14 ++++++- tests/ginkgo/README.md | 19 +++++++-- tests/ginkgo/internal/testrun/labels.go | 29 ++++++++++++++ tests/ginkgo/internal/testrun/labels.json | 4 ++ tests/ginkgo/internal/testrun/labels_test.go | 42 ++++++++++++++++++++ tests/ginkgo/internal/testrun/run.go | 5 +++ tests/ginkgo/iscsi/acg_reassign_test.go | 2 +- 10 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 tests/ginkgo/internal/testrun/labels.go create mode 100644 tests/ginkgo/internal/testrun/labels.json create mode 100644 tests/ginkgo/internal/testrun/labels_test.go diff --git a/.github/workflows/ginkgo.yml b/.github/workflows/ginkgo.yml index acf6b5223..4f81d77e3 100644 --- a/.github/workflows/ginkgo.yml +++ b/.github/workflows/ginkgo.yml @@ -36,7 +36,7 @@ jobs: run: bash scripts/ci/build-image ginkgo "$RUNNER_TEMP/ginkgo-image" - name: Run local and iSCSI tests against the compiled SCST env: - GINKGO_LABEL: ${{ github.event_name == 'pull_request' && 'basic || smoke' || '' }} + GINKGO_LABEL: ${{ github.event_name == 'pull_request' && 'ci-pr' || github.event_name == 'push' && 'ci-push' || '' }} run: | python3 scripts/ci/vm.py ginkgo "$RUNNER_TEMP/ginkgo-image" \ "$RUNNER_TEMP/ginkgo-results" --timeout 10000 --ginkgo-label "$GINKGO_LABEL" diff --git a/doc/CI.md b/doc/CI.md index 1a531257d..6588bf359 100644 --- a/doc/CI.md +++ b/doc/CI.md @@ -7,7 +7,10 @@ and runs tests in disposable QEMU guests within a single job: success, panic and timeout, then `make kunit-run`. - `.github/workflows/ginkgo.yml`: ordinary SCST modules, `make ginkgo-check`, then local and TCP iSCSI tests through `make ginkgo-test`. Pull requests use - `basic || smoke`; pushes to `master` and manual runs select all tests. + the `ci-pr` label set; pushes to `master` use `ci-push`. Define these sets + in [labels.json](../tests/ginkgo/internal/testrun/labels.json) by combining + ordinary Ginkgo labels. Manual runs select all tests, including + timing-dependent race scenarios. Both run on pushes and pull requests to `master`, or via `workflow_dispatch`. The Debian package build in `.github/workflows/ci.yml` remains independent. @@ -56,9 +59,12 @@ bash scripts/ci/build-image ginkgo /tmp/scst-ginkgo-image python3 scripts/ci/vm.py ginkgo /tmp/scst-ginkgo-image /tmp/scst-ginkgo-results --timeout 10000 ``` -Add `--ginkgo-label 'basic || smoke'` to reproduce the pull-request selection. -The selected filter is printed in stdout and recorded in `result.json`; -both transport reports must confirm that filter. +Add `--ginkgo-label ci-pr` to reproduce the pull-request selection. +Use `--ginkgo-label ci-push` for the master push selection; omit the option +for the full manual selection. +Ordinary Ginkgo expressions are also accepted. The resolved filter is printed +in stdout and recorded in `result.json`; both transport reports must confirm +that filter. Supervisor checks without VM or SCST access: diff --git a/scripts/ci/test_vm.py b/scripts/ci/test_vm.py index ed6443bf5..c0a0981dd 100644 --- a/scripts/ci/test_vm.py +++ b/scripts/ci/test_vm.py @@ -232,11 +232,15 @@ class ConsoleTests(unittest.TestCase): self.assertIn('Label filter: ', output) self.assertEqual(result['ginkgo_label'], '') - def test_ginkgo_pr_filter_reaches_both_suites_and_is_reported(self): - result, output = self.run_guest('ginkgo', ginkgo_label='basic || smoke') - self.assertEqual(result['status'], 'PASS') - self.assertEqual(result['ginkgo_label'], 'basic || smoke') - self.assertIn('Label filter: basic || smoke', output) + def test_ginkgo_label_sets_reach_both_suites_and_are_reported(self): + for label, expression in (('basic || smoke', 'basic || smoke'), + ('ci-pr', 'basic || smoke'), ('ci-push', '!race')): + with self.subTest(label=label): + self.output = self.root / label + result, output = self.run_guest('ginkgo', ginkgo_label=label) + self.assertEqual(result['status'], 'PASS') + self.assertEqual(result['ginkgo_label'], expression) + self.assertIn(f'Label filter: {expression}', output) def test_failed_tests_print_assertions_and_preserve_failed_verdict(self): for mode in ('kunit', 'ginkgo'): diff --git a/scripts/ci/vm.py b/scripts/ci/vm.py index 52973b057..b54b0410a 100644 --- a/scripts/ci/vm.py +++ b/scripts/ci/vm.py @@ -24,6 +24,15 @@ class VMError(Exception): pass +def resolve_ginkgo_label(label): + path = Path(__file__).resolve().parents[2] / 'tests/ginkgo/internal/testrun/labels.json' + sets = json.loads(path.read_text()) + expression = sets.get(label.strip(), label) + if label.strip() in sets and (not isinstance(expression, str) or not expression.strip()): + raise VMError(f'Ginkgo label set {label!r} must be a nonempty expression') + return expression + + def stop(process): """Kill only our own QEMU process group, including on interruption.""" if process.poll() is None: @@ -220,6 +229,8 @@ def run(image, output, mode, timeout, accel, ginkgo_label=''): manifest = json.loads((image / 'image.json').read_text()) if mode in ('kunit', 'ginkgo') and manifest['suite'] != mode: raise VMError('image was built for a different test suite') + if mode == 'ginkgo': + ginkgo_label = resolve_ginkgo_label(ginkgo_label) output.mkdir(parents=True, exist_ok=False) token = secrets.token_hex(16) result = dict(mode=mode, image=manifest, status='ERROR', error=None) @@ -265,7 +276,8 @@ def main(): parser.add_argument('output', type=Path) parser.add_argument('--timeout', type=int, default=10000) parser.add_argument('--accel', choices=('auto', 'tcg', 'kvm'), default='auto') - parser.add_argument('--ginkgo-label', default='', help='Ginkgo label expression (default: all tests)') + parser.add_argument('--ginkgo-label', default='', + help='Ginkgo label set or expression (default: all tests)') args = parser.parse_args() if args.timeout <= 0: parser.error('--timeout must be positive') diff --git a/tests/ginkgo/README.md b/tests/ginkgo/README.md index 78aa1e311..94fec9af5 100644 --- a/tests/ginkgo/README.md +++ b/tests/ginkgo/README.md @@ -51,12 +51,25 @@ with `basic` (management), `scsi` or `io`; add `fileio` or `blockio` to narrow the backend. For example, `local && io && fileio` needs no physical disk. `GINKGO_FOCUS` additionally filters spec names by regexp. -GitHub pull requests use `basic || smoke` on both transports. -Pushes to `master` run all specs, including stress and the full parameter tables. +GitHub pull requests use the `ci-pr` label set; pushes to `master` use `ci-push`. +Their definitions live in [labels.json](internal/testrun/labels.json), shared +by the local runner and VM supervisor: + +| Label set | Ginkgo expression | +| --- | --- | +| `ci-pr` | `basic || smoke` | +| `ci-push` | `!race` | + +Each set aliases a whole filter; its definition combines ordinary Ginkgo labels. +Pass either a set name or a Ginkgo expression as `GINKGO_LABEL`. Manual workflow +runs use an empty filter and select all specs, including timing-dependent +`race` scenarios. Other stress tests remain in `ci-push`. ```sh # List the pull-request selection: -make ginkgo-list GINKGO_LABEL='basic || smoke' +make ginkgo-list GINKGO_LABEL=ci-pr +# List the master push selection: +make ginkgo-list GINKGO_LABEL=ci-push # List every spec and its labels without accessing either host: make ginkgo-list GINKGO_LABEL='' # All specs; prepare both hosts and a block device first: diff --git a/tests/ginkgo/internal/testrun/labels.go b/tests/ginkgo/internal/testrun/labels.go new file mode 100644 index 000000000..4c43e9b6a --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +package testrun + +import ( + _ "embed" + "encoding/json" + "fmt" + "strings" +) + +// Shared with scripts/ci/vm.py so guest reports use the expected filter. +// +//go:embed labels.json +var labelSetsJSON []byte + +// Named sets alias a whole filter; ordinary Ginkgo expressions pass through. +func resolveLabelFilter(filter string) (string, error) { + var sets map[string]string + if err := json.Unmarshal(labelSetsJSON, &sets); err != nil { + return "", fmt.Errorf("Ginkgo label sets: %w", err) + } + if expression, ok := sets[strings.TrimSpace(filter)]; ok { + if strings.TrimSpace(expression) == "" { + return "", fmt.Errorf("Ginkgo label set %q is empty", filter) + } + return expression, nil + } + return filter, nil +} diff --git a/tests/ginkgo/internal/testrun/labels.json b/tests/ginkgo/internal/testrun/labels.json new file mode 100644 index 000000000..810906318 --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels.json @@ -0,0 +1,4 @@ +{ + "ci-pr": "basic || smoke", + "ci-push": "!race" +} diff --git a/tests/ginkgo/internal/testrun/labels_test.go b/tests/ginkgo/internal/testrun/labels_test.go new file mode 100644 index 000000000..29721989e --- /dev/null +++ b/tests/ginkgo/internal/testrun/labels_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only +package testrun + +import ( + "testing" + + "github.com/onsi/ginkgo/v2/types" +) + +func TestLabelSetSelection(t *testing.T) { + for _, tc := range []struct { + filter string + labels []string + want bool + }{ + {"ci-pr", []string{"local", "basic"}, true}, + {"ci-pr", []string{"iscsi", "scsi", "smoke"}, true}, + {"ci-pr", []string{"iscsi", "stress", "race", "issue-367"}, false}, + {"ci-pr", []string{"local", "stress"}, false}, + {"ci-push", []string{"local", "stress"}, true}, + {" ci-push ", []string{"iscsi", "scsi"}, true}, + {"ci-push", []string{"iscsi", "stress", "race", "issue-367"}, false}, + {"", []string{"iscsi", "race"}, true}, + {"iscsi && race", []string{"iscsi", "race"}, true}, + {"iscsi && race", []string{"local", "stress"}, false}, + } { + t.Run(tc.filter, func(t *testing.T) { + expression, err := resolveLabelFilter(tc.filter) + if err != nil { + t.Fatal(err) + } + matches, err := types.ParseLabelFilter(expression) + if err != nil { + t.Fatal(err) + } + if got := matches(tc.labels); got != tc.want { + t.Fatalf("filter %q (%q) matches %v: got %t, want %t", + tc.filter, expression, tc.labels, got, tc.want) + } + }) + } +} diff --git a/tests/ginkgo/internal/testrun/run.go b/tests/ginkgo/internal/testrun/run.go index 81d224bf0..ba28abb13 100644 --- a/tests/ginkgo/internal/testrun/run.go +++ b/tests/ginkgo/internal/testrun/run.go @@ -29,6 +29,11 @@ func Run(t *testing.T, name string, iscsi bool, assign func(Environment)) { var cli *admin.Client RegisterFailHandler(Fail) suiteConfig, reporterConfig := GinkgoConfiguration() + filter, err := resolveLabelFilter(suiteConfig.LabelFilter) + if err != nil { + t.Fatal(err) + } + suiteConfig.LabelFilter = filter if suiteConfig.ParallelTotal != 1 { t.Fatal("SCST component tests share host state; parallel execution is unsupported") } diff --git a/tests/ginkgo/iscsi/acg_reassign_test.go b/tests/ginkgo/iscsi/acg_reassign_test.go index 2d0d18d77..68b073c51 100644 --- a/tests/ginkgo/iscsi/acg_reassign_test.go +++ b/tests/ginkgo/iscsi/acg_reassign_test.go @@ -90,7 +90,7 @@ func (f *fixture) raceACG(ctx context.Context, kernel *testhost.KernelLog, from, "ACG move completed before teardown started; this round is inconclusive") } -var _ = DescribeTable("ACG reassignment during iSCSI teardown", Label("iscsi", "stress", "sessions", "issue-367"), +var _ = DescribeTable("ACG reassignment during iSCSI teardown", Label("iscsi", "stress", "race", "sessions", "issue-367"), func(ctx SpecContext, recoverTCP bool) { path := filepath.Join(artifactDir, fmt.Sprintf("issue-367-%t-kernel.log", recoverTCP)) kernel := must(testhost.StartKernelLog(path))