Merge 'test.py: support boost labels in test.py' from Artsiom Mishuta

related PR: https://github.com/scylladb/scylladb/pull/27527

This PR changes test.py logic of parsing boost test cases to use -- --list_json_content
and pass boost labels as pytests markers

using  -- --list_json_content is not ideal and currenly require to implement severall [workarounds](https://github.com/scylladb/scylladb/pull/27527#issuecomment-3765499812), but having the ability to support boost labels in pytest is worth it. because now we can apply the tiering mechanism for the boost tests as well

Fixes SCYLLADB-246

Closes scylladb/scylladb#28232

* github.com:scylladb/scylladb:
  test: add nightly label
  test.py: support boost labels in test.py

test/boost/mutation_reader_test.cc

@@ -42,6 +42,7 @@
 #include "test/lib/key_utils.hh"
 #include "test/lib/test_utils.hh"
 
+#include <boost/test/unit_test.hpp>
 #include "dht/sharder.hh"
 #include "schema/schema_builder.hh"
 #include "replica/cell_locking.hh"
@@ -69,6 +70,8 @@
 
 BOOST_AUTO_TEST_SUITE(mutation_reader_test)
 
+namespace test_label = boost::unit_test;
+
 static schema_ptr make_schema() {
     return schema_builder("ks", "cf")
         .with_column("pk", bytes_type, column_kind::partition_key)
@@ -1239,7 +1242,7 @@ SEASTAR_TEST_CASE(test_combined_mutation_source_is_a_mutation_source) {
 }
 
 // Best run with SMP >= 2
-SEASTAR_THREAD_TEST_CASE(test_foreign_reader_as_mutation_source) {
+SEASTAR_THREAD_TEST_CASE(test_foreign_reader_as_mutation_source, *test_label::label("nightly")) {
     if (smp::count < 2) {
         std::cerr << "Cannot run test " << get_name() << " with smp::count < 2" << std::endl;
         return;

test/pylib/cpp/base.py

@@ -128,6 +128,11 @@ class CppFile(pytest.File, ABC):
         custom_args = self.suite_config.get("custom_args", {}).get(self.test_name, DEFAULT_CUSTOM_ARGS)
 
         for test_case in self.list_test_cases():
+            if isinstance(test_case, list):
+                test_labels = test_case[1]
+                test_case = test_case[0]
+            else:
+                test_labels = []
             # Start `index` from 1 if there are more than one custom_args item.  This allows us to create
             # test cases with unique names for each custom_args item and don't add any additional suffixes
             # if there is only one item (in this case `index` is 0.)
@@ -137,6 +142,7 @@ class CppFile(pytest.File, ABC):
                     name=f"{test_case}.{index}" if index else test_case,
                     test_case_name=test_case,
                     test_custom_args=shlex.split(args),
+                    own_markers=test_labels,
                 )
 
     @classmethod
@@ -149,14 +155,14 @@ class CppFile(pytest.File, ABC):
 class CppTestCase(pytest.Item):
     parent: CppFile
 
-    def __init__(self, *, test_case_name: str, test_custom_args: list[str], **kwargs: Any):
+    def __init__(self, *, test_case_name: str, test_custom_args: list[str], own_markers: list[str] | set[str], **kwargs: Any):
         super().__init__(**kwargs)
 
         self.test_case_name = test_case_name
         self.test_custom_args = test_custom_args
 
         self.fixturenames = []
-        self.own_markers = []
+        self.own_markers = [getattr(pytest.mark, mark_name) for mark_name in own_markers]
         self.add_marker(pytest.mark.cpp)
 
     def get_artifact_path(self, extra: str = "", suffix: str = "") -> pathlib.Path:

test/pylib/cpp/boost.py

@@ -11,6 +11,7 @@ import logging
 import subprocess
 import tempfile
 import pathlib
+import json
 from functools import cache, cached_property
 from itertools import chain
 from textwrap import dedent
@@ -57,7 +58,7 @@ class BoostTestFile(CppFile):
     def list_test_cases(self) -> list[str]:
         if self.no_parallel:
             return [self.test_name]
-        return get_boost_test_list_content(executable=self.exe_path, combined=self.combined)[self.test_name]
+        return get_boost_test_list_json_content(executable=self.exe_path,combined=self.combined).get(self.test_name, [])
 
     def run_test_case(self, test_case: CppTestCase) -> tuple[None | list[CppTestFailure], str]:
         run_test = f"{self.test_name}/{test_case.test_case_name}" if self.combined else test_case.test_case_name
@@ -110,6 +111,54 @@ class BoostTestFile(CppFile):
 
 pytest_collect_file = BoostTestFile.pytest_collect_file
 
+@cache
+def get_boost_test_list_json_content(executable: pathlib.Path, combined: bool = False)-> dict[str, list[list[str, set[str]]]]:
+    """
+    mimic get_boost_test_list_content but using --list_json_content which provides more structured data including test labels
+
+    List the content of test tree in an executable.
+    Return a dict where key is the name of test file and value is a list of tests in this file with their labels.
+    In case of combined tests the dict will have multiple items, otherwise we assume that name of the executable is the same
+    as the source test file (.cc)
+    """
+    try:
+        output = subprocess.check_output(
+            [executable, "--","--list_json_content"],
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
+    except subprocess.CalledProcessError as e:
+        if "Test setup error: test tree is empty" in e.output:
+            return {executable.name: []}
+        raise e
+
+    data = json.loads(output)
+    test_tree = {}
+
+    def parse_suite(key, suite, suite_str=""):
+        for _suite in suite["suites"]:
+            parse_suite(key, _suite, f'{suite_str}/{_suite["name"]}')
+        for test in suite["tests"]:
+            if test["name"].startswith("_"):
+                test_tree[key].append([suite["name"], []])
+                break
+            test_name = f'{suite_str}/{test["name"]}' if suite_str else test["name"]
+            labels = set(test["labels"].split(",")) if "labels" in test and test["labels"] else set()
+            test_tree[key].append([test_name, labels])
+
+    for file in data:
+        for s in file["content"]["suites"]:
+            k = s["name"] if combined else executable.name
+            if k not in test_tree:
+                test_tree[k] = []
+            parse_suite(k, s, suite_str=s["name"] if not combined else "")
+        if file["content"]["tests"]:
+            if executable.name not in test_tree:
+                test_tree[executable.name] = []
+            for test in file["content"]["tests"]:
+                labels = set(test["labels"].split(",")) if "labels" in test and test["labels"] else set()
+                test_tree[executable.name].append([test["name"], labels])
+    return test_tree
 
 @cache
 def get_boost_test_list_content(executable: pathlib.Path, combined: bool = False) -> dict[str, list[str]]: