diff --git a/test/pylib/cpp/base.py b/test/pylib/cpp/base.py
index df2445af51..00906b82f2 100644
--- a/test/pylib/cpp/base.py
+++ b/test/pylib/cpp/base.py
@@ -126,6 +126,9 @@ class CppFile(pytest.File, ABC):
         return args
 
     def collect(self) -> Iterator[CppTestCase]:
+        if BUILD_MODE not in self.stash:
+            return
+
         custom_args = self.suite_config.get("custom_args", {}).get(self.test_name, DEFAULT_CUSTOM_ARGS)
 
         for test_case in self.list_test_cases():
diff --git a/test/pylib/runner.py b/test/pylib/runner.py
index 147e31700d..ca3fc3f76d 100644
--- a/test/pylib/runner.py
+++ b/test/pylib/runner.py
@@ -163,6 +163,11 @@ def scylla_binary(testpy_test) -> Path:
 
 
 def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    items[:] = [
+        item for item in items
+        if (parent_file := item.getparent(cls=pytest.File)) is not None
+           and BUILD_MODE in parent_file.stash
+    ]
     for item in items:
         modify_pytest_item(item=item)
 
@@ -340,7 +345,8 @@ def pytest_collect_file(file_path: pathlib.Path,
         repeats = list(product(build_modes, parent.config.run_ids))
 
         if not repeats:
-            return []
+            parent.stash[REPEATING_FILES].remove(file_path)
+            return collectors
 
         ihook = parent.ihook
         collectors = list(chain(collectors, chain.from_iterable(