Revert "Merge 'vector_search: add validator tests' from Pawel Pery"

This reverts commit bcd1758911, reversing changes made to b2c2a99741. There is a design decision to not introduce additional test orchestration tool for scylladb.git (see comments for #27499). One commit has already been reverted in 55c7bc7. Last CI runs made validator test flaky, so it is a time to remove all remaining validator tests. It needs a backport to 2026.1 to remove remaining validator tests from there. Fixes: VECTOR-497 Closes scylladb/scylladb#28568
2026-02-06 11:16:33 +01:00
parent bb99bfe815
commit 81d11a23ce
21 changed files with 5 additions and 412 deletions
--- a/configure.py
+++ b/configure.py
@@ -730,28 +730,6 @@ vector_search_tests = set([
    'test/vector_search/rescoring_test'
 ])

-vector_search_validator_bin = 'vector-search-validator/bin/vector-search-validator'
-vector_search_validator_deps = set([
-    'test/vector_search_validator/build-validator',
-    'test/vector_search_validator/Cargo.toml',
-    'test/vector_search_validator/crates/validator/Cargo.toml',
-    'test/vector_search_validator/crates/validator/src/main.rs',
-    'test/vector_search_validator/crates/validator-scylla/Cargo.toml',
-    'test/vector_search_validator/crates/validator-scylla/src/lib.rs',
-    'test/vector_search_validator/crates/validator-scylla/src/cql.rs',
-])
-
-vector_store_bin = 'vector-search-validator/bin/vector-store'
-vector_store_deps = set([
-    'test/vector_search_validator/build-env',
-    'test/vector_search_validator/build-vector-store',
-])
-
-vector_search_validator_bins = set([
-    vector_search_validator_bin,
-    vector_store_bin,
-])
-
 wasms = set([
    'wasm/return_input.wat',
    'wasm/test_complex_null_values.wat',
@@ -785,7 +763,7 @@ other = set([
    'iotune',
 ])

-all_artifacts = apps | cpp_apps | tests | other | wasms | vector_search_validator_bins
+all_artifacts = apps | cpp_apps | tests | other | wasms

 arg_parser = argparse.ArgumentParser('Configure scylla', add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 arg_parser.add_argument('--out', dest='buildfile', action='store', default='build.ninja',
@@ -2585,11 +2563,10 @@ def write_build_file(f,
              description = RUST_LIB $out
            ''').format(mode=mode, antlr3_exec=args.antlr3_exec, fmt_lib=fmt_lib, test_repeat=args.test_repeat, test_timeout=args.test_timeout, rustc_wrapper=rustc_wrapper, **modeval))
        f.write(
-            'build {mode}-build: phony {artifacts} {wasms} {vector_search_validator_bins}\n'.format(
+            'build {mode}-build: phony {artifacts} {wasms}\n'.format(
                mode=mode,
-                artifacts=str.join(' ', ['$builddir/' + mode + '/' + x for x in sorted(build_artifacts - wasms - vector_search_validator_bins)]),
+                artifacts=str.join(' ', ['$builddir/' + mode + '/' + x for x in sorted(build_artifacts - wasms)]),
                wasms = str.join(' ', ['$builddir/' + x for x in sorted(build_artifacts & wasms)]),
-                vector_search_validator_bins=str.join(' ', ['$builddir/' + x for x in sorted(build_artifacts & vector_search_validator_bins)]),
            )
        )
        if profile_recipe := modes[mode].get('profile_recipe'):
@@ -2619,7 +2596,7 @@ def write_build_file(f,
                continue
            profile_dep = modes[mode].get('profile_target', "")

-            if binary in other or binary in wasms or binary in vector_search_validator_bins:
+            if binary in other or binary in wasms:
                continue
            srcs = deps[binary]
            # 'scylla'
@@ -2730,11 +2707,10 @@ def write_build_file(f,
        )

        f.write(
-            'build {mode}-test: test.{mode} {test_executables} $builddir/{mode}/scylla {wasms} {vector_search_validator_bins} \n'.format(
+            'build {mode}-test: test.{mode} {test_executables} $builddir/{mode}/scylla {wasms}\n'.format(
                mode=mode,
                test_executables=' '.join(['$builddir/{}/{}'.format(mode, binary) for binary in sorted(tests)]),
                wasms=' '.join([f'$builddir/{binary}' for binary in sorted(wasms)]),
-                vector_search_validator_bins=' '.join([f'$builddir/{binary}' for binary in sorted(vector_search_validator_bins)]),
            )
        )
        f.write(
@@ -2902,19 +2878,6 @@ def write_build_file(f,
            'build compiler-training: phony {}\n'.format(' '.join(['{mode}-compiler-training'.format(mode=mode) for mode in default_modes]))
    )

-    f.write(textwrap.dedent(f'''\
-        rule build-vector-search-validator
-            command = test/vector_search_validator/build-validator $builddir
-        rule build-vector-store
-            command = test/vector_search_validator/build-vector-store $builddir
-        '''))
-    f.write(
-            'build $builddir/{vector_search_validator_bin}: build-vector-search-validator {}\n'.format(' '.join([dep for dep in sorted(vector_search_validator_deps)]), vector_search_validator_bin=vector_search_validator_bin)
-    )
-    f.write(
-            'build $builddir/{vector_store_bin}: build-vector-store {}\n'.format(' '.join([dep for dep in sorted(vector_store_deps)]), vector_store_bin=vector_store_bin)
-    )
-
    f.write(textwrap.dedent(f'''\
        build dist-unified-tar: phony {' '.join([f'$builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz' for mode in default_modes])}
        build dist-unified: phony dist-unified-tar
--- a/test.py
+++ b/test.py
@@ -61,7 +61,6 @@ PYTEST_RUNNER_DIRECTORIES = [
    TEST_DIR / 'raft',
    TEST_DIR / 'unit',
    TEST_DIR / 'vector_search',
-    TEST_DIR / 'vector_search_validator',
    TEST_DIR / 'alternator',
    TEST_DIR / 'broadcast_tables',
    TEST_DIR / 'cql',
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -103,7 +103,6 @@ if(BUILD_TESTING)
    add_subdirectory(raft)
    add_subdirectory(resource/wasm)
    add_subdirectory(vector_search)
-    add_subdirectory(vector_search_validator)

    if(CMAKE_CONFIGURATION_TYPES)
      foreach(config ${CMAKE_CONFIGURATION_TYPES})
--- a/test/pylib/suite/base.py
+++ b/test/pylib/suite/base.py
@@ -537,7 +537,6 @@ def prepare_dirs(tempdir_base: pathlib.Path, modes: list[str], gather_metrics: b
        prepare_dir(tempdir_base / mode / "xml", "*.xml", save_log_on_success)
        prepare_dir(tempdir_base / mode / "failed_test", "*", save_log_on_success)
        prepare_dir(tempdir_base / mode / "allure", "*.xml", save_log_on_success)
-        prepare_dir(tempdir_base / mode / "vector-search-validator", "*", save_log_on_success)
        if TEST_RUNNER != "pytest":
            prepare_dir(tempdir_base / mode / "pytest", "*", save_log_on_success)

--- a/test/vector_search_validator/.gitignore
+++ b/test/vector_search_validator/.gitignore
@@ -1,2 +0,0 @@
-Cargo.lock
-target/
--- a/test/vector_search_validator/CMakeLists.txt
+++ b/test/vector_search_validator/CMakeLists.txt
@@ -1,27 +0,0 @@
-set(VALIDATOR_BIN "${CMAKE_BINARY_DIR}/vector-search-validator/bin/vector-search-validator")
-set(VECTOR_STORE_BIN "${CMAKE_BINARY_DIR}/vector-search-validator/bin/vector-store")
-
-add_custom_command(
-    OUTPUT ${VALIDATOR_BIN}
-    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build-validator ${CMAKE_BINARY_DIR}
-    DEPENDS
-        build-validator
-        Cargo.toml
-        crates/validator/Cargo.toml
-        crates/validator/src/main.rs
-        crates/validator-scylla/Cargo.toml
-        crates/validator-scylla/src/lib.rs
-        crates/validator-scylla/src/cql.rs
-)
-
-add_custom_command(
-    OUTPUT ${VECTOR_STORE_BIN}
-    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build-vector-store ${CMAKE_BINARY_DIR}
-    DEPENDS
-        build-env
-        build-vector-store
-)
-
-add_custom_target(vector_search_validator DEPENDS ${VALIDATOR_BIN} ${VECTOR_STORE_BIN})
-add_dependencies(tests vector_search_validator)
-
--- a/test/vector_search_validator/Cargo.toml
+++ b/test/vector_search_validator/Cargo.toml
@@ -1,24 +0,0 @@
-## Copyright 2025-present ScyllaDB
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-
-[workspace]
-members = ["crates/*"]
-default-members = ["crates/validator"]
-resolver = "3"
-
-[workspace.package]
-version = "0.1.0"
-edition = "2024"
-
-[workspace.dependencies]
-anyhow = "1.0.97"
-futures = "0.3.31"
-scylla = { version = "1.2.0", features = ["time-03"] }
-tokio = { version = "1.44.1", features = ["full"] }
-tracing = "0.1.41"
-uuid = "1.16.0"
-vector-search-validator-engine = { git = "https://github.com/scylladb/vector-store.git", rev = "3ee46a5" }
-vector-search-validator-tests = { git = "https://github.com/scylladb/vector-store.git", rev = "3ee46a5" }
-
-[patch.'https://github.com/scylladb/vector-store.git']
-vector-search-validator-scylla = { path = "crates/validator-scylla" }
--- a/test/vector_search_validator/README.md
+++ b/test/vector_search_validator/README.md
@@ -1,42 +0,0 @@
-# `vector-search-validator` tests for Scylla and Vector Store
-
-`vector-search-validator` is a testing tool for validating the functionality of
-integration between Scylla and Vector Store. Such integration depends on the
-Scylla cluster and the Vector Store nodes and also on the DNS service. For this
-reason we run `vector-search-validator` in a network and storage linux
-namespace to separate it from the host environment. `vector-search-validator`
-contains DNS server and all tests in one binary. It uses external scylla and
-vector-store binaries.
-
-The `test_validator.py::test_validator[test-case]` is the entry point for
-running the tests. It is parametrized with name of the test case.  Available
-test cases are taken dynamically from the `vector-search-validator` binary.
-
-To run test with `dev` Scylla and test case `cql` run the following command in
-the dbuild environment (non dbuild environment is not supported currently, as
-test needs to have `sudo` permissions without password):
-
-```bash
-$ pytest --mode=dev test/vector_search_validator/test_validator.py::test_validator[cql]
-```
-
-To run all tests with `dev` Scylla run the following command:
-
-```bash
-$ pytest --mode=dev test/vector_search_validator/test_validator.py
-$ pytest --mode=dev test/vector_search_validator/test_validator.py::test_validator
-```
-
-You can tests with custom filters supported by validator. To run filtered tests
-with `dev` Scylla run the following command:
-
-```bash
-$ pytest --mode=dev test/vector_search_validator/test_validator.py --filters filter1,filter2,...
-```
-
-Logs are stored in
-`testlog/{mode}/vector_search_validator/{test-case}-{run_id}/` directory.
-
-Implementing new test cases on the Scylla repository side means adding new test
-in crate `crates/validator-scylla`.
-
--- a/test/vector_search_validator/build-env
+++ b/test/vector_search_validator/build-env
@@ -1,2 +0,0 @@
-VECTOR_STORE_GIT=https://github.com/scylladb/vector-store.git
-VECTOR_STORE_REV=3ee46a5
--- a/test/vector_search_validator/build-validator
+++ b/test/vector_search_validator/build-validator
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e
-
-usage() {
-    echo "usage: $0 <build-dir>"
-    exit 1
-}
-
-build_dir=$1
-[[ -z $build_dir ]] && usage
-build_dir=$(realpath $build_dir/vector-search-validator)
-
-proj_dir=$(dirname "${BASH_SOURCE[0]}")
-proj_dir=$(realpath $proj_dir)
-
-mkdir -p "$build_dir"
-cd "$build_dir"
-
-rm bin/vector-search-validator release/deps/vector_search_validator* -f
-cargo install --path "$proj_dir/crates/validator" --root . --target-dir . --no-track vector-search-validator
--- a/test/vector_search_validator/build-vector-store
+++ b/test/vector_search_validator/build-vector-store
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-
-usage() {
-    echo "usage: $0 <build-dir>"
-    exit 1
-}
-
-build_dir=$1
-[[ -z $build_dir ]] && usage
-build_dir=$(realpath $build_dir/vector-search-validator)
-
-proj_dir=$(dirname "${BASH_SOURCE[0]}")
-proj_dir=$(realpath $proj_dir)
-
-source "$proj_dir/build-env"
-
-mkdir -p "$build_dir"
-cd "$build_dir"
-
-rm bin/vector-store release/deps/vector_store* -f
-cargo install --git "$VECTOR_STORE_GIT" --rev "$VECTOR_STORE_REV" --root . --target-dir . --no-track vector-store
--- a/test/vector_search_validator/cargo-toml-template
+++ b/test/vector_search_validator/cargo-toml-template
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-set -e
-
-proj_dir=$(dirname "${BASH_SOURCE[0]}")
-proj_dir=$(realpath $proj_dir)
-
-source "$proj_dir/build-env"
-
-cat << EOF
-## Copyright 2025-present ScyllaDB
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-
-[workspace]
-members = ["crates/*"]
-default-members = ["crates/validator"]
-resolver = "3"
-
-[workspace.package]
-version = "0.1.0"
-edition = "2024"
-
-[workspace.dependencies]
-anyhow = "1.0.97"
-futures = "0.3.31"
-scylla = { version = "1.2.0", features = ["time-03"] }
-tokio = { version = "1.44.1", features = ["full"] }
-tracing = "0.1.41"
-uuid = "1.16.0"
-vector-search-validator-engine = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
-vector-search-validator-tests = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
-
-[patch.'$VECTOR_STORE_GIT']
-vector-search-validator-scylla = { path = "crates/validator-scylla" }
-EOF
--- a/test/vector_search_validator/conftest.py
+++ b/test/vector_search_validator/conftest.py
@@ -1,37 +0,0 @@
-#
-# Copyright (C) 2025-present ScyllaDB
-#
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-#
-
-from pathlib import Path
-import pytest
-from test import path_to, BUILD_DIR, TEST_DIR
-from test.pylib.runner import RUN_ID
-
-
-def pytest_addoption(parser: pytest.Parser) -> None:
-    parser.addoption(
-        "--filters",
-        action="store",
-        default="",
-        help="vector-search-validator test filter",
-    )
-
-
-@pytest.fixture
-def logdir_path(request: pytest.FixtureRequest, build_mode: str) -> Path:
-    return Path(request.config.getoption("--tmpdir")).absolute() / build_mode / "vector-search-validator"
-
-@pytest.fixture(scope="module")
-def scylla_path(build_mode: str) -> str:
-    return path_to(build_mode, "scylla")
-
-@pytest.fixture
-def filters(request: pytest.FixtureRequest) -> str:
-    return request.config.getoption("--filters").replace(",", " ")
-
-
-@pytest.fixture
-def run_id(request: pytest.FixtureRequest) -> int:
-    return request.node.stash[RUN_ID]
--- a/test/vector_search_validator/crates/validator-scylla/Cargo.toml
+++ b/test/vector_search_validator/crates/validator-scylla/Cargo.toml
@@ -1,8 +0,0 @@
-[package]
-name = "vector-search-validator-scylla"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-tracing.workspace = true
-vector-search-validator-tests.workspace = true
--- a/test/vector_search_validator/crates/validator-scylla/src/cql.rs
+++ b/test/vector_search_validator/crates/validator-scylla/src/cql.rs
@@ -1,15 +0,0 @@
-/*
- * Copyright 2025-present ScyllaDB
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-use std::time::Duration;
-use vector_search_validator_tests::common;
-use vector_search_validator_tests::*;
-
-pub(crate) async fn new() -> TestCase {
-    let timeout = Duration::from_secs(30);
-    TestCase::empty()
-        .with_init(timeout, common::init)
-        .with_cleanup(timeout, common::cleanup)
-}
--- a/test/vector_search_validator/crates/validator-scylla/src/lib.rs
+++ b/test/vector_search_validator/crates/validator-scylla/src/lib.rs
@@ -1,14 +0,0 @@
-/*
- * Copyright 2025-present ScyllaDB
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-mod cql;
-
-use vector_search_validator_tests::TestCase;
-
-pub async fn test_cases() -> impl Iterator<Item = (String, TestCase)> {
-    vec![("cql", cql::new().await)]
-        .into_iter()
-        .map(|(name, test_case)| (name.to_string(), test_case))
-}
--- a/test/vector_search_validator/crates/validator/Cargo.toml
+++ b/test/vector_search_validator/crates/validator/Cargo.toml
@@ -1,7 +0,0 @@
-[package]
-name = "vector-search-validator"
-version.workspace = true
-edition.workspace = true
-
-[dependencies]
-vector-search-validator-engine.workspace = true
--- a/test/vector_search_validator/crates/validator/src/main.rs
+++ b/test/vector_search_validator/crates/validator/src/main.rs
@@ -1,8 +0,0 @@
-/*
- * Copyright 2025-present ScyllaDB
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-fn main() -> Result<(), &'static str> {
-    vector_search_validator_engine::run()
-}
--- a/test/vector_search_validator/test_cargo_toml.py
+++ b/test/vector_search_validator/test_cargo_toml.py
@@ -1,19 +0,0 @@
-#
-# Copyright (C) 2025-present ScyllaDB
-#
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-#
-
-import subprocess
-from test import TEST_DIR
-
-def test_cargo_toml():
-    dir_path = TEST_DIR / "vector_search_validator"
-    cargo_toml_path = dir_path / "Cargo.toml"
-
-    with cargo_toml_path.open("r") as f:
-        cargo_toml = f.read()
-
-    result = subprocess.run([f"{dir_path}/cargo-toml-template"], text=True, capture_output=True, check=True)
-
-    assert result.stdout == cargo_toml, "Cargo.toml does not match the template output"
--- a/test/vector_search_validator/test_validator.py
+++ b/test/vector_search_validator/test_validator.py
@@ -1,82 +0,0 @@
-#
-# Copyright (C) 2025-present ScyllaDB
-#
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-#
-
-import os
-from pathlib import Path
-import pytest
-import subprocess
-from test import BUILD_DIR
-
-# IP address for custom DNS server inside linux namespace
-DNS_IP = "127.0.1.1"
-# 3 tuple IP address as a base for scylla or vector-store nodes
-BASE_IP = "127.0.2"
-
-VALIDATOR_PATH = BUILD_DIR / "vector-search-validator/bin/vector-search-validator"
-VECTOR_STORE_PATH = BUILD_DIR / "vector-search-validator/bin/vector-store"
-
-
-def create_resolv_conf(tmp_path: Path) -> Path:
-    """Create a custom resolv.conf file in the temporary directory."""
-
-    path = tmp_path / "resolv.conf"
-
-    with path.open("w") as f:
-        f.write(f"nameserver {DNS_IP}\n")
-
-    return path
-
-
-def create_unshare_sh(tmp_path: Path, resolv_conf_path: Path, scylla_path: str, filters: str) -> Path:
-    """Create a custom resolv.conf file in the temporary directory."""
-
-    path = tmp_path / "unshare.sh"
-
-    with path.open("w") as f:
-        f.write(f"""
-mount --bind {resolv_conf_path} /etc/resolv.conf
-ip link set lo up
-ip addr add {DNS_IP}/32 dev lo
-for i in {{1..10}}; do
-    ip addr add {BASE_IP}.$i/32 dev lo
-done
-{VALIDATOR_PATH} run --dns-ip {DNS_IP} --base-ip {BASE_IP}.1 --duplicate-errors --verbose --disable-colors {scylla_path} {VECTOR_STORE_PATH} {filters}
-""")
-
-    return path
-
-
-def list_test_case_names() -> list[str]:
-    result = subprocess.run([VALIDATOR_PATH, "list"], text=True, capture_output=True, check=True)
-    return sorted(list(set(map(lambda name: name.split("::", 1)[0], result.stdout.splitlines()))))
-
-
-@pytest.mark.skipif(os.getenv("DBUILD_TOOLCHAIN") != "1", reason="Requires running test inside dbuild toolchain container")
-@pytest.mark.parametrize("test_case_name", list_test_case_names())
-def test_validator(logdir_path: Path, run_id: int, scylla_path: str, test_case_name: str, filters: str):
-    logdir_path = logdir_path / f"{test_case_name}-{run_id}"
-    logdir_path.mkdir(parents=True, exist_ok=True)
-    resolve_conf_path = create_resolv_conf(logdir_path)
-    unshare_sh_path = create_unshare_sh(
-            logdir_path,
-            resolve_conf_path,
-            scylla_path,
-            filters if filters else test_case_name + "::"
-            )
-
-    stdout_log = logdir_path / "stdout.log"
-
-    with open(stdout_log, "w") as f:
-        result = subprocess.run(
-                ["sudo", "unshare", "-n", "-m", "/bin/bash", unshare_sh_path],
-                text=True, stdout=f, stderr=subprocess.PIPE)
-
-    assert result.returncode == 0, f"""
-vector-search-validator tests failed:
-{result.stderr}
-See {stdout_log} for details.
-"""
-
--- a/tools/toolchain/dbuild
+++ b/tools/toolchain/dbuild
@@ -259,7 +259,6 @@ docker_common_args+=(
       -v "${XDG_CONFIG_HOME}:${XDG_CONFIG_HOME}" \
       -w "$PWD" \
       -e HOME="$HOME" \
-       --env DBUILD_TOOLCHAIN=1 \
       "${docker_args[@]}" \
       "$image" \
       "${args[@]}"