mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "vector_search_validator: move high availability tests from vector-store.git"

Browse Source 
This reverts commit caa0cbe328. It is
either extremely slow or broken. I was never able to get it to
run on an r8gd.8xlarge (on the NVMe disk). Even when it passes,
it is very slow.

Test script:

```

git submodule update --recursive || exit 125

rm -rf build

d() { ./tools/toolchain/dbuild -it -- "$@"; }

d ./configure.py --mode release || exit 125
d ninja release-build || exit 125
d ./test.py --mode release
```

Ref #27858
Ref #27859
Ref #27860
This commit is contained in:
Avi Kivity
2025-12-25 10:59:37 +00:00
parent ebb101f8ae
commit 55c7bc746e
9 changed files with 14 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
test/vector_search_validator/Cargo.toml
View File

@@ -1,10 +1,6 @@
## Copyright 2025-present ScyllaDB
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
# This file is generated by cargo-toml-template. Do not edit directly.
# To make changes, edit the template and regenerate with command:
# "$ cargo-toml-template > Cargo.toml".
[workspace]
members = ["crates/*"]
default-members = ["crates/validator"]
@@ -16,16 +12,13 @@ edition = "2024"
[workspace.dependencies]
anyhow = "1.0.97"
async-backtrace = "0.2.7"
futures = "0.3.31"
scylla = { version = "1.2.0", features = ["time-03"] }
tokio = { version = "1.44.1", features = ["full"] }
tracing = "0.1.41"
uuid = "1.16.0"
httpclient = { git = "https://github.com/scylladb/vector-store.git", rev = "d79ee80" }
vector-search-validator-engine = { git = "https://github.com/scylladb/vector-store.git", rev = "d79ee80" }
vector-search-validator-tests = { git = "https://github.com/scylladb/vector-store.git", rev = "d79ee80" }
vector-store = { git = "https://github.com/scylladb/vector-store.git", rev = "d79ee80" }
vector-search-validator-engine = { git = "https://github.com/scylladb/vector-store.git", rev = "3ee46a5" }
vector-search-validator-tests = { git = "https://github.com/scylladb/vector-store.git", rev = "3ee46a5" }
[patch.'https://github.com/scylladb/scylladb.git']
[patch.'https://github.com/scylladb/vector-store.git']
vector-search-validator-scylla = { path = "crates/validator-scylla" }

22
test/vector_search_validator/README.md
View File

@@ -8,8 +8,6 @@ namespace to separate it from the host environment. `vector-search-validator`
contains DNS server and all tests in one binary. It uses external scylla and
vector-store binaries.
## Running tests
The `test_validator.py::test_validator[test-case]` is the entry point for
running the tests. It is parametrized with name of the test case. Available
test cases are taken dynamically from the `vector-search-validator` binary.
@@ -39,22 +37,6 @@ $ pytest --mode=dev test/vector_search_validator/test_validator.py --filters fil
Logs are stored in
`testlog/{mode}/vector_search_validator/{test-case}-{run_id}/` directory.
## Development of test cases
`vector-search-validator` (in short `validator`) is divided into multiple
crates:
- `validator` - a main crate that contains only the entry point
- `validator-scylla` - contains implementation of the validator tests on the
scylladb.git side. If you want to add/modify the tests implemented in the
scylladb.git, you will work in this crate.
- `vector-store.git/validator-engine` - contains the core logic of the
validator - overall test runner and implementation of actors for tests (dns
server, scylla cluster, vector store cluster)
- `vector-store.git/validator-tests` - contains the core logic of the framework
tests, provides base structures for tests and actor interfaces. In the
future we should check if it is possible to merge it with `validator-engine`
crate.
- `vector-store.git/validator-vector-store` - contains implementation of the
validator tests on the vector-store.git side. If you want to add/modify the
tests implemented in the vector-store.git, you will work in this crate.
Implementing new test cases on the Scylla repository side means adding new test
in crate `crates/validator-scylla`.

2
test/vector_search_validator/build-env
View File

@@ -1,2 +1,2 @@
VECTOR_STORE_GIT=https://github.com/scylladb/vector-store.git
VECTOR_STORE_REV=d79ee80
VECTOR_STORE_REV=3ee46a5

9
test/vector_search_validator/cargo-toml-template
View File

@@ -11,10 +11,6 @@ cat << EOF
## Copyright 2025-present ScyllaDB
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
# This file is generated by cargo-toml-template. Do not edit directly.
# To make changes, edit the template and regenerate with command:
# "\$ cargo-toml-template > Cargo.toml".
[workspace]
members = ["crates/*"]
default-members = ["crates/validator"]
@@ -26,17 +22,14 @@ edition = "2024"
[workspace.dependencies]
anyhow = "1.0.97"
async-backtrace = "0.2.7"
futures = "0.3.31"
scylla = { version = "1.2.0", features = ["time-03"] }
tokio = { version = "1.44.1", features = ["full"] }
tracing = "0.1.41"
uuid = "1.16.0"
httpclient = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
vector-search-validator-engine = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
vector-search-validator-tests = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
vector-store = { git = "$VECTOR_STORE_GIT", rev = "$VECTOR_STORE_REV" }
[patch.'https://github.com/scylladb/scylladb.git']
[patch.'$VECTOR_STORE_GIT']
vector-search-validator-scylla = { path = "crates/validator-scylla" }
EOF

6
test/vector_search_validator/crates/validator-scylla/Cargo.toml
View File

@@ -4,11 +4,5 @@ version = "0.1.0"
edition = "2024"
[dependencies]
async-backtrace.workspace = true
httpclient.workspace = true
scylla.workspace = true
tokio.workspace = true
tracing.workspace = true
uuid.workspace = true
vector-search-validator-tests.workspace = true
vector-store.workspace = true

302
test/vector_search_validator/crates/validator-scylla/src/common.rs
View File

@@ -1,302 +0,0 @@
/*
* Copyright 2025-present ScyllaDB
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
*/
use async_backtrace::framed;
use httpclient::HttpClient;
use scylla::client::session::Session;
use scylla::client::session_builder::SessionBuilder;
use scylla::response::query_result::QueryRowsResult;
use std::collections::HashMap;
use std::net::Ipv4Addr;
use std::sync::Arc;
use std::time::Duration;
use tokio::time;
use tracing::info;
use uuid::Uuid;
use vector_search_validator_tests::DnsExt;
use vector_search_validator_tests::ScyllaClusterExt;
use vector_search_validator_tests::ScyllaNodeConfig;
use vector_search_validator_tests::TestActors;
use vector_search_validator_tests::VectorStoreClusterExt;
use vector_search_validator_tests::VectorStoreNodeConfig;
use vector_store::httproutes::IndexStatus;
use vector_store::IndexInfo;
pub(crate) const DEFAULT_TEST_TIMEOUT: Duration = Duration::from_secs(120);
pub(crate) const VS_NAMES: [&str; 3] = ["vs1", "vs2", "vs3"];
pub(crate) const VS_PORT: u16 = 6080;
pub(crate) const DB_OCTET_1: u8 = 1;
pub(crate) const DB_OCTET_2: u8 = 2;
pub(crate) const DB_OCTET_3: u8 = 3;
pub(crate) const VS_OCTET_1: u8 = 128;
pub(crate) const VS_OCTET_2: u8 = 129;
pub(crate) const VS_OCTET_3: u8 = 130;
#[framed]
pub(crate) async fn get_default_vs_urls(actors: &TestActors) -> Vec<String> {
let domain = actors.dns.domain().await;
VS_NAMES
.iter()
.map(|name| format!("http://{name}.{domain}:{VS_PORT}"))
.collect()
}
pub(crate) fn get_default_vs_ips(actors: &TestActors) -> Vec<Ipv4Addr> {
vec![
actors.services_subnet.ip(VS_OCTET_1),
actors.services_subnet.ip(VS_OCTET_2),
actors.services_subnet.ip(VS_OCTET_3),
]
}
pub(crate) fn get_default_db_ips(actors: &TestActors) -> Vec<Ipv4Addr> {
vec![
actors.services_subnet.ip(DB_OCTET_1),
actors.services_subnet.ip(DB_OCTET_2),
actors.services_subnet.ip(DB_OCTET_3),
]
}
#[framed]
pub(crate) async fn get_default_scylla_node_configs(actors: &TestActors) -> Vec<ScyllaNodeConfig> {
let default_vs_urls = get_default_vs_urls(actors).await;
get_default_db_ips(actors)
.iter()
.enumerate()
.map(|(i, &ip)| {
let mut vs_urls = default_vs_urls.clone();
ScyllaNodeConfig {
db_ip: ip,
primary_vs_uris: vec![vs_urls.remove(i)],
secondary_vs_uris: vs_urls,
}
})
.collect()
}
pub(crate) fn get_default_vs_node_configs(actors: &TestActors) -> Vec<VectorStoreNodeConfig> {
let db_ips = get_default_db_ips(actors);
get_default_vs_ips(actors)
.iter()
.zip(db_ips.iter())
.map(|(&vs_ip, &db_ip)| VectorStoreNodeConfig {
vs_ip,
db_ip,
envs: HashMap::new(),
})
.collect()
}
#[framed]
pub(crate) async fn init(actors: TestActors) {
info!("started");
let scylla_configs = get_default_scylla_node_configs(&actors).await;
let vs_configs = get_default_vs_node_configs(&actors);
init_with_config(actors, scylla_configs, vs_configs).await;
info!("finished");
}
#[framed]
pub(crate) async fn init_with_config(
actors: TestActors,
scylla_configs: Vec<ScyllaNodeConfig>,
vs_configs: Vec<VectorStoreNodeConfig>,
) {
let vs_ips = get_default_vs_ips(&actors);
for (name, ip) in VS_NAMES.iter().zip(vs_ips.iter()) {
actors.dns.upsert(name.to_string(), *ip).await;
}
actors.db.start(scylla_configs, None).await;
assert!(actors.db.wait_for_ready().await);
actors.vs.start(vs_configs).await;
assert!(actors.vs.wait_for_ready().await);
}
#[framed]
pub(crate) async fn cleanup(actors: TestActors) {
info!("started");
for name in VS_NAMES.iter() {
actors.dns.remove(name.to_string()).await;
}
actors.vs.stop().await;
actors.db.stop().await;
info!("finished");
}
#[framed]
pub(crate) async fn prepare_connection_with_custom_vs_ips(
actors: &TestActors,
vs_ips: Vec<Ipv4Addr>,
) -> (Arc<Session>, Vec<HttpClient>) {
let session = Arc::new(
SessionBuilder::new()
.known_node(actors.services_subnet.ip(DB_OCTET_1).to_string())
.build()
.await
.expect("failed to create session"),
);
let clients = vs_ips
.iter()
.map(|&ip| HttpClient::new((ip, VS_PORT).into()))
.collect();
(session, clients)
}
#[framed]
pub(crate) async fn wait_for<F, Fut>(mut condition: F, msg: &str, timeout: Duration)
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = bool>,
{
time::timeout(timeout, async {
while !condition().await {
time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap_or_else(|_| panic!("Timeout on: {msg}"))
}
#[framed]
pub(crate) async fn wait_for_value<F, Fut, T>(mut poll_fn: F, msg: &str, timeout: Duration) -> T
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Option<T>>,
{
time::timeout(timeout, async {
loop {
if let Some(value) = poll_fn().await {
return value;
}
time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap_or_else(|_| panic!("Timeout on: {msg}"))
}
#[framed]
pub(crate) async fn wait_for_index(
client: &HttpClient,
index: &IndexInfo,
) -> vector_store::httproutes::IndexStatusResponse {
wait_for_value(
|| async {
match client.index_status(&index.keyspace, &index.index).await {
Ok(resp) if resp.status == IndexStatus::Serving => Some(resp),
_ => None,
}
},
"Waiting for index to be SERVING",
Duration::from_secs(20),
)
.await
}
#[framed]
pub(crate) async fn get_query_results(query: String, session: &Session) -> QueryRowsResult {
session
.query_unpaged(query, ())
.await
.expect("failed to run query")
.into_rows_result()
.expect("failed to get rows")
}
#[framed]
pub(crate) async fn create_keyspace(session: &Session) -> String {
let keyspace = format!("ks_{}", Uuid::new_v4().simple());
// Create keyspace with replication factor of 3 for the 3-node cluster
session.query_unpaged(
format!("CREATE KEYSPACE {keyspace} WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': 3}}"),
(),
).await.expect("failed to create a keyspace");
// Use keyspace
session
.use_keyspace(&keyspace, false)
.await
.expect("failed to use a keyspace");
keyspace
}
#[framed]
pub(crate) async fn create_table(
session: &Session,
columns: &str,
options: Option<&str>,
) -> String {
let table = format!("tbl_{}", Uuid::new_v4().simple());
let extra = if let Some(options) = options {
format!("WITH {options}")
} else {
String::new()
};
// Create table
session
.query_unpaged(format!("CREATE TABLE {table} ({columns}) {extra}"), ())
.await
.expect("failed to create a table");
table
}
#[framed]
pub(crate) async fn create_index(
session: &Session,
clients: &[HttpClient],
table: &str,
column: &str,
) -> IndexInfo {
let index = format!("idx_{}", Uuid::new_v4().simple());
// Create index
session
.query_unpaged(
format!("CREATE INDEX {index} ON {table}({column}) USING 'vector_index'"),
(),
)
.await
.expect("failed to create an index");
// Wait for the index to be created
wait_for(
|| async {
for client in clients.iter() {
if !client
.indexes()
.await
.iter()
.any(|idx| idx.index.to_string() == index)
{
return false;
}
}
true
},
"Waiting for the first index to be created",
Duration::from_secs(10),
)
.await;
clients
.first()
.expect("No vector store clients provided")
.indexes()
.await
.into_iter()
.find(|idx| idx.index.to_string() == index)
.expect("index not found")
}

9
test/vector_search_validator/crates/validator-scylla/src/cql.rs
View File

@@ -3,13 +3,12 @@
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
*/
use crate::common;
use async_backtrace::framed;
use vector_search_validator_tests::TestCase;
use std::time::Duration;
use vector_search_validator_tests::common;
use vector_search_validator_tests::*;
#[framed]
pub(crate) async fn new() -> TestCase {
let timeout = common::DEFAULT_TEST_TIMEOUT;
let timeout = Duration::from_secs(30);
TestCase::empty()
.with_init(timeout, common::init)
.with_cleanup(timeout, common::cleanup)

115
test/vector_search_validator/crates/validator-scylla/src/high_availability.rs
View File

@@ -1,115 +0,0 @@
/*
* Copyright 2025-present ScyllaDB
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
*/
use crate::common;
use async_backtrace::framed;
use tracing::info;
use vector_search_validator_tests::ScyllaClusterExt;
use vector_search_validator_tests::ScyllaNodeConfig;
use vector_search_validator_tests::TestActors;
use vector_search_validator_tests::TestCase;
use vector_search_validator_tests::VectorStoreNodeConfig;
#[framed]
pub(crate) async fn new() -> TestCase {
let timeout = common::DEFAULT_TEST_TIMEOUT;
TestCase::empty()
.with_cleanup(timeout, common::cleanup)
.with_test(
"secondary_uri_works_correctly",
timeout,
test_secondary_uri_works_correctly,
)
}
#[framed]
async fn test_secondary_uri_works_correctly(actors: TestActors) {
info!("started");
let vs_urls = common::get_default_vs_urls(&actors).await;
let vs_url = &vs_urls[0];
let scylla_configs: Vec<ScyllaNodeConfig> = vec![
ScyllaNodeConfig {
db_ip: actors.services_subnet.ip(common::DB_OCTET_1),
primary_vs_uris: vec![vs_url.clone()],
secondary_vs_uris: vec![],
},
ScyllaNodeConfig {
db_ip: actors.services_subnet.ip(common::DB_OCTET_2),
primary_vs_uris: vec![],
secondary_vs_uris: vec![vs_url.clone()],
},
ScyllaNodeConfig {
db_ip: actors.services_subnet.ip(common::DB_OCTET_3),
primary_vs_uris: vec![],
secondary_vs_uris: vec![vs_url.clone()],
},
];
let vs_configs = vec![VectorStoreNodeConfig {
vs_ip: actors.services_subnet.ip(common::VS_OCTET_1),
db_ip: actors.services_subnet.ip(common::DB_OCTET_1),
envs: Default::default(),
}];
common::init_with_config(actors.clone(), scylla_configs, vs_configs).await;
let vs_ips = vec![actors.services_subnet.ip(common::VS_OCTET_1)];
let (session, clients) = common::prepare_connection_with_custom_vs_ips(&actors, vs_ips).await;
let keyspace = common::create_keyspace(&session).await;
let table =
common::create_table(&session, "pk INT PRIMARY KEY, v VECTOR<FLOAT, 3>", None).await;
// Insert vectors
for i in 0..100 {
let embedding = vec![i as f32, (i * 2) as f32, (i * 3) as f32];
session
.query_unpaged(
format!("INSERT INTO {table} (pk, v) VALUES (?, ?)"),
(i, &embedding),
)
.await
.expect("failed to insert data");
}
let index = common::create_index(&session, &clients, &table, "v").await;
for client in &clients {
let index_status = common::wait_for_index(&client, &index).await;
assert_eq!(
index_status.count, 100,
"Expected 100 vectors to be indexed"
);
}
// Down the first node with primary URI
let first_node_ip = actors.services_subnet.ip(common::DB_OCTET_1);
info!("Bringing down node {first_node_ip}");
actors.db.down_node(first_node_ip).await;
// Should work via secondary URIs
let results = common::get_query_results(
format!("SELECT pk FROM {table} ORDER BY v ANN OF [0.0, 0.0, 0.0] LIMIT 10"),
&session,
)
.await;
let rows = results
.rows::<(i32,)>()
.expect("failed to get rows after node down");
assert!(
rows.rows_remaining() <= 10,
"Expected at most 10 results from ANN query after node down"
);
// Drop keyspace
session
.query_unpaged(format!("DROP KEYSPACE {keyspace}"), ())
.await
.expect("failed to drop a keyspace");
info!("finished");
}

13
test/vector_search_validator/crates/validator-scylla/src/lib.rs
View File

@@ -3,19 +3,12 @@
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
*/
mod common;
mod cql;
mod high_availability;
use async_backtrace::framed;
use vector_search_validator_tests::TestCase;
#[framed]
pub async fn test_cases() -> impl Iterator<Item = (String, TestCase)> {
vec![
("scylla_cql", cql::new().await),
("scylla_high_availability", high_availability::new().await),
]
.into_iter()
.map(|(name, test_case)| (name.to_string(), test_case))
vec![("cql", cql::new().await)]
.into_iter()
.map(|(name, test_case)| (name.to_string(), test_case))
}