test: Add test_node_ops_metrics.py

It tests the node_ops_metrics_done metric reaches 100% when a node ops
is done.

Refs: #21174
(cherry picked from commit 9868ccbac0)
This commit is contained in:
Asias He
2024-10-24 10:32:42 +08:00
committed by Mergify
parent 6847b9b099
commit 2cd4869c7c

View File

@@ -0,0 +1,35 @@
#
# Copyright (C) 2024-present ScyllaDB
#
# SPDX-License-Identifier: AGPL-3.0-or-later
#
import asyncio
import pytest
import logging
logger = logging.getLogger(__name__)
@pytest.mark.asyncio
async def test_bootstrap_removenode_metrics(manager):
cfg = {'enable_repair_based_node_ops': True}
servers = [await manager.server_add(config=cfg),
await manager.server_add(config=cfg),
await manager.server_add(config=cfg)]
await manager.server_stop_gracefully(servers[2].server_id)
await manager.remove_node(servers[0].server_id, servers[2].server_id)
def check_ops(metrics, ops):
metric_name = "scylla_node_ops_finished_percentage"
shard = 0
while True:
cnt = metrics.get(name=metric_name, labels={'ops': ops}, shard=str(shard))
if cnt == None:
break
logger.info(f"Checking {shard=} {cnt=}")
assert int(cnt) == 1
shard = shard + 1
for s in servers[:2]:
metrics = await manager.metrics.query(s.ip_addr)
check_ops(metrics, 'bootstrap')
check_ops(metrics, 'removenode')