mirror of
https://github.com/scylladb/scylladb.git
synced 2026-04-21 00:50:35 +00:00
message, db: extract RPC compression config types to a lightweight header
Move the config-facing RPC compression and dict-training types into message/rpc_compression_types.hh so db/config.hh and the compression protocol declarations do not need the full compressor and trainer implementation headers. Update dictionary_service to use the same top-level dict_training_when type and add the direct <filesystem> include now required by db/config.hh.
This commit is contained in:
16
db/config.cc
16
db/config.cc
@@ -330,14 +330,14 @@ const config_type& config_type_for<std::vector<db::config::error_injection_at_st
|
||||
}
|
||||
|
||||
template <>
|
||||
const config_type& config_type_for<enum_option<netw::dict_training_loop::when>>() {
|
||||
const config_type& config_type_for<enum_option<netw::dict_training_when>>() {
|
||||
static config_type ct(
|
||||
"dictionary training conditions", printable_to_json<enum_option<netw::dict_training_loop::when>>);
|
||||
"dictionary training conditions", printable_to_json<enum_option<netw::dict_training_when>>);
|
||||
return ct;
|
||||
}
|
||||
|
||||
template <>
|
||||
const config_type& config_type_for<netw::advanced_rpc_compressor::tracker::algo_config>() {
|
||||
const config_type& config_type_for<netw::algo_config>() {
|
||||
static config_type ct(
|
||||
"advanced rpc compressor config", printable_vector_to_json<enum_option<netw::compression_algorithm>>);
|
||||
return ct;
|
||||
@@ -530,9 +530,9 @@ struct convert<db::config::error_injection_at_startup> {
|
||||
|
||||
|
||||
template <>
|
||||
class convert<enum_option<netw::dict_training_loop::when>> {
|
||||
class convert<enum_option<netw::dict_training_when>> {
|
||||
public:
|
||||
static bool decode(const Node& node, enum_option<netw::dict_training_loop::when>& rhs) {
|
||||
static bool decode(const Node& node, enum_option<netw::dict_training_when>& rhs) {
|
||||
std::string name;
|
||||
if (!convert<std::string>::decode(node, name)) {
|
||||
return false;
|
||||
@@ -1110,7 +1110,7 @@ db::config::config(std::shared_ptr<db::extensions> exts)
|
||||
"Specifies RPC compression algorithms supported by this node. ")
|
||||
, internode_compression_enable_advanced(this, "internode_compression_enable_advanced", liveness::MustRestart, value_status::Used, false,
|
||||
"Enables the new implementation of RPC compression. If disabled, Scylla will fall back to the old implementation.")
|
||||
, rpc_dict_training_when(this, "rpc_dict_training_when", liveness::LiveUpdate, value_status::Used, netw::dict_training_loop::when::type::NEVER,
|
||||
, rpc_dict_training_when(this, "rpc_dict_training_when", liveness::LiveUpdate, value_status::Used, netw::dict_training_when::type::NEVER,
|
||||
"Specifies when RPC compression dictionary training is performed by this node.\n"
|
||||
"* `never` disables it unconditionally.\n"
|
||||
"* `when_leader` enables it only whenever the node is the Raft leader.\n"
|
||||
@@ -2025,8 +2025,8 @@ template struct utils::config_file::named_value<enum_option<db::experimental_fea
|
||||
template struct utils::config_file::named_value<enum_option<db::replication_strategy_restriction_t>>;
|
||||
template struct utils::config_file::named_value<enum_option<db::consistency_level_restriction_t>>;
|
||||
template struct utils::config_file::named_value<enum_option<db::tablets_mode_t>>;
|
||||
template struct utils::config_file::named_value<enum_option<netw::dict_training_loop::when>>;
|
||||
template struct utils::config_file::named_value<netw::advanced_rpc_compressor::tracker::algo_config>;
|
||||
template struct utils::config_file::named_value<enum_option<netw::dict_training_when>>;
|
||||
template struct utils::config_file::named_value<netw::algo_config>;
|
||||
template struct utils::config_file::named_value<std::vector<enum_option<db::experimental_features_t>>>;
|
||||
template struct utils::config_file::named_value<std::vector<enum_option<db::replication_strategy_restriction_t>>>;
|
||||
template struct utils::config_file::named_value<std::vector<enum_option<db::consistency_level_restriction_t>>>;
|
||||
|
||||
12
db/config.hh
12
db/config.hh
@@ -9,6 +9,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <seastar/core/sstring.hh>
|
||||
@@ -23,8 +24,7 @@
|
||||
#include "gms/inet_address.hh"
|
||||
#include "db/hints/host_filter.hh"
|
||||
#include "utils/error_injection.hh"
|
||||
#include "message/dict_trainer.hh"
|
||||
#include "message/advanced_rpc_compressor.hh"
|
||||
#include "message/rpc_compression_types.hh"
|
||||
#include "db/consistency_level_type.hh"
|
||||
#include "db/tri_mode_restriction.hh"
|
||||
#include "sstables/compressor.hh"
|
||||
@@ -325,9 +325,9 @@ public:
|
||||
named_value<uint32_t> internode_compression_zstd_min_message_size;
|
||||
named_value<uint32_t> internode_compression_zstd_max_message_size;
|
||||
named_value<bool> internode_compression_checksumming;
|
||||
named_value<netw::advanced_rpc_compressor::tracker::algo_config> internode_compression_algorithms;
|
||||
named_value<netw::algo_config> internode_compression_algorithms;
|
||||
named_value<bool> internode_compression_enable_advanced;
|
||||
named_value<enum_option<netw::dict_training_loop::when>> rpc_dict_training_when;
|
||||
named_value<enum_option<netw::dict_training_when>> rpc_dict_training_when;
|
||||
named_value<uint32_t> rpc_dict_training_min_time_seconds;
|
||||
named_value<uint64_t> rpc_dict_training_min_bytes;
|
||||
named_value<bool> inter_dc_tcp_nodelay;
|
||||
@@ -739,8 +739,8 @@ extern template struct utils::config_file::named_value<enum_option<db::experimen
|
||||
extern template struct utils::config_file::named_value<enum_option<db::replication_strategy_restriction_t>>;
|
||||
extern template struct utils::config_file::named_value<enum_option<db::consistency_level_restriction_t>>;
|
||||
extern template struct utils::config_file::named_value<enum_option<db::tablets_mode_t>>;
|
||||
extern template struct utils::config_file::named_value<enum_option<netw::dict_training_loop::when>>;
|
||||
extern template struct utils::config_file::named_value<netw::advanced_rpc_compressor::tracker::algo_config>;
|
||||
extern template struct utils::config_file::named_value<enum_option<netw::dict_training_when>>;
|
||||
extern template struct utils::config_file::named_value<netw::algo_config>;
|
||||
extern template struct utils::config_file::named_value<std::vector<enum_option<db::experimental_features_t>>>;
|
||||
extern template struct utils::config_file::named_value<std::vector<enum_option<db::replication_strategy_restriction_t>>>;
|
||||
extern template struct utils::config_file::named_value<std::vector<enum_option<db::consistency_level_restriction_t>>>;
|
||||
|
||||
@@ -11,9 +11,10 @@
|
||||
#include <seastar/core/condition-variable.hh>
|
||||
#include <seastar/rpc/rpc_types.hh>
|
||||
#include <utility>
|
||||
|
||||
#include "rpc_compression_types.hh"
|
||||
#include "utils/refcounted.hh"
|
||||
#include "utils/updateable_value.hh"
|
||||
#include "utils/enum_option.hh"
|
||||
#include "shared_dict.hh"
|
||||
|
||||
namespace netw {
|
||||
@@ -28,103 +29,6 @@ class dict_sampler;
|
||||
using dict_ptr = lw_shared_ptr<foreign_ptr<lw_shared_ptr<shared_dict>>>;
|
||||
class control_protocol_frame;
|
||||
|
||||
// An enum wrapper, describing supported RPC compression algorithms.
|
||||
// Always contains a valid value —- the constructors won't allow
|
||||
// an invalid/unknown enum variant to be constructed.
|
||||
struct compression_algorithm {
|
||||
using underlying = uint8_t;
|
||||
enum class type : underlying {
|
||||
RAW,
|
||||
LZ4,
|
||||
ZSTD,
|
||||
COUNT,
|
||||
} _value;
|
||||
// Construct from an integer.
|
||||
// Used to deserialize the algorithm from the first byte of the frame.
|
||||
constexpr compression_algorithm(underlying x) {
|
||||
if (x < std::to_underlying(type::RAW) || x >= std::to_underlying(type::COUNT)) {
|
||||
throw std::runtime_error(fmt::format("Invalid value {} for enum compression_algorithm", static_cast<int>(x)));
|
||||
}
|
||||
_value = static_cast<type>(x);
|
||||
}
|
||||
// Construct from `type`. Makes sure that `type` has a valid value.
|
||||
constexpr compression_algorithm(type x) : compression_algorithm(std::to_underlying(x)) {}
|
||||
|
||||
// These names are used in multiple places:
|
||||
// RPC negotiation, in metric labels, and config.
|
||||
static constexpr std::string_view names[] = {
|
||||
"raw",
|
||||
"lz4",
|
||||
"zstd",
|
||||
};
|
||||
static_assert(std::size(names) == static_cast<int>(compression_algorithm::type::COUNT));
|
||||
|
||||
// Implements enum_option.
|
||||
static auto map() {
|
||||
std::unordered_map<std::string, type> ret;
|
||||
for (size_t i = 0; i < std::size(names); ++i) {
|
||||
ret.insert(std::make_pair<std::string, type>(std::string(names[i]), compression_algorithm(i).get()));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
constexpr std::string_view name() const noexcept { return names[idx()]; }
|
||||
constexpr underlying idx() const noexcept { return std::to_underlying(_value); }
|
||||
constexpr type get() const noexcept { return _value; }
|
||||
constexpr static size_t count() { return static_cast<size_t>(type::COUNT); };
|
||||
bool operator<=>(const compression_algorithm &) const = default;
|
||||
};
|
||||
|
||||
|
||||
// Represents a set of compression algorithms.
|
||||
// Backed by a bitset.
|
||||
// Used for convenience during algorithm negotiations.
|
||||
class compression_algorithm_set {
|
||||
uint8_t _bitset;
|
||||
static_assert(std::numeric_limits<decltype(_bitset)>::digits > compression_algorithm::count());
|
||||
constexpr compression_algorithm_set(uint8_t v) noexcept : _bitset(v) {}
|
||||
public:
|
||||
// Returns a set containing the given algorithm and all algorithms weaker (smaller in the enum order)
|
||||
// than it.
|
||||
constexpr static compression_algorithm_set this_or_lighter(compression_algorithm algo) noexcept {
|
||||
auto x = 1 << (algo.idx());
|
||||
return {x + (x - 1)};
|
||||
}
|
||||
// Returns the strongest (greatest in the enum order) algorithm in the set.
|
||||
constexpr compression_algorithm heaviest() const {
|
||||
return {std::bit_width(_bitset) - 1};
|
||||
}
|
||||
// The usual set operations.
|
||||
constexpr static compression_algorithm_set singleton(compression_algorithm algo) noexcept {
|
||||
return {1 << algo.idx()};
|
||||
}
|
||||
constexpr compression_algorithm_set intersection(compression_algorithm_set o) const noexcept {
|
||||
return {_bitset & o._bitset};
|
||||
}
|
||||
constexpr compression_algorithm_set difference(compression_algorithm_set o) const noexcept {
|
||||
return {_bitset &~ o._bitset};
|
||||
}
|
||||
constexpr compression_algorithm_set sum(compression_algorithm_set o) const noexcept {
|
||||
return {_bitset | o._bitset};
|
||||
}
|
||||
constexpr bool contains(compression_algorithm algo) const noexcept {
|
||||
return _bitset & (1 << algo.idx());
|
||||
}
|
||||
constexpr bool operator==(const compression_algorithm_set&) const = default;
|
||||
// Returns the contained bitset. Used for serialization.
|
||||
constexpr uint8_t value() const noexcept {
|
||||
return _bitset;
|
||||
}
|
||||
// Reconstructs the set from the output of `value()`. Used for deserialization.
|
||||
constexpr static compression_algorithm_set from_value(uint8_t bitset) {
|
||||
compression_algorithm_set x = bitset;
|
||||
x.heaviest(); // This is a validation check. It will throw if the bitset contains some illegal/unknown bits.
|
||||
return x;
|
||||
}
|
||||
};
|
||||
|
||||
using algo_config = std::vector<enum_option<compression_algorithm>>;
|
||||
|
||||
// See docs/dev/advanced_rpc_compression.md,
|
||||
// section `Negotiation` for more information about the protocol.
|
||||
struct control_protocol {
|
||||
@@ -248,7 +152,7 @@ struct per_algorithm_stats {
|
||||
// prevent a misuse of the API (dangling references).
|
||||
class advanced_rpc_compressor::tracker : public utils::refcounted {
|
||||
public:
|
||||
using algo_config = algo_config;
|
||||
using algo_config = netw::algo_config;
|
||||
struct config {
|
||||
utils::updateable_value<uint32_t> zstd_min_msg_size{0};
|
||||
utils::updateable_value<uint32_t> zstd_max_msg_size{std::numeric_limits<uint32_t>::max()};
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "shared_dict.hh"
|
||||
#include "advanced_rpc_compressor.hh"
|
||||
#include "rpc_compression_types.hh"
|
||||
|
||||
namespace netw {
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "rpc_compression_types.hh"
|
||||
#include "utils/reservoir_sampling.hh"
|
||||
#include "utils/updateable_value.hh"
|
||||
#include <seastar/core/future.hh>
|
||||
@@ -88,28 +89,7 @@ class dict_training_loop {
|
||||
seastar::semaphore _pause{0};
|
||||
seastar::abort_source _pause_as;
|
||||
public:
|
||||
struct when {
|
||||
enum class type {
|
||||
NEVER,
|
||||
WHEN_LEADER,
|
||||
ALWAYS,
|
||||
COUNT,
|
||||
};
|
||||
static constexpr std::string_view names[] = {
|
||||
"never",
|
||||
"when_leader",
|
||||
"always",
|
||||
};
|
||||
static_assert(std::size(names) == static_cast<size_t>(type::COUNT));
|
||||
// Implements enum_option.
|
||||
static std::unordered_map<std::string, type> map() {
|
||||
std::unordered_map<std::string, type> ret;
|
||||
for (size_t i = 0; i < std::size(names); ++i) {
|
||||
ret.insert({std::string(names[i]), type(i)});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
using when = netw::dict_training_when;
|
||||
void pause();
|
||||
void unpause();
|
||||
void cancel() noexcept;
|
||||
|
||||
@@ -54,11 +54,11 @@ dictionary_service::dictionary_service(
|
||||
void dictionary_service::maybe_toggle_dict_training() {
|
||||
auto when = _rpc_dict_training_when();
|
||||
netw::dict_trainer_logger.debug("dictionary_service::maybe_toggle_dict_training(), called, _is_leader={}, when={}", _is_leader, when);
|
||||
if (when == netw::dict_training_loop::when::type::NEVER) {
|
||||
if (when == netw::dict_training_when::type::NEVER) {
|
||||
_training_fiber.pause();
|
||||
} else if (when == netw::dict_training_loop::when::type::ALWAYS) {
|
||||
} else if (when == netw::dict_training_when::type::ALWAYS) {
|
||||
_training_fiber.unpause();
|
||||
} else if (when == netw::dict_training_loop::when::type::WHEN_LEADER) {
|
||||
} else if (when == netw::dict_training_when::type::WHEN_LEADER) {
|
||||
_is_leader ? _training_fiber.unpause() : _training_fiber.pause();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -40,7 +40,7 @@ namespace gms {
|
||||
class dictionary_service {
|
||||
db::system_keyspace& _sys_ks;
|
||||
locator::host_id _our_host_id;
|
||||
utils::updateable_value<enum_option<netw::dict_training_loop::when>> _rpc_dict_training_when;
|
||||
utils::updateable_value<enum_option<netw::dict_training_when>> _rpc_dict_training_when;
|
||||
service::raft_group0_client& _raft_group0_client;
|
||||
abort_source& _as;
|
||||
netw::dict_training_loop _training_fiber;
|
||||
@@ -48,7 +48,7 @@ class dictionary_service {
|
||||
|
||||
bool _is_leader = false;
|
||||
utils::observer<bool> _leadership_observer;
|
||||
utils::observer<enum_option<netw::dict_training_loop::when>> _when_observer;
|
||||
utils::observer<enum_option<netw::dict_training_when>> _when_observer;
|
||||
std::optional<std::any> _feature_observer;
|
||||
|
||||
void maybe_toggle_dict_training();
|
||||
@@ -61,7 +61,7 @@ public:
|
||||
locator::host_id our_host_id = Uninitialized();
|
||||
utils::updateable_value<uint32_t> rpc_dict_training_min_time_seconds = Uninitialized();
|
||||
utils::updateable_value<uint64_t> rpc_dict_training_min_bytes = Uninitialized();
|
||||
utils::updateable_value<enum_option<netw::dict_training_loop::when>> rpc_dict_training_when = Uninitialized();
|
||||
utils::updateable_value<enum_option<netw::dict_training_when>> rpc_dict_training_when = Uninitialized();
|
||||
};
|
||||
// Note: the training fiber will start as soon as the relevant cluster feature is enabled.
|
||||
dictionary_service(
|
||||
|
||||
155
message/rpc_compression_types.hh
Normal file
155
message/rpc_compression_types.hh
Normal file
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* Copyright (C) 2026-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.1
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <compare>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "utils/enum_option.hh"
|
||||
|
||||
namespace netw {
|
||||
|
||||
// An enum wrapper, describing supported RPC compression algorithms.
|
||||
// Always contains a valid value -- the constructors won't allow
|
||||
// an invalid/unknown enum variant to be constructed.
|
||||
struct compression_algorithm {
|
||||
using underlying = uint8_t;
|
||||
enum class type : underlying {
|
||||
RAW,
|
||||
LZ4,
|
||||
ZSTD,
|
||||
COUNT,
|
||||
} _value;
|
||||
|
||||
// Construct from an integer.
|
||||
// Used to deserialize the algorithm from the first byte of the frame.
|
||||
constexpr compression_algorithm(underlying x) {
|
||||
if (x < std::to_underlying(type::RAW) || x >= std::to_underlying(type::COUNT)) {
|
||||
throw std::runtime_error(std::string("Invalid value ") + std::to_string(unsigned(x)) + " for enum compression_algorithm");
|
||||
}
|
||||
_value = static_cast<type>(x);
|
||||
}
|
||||
|
||||
// Construct from `type`. Makes sure that `type` has a valid value.
|
||||
constexpr compression_algorithm(type x) : compression_algorithm(std::to_underlying(x)) {}
|
||||
|
||||
// These names are used in multiple places:
|
||||
// RPC negotiation, in metric labels, and config.
|
||||
static constexpr std::string_view names[] = {
|
||||
"raw",
|
||||
"lz4",
|
||||
"zstd",
|
||||
};
|
||||
static_assert(std::size(names) == static_cast<int>(compression_algorithm::type::COUNT));
|
||||
|
||||
// Implements enum_option.
|
||||
static auto map() {
|
||||
std::unordered_map<std::string, type> ret;
|
||||
for (size_t i = 0; i < std::size(names); ++i) {
|
||||
ret.insert(std::make_pair(std::string(names[i]), compression_algorithm(i).get()));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
constexpr std::string_view name() const noexcept { return names[idx()]; }
|
||||
constexpr underlying idx() const noexcept { return std::to_underlying(_value); }
|
||||
constexpr type get() const noexcept { return _value; }
|
||||
constexpr static size_t count() { return static_cast<size_t>(type::COUNT); }
|
||||
bool operator<=>(const compression_algorithm&) const = default;
|
||||
};
|
||||
|
||||
// Represents a set of compression algorithms.
|
||||
// Backed by a bitset.
|
||||
// Used for convenience during algorithm negotiations.
|
||||
class compression_algorithm_set {
|
||||
uint8_t _bitset;
|
||||
static_assert(std::numeric_limits<decltype(_bitset)>::digits > compression_algorithm::count());
|
||||
constexpr compression_algorithm_set(uint8_t v) noexcept : _bitset(v) {}
|
||||
public:
|
||||
// Returns a set containing the given algorithm and all algorithms weaker (smaller in the enum order)
|
||||
// than it.
|
||||
constexpr static compression_algorithm_set this_or_lighter(compression_algorithm algo) noexcept {
|
||||
auto x = 1 << algo.idx();
|
||||
return {uint8_t(x + (x - 1))};
|
||||
}
|
||||
|
||||
// Returns the strongest (greatest in the enum order) algorithm in the set.
|
||||
constexpr compression_algorithm heaviest() const {
|
||||
return {compression_algorithm::underlying(std::bit_width(_bitset) - 1)};
|
||||
}
|
||||
|
||||
// The usual set operations.
|
||||
constexpr static compression_algorithm_set singleton(compression_algorithm algo) noexcept {
|
||||
return {uint8_t(1 << algo.idx())};
|
||||
}
|
||||
constexpr compression_algorithm_set intersection(compression_algorithm_set o) const noexcept {
|
||||
return {uint8_t(_bitset & o._bitset)};
|
||||
}
|
||||
constexpr compression_algorithm_set difference(compression_algorithm_set o) const noexcept {
|
||||
return {uint8_t(_bitset &~ o._bitset)};
|
||||
}
|
||||
constexpr compression_algorithm_set sum(compression_algorithm_set o) const noexcept {
|
||||
return {uint8_t(_bitset | o._bitset)};
|
||||
}
|
||||
constexpr bool contains(compression_algorithm algo) const noexcept {
|
||||
return _bitset & (1 << algo.idx());
|
||||
}
|
||||
constexpr bool operator==(const compression_algorithm_set&) const = default;
|
||||
|
||||
// Returns the contained bitset. Used for serialization.
|
||||
constexpr uint8_t value() const noexcept {
|
||||
return _bitset;
|
||||
}
|
||||
|
||||
// Reconstructs the set from the output of `value()`. Used for deserialization.
|
||||
constexpr static compression_algorithm_set from_value(uint8_t bitset) {
|
||||
compression_algorithm_set x = bitset;
|
||||
x.heaviest(); // Validation: throws on illegal/unknown bits.
|
||||
return x;
|
||||
}
|
||||
};
|
||||
|
||||
using algo_config = std::vector<enum_option<compression_algorithm>>;
|
||||
|
||||
struct dict_training_when {
|
||||
enum class type {
|
||||
NEVER,
|
||||
WHEN_LEADER,
|
||||
ALWAYS,
|
||||
COUNT,
|
||||
};
|
||||
|
||||
static constexpr std::string_view names[] = {
|
||||
"never",
|
||||
"when_leader",
|
||||
"always",
|
||||
};
|
||||
static_assert(std::size(names) == static_cast<size_t>(type::COUNT));
|
||||
|
||||
// Implements enum_option.
|
||||
static std::unordered_map<std::string, type> map() {
|
||||
std::unordered_map<std::string, type> ret;
|
||||
for (size_t i = 0; i < std::size(names); ++i) {
|
||||
ret.insert({std::string(names[i]), type(i)});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace netw
|
||||
Reference in New Issue
Block a user