Fix version numbers in upgrade page title

Closes #11873

.gitignore

@@ -31,3 +31,5 @@ docs/poetry.lock
 compile_commands.json
 .ccls-cache/
 .mypy_cache
+.envrc
+rust/Cargo.lock

CMakeLists.txt

@@ -189,6 +189,8 @@ set(swagger_files
     api/api-doc/storage_service.json
     api/api-doc/stream_manager.json
     api/api-doc/system.json
+    api/api-doc/task_manager.json
+    api/api-doc/task_manager_test.json
     api/api-doc/utils.json)
 
 set(swagger_gen_files)
@@ -301,6 +303,8 @@ set(scylla_sources
     api/storage_service.cc
     api/stream_manager.cc
     api/system.cc
+    api/task_manager.cc
+    api/task_manager_test.cc
     atomic_cell.cc
     auth/allow_all_authenticator.cc
     auth/allow_all_authorizer.cc
@@ -424,6 +428,8 @@ set(scylla_sources
     cql3/statements/sl_prop_defs.cc
     cql3/statements/truncate_statement.cc
     cql3/statements/update_statement.cc
+    cql3/statements/strongly_consistent_modification_statement.cc
+    cql3/statements/strongly_consistent_select_statement.cc
     cql3/statements/use_statement.cc
     cql3/type_json.cc
     cql3/untyped_result_set.cc
@@ -533,6 +539,7 @@ set(scylla_sources
     raft/raft.cc
     raft/server.cc
     raft/tracker.cc
+    service/broadcast_tables/experimental/lang.cc
     range_tombstone.cc
     range_tombstone_list.cc
     tombstone_gc_options.cc
@@ -612,6 +619,7 @@ set(scylla_sources
     streaming/stream_task.cc
     streaming/stream_transfer_task.cc
     table_helper.cc
+    tasks/task_manager.cc
     thrift/controller.cc
     thrift/handler.cc
     thrift/server.cc

SCYLLA-VERSION-GEN

@@ -1,11 +1,12 @@
 #!/bin/sh
 
 USAGE=$(cat <<-END
-Usage: $(basename "$0") [-h|--help] [-o|--output-dir PATH] -- generate Scylla version and build information files.
+Usage: $(basename "$0") [-h|--help] [-o|--output-dir PATH] [--date-stamp DATE] -- generate Scylla version and build information files.
 
 Options:
   -h|--help show this help message.
   -o|--output-dir PATH specify destination path at which the version files are to be created.
+  -d|--date-stamp DATE manually set date for release parameter
 
 By default, the script will attempt to parse 'version' file
 in the current directory, which should contain a string of
@@ -31,6 +32,8 @@ using '-o PATH' option.
 END
 )
 
+DATE=""
+
 while [[ $# -gt 0 ]]; do
 	opt="$1"
 	case $opt in
@@ -43,6 +46,11 @@ while [[ $# -gt 0 ]]; do
 			shift
 			shift
 			;;
+		--date-stamp)
+			DATE="$2"
+			shift
+			shift
+			;;
 		*)
 			echo "Unexpected argument found: $1"
 			echo
@@ -58,24 +66,33 @@ if [ -z "$OUTPUT_DIR" ]; then
 	OUTPUT_DIR="$SCRIPT_DIR/build"
 fi
 
+if [ -z "$DATE" ]; then
+  DATE=$(date --utc +%Y%m%d)
+fi
+
 # Default scylla product/version tags
 PRODUCT=scylla
-VERSION=5.1.0-dev
+VERSION=5.2.0-dev
 
 if test -f version
 then
 	SCYLLA_VERSION=$(cat version | awk -F'-' '{print $1}')
 	SCYLLA_RELEASE=$(cat version | awk -F'-' '{print $2}')
 else
-	DATE=$(date --utc +%Y%m%d)
-	GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
 	SCYLLA_VERSION=$VERSION
-	# For custom package builds, replace "0" with "counter.your_name",
-	# where counter starts at 1 and increments for successive versions.
-	# This ensures that the package manager will select your custom
-	# package over the standard release.
-	SCYLLA_BUILD=0
-	SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
+	if [ -z "$SCYLLA_RELEASE" ]; then
+		DATE=$(date --utc +%Y%m%d)
+		GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
+		# For custom package builds, replace "0" with "counter.your_name",
+		# where counter starts at 1 and increments for successive versions.
+		# This ensures that the package manager will select your custom
+		# package over the standard release.
+		SCYLLA_BUILD=0
+		SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
+	elif [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
+		echo "setting SCYLLA_RELEASE only makes sense in clean builds" 1>&2
+		exit 1
+	fi
 fi
 
 if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then

alternator/auth.cc

@@ -133,7 +133,8 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, std::strin
     }
     auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col});
     auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id}, selection->get_query_options());
-    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
+    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice,
+            proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
     auto cl = auth::password_authenticator::consistency_for_user(username);
 
     service::client_state client_state{service::client_state::internal_tag()};

alternator/controller.cc

@@ -14,6 +14,8 @@
 #include "db/config.hh"
 #include "cdc/generation_service.hh"
 #include "service/memory_limiter.hh"
+#include "auth/service.hh"
+#include "service/qos/service_level_controller.hh"
 
 using namespace seastar;
 
@@ -28,6 +30,8 @@ controller::controller(
         sharded<db::system_distributed_keyspace>& sys_dist_ks,
         sharded<cdc::generation_service>& cdc_gen_svc,
         sharded<service::memory_limiter>& memory_limiter,
+        sharded<auth::service>& auth_service,
+        sharded<qos::service_level_controller>& sl_controller,
         const db::config& config)
     : _gossiper(gossiper)
     , _proxy(proxy)
@@ -35,6 +39,8 @@ controller::controller(
     , _sys_dist_ks(sys_dist_ks)
     , _cdc_gen_svc(cdc_gen_svc)
     , _memory_limiter(memory_limiter)
+    , _auth_service(auth_service)
+    , _sl_controller(sl_controller)
     , _config(config)
 {
 }
@@ -77,7 +83,7 @@ future<> controller::start_server() {
         auto get_cdc_metadata = [] (cdc::generation_service& svc) { return std::ref(svc.get_cdc_metadata()); };
 
         _executor.start(std::ref(_gossiper), std::ref(_proxy), std::ref(_mm), std::ref(_sys_dist_ks), sharded_parameter(get_cdc_metadata, std::ref(_cdc_gen_svc)), _ssg.value()).get();
-        _server.start(std::ref(_executor), std::ref(_proxy), std::ref(_gossiper)).get();
+        _server.start(std::ref(_executor), std::ref(_proxy), std::ref(_gossiper), std::ref(_auth_service), std::ref(_sl_controller)).get();
         // Note: from this point on, if start_server() throws for any reason,
         // it must first call stop_server() to stop the executor and server
         // services we just started - or Scylla will cause an assertion

alternator/controller.hh

@@ -34,6 +34,14 @@ class gossiper;
 
 }
 
+namespace auth {
+class service;
+}
+
+namespace qos {
+class service_level_controller;
+}
+
 namespace alternator {
 
 // This is the official DynamoDB API version.
@@ -53,6 +61,8 @@ class controller : public protocol_server {
     sharded<db::system_distributed_keyspace>& _sys_dist_ks;
     sharded<cdc::generation_service>& _cdc_gen_svc;
     sharded<service::memory_limiter>& _memory_limiter;
+    sharded<auth::service>& _auth_service;
+    sharded<qos::service_level_controller>& _sl_controller;
     const db::config& _config;
 
     std::vector<socket_address> _listen_addresses;
@@ -68,6 +78,8 @@ public:
         sharded<db::system_distributed_keyspace>& sys_dist_ks,
         sharded<cdc::generation_service>& cdc_gen_svc,
         sharded<service::memory_limiter>& memory_limiter,
+        sharded<auth::service>& auth_service,
+        sharded<qos::service_level_controller>& sl_controller,
         const db::config& config);
 
     virtual sstring name() const override;

alternator/executor.cc

@@ -438,6 +438,11 @@ future<executor::request_return_type> executor::describe_table(client_state& cli
     rjson::add(table_description, "BillingModeSummary", rjson::empty_object());
     rjson::add(table_description["BillingModeSummary"], "BillingMode", "PAY_PER_REQUEST");
     rjson::add(table_description["BillingModeSummary"], "LastUpdateToPayPerRequestDateTime", rjson::value(creation_date_seconds));
+    // In PAY_PER_REQUEST billing mode, provisioned capacity should return 0
+    rjson::add(table_description, "ProvisionedThroughput", rjson::empty_object());
+    rjson::add(table_description["ProvisionedThroughput"], "ReadCapacityUnits", 0);
+    rjson::add(table_description["ProvisionedThroughput"], "WriteCapacityUnits", 0);
+    rjson::add(table_description["ProvisionedThroughput"], "NumberOfDecreasesToday", 0);
 
     std::unordered_map<std::string,std::string> key_attribute_types;
     // Add base table's KeySchema and collect types for AttributeDefinitions:
@@ -460,6 +465,11 @@ future<executor::request_return_type> executor::describe_table(client_state& cli
             rjson::add(view_entry, "IndexArn", generate_arn_for_index(*schema, index_name));
             // Add indexes's KeySchema and collect types for AttributeDefinitions:
             describe_key_schema(view_entry, *vptr, key_attribute_types);
+            // Add projection type
+            rjson::value projection = rjson::empty_object();
+            rjson::add(projection, "ProjectionType", "ALL");
+            // FIXME: we have to get ProjectionType from the schema when it is added
+            rjson::add(view_entry, "Projection", std::move(projection));
             // Local secondary indexes are marked by an extra '!' sign occurring before the ':' delimiter
             rjson::value& index_array = (delim_it > 1 && cf_name[delim_it-1] == '!') ? lsi_array : gsi_array;
             rjson::push_back(index_array, std::move(view_entry));
@@ -1082,7 +1092,6 @@ future<executor::request_return_type> executor::update_table(client_state& clien
     elogger.trace("Updating table {}", request);
 
     static const std::vector<sstring> unsupported = {
-        "AttributeDefinitions", 
         "GlobalSecondaryIndexUpdates", 
         "ProvisionedThroughput",
         "ReplicaUpdates",
@@ -1255,6 +1264,22 @@ put_or_delete_item::put_or_delete_item(const rjson::value& key, schema_ptr schem
     check_key(key, schema);
 }
 
+// find_attribute() checks whether the named attribute is stored in the
+// schema as a real column (we do this for key attribute, and for a GSI key)
+// and if so, returns that column. If not, the function returns nullptr,
+// telling the caller that the attribute is stored serialized in the
+// ATTRS_COLUMN_NAME map - not in a stand-alone column in the schema.
+static inline const column_definition* find_attribute(const schema& schema, const bytes& attribute_name) {
+    const column_definition* cdef = schema.get_column_definition(attribute_name);
+    // Although ATTRS_COLUMN_NAME exists as an actual column, when used as an
+    // attribute name it should refer to an attribute inside ATTRS_COLUMN_NAME
+    // not to ATTRS_COLUMN_NAME itself. This if() is needed for #5009.
+    if (cdef && cdef->name() == executor::ATTRS_COLUMN_NAME) {
+        return nullptr;
+    }
+    return cdef;
+}
+
 put_or_delete_item::put_or_delete_item(const rjson::value& item, schema_ptr schema, put_item)
         : _pk(pk_from_json(item, schema)), _ck(ck_from_json(item, schema)) {
     _cells = std::vector<cell>();
@@ -1262,7 +1287,7 @@ put_or_delete_item::put_or_delete_item(const rjson::value& item, schema_ptr sche
     for (auto it = item.MemberBegin(); it != item.MemberEnd(); ++it) {
         bytes column_name = to_bytes(it->name.GetString());
         validate_value(it->value, "PutItem");
-        const column_definition* cdef = schema->get_column_definition(column_name);
+        const column_definition* cdef = find_attribute(*schema, column_name);
         if (!cdef) {
             bytes value = serialize_item(it->value);
             _cells->push_back({std::move(column_name), serialize_item(it->value)});
@@ -1294,7 +1319,7 @@ mutation put_or_delete_item::build(schema_ptr schema, api::timestamp_type ts) co
     auto& row = m.partition().clustered_row(*schema, _ck);
     attribute_collector attrs_collector;
     for (auto& c : *_cells) {
-        const column_definition* cdef = schema->get_column_definition(c.column_name);
+        const column_definition* cdef = find_attribute(*schema, c.column_name);
         if (!cdef) {
             attrs_collector.put(c.column_name, c.value, ts);
         } else {
@@ -1359,7 +1384,8 @@ static lw_shared_ptr<query::read_command> previous_item_read_command(service::st
     auto regular_columns = boost::copy_range<query::column_id_vector>(
             schema->regular_columns() | boost::adaptors::transformed([] (const column_definition& cdef) { return cdef.id; }));
     auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
-    return ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
+    return ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice),
+            query::tombstone_limit(proxy.get_tombstone_limit()));
 }
 
 static dht::partition_range_vector to_partition_ranges(const schema& schema, const partition_key& pk) {
@@ -2748,7 +2774,7 @@ update_item_operation::apply(std::unique_ptr<rjson::value> previous_item, api::t
                 }
             }
         }
-        const column_definition* cdef = _schema->get_column_definition(column_name);
+        const column_definition* cdef = find_attribute(*_schema, column_name);
         if (cdef) {
             bytes column_value = get_key_from_typed_value(json_value, *cdef);
             row.cells().apply(*cdef, atomic_cell::make_live(*cdef->type, ts, column_value));
@@ -2770,7 +2796,7 @@ update_item_operation::apply(std::unique_ptr<rjson::value> previous_item, api::t
                 rjson::add_with_string_name(_return_attributes, cn, rjson::copy(*col));
             }
         }
-        const column_definition* cdef = _schema->get_column_definition(column_name);
+        const column_definition* cdef = find_attribute(*_schema, column_name);
         if (cdef) {
             row.cells().apply(*cdef, atomic_cell::make_dead(ts, gc_clock::now()));
         } else {
@@ -3063,7 +3089,8 @@ future<executor::request_return_type> executor::get_item(client_state& client_st
     auto selection = cql3::selection::selection::wildcard(schema);
 
     auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
-    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice));
+    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
+            query::tombstone_limit(_proxy.get_tombstone_limit()));
 
     std::unordered_set<std::string> used_attribute_names;
     auto attrs_to_get = calculate_attrs_to_get(request, used_attribute_names);
@@ -3217,7 +3244,8 @@ future<executor::request_return_type> executor::batch_get_item(client_state& cli
                     rs.schema->regular_columns() | boost::adaptors::transformed([] (const column_definition& cdef) { return cdef.id; }));
             auto selection = cql3::selection::selection::wildcard(rs.schema);
             auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
-            auto command = ::make_lw_shared<query::read_command>(rs.schema->id(), rs.schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice));
+            auto command = ::make_lw_shared<query::read_command>(rs.schema->id(), rs.schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
+                    query::tombstone_limit(_proxy.get_tombstone_limit()));
             command->allow_limit = db::allow_per_partition_rate_limit::yes;
             future<std::vector<rjson::value>> f = _proxy.query(rs.schema, std::move(command), std::move(partition_ranges), rs.cl,
                     service::storage_proxy::coordinator_query_options(executor::default_timeout(), permit, client_state, trace_state)).then(
@@ -3618,7 +3646,7 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
         if (schema->clustering_key_size() > 0) {
             pos = pos_from_json(*exclusive_start_key, schema);
         }
-        paging_state = make_lw_shared<service::pager::paging_state>(pk, pos, query::max_partitions, utils::UUID(), service::pager::paging_state::replicas_per_token_range{}, std::nullopt, 0);
+        paging_state = make_lw_shared<service::pager::paging_state>(pk, pos, query::max_partitions, query_id::create_null_id(), service::pager::paging_state::replicas_per_token_range{}, std::nullopt, 0);
     }
 
     auto regular_columns = boost::copy_range<query::column_id_vector>(
@@ -3629,7 +3657,8 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
     query::partition_slice::option_set opts = selection->get_query_options();
     opts.add(custom_opts);
     auto partition_slice = query::partition_slice(std::move(ck_bounds), std::move(static_columns), std::move(regular_columns), opts);
-    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
+    auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice),
+        query::tombstone_limit(proxy.get_tombstone_limit()));
 
     auto query_state_ptr = std::make_unique<service::query_state>(client_state, trace_state, std::move(permit));
 

alternator/server.cc

@@ -16,6 +16,7 @@
 #include <seastar/util/short_streams.hh>
 #include "seastarx.hh"
 #include "error.hh"
+#include "service/qos/service_level_controller.hh"
 #include "utils/rjson.hh"
 #include "auth.hh"
 #include <cctype>
@@ -234,7 +235,7 @@ protected:
 future<std::string> server::verify_signature(const request& req, const chunked_content& content) {
     if (!_enforce_authorization) {
         slogger.debug("Skipping authorization");
-        return make_ready_future<std::string>("<unauthenticated request>");
+        return make_ready_future<std::string>();
     }
     auto host_it = req._headers.find("Host");
     if (host_it == req._headers.end()) {
@@ -364,7 +365,9 @@ static tracing::trace_state_ptr maybe_trace_query(service::client_state& client_
         tracing::add_session_param(trace_state, "alternator_op", op);
         tracing::add_query(trace_state, truncated_content_view(query, buf));
         tracing::begin(trace_state, format("Alternator {}", op), client_state.get_client_address());
-        tracing::set_username(trace_state, auth::authenticated_user(username));
+        if (!username.empty()) {
+            tracing::set_username(trace_state, auth::authenticated_user(username));
+        }
     }
     return trace_state;
 }
@@ -407,7 +410,11 @@ future<executor::request_return_type> server::handle_api_request(std::unique_ptr
     auto leave = defer([this] () noexcept { _pending_requests.leave(); });
     //FIXME: Client state can provide more context, e.g. client's endpoint address
     // We use unique_ptr because client_state cannot be moved or copied
-    executor::client_state client_state{executor::client_state::internal_tag()};
+    executor::client_state client_state = username.empty()
+        ? service::client_state{service::client_state::internal_tag()}
+        : service::client_state{service::client_state::internal_tag(), _auth_service, _sl_controller, username};
+    co_await client_state.maybe_update_per_service_level_params();
+
     tracing::trace_state_ptr trace_state = maybe_trace_query(client_state, username, op, content);
     tracing::trace(trace_state, op);
     rjson::value json_request = co_await _json_parser.parse(std::move(content));
@@ -440,12 +447,14 @@ void server::set_routes(routes& r) {
 //FIXME: A way to immediately invalidate the cache should be considered,
 // e.g. when the system table which stores the keys is changed.
 // For now, this propagation may take up to 1 minute.
-server::server(executor& exec, service::storage_proxy& proxy, gms::gossiper& gossiper)
+server::server(executor& exec, service::storage_proxy& proxy, gms::gossiper& gossiper, auth::service& auth_service, qos::service_level_controller& sl_controller)
         : _http_server("http-alternator")
         , _https_server("https-alternator")
         , _executor(exec)
         , _proxy(proxy)
         , _gossiper(gossiper)
+        , _auth_service(auth_service)
+        , _sl_controller(sl_controller)
         , _key_cache(1024, 1min, slogger)
         , _enforce_authorization(false)
         , _enabled_servers{}

alternator/server.hh

@@ -15,6 +15,7 @@
 #include <seastar/net/tls.hh>
 #include <optional>
 #include "alternator/auth.hh"
+#include "service/qos/service_level_controller.hh"
 #include "utils/small_vector.hh"
 #include "utils/updateable_value.hh"
 #include <seastar/core/units.hh>
@@ -34,6 +35,8 @@ class server {
     executor& _executor;
     service::storage_proxy& _proxy;
     gms::gossiper& _gossiper;
+    auth::service& _auth_service;
+    qos::service_level_controller& _sl_controller;
 
     key_cache _key_cache;
     bool _enforce_authorization;
@@ -65,7 +68,7 @@ class server {
     json_parser _json_parser;
 
 public:
-    server(executor& executor, service::storage_proxy& proxy, gms::gossiper& gossiper);
+    server(executor& executor, service::storage_proxy& proxy, gms::gossiper& gossiper, auth::service& service, qos::service_level_controller& sl_controller);
 
     future<> init(net::inet_address addr, std::optional<uint16_t> port, std::optional<uint16_t> https_port, std::optional<tls::credentials_builder> creds,
             bool enforce_authorization, semaphore* memory_limiter, utils::updateable_value<uint32_t> max_concurrent_requests);

alternator/streams.cc

@@ -74,8 +74,8 @@ struct rapidjson::internal::TypeHelper<ValueType, utils::UUID>
     : public from_string_helper<ValueType, utils::UUID>
 {};
 
-static db_clock::time_point as_timepoint(const utils::UUID& uuid) {
-    return db_clock::time_point{utils::UUID_gen::unix_timestamp(uuid)};
+static db_clock::time_point as_timepoint(const table_id& tid) {
+    return db_clock::time_point{utils::UUID_gen::unix_timestamp(tid.uuid())};
 }
 
 /**
@@ -106,6 +106,9 @@ public:
     stream_arn(const UUID& uuid)
         : UUID(uuid)
     {}
+    stream_arn(const table_id& tid)
+        : UUID(tid.uuid())
+    {}
     stream_arn(std::string_view v)
         : UUID(v.substr(1))
     {
@@ -155,7 +158,7 @@ future<alternator::executor::request_return_type> alternator::executor::list_str
     // and we can probably expect this to be a single call.
     if (streams_start) {
         i = std::find_if(i, e, [&](data_dictionary::table t) {
-            return t.schema()->id() == streams_start 
+            return t.schema()->id().uuid() == streams_start
                 && cdc::get_base_table(db.real_database(), *t.schema())
                 && is_alternator_keyspace(t.schema()->ks_name())
                 ;
@@ -430,7 +433,7 @@ future<executor::request_return_type> executor::describe_stream(client_state& cl
     auto db = _proxy.data_dictionary();
 
     try {
-        auto cf = db.find_column_family(stream_arn);
+        auto cf = db.find_column_family(table_id(stream_arn));
         schema = cf.schema();
         bs = cdc::get_base_table(db.real_database(), *schema);
     } catch (...) {        
@@ -717,7 +720,7 @@ future<executor::request_return_type> executor::get_shard_iterator(client_state&
     std::optional<shard_id> sid;
 
     try {
-        auto cf = db.find_column_family(stream_arn);
+        auto cf = db.find_column_family(table_id(stream_arn));
         schema = cf.schema();
         sid = rjson::get<shard_id>(request, "ShardId");
     } catch (...) {
@@ -802,7 +805,7 @@ future<executor::request_return_type> executor::get_records(client_state& client
     auto db = _proxy.data_dictionary();
     schema_ptr schema, base;
     try {
-        auto log_table = db.find_column_family(iter.table);
+        auto log_table = db.find_column_family(table_id(iter.table));
         schema = log_table.schema();
         base = cdc::get_base_table(db.real_database(), *schema);
     } catch (...) {        
@@ -876,7 +879,7 @@ future<executor::request_return_type> executor::get_records(client_state& client
         ++mul;
     }
     auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
-            query::row_limit(limit * mul));
+            query::tombstone_limit(_proxy.get_tombstone_limit()), query::row_limit(limit * mul));
 
     return _proxy.query(schema, std::move(command), std::move(partition_ranges), cl, service::storage_proxy::coordinator_query_options(default_timeout(), std::move(permit), client_state)).then(
             [this, schema, partition_slice = std::move(partition_slice), selection = std::move(selection), start_time = std::move(start_time), limit, key_names = std::move(key_names), attr_names = std::move(attr_names), type, iter, high_ts] (service::storage_proxy::coordinator_query_result qr) mutable {       

alternator/ttl.cc

@@ -136,7 +136,7 @@ future<executor::request_return_type> executor::describe_time_to_live(client_sta
 
 // expiration_service is a sharded service responsible for cleaning up expired
 // items in all tables with per-item expiration enabled. Currently, this means
-// Alternator tables with TTL configured via a UpdateTimeToLeave request.
+// Alternator tables with TTL configured via a UpdateTimeToLive request.
 //
 // Here is a brief overview of how the expiration service works:
 //
@@ -150,25 +150,25 @@ future<executor::request_return_type> executor::describe_time_to_live(client_sta
 // To avoid scanning the same items RF times in RF replicas, only one node is
 // responsible for scanning a token range at a time. Normally, this is the
 // node owning this range as a "primary range" (the first node in the ring
-// with this range), but when this node is down, other nodes may take over
-// (FIXME: this is not implemented yet).
+// with this range), but when this node is down, the secondary owner (the
+// second in the ring) may take over.
 // An expiration thread is reponsible for all tables which need expiration
-// scans. FIXME: explain how this is done with multiple tables - parallel,
-// staggered, or what?
+// scans. Currently, the different tables are scanned sequentially (not in
+// parallel).
 // The expiration thread scans item using CL=QUORUM to ensures that it reads
 // a consistent expiration-time attribute. This means that the items are read
 // locally and in addition QUORUM-1 additional nodes (one additional node
 // when RF=3) need to read the data and send digests.
-// FIXME: explain if we can read the exact attribute or the entire map.
 // When the expiration thread decides that an item has expired and wants
 // to delete it, it does it using a CL=QUORUM write. This allows this
 // deletion to be visible for consistent (quorum) reads. The deletion,
 // like user deletions, will also appear on the CDC log and therefore
-// Alternator Streams if enabled (FIXME: explain how we mark the
-// deletion different from user deletes. We don't do it yet.).
-expiration_service::expiration_service(data_dictionary::database db, service::storage_proxy& proxy)
+// Alternator Streams if enabled - currently as ordinary deletes (the
+// userIdentity flag is currently missing this is issue #11523).
+expiration_service::expiration_service(data_dictionary::database db, service::storage_proxy& proxy, gms::gossiper& g)
         : _db(db)
         , _proxy(proxy)
+        , _gossiper(g)
 {
 }
 
@@ -282,7 +282,9 @@ static future<> expire_item(service::storage_proxy& proxy,
         auto ck = clustering_key::from_exploded(exploded_ck);
         m.partition().clustered_row(*schema, ck).apply(tombstone(ts, gc_clock::now()));
     }
-    return proxy.mutate(std::vector<mutation>{std::move(m)},
+    std::vector<mutation> mutations;
+    mutations.push_back(std::move(m));
+    return proxy.mutate(std::move(mutations),
         db::consistency_level::LOCAL_QUORUM,
         executor::default_timeout(), // FIXME - which timeout?
         qs.get_trace_state(), qs.get_permit(),
@@ -365,7 +367,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
 // 2. The primary replica for this token is currently marked down.
 // 3. In this node, this shard is responsible for this token.
 // We use the <secondary> case to handle the possibility that some of the
-// nodes in the system are down. A dead node will not be expiring expiring
+// nodes in the system are down. A dead node will not be expiring
 // the tokens owned by it, so we want the secondary owner to take over its
 // primary ranges.
 //
@@ -511,7 +513,7 @@ struct scan_ranges_context {
         opts.set<query::partition_slice::option::bypass_cache>();
         std::vector<query::clustering_range> ck_bounds{query::clustering_range::make_open_ended_both_sides()};
         auto partition_slice = query::partition_slice(std::move(ck_bounds), {}, std::move(regular_columns), opts);
-        command = ::make_lw_shared<query::read_command>(s->id(), s->version(), partition_slice, proxy.get_max_result_size(partition_slice));
+        command = ::make_lw_shared<query::read_command>(s->id(), s->version(), partition_slice, proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
         executor::client_state client_state{executor::client_state::internal_tag()};
         tracing::trace_state_ptr trace_state;
         // NOTICE: empty_service_permit is used because the TTL service has fixed parallelism
@@ -637,6 +639,7 @@ static future<> scan_table_ranges(
 static future<bool> scan_table(
     service::storage_proxy& proxy,
     data_dictionary::database db,
+    gms::gossiper& gossiper,
     schema_ptr s,
     abort_source& abort_source,
     named_semaphore& page_sem,
@@ -689,7 +692,7 @@ static future<bool> scan_table(
     expiration_stats.scan_table++;
     // FIXME: need to pace the scan, not do it all at once.
     scan_ranges_context scan_ctx{s, proxy, std::move(column_name), std::move(member)};
-    token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), proxy.gossiper(), s);
+    token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), gossiper, s);
     while (std::optional<dht::partition_range> range = my_ranges.next_partition_range()) {
         // Note that because of issue #9167 we need to run a separate
         // query on each partition range, and can't pass several of
@@ -709,7 +712,7 @@ static future<bool> scan_table(
     // by tasking another node to take over scanning of the dead node's primary
     // ranges. What we do here is that this node will also check expiration
     // on its *secondary* ranges - but only those whose primary owner is down.
-    token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), proxy.gossiper(), s);
+    token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), gossiper, s);
     while (std::optional<dht::partition_range> range = my_secondary_ranges.next_partition_range()) {
         expiration_stats.secondary_ranges_scanned++;
         dht::partition_range_vector partition_ranges;
@@ -741,7 +744,7 @@ future<> expiration_service::run() {
                 co_return;
             }
             try {
-                co_await scan_table(_proxy, _db, s, _abort_source, _page_sem, _expiration_stats);
+                co_await scan_table(_proxy, _db, _gossiper, s, _abort_source, _page_sem, _expiration_stats);
             } catch (...) {
                 // The scan of a table may fail in the middle for many
                 // reasons, including network failure and even the table
@@ -767,13 +770,15 @@ future<> expiration_service::run() {
         // in the next iteration by reducing the scanner's scheduling-group
         // share (if using a separate scheduling group), or introduce
         // finer-grain sleeps into the scanning code.
-        std::chrono::seconds scan_duration(std::chrono::duration_cast<std::chrono::seconds>(lowres_clock::now() - start));
-        std::chrono::seconds period(_db.get_config().alternator_ttl_period_in_seconds());
+        std::chrono::milliseconds scan_duration(std::chrono::duration_cast<std::chrono::milliseconds>(lowres_clock::now() - start));
+        std::chrono::milliseconds period(long(_db.get_config().alternator_ttl_period_in_seconds() * 1000));
         if (scan_duration < period) {
             try {
-                tlogger.info("sleeping {} seconds until next period", (period - scan_duration).count());
+                tlogger.info("sleeping {} seconds until next period", (period - scan_duration).count()/1000.0);
                 co_await seastar::sleep_abortable(period - scan_duration, _abort_source);
             } catch(seastar::sleep_aborted&) {}
+        } else {
+                tlogger.warn("scan took {} seconds, longer than period - not sleeping", scan_duration.count()/1000.0);
         }
     }
 }

alternator/ttl.hh

@@ -14,6 +14,10 @@
 #include <seastar/core/semaphore.hh>
 #include "data_dictionary/data_dictionary.hh"
 
+namespace gms {
+class gossiper;
+}
+
 namespace replica {
 class database;
 }
@@ -47,6 +51,7 @@ public:
 private:
     data_dictionary::database _db;
     service::storage_proxy& _proxy;
+    gms::gossiper& _gossiper;
     // _end is set by start(), and resolves when the the background service
     // started by it ends. To ask the background service to end, _abort_source
     // should be triggered. stop() below uses both _abort_source and _end.
@@ -60,7 +65,7 @@ public:
     // sharded_service<expiration_service>::start() creates this object on
     // all shards, so calls this constructor on each shard. Later, the
     // additional start() function should be invoked on all shards.
-    expiration_service(data_dictionary::database, service::storage_proxy&);
+    expiration_service(data_dictionary::database, service::storage_proxy&, gms::gossiper&);
     future<> start();
     future<> run();
     // sharded_service<expiration_service>::stop() calls the following stop()

api/api-doc/storage_service.json

@@ -1228,7 +1228,7 @@
          "operations":[
             {
                "method":"POST",
-               "summary":"Removes token (and all data associated with enpoint that had it) from the ring",
+               "summary":"Removes a node from the cluster. Replicated data that logically belonged to this node is redistributed among the remaining nodes.",
                "type":"void",
                "nickname":"remove_node",
                "produces":[
@@ -1245,7 +1245,7 @@
                   },
                   {
                      "name":"ignore_nodes",
-                     "description":"List of dead nodes to ingore in removenode operation",
+                     "description":"Comma-separated list of dead nodes to ignore in removenode operation. Use the same method for all nodes to ignore: either Host IDs or ip addresses.",
                      "required":false,
                      "allowMultiple":false,
                      "type":"string",

api/api-doc/system.json

@@ -52,6 +52,45 @@
             }
          ]
       },
+      {
+         "path":"/system/log",
+         "operations":[
+            {
+               "method":"POST",
+               "summary":"Write a message to the Scylla log",
+               "type":"void",
+               "nickname":"write_log_message",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"message",
+                     "description":"The message to write to the log",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  },
+                  {
+                     "name":"level",
+                     "description":"The logging level to use",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "enum":[
+                        "error",
+                        "warn",
+                        "info",
+                        "debug",
+                        "trace"
+                     ],
+                     "paramType":"query"
+                  }
+               ]
+            }
+         ]
+      },
       {
          "path":"/system/drop_sstable_caches",
          "operations":[

api/api-doc/task_manager.json

@@ -0,0 +1,251 @@
+{
+    "apiVersion":"0.0.1",
+    "swaggerVersion":"1.2",
+    "basePath":"{{Protocol}}://{{Host}}",
+    "resourcePath":"/task_manager",
+    "produces":[
+       "application/json"
+    ],
+    "apis":[
+       {
+          "path":"/task_manager/list_modules",
+          "operations":[
+             {
+                "method":"GET",
+                "summary":"Get all modules names",
+                "type":"array",
+                "items":{
+                   "type":"string"
+                },
+                "nickname":"get_modules",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                ]
+             }
+          ]
+       },
+       {
+          "path":"/task_manager/list_module_tasks/{module}",
+          "operations":[
+             {
+                "method":"GET",
+                "summary":"Get a list of tasks",
+                "type":"array",
+                "items":{
+                    "type":"task_stats"
+                },
+                "nickname":"get_tasks",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                    {
+                        "name":"module",
+                        "description":"The module to query about",
+                        "required":true,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"path"
+                    },
+                    {
+                        "name":"internal",
+                        "description":"Boolean flag indicating whether internal tasks should be shown (false by default)",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"boolean",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"keyspace",
+                        "description":"The keyspace to query about",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"table",
+                        "description":"The table to query about",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    }
+                ]
+             }
+          ]
+       },
+       {
+          "path":"/task_manager/task_status/{task_id}",
+          "operations":[
+             {
+                "method":"GET",
+                "summary":"Get task status",
+                "type":"task_status",
+                "nickname":"get_task_status",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                    {
+                        "name":"task_id",
+                        "description":"The uuid of a task to query about",
+                        "required":true,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"path"
+                    }
+                ]
+             }
+          ]
+       },
+       {
+          "path":"/task_manager/abort_task/{task_id}",
+          "operations":[
+             {
+                "method":"POST",
+                "summary":"Abort running task and its descendants",
+                "type":"void",
+                "nickname":"abort_task",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                   {
+                      "name":"task_id",
+                      "description":"The uuid of a task to abort",
+                      "required":true,
+                      "allowMultiple":false,
+                      "type":"string",
+                      "paramType":"path"
+                   }
+                ]
+             }
+          ]
+       },
+       {
+        "path":"/task_manager/wait_task/{task_id}",
+        "operations":[
+           {
+              "method":"GET",
+              "summary":"Wait for a task to complete",
+              "type":"task_status",
+              "nickname":"wait_task",
+              "produces":[
+                 "application/json"
+              ],
+              "parameters":[
+                 {
+                    "name":"task_id",
+                    "description":"The uuid of a task to wait for",
+                    "required":true,
+                    "allowMultiple":false,
+                    "type":"string",
+                    "paramType":"path"
+                 }
+              ]
+           }
+        ]
+     }
+    ],
+    "models":{
+       "task_stats" :{
+           "id": "task_stats",
+           "description":"A task statistics object",
+           "properties":{
+             "task_id":{
+                "type":"string",
+                "description":"The uuid of a task"
+             },
+             "state":{
+                "type":"string",
+                "enum":[
+                  "created",
+                  "running",
+                  "done",
+                  "failed"
+                ],
+                "description":"The state of a task"
+             }
+           }
+       },
+       "task_status":{
+          "id":"task_status",
+          "description":"A task status object",
+          "properties":{
+             "id":{
+                "type":"string",
+                "description":"The uuid of the task"
+             },
+             "type":{
+                "type":"string",
+                "description":"The description of the task"
+             },
+             "state":{
+               "type":"string",
+               "enum":[
+                 "created",
+                 "running",
+                 "done",
+                 "failed"
+               ],
+                "description":"The state of the task"
+             },
+             "is_abortable":{
+                "type":"boolean",
+                "description":"Boolean flag indicating whether the task can be aborted"
+             },
+             "start_time":{
+                "type":"datetime",
+                "description":"The start time of the task"
+             },
+             "end_time":{
+                "type":"datetime",
+                "description":"The end time of the task (unspecified when the task is not completed)"
+             },
+             "error":{
+                "type":"string",
+                "description":"Error string, if the task failed"
+             },
+             "parent_id":{
+               "type":"string",
+               "description":"The uuid of the parent task"
+            },
+            "sequence_number":{
+               "type":"long",
+               "description":"The running sequence number of the task"
+            },
+            "shard":{
+               "type":"long",
+               "description":"The number of a shard the task is running on"
+            },
+            "keyspace":{
+               "type":"string",
+               "description":"The keyspace the task is working on (if applicable)"
+            },
+            "table":{
+               "type":"string",
+               "description":"The table the task is working on (if applicable)"
+            },
+            "entity":{
+               "type":"string",
+               "description":"Task-specific entity description"
+            },
+            "progress_units":{
+               "type":"string",
+               "description":"A description of the progress units"
+            },
+            "progress_total":{
+               "type":"double",
+               "description":"The total number of units to complete for the task"
+            },
+            "progress_completed":{
+               "type":"double",
+               "description":"The number of units completed so far"
+            }
+          }
+       }
+    }
+ }

api/api-doc/task_manager_test.json

@@ -0,0 +1,185 @@
+{
+    "apiVersion":"0.0.1",
+    "swaggerVersion":"1.2",
+    "basePath":"{{Protocol}}://{{Host}}",
+    "resourcePath":"/task_manager_test",
+    "produces":[
+       "application/json"
+    ],
+    "apis":[
+       {
+          "path":"/task_manager_test/test_module",
+          "operations":[
+             {
+                "method":"POST",
+                "summary":"Register test module in task manager",
+                "type":"void",
+                "nickname":"register_test_module",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                ]
+             },
+             {
+                "method":"DELETE",
+                "summary":"Unregister test module in task manager",
+                "type":"void",
+                "nickname":"unregister_test_module",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                ]
+             }
+          ]
+       },
+       {
+          "path":"/task_manager_test/test_task",
+          "operations":[
+             {
+                "method":"POST",
+                "summary":"Register test task",
+                "type":"string",
+                "nickname":"register_test_task",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                    {
+                        "name":"task_id",
+                        "description":"The uuid of a task to register",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"shard",
+                        "description":"The shard of the task",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"long",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"parent_id",
+                        "description":"The uuid of a parent task",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"keyspace",
+                        "description":"The keyspace the task is working on",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"table",
+                        "description":"The table the task is working on",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"type",
+                        "description":"The type of the task",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    },
+                    {
+                        "name":"entity",
+                        "description":"Task-specific entity description",
+                        "required":false,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    }
+                ]
+             },
+             {
+                "method":"DELETE",
+                "summary":"Unregister test task",
+                "type":"void",
+                "nickname":"unregister_test_task",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                    {
+                        "name":"task_id",
+                        "description":"The uuid of a task to register",
+                        "required":true,
+                        "allowMultiple":false,
+                        "type":"string",
+                        "paramType":"query"
+                    }
+                ]
+             }
+          ]
+       },
+       {
+          "path":"/task_manager_test/finish_test_task/{task_id}",
+          "operations":[
+             {
+                "method":"POST",
+                "summary":"Finish test task",
+                "type":"void",
+                "nickname":"finish_test_task",
+                "produces":[
+                   "application/json"
+                ],
+                "parameters":[
+                   {
+                      "name":"task_id",
+                      "description":"The uuid of a task to finish",
+                      "required":true,
+                      "allowMultiple":false,
+                      "type":"string",
+                      "paramType":"path"
+                   },
+                   {
+                      "name":"error",
+                      "description":"The error with which task fails (if it does)",
+                      "required":false,
+                      "allowMultiple":false,
+                      "type":"string",
+                      "paramType":"query"
+                   }
+                ]
+             }
+          ]
+       },
+       {
+         "path":"/task_manager_test/ttl",
+         "operations":[
+            {
+               "method":"POST",
+               "summary":"Set ttl in seconds and get last value",
+               "type":"long",
+               "nickname":"get_and_update_ttl",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"ttl",
+                     "description":"The number of seconds for which the tasks will be kept in memory after it finishes",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"long",
+                     "paramType":"query"
+                  }
+               ]
+            }
+         ]
+      }
+    ]
+ }

api/api.cc

@@ -29,6 +29,8 @@
 #include "stream_manager.hh"
 #include "system.hh"
 #include "api/config.hh"
+#include "task_manager.hh"
+#include "task_manager_test.hh"
 
 logging::logger apilog("api");
 
@@ -146,8 +148,14 @@ future<> unset_server_snapshot(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_snapshot(ctx, r); });
 }
 
-future<> set_server_snitch(http_context& ctx) {
-    return register_api(ctx, "endpoint_snitch_info", "The endpoint snitch info API", set_endpoint_snitch);
+future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch) {
+    return register_api(ctx, "endpoint_snitch_info", "The endpoint snitch info API", [&snitch] (http_context& ctx, routes& r) {
+        set_endpoint_snitch(ctx, r, snitch);
+    });
+}
+
+future<> unset_server_snitch(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_endpoint_snitch(ctx, r); });
 }
 
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g) {
@@ -245,6 +253,30 @@ future<> set_server_done(http_context& ctx) {
     });
 }
 
+future<> set_server_task_manager(http_context& ctx) {
+    auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
+
+    return ctx.http_server.set_routes([rb, &ctx](routes& r) {
+        rb->register_function(r, "task_manager",
+                "The task manager API");
+        set_task_manager(ctx, r);
+    });
+}
+
+#ifndef SCYLLA_BUILD_MODE_RELEASE
+
+future<> set_server_task_manager_test(http_context& ctx, lw_shared_ptr<db::config> cfg) {
+    auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
+
+    return ctx.http_server.set_routes([rb, &ctx, &cfg = *cfg](routes& r) mutable {
+        rb->register_function(r, "task_manager_test",
+                "The task manager test API");
+        set_task_manager_test(ctx, r, cfg);
+    });
+}
+
+#endif
+
 void req_params::process(const request& req) {
     // Process mandatory parameters
     for (auto& [name, ent] : params) {

api/api.hh

@@ -137,6 +137,14 @@ future<json::json_return_type>  sum_timer_stats(distributed<T>& d, utils::timed_
     });
 }
 
+template<class T, class F>
+future<json::json_return_type>  sum_timer_stats(distributed<T>& d, utils::timed_rate_moving_average_summary_and_histogram F::*f) {
+    return d.map_reduce0([f](const T& p) {return (p.get_stats().*f).rate();}, utils::rate_moving_average_and_histogram(),
+            std::plus<utils::rate_moving_average_and_histogram>()).then([](const utils::rate_moving_average_and_histogram& val) {
+        return make_ready_future<json::json_return_type>(timer_to_json(val));
+    });
+}
+
 inline int64_t min_int64(int64_t a, int64_t b) {
     return std::min(a,b);
 }

api/api_init.hh

@@ -11,6 +11,7 @@
 #include <seastar/core/future.hh>
 
 #include "replica/database_fwd.hh"
+#include "tasks/task_manager.hh"
 #include "seastarx.hh"
 
 namespace service {
@@ -31,6 +32,7 @@ namespace locator {
 
 class token_metadata;
 class shared_token_metadata;
+class snitch_ptr;
 
 } // namespace locator
 
@@ -66,11 +68,12 @@ struct http_context {
     distributed<service::storage_proxy>& sp;
     service::load_meter& lmeter;
     const sharded<locator::shared_token_metadata>& shared_token_metadata;
+    sharded<tasks::task_manager>& tm;
 
     http_context(distributed<replica::database>& _db,
             distributed<service::storage_proxy>& _sp,
-            service::load_meter& _lm, const sharded<locator::shared_token_metadata>& _stm)
-            : db(_db), sp(_sp), lmeter(_lm), shared_token_metadata(_stm) {
+            service::load_meter& _lm, const sharded<locator::shared_token_metadata>& _stm, sharded<tasks::task_manager>& _tm)
+            : db(_db), sp(_sp), lmeter(_lm), shared_token_metadata(_stm), tm(_tm) {
     }
 
     const locator::token_metadata& get_token_metadata();
@@ -78,7 +81,8 @@ struct http_context {
 
 future<> set_server_init(http_context& ctx);
 future<> set_server_config(http_context& ctx, const db::config& cfg);
-future<> set_server_snitch(http_context& ctx);
+future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch);
+future<> unset_server_snitch(http_context& ctx);
 future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, sharded<gms::gossiper>& g, sharded<cdc::generation_service>& cdc_gs, sharded<db::system_keyspace>& sys_ks);
 future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader);
 future<> unset_server_sstables_loader(http_context& ctx);
@@ -107,5 +111,7 @@ future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g);
 future<> set_server_cache(http_context& ctx);
 future<> set_server_compaction_manager(http_context& ctx);
 future<> set_server_done(http_context& ctx);
+future<> set_server_task_manager(http_context& ctx);
+future<> set_server_task_manager_test(http_context& ctx, lw_shared_ptr<db::config> cfg);
 
 }

api/column_family.cc

@@ -14,7 +14,7 @@
 #include "sstables/metadata_collector.hh"
 #include "utils/estimated_histogram.hh"
 #include <algorithm>
-#include "db/system_keyspace_view_types.hh"
+#include "db/system_keyspace.hh"
 #include "db/data_listeners.hh"
 #include "storage_service.hh"
 #include "unimplemented.hh"
@@ -43,7 +43,7 @@ std::tuple<sstring, sstring> parse_fully_qualified_cf_name(sstring name) {
     return std::make_tuple(name.substr(0, pos), name.substr(end));
 }
 
-const utils::UUID& get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
+const table_id& get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
     try {
         return db.find_uuid(ks, cf);
     } catch (replica::no_such_column_family& e) {
@@ -51,7 +51,7 @@ const utils::UUID& get_uuid(const sstring& ks, const sstring& cf, const replica:
     }
 }
 
-const utils::UUID& get_uuid(const sstring& name, const replica::database& db) {
+const table_id& get_uuid(const sstring& name, const replica::database& db) {
     auto [ks, cf] = parse_fully_qualified_cf_name(name);
     return get_uuid(ks, cf, db);
 }
@@ -110,7 +110,7 @@ static future<json::json_return_type>  get_cf_stats_count(http_context& ctx,
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_and_histogram replica::column_family_stats::*f) {
-    utils::UUID uuid = get_uuid(name, ctx.db.local());
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -122,7 +122,7 @@ static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    utils::UUID uuid = get_uuid(name, ctx.db.local());
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -149,7 +149,7 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils:
 
 static future<json::json_return_type>  get_cf_rate_and_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    utils::UUID uuid = get_uuid(name, ctx.db.local());
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).rate();},
             utils::rate_moving_average_and_histogram(),
@@ -394,7 +394,7 @@ void set_column_family(http_context& ctx, routes& r) {
     cf::get_all_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
         warn(unimplemented::cause::INDEXES);
         return ctx.db.map_reduce0([](const replica::database& db){
-            return db.dirty_memory_region_group().memory_used();
+            return db.dirty_memory_region_group().real_memory_used();
         }, int64_t(0), std::plus<int64_t>()).then([](int res) {
             return make_ready_future<json::json_return_type>(res);
         });
@@ -855,7 +855,7 @@ void set_column_family(http_context& ctx, routes& r) {
     });
 
     cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
-        const utils::UUID& uuid = get_uuid(req.param["name"], ctx.db.local());
+        auto uuid = get_uuid(req.param["name"], ctx.db.local());
         replica::column_family& cf = ctx.db.local().find_column_family(uuid);
         return !cf.is_auto_compaction_disabled_by_user();
     });

api/column_family.hh

@@ -18,7 +18,7 @@ namespace api {
 
 void set_column_family(http_context& ctx, routes& r);
 
-const utils::UUID& get_uuid(const sstring& name, const replica::database& db);
+const table_id& get_uuid(const sstring& name, const replica::database& db);
 future<> foreach_column_family(http_context& ctx, const sstring& name, std::function<void(replica::column_family&)> f);
 
 
@@ -63,7 +63,7 @@ struct map_reduce_column_families_locally {
     std::function<std::unique_ptr<std::any>(std::unique_ptr<std::any>, std::unique_ptr<std::any>)> reducer;
     future<std::unique_ptr<std::any>> operator()(replica::database& db) const {
         auto res = seastar::make_lw_shared<std::unique_ptr<std::any>>(std::make_unique<std::any>(init));
-        return do_for_each(db.get_column_families(), [res, this](const std::pair<utils::UUID, seastar::lw_shared_ptr<replica::table>>& i) {
+        return do_for_each(db.get_column_families(), [res, this](const std::pair<table_id, seastar::lw_shared_ptr<replica::table>>& i) {
             *res = reducer(std::move(*res), mapper(*i.second.get()));
         }).then([res] {
             return std::move(*res);

api/compaction_manager.cc

@@ -68,7 +68,7 @@ void set_compaction_manager(http_context& ctx, routes& r) {
     cm::get_pending_tasks_by_table.set(r, [&ctx] (std::unique_ptr<request> req) {
         return ctx.db.map_reduce0([&ctx](replica::database& db) {
             return do_with(std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>(), [&ctx, &db](std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>& tasks) {
-                return do_for_each(db.get_column_families(), [&tasks](const std::pair<utils::UUID, seastar::lw_shared_ptr<replica::table>>& i) {
+                return do_for_each(db.get_column_families(), [&tasks](const std::pair<table_id, seastar::lw_shared_ptr<replica::table>>& i) {
                     replica::table& cf = *i.second.get();
                     tasks[std::make_pair(cf.schema()->ks_name(), cf.schema()->cf_name())] = cf.get_compaction_strategy().estimated_pending_compactions(cf.as_table_state());
                     return make_ready_future<>();

api/endpoint_snitch.cc

@@ -8,13 +8,15 @@
 
 #include "locator/token_metadata.hh"
 #include "locator/snitch_base.hh"
+#include "locator/production_snitch_base.hh"
 #include "endpoint_snitch.hh"
 #include "api/api-doc/endpoint_snitch_info.json.hh"
+#include "api/api-doc/storage_service.json.hh"
 #include "utils/fb_utilities.hh"
 
 namespace api {
 
-void set_endpoint_snitch(http_context& ctx, routes& r) {
+void set_endpoint_snitch(http_context& ctx, routes& r, sharded<locator::snitch_ptr>& snitch) {
     static auto host_or_broadcast = [](const_req req) {
         auto host = req.get_query_param("host");
         return host.empty() ? gms::inet_address(utils::fb_utilities::get_broadcast_address()) : gms::inet_address(host);
@@ -22,17 +24,45 @@ void set_endpoint_snitch(http_context& ctx, routes& r) {
 
     httpd::endpoint_snitch_info_json::get_datacenter.set(r, [&ctx](const_req req) {
         auto& topology = ctx.shared_token_metadata.local().get()->get_topology();
-        return topology.get_datacenter(host_or_broadcast(req));
+        auto ep = host_or_broadcast(req);
+        if (!topology.has_endpoint(ep, locator::topology::pending::yes)) {
+            // Cannot return error here, nodetool status can race, request
+            // info about just-left node and not handle it nicely
+            return sstring(locator::production_snitch_base::default_dc);
+        }
+        return topology.get_datacenter(ep);
     });
 
     httpd::endpoint_snitch_info_json::get_rack.set(r, [&ctx](const_req req) {
         auto& topology = ctx.shared_token_metadata.local().get()->get_topology();
-        return topology.get_rack(host_or_broadcast(req));
+        auto ep = host_or_broadcast(req);
+        if (!topology.has_endpoint(ep, locator::topology::pending::yes)) {
+            // Cannot return error here, nodetool status can race, request
+            // info about just-left node and not handle it nicely
+            return sstring(locator::production_snitch_base::default_rack);
+        }
+        return topology.get_rack(ep);
     });
 
-    httpd::endpoint_snitch_info_json::get_snitch_name.set(r, [] (const_req req) {
-        return locator::i_endpoint_snitch::get_local_snitch_ptr()->get_name();
+    httpd::endpoint_snitch_info_json::get_snitch_name.set(r, [&snitch] (const_req req) {
+        return snitch.local()->get_name();
     });
+
+    httpd::storage_service_json::update_snitch.set(r, [&snitch](std::unique_ptr<request> req) {
+        locator::snitch_config cfg;
+        cfg.name = req->get_query_param("ep_snitch_class_name");
+        return locator::i_endpoint_snitch::reset_snitch(snitch, cfg).then([] {
+            return make_ready_future<json::json_return_type>(json::json_void());
+        });
+    });
+
+}
+
+void unset_endpoint_snitch(http_context& ctx, routes& r) {
+    httpd::endpoint_snitch_info_json::get_datacenter.unset(r);
+    httpd::endpoint_snitch_info_json::get_rack.unset(r);
+    httpd::endpoint_snitch_info_json::get_snitch_name.unset(r);
+    httpd::storage_service_json::update_snitch.unset(r);
 }
 
 }

api/endpoint_snitch.hh

@@ -10,8 +10,13 @@
 
 #include "api.hh"
 
+namespace locator {
+class snitch_ptr;
+}
+
 namespace api {
 
-void set_endpoint_snitch(http_context& ctx, routes& r);
+void set_endpoint_snitch(http_context& ctx, routes& r, sharded<locator::snitch_ptr>&);
+void unset_endpoint_snitch(http_context& ctx, routes& r);
 
 }

api/storage_proxy.cc

@@ -22,6 +22,9 @@ namespace sp = httpd::storage_proxy_json;
 using proxy = service::storage_proxy;
 using namespace json;
 
+utils::time_estimated_histogram timed_rate_moving_average_summary_merge(utils::time_estimated_histogram a, const utils::timed_rate_moving_average_summary_and_histogram& b) {
+    return a.merge(b.histogram());
+}
 
 /**
  * This function implement a two dimentional map reduce where
@@ -55,10 +58,10 @@ future<V> two_dimensional_map_reduce(distributed<service::storage_proxy>& d,
  * @param initial_value - the initial value to use for both aggregations* @return
  * @return A future that resolves to the result of the aggregation.
  */
-template<typename V, typename Reducer, typename F>
+template<typename V, typename Reducer, typename F, typename C>
 future<V> two_dimensional_map_reduce(distributed<service::storage_proxy>& d,
-        V F::*f, Reducer reducer, V initial_value) {
-    return two_dimensional_map_reduce(d, [f] (F& stats) {
+        C F::*f, Reducer reducer, V initial_value) {
+    return two_dimensional_map_reduce(d, [f] (F& stats) -> V {
         return stats.*f;
     }, reducer, initial_value);
 }
@@ -112,10 +115,10 @@ utils_json::estimated_histogram time_to_json_histogram(const utils::time_estimat
     return res;
 }
 
-static future<json::json_return_type>  sum_estimated_histogram(http_context& ctx, utils::time_estimated_histogram service::storage_proxy_stats::stats::*f) {
-
-    return two_dimensional_map_reduce(ctx.sp, f, utils::time_estimated_histogram_merge,
-            utils::time_estimated_histogram()).then([](const utils::time_estimated_histogram& val) {
+static future<json::json_return_type>  sum_estimated_histogram(http_context& ctx, utils::timed_rate_moving_average_summary_and_histogram service::storage_proxy_stats::stats::*f) {
+    return two_dimensional_map_reduce(ctx.sp, [f] (service::storage_proxy_stats::stats& stats) {
+        return (stats.*f).histogram();
+    }, utils::time_estimated_histogram_merge, utils::time_estimated_histogram()).then([](const utils::time_estimated_histogram& val) {
         return make_ready_future<json::json_return_type>(time_to_json_histogram(val));
     });
 }
@@ -130,7 +133,7 @@ static future<json::json_return_type>  sum_estimated_histogram(http_context& ctx
     });
 }
 
-static future<json::json_return_type>  total_latency(http_context& ctx, utils::timed_rate_moving_average_and_histogram service::storage_proxy_stats::stats::*f) {
+static future<json::json_return_type>  total_latency(http_context& ctx, utils::timed_rate_moving_average_summary_and_histogram service::storage_proxy_stats::stats::*f) {
     return two_dimensional_map_reduce(ctx.sp, [f] (service::storage_proxy_stats::stats& stats) {
             return (stats.*f).hist.mean * (stats.*f).hist.count;
         }, std::plus<double>(), 0.0).then([](double val) {
@@ -150,7 +153,7 @@ static future<json::json_return_type>  total_latency(http_context& ctx, utils::t
 template<typename F>
 future<json::json_return_type>
 sum_histogram_stats_storage_proxy(distributed<proxy>& d,
-        utils::timed_rate_moving_average_and_histogram F::*f) {
+        utils::timed_rate_moving_average_summary_and_histogram F::*f) {
     return two_dimensional_map_reduce(d, [f] (service::storage_proxy_stats::stats& stats) {
         return (stats.*f).hist;
     }, std::plus<utils::ihistogram>(), utils::ihistogram()).
@@ -170,7 +173,7 @@ sum_histogram_stats_storage_proxy(distributed<proxy>& d,
 template<typename F>
 future<json::json_return_type>
 sum_timer_stats_storage_proxy(distributed<proxy>& d,
-        utils::timed_rate_moving_average_and_histogram F::*f) {
+        utils::timed_rate_moving_average_summary_and_histogram F::*f) {
 
     return two_dimensional_map_reduce(d, [f] (service::storage_proxy_stats::stats& stats) {
         return (stats.*f).rate();
@@ -491,14 +494,14 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_se
     });
 
     sp::get_read_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
-        return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::estimated_read);
+        return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::read);
     });
 
     sp::get_read_latency.set(r, [&ctx](std::unique_ptr<request> req) {
         return total_latency(ctx, &service::storage_proxy_stats::stats::read);
     });
     sp::get_write_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
-        return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::estimated_write);
+        return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::write);
     });
 
     sp::get_write_latency.set(r, [&ctx](std::unique_ptr<request> req) {

api/storage_service.cc

@@ -69,6 +69,11 @@ sstring validate_keyspace(http_context& ctx, const parameters& param) {
     return validate_keyspace(ctx, param["keyspace"]);
 }
 
+locator::host_id validate_host_id(const sstring& param) {
+    auto hoep = locator::host_id_or_endpoint(param, locator::host_id_or_endpoint::param_type::host_id);
+    return hoep.id;
+}
+
 // splits a request parameter assumed to hold a comma-separated list of table names
 // verify that the tables are found, otherwise a bad_param_exception exception is thrown
 // containing the description of the respective no_such_column_family error.
@@ -548,7 +553,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
 
     ss::describe_any_ring.set(r, [&ctx, &ss](std::unique_ptr<request> req) {
         // Find an arbitrary non-system keyspace.
-        auto keyspaces = ctx.db.local().get_non_system_keyspaces();
+        auto keyspaces = ctx.db.local().get_non_local_strategy_keyspaces();
         if (keyspaces.empty()) {
             throw std::runtime_error("No keyspace provided and no non system kespace exist");
         }
@@ -611,11 +616,11 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
             column_families = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
         }
         return ctx.db.invoke_on_all([keyspace, column_families] (replica::database& db) -> future<> {
-            auto table_ids = boost::copy_range<std::vector<utils::UUID>>(column_families | boost::adaptors::transformed([&] (auto& cf_name) {
+            auto table_ids = boost::copy_range<std::vector<table_id>>(column_families | boost::adaptors::transformed([&] (auto& cf_name) {
                 return db.find_uuid(keyspace, cf_name);
             }));
             // major compact smaller tables first, to increase chances of success if low on space.
-            std::ranges::sort(table_ids, std::less<>(), [&] (const utils::UUID& id) {
+            std::ranges::sort(table_ids, std::less<>(), [&] (const table_id& id) {
                 return db.find_column_family(id).get_stats().live_disk_space_used;
             });
             // as a table can be dropped during loop below, let's find it before issuing major compaction request.
@@ -641,11 +646,11 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
                         std::runtime_error("Can not perform cleanup operation when topology changes"));
             }
             return ctx.db.invoke_on_all([keyspace, column_families] (replica::database& db) -> future<> {
-                auto table_ids = boost::copy_range<std::vector<utils::UUID>>(column_families | boost::adaptors::transformed([&] (auto& table_name) {
+                auto table_ids = boost::copy_range<std::vector<table_id>>(column_families | boost::adaptors::transformed([&] (auto& table_name) {
                     return db.find_uuid(keyspace, table_name);
                 }));
                 // cleanup smaller tables first, to increase chances of success if low on space.
-                std::ranges::sort(table_ids, std::less<>(), [&] (const utils::UUID& id) {
+                std::ranges::sort(table_ids, std::less<>(), [&] (const table_id& id) {
                     return db.find_column_family(id).get_stats().live_disk_space_used;
                 });
                 auto& cm = db.get_compaction_manager();
@@ -691,11 +696,11 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     ss::force_keyspace_flush.set(r, [&ctx](std::unique_ptr<request> req) -> future<json::json_return_type> {
         auto keyspace = validate_keyspace(ctx, req->param);
         auto column_families = parse_tables(keyspace, ctx, req->query_parameters, "cf");
-        auto &db = ctx.db.local();
+        auto& db = ctx.db;
         if (column_families.empty()) {
-            co_await db.flush_on_all(keyspace);
+            co_await replica::database::flush_keyspace_on_all_shards(db, keyspace);
         } else {
-            co_await db.flush_on_all(keyspace, std::move(column_families));
+            co_await replica::database::flush_tables_on_all_shards(db, keyspace, std::move(column_families));
         }
         co_return json_void();
     });
@@ -715,20 +720,23 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::remove_node.set(r, [&ss](std::unique_ptr<request> req) {
-        auto host_id = req->get_query_param("host_id");
+        auto host_id = validate_host_id(req->get_query_param("host_id"));
         std::vector<sstring> ignore_nodes_strs= split(req->get_query_param("ignore_nodes"), ",");
-        auto ignore_nodes = std::list<gms::inet_address>();
+        auto ignore_nodes = std::list<locator::host_id_or_endpoint>();
         for (std::string n : ignore_nodes_strs) {
             try {
                 std::replace(n.begin(), n.end(), '\"', ' ');
                 std::replace(n.begin(), n.end(), '\'', ' ');
                 boost::trim_all(n);
                 if (!n.empty()) {
-                    auto node = gms::inet_address(n);
-                    ignore_nodes.push_back(node);
+                    auto hoep = locator::host_id_or_endpoint(n);
+                    if (!ignore_nodes.empty() && hoep.has_host_id() != ignore_nodes.front().has_host_id()) {
+                        throw std::runtime_error("All nodes should be identified using the same method: either Host IDs or ip addresses.");
+                    }
+                    ignore_nodes.push_back(std::move(hoep));
                 }
             } catch (...) {
-                throw std::runtime_error(format("Failed to parse ignore_nodes parameter: ignore_nodes={}, node={}", ignore_nodes_strs, n));
+                throw std::runtime_error(format("Failed to parse ignore_nodes parameter: ignore_nodes={}, node={}: {}", ignore_nodes_strs, n, std::current_exception()));
             }
         }
         return ss.local().removenode(host_id, std::move(ignore_nodes)).then([] {
@@ -806,21 +814,11 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         if (type == "user") {
             return ctx.db.local().get_user_keyspaces();
         } else if (type == "non_local_strategy") {
-            return map_keys(ctx.db.local().get_keyspaces() | boost::adaptors::filtered([](const auto& p) {
-                return p.second.get_replication_strategy().get_type() != locator::replication_strategy_type::local;
-            }));
+            return ctx.db.local().get_non_local_strategy_keyspaces();
         }
         return map_keys(ctx.db.local().get_keyspaces());
     });
 
-    ss::update_snitch.set(r, [](std::unique_ptr<request> req) {
-        locator::snitch_config cfg;
-        cfg.name = req->get_query_param("ep_snitch_class_name");
-        return locator::i_endpoint_snitch::reset_snitch(cfg).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
-    });
-
     ss::stop_gossiping.set(r, [&ss](std::unique_ptr<request> req) {
         return ss.local().stop_gossiping().then([] {
             return make_ready_future<json::json_return_type>(json_void());
@@ -1314,7 +1312,7 @@ void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_
         });
     });
 
-    ss::take_snapshot.set(r, [&snap_ctl](std::unique_ptr<request> req) -> future<json::json_return_type> {
+    ss::take_snapshot.set(r, [&ctx, &snap_ctl](std::unique_ptr<request> req) -> future<json::json_return_type> {
         apilog.info("take_snapshot: {}", req->query_parameters);
         auto tag = req->get_query_param("tag");
         auto column_families = split(req->get_query_param("cf"), ",");
@@ -1332,7 +1330,13 @@ void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_
                 if (keynames.size() > 1) {
                     throw httpd::bad_param_exception("Only one keyspace allowed when specifying a column family");
                 }
-                co_await snap_ctl.local().take_column_family_snapshot(keynames[0], column_families, tag, sf);
+                for (const auto& table_name : column_families) {
+                    auto& t = ctx.db.local().find_column_family(keynames[0], table_name);
+                    if (t.schema()->is_view()) {
+                        throw std::invalid_argument("Do not take a snapshot of a materialized view or a secondary index by itself. Run snapshot on the base table instead.");
+                    }
+                }
+                co_await snap_ctl.local().take_column_family_snapshot(keynames[0], column_families, tag, db::snapshot_ctl::snap_views::yes, sf);
             }
             co_return json_void();
         } catch (...) {
@@ -1402,7 +1406,10 @@ void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_
         if (!req_param<bool>(*req, "disable_snapshot", false)) {
             auto tag = format("pre-scrub-{:d}", db_clock::now().time_since_epoch().count());
             f = parallel_for_each(column_families, [&snap_ctl, keyspace, tag](sstring cf) {
-                return snap_ctl.local().take_column_family_snapshot(keyspace, cf, tag, db::snapshot_ctl::skip_flush::no, db::snapshot_ctl::allow_view_snapshots::yes);
+                // We always pass here db::snapshot_ctl::snap_views::no since:
+                // 1. When scrubbing particular tables, there's no need to auto-snapshot their views.
+                // 2. When scrubbing the whole keyspace, column_families will contain both base tables and views.
+                return snap_ctl.local().take_column_family_snapshot(keyspace, cf, tag, db::snapshot_ctl::snap_views::no, db::snapshot_ctl::skip_flush::no);
             });
         }
 

api/system.cc

@@ -61,6 +61,16 @@ void set_system(http_context& ctx, routes& r) {
         return json::json_void();
     });
 
+    hs::write_log_message.set(r, [](const_req req) {
+        try {
+            logging::log_level level = boost::lexical_cast<logging::log_level>(std::string(req.get_query_param("level")));
+            apilog.log(level, "/system/log: {}", std::string(req.get_query_param("message")));
+        } catch (boost::bad_lexical_cast& e) {
+            throw bad_param_exception("Unknown logging level " + req.get_query_param("level"));
+        }
+        return json::json_void();
+    });
+
     hs::drop_sstable_caches.set(r, [&ctx](std::unique_ptr<request> req) {
         apilog.info("Dropping sstable caches");
         return ctx.db.invoke_on_all([] (replica::database& db) {

api/task_manager.cc

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#include <seastar/core/coroutine.hh>
+
+#include "task_manager.hh"
+#include "api/api-doc/task_manager.json.hh"
+#include "db/system_keyspace.hh"
+#include "column_family.hh"
+#include "unimplemented.hh"
+#include "storage_service.hh"
+
+#include <utility>
+#include <boost/range/adaptors.hpp>
+
+namespace api {
+
+namespace tm = httpd::task_manager_json;
+using namespace json;
+
+inline bool filter_tasks(tasks::task_manager::task_ptr task, std::unordered_map<sstring, sstring>& query_params) {
+    return (!query_params.contains("keyspace") || query_params["keyspace"] == task->get_status().keyspace) &&
+        (!query_params.contains("table") || query_params["table"] == task->get_status().table);
+}
+
+struct full_task_status {
+    tasks::task_manager::task::status task_status;
+    tasks::task_manager::task::progress progress;
+    std::string module;
+    tasks::task_id parent_id;
+    tasks::is_abortable abortable;
+};
+
+struct task_stats {
+    task_stats(tasks::task_manager::task_ptr task) : task_id(task->id().to_sstring()), state(task->get_status().state) {}
+
+    sstring task_id;
+    tasks::task_manager::task_state state;
+};
+
+tm::task_status make_status(full_task_status status) {
+    auto start_time = db_clock::to_time_t(status.task_status.start_time);
+    auto end_time = db_clock::to_time_t(status.task_status.end_time);
+    ::tm st, et;
+    ::gmtime_r(&end_time, &et);
+    ::gmtime_r(&start_time, &st);
+
+    tm::task_status res{};
+    res.id = status.task_status.id.to_sstring();
+    res.type = status.task_status.type;
+    res.state = status.task_status.state;
+    res.is_abortable = bool(status.abortable);
+    res.start_time = st;
+    res.end_time = et;
+    res.error = status.task_status.error;
+    res.parent_id = status.parent_id.to_sstring();
+    res.sequence_number = status.task_status.sequence_number;
+    res.shard = status.task_status.shard;
+    res.keyspace = status.task_status.keyspace;
+    res.table = status.task_status.table;
+    res.entity = status.task_status.entity;
+    res.progress_units = status.task_status.progress_units;
+    res.progress_total = status.progress.total;
+    res.progress_completed = status.progress.completed;
+    return res;
+}
+
+future<json::json_return_type> retrieve_status(tasks::task_manager::foreign_task_ptr task) {
+    if (task.get() == nullptr) {
+        co_return coroutine::return_exception(httpd::bad_param_exception("Task not found"));
+    }
+    auto progress = co_await task->get_progress();
+    full_task_status s;
+    s.task_status = task->get_status();
+    s.parent_id = task->get_parent_id();
+    s.abortable = task->is_abortable();
+    s.module = task->get_module_name();
+    s.progress.completed = progress.completed;
+    s.progress.total = progress.total;
+    co_return make_status(s);
+}
+
+void set_task_manager(http_context& ctx, routes& r) {
+    tm::get_modules.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        std::vector<std::string> v = boost::copy_range<std::vector<std::string>>(ctx.tm.local().get_modules() | boost::adaptors::map_keys);
+        co_return v;
+    });
+
+    tm::get_tasks.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        using chunked_stats = utils::chunked_vector<task_stats>;
+        auto internal = tasks::is_internal{req_param<bool>(*req, "internal", false)};
+        std::vector<chunked_stats> res = co_await ctx.tm.map([&req, internal] (tasks::task_manager& tm) {
+            chunked_stats local_res;
+            auto module = tm.find_module(req->param["module"]);
+            const auto& filtered_tasks = module->get_tasks() | boost::adaptors::filtered([&params = req->query_parameters, internal] (const auto& task) {
+                return (internal || !task.second->is_internal()) && filter_tasks(task.second, params);
+            });
+            for (auto& [task_id, task] : filtered_tasks) {
+                local_res.push_back(task_stats{task});
+            }
+            return local_res;
+        });
+
+        std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
+            auto s = std::move(os);
+            auto res = std::move(r);
+            co_await s.write("[");
+            std::string delim = "";
+            for (auto& v: res) {
+                for (auto& stats: v) {
+                    co_await s.write(std::exchange(delim, ", "));
+                    tm::task_stats ts;
+                    ts = stats;
+                    co_await formatter::write(s, ts);
+                }
+            }
+            co_await s.write("]");
+            co_await s.close();
+        };
+        co_return std::move(f);
+    });
+
+    tm::get_task_status.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto task = co_await tasks::task_manager::invoke_on_task(ctx.tm, id, std::function([] (tasks::task_manager::task_ptr task) -> future<tasks::task_manager::foreign_task_ptr> {
+            auto state = task->get_status().state;
+            if (state == tasks::task_manager::task_state::done || state == tasks::task_manager::task_state::failed) {
+                task->unregister_task();
+            }
+            co_return std::move(task);
+        }));
+        co_return co_await retrieve_status(std::move(task));
+    });
+
+    tm::abort_task.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        co_await tasks::task_manager::invoke_on_task(ctx.tm, id, [] (tasks::task_manager::task_ptr task) -> future<> {
+            if (!task->is_abortable()) {
+                co_await coroutine::return_exception(std::runtime_error("Requested task cannot be aborted"));
+            }
+            co_await task->abort();
+        });
+        co_return json_void();
+    });
+
+    tm::wait_task.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto task = co_await tasks::task_manager::invoke_on_task(ctx.tm, id, std::function([] (tasks::task_manager::task_ptr task) {
+            return task->done().then_wrapped([task] (auto f) {
+                task->unregister_task();
+                f.get();
+                return make_foreign(task);
+            });
+        }));
+        co_return co_await retrieve_status(std::move(task));
+    });
+}
+
+}

api/task_manager.hh

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#pragma once
+
+#include "api.hh"
+
+namespace api {
+
+void set_task_manager(http_context& ctx, routes& r);
+
+}

api/task_manager_test.cc

@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#ifndef SCYLLA_BUILD_MODE_RELEASE
+
+#include <seastar/core/coroutine.hh>
+
+#include "task_manager_test.hh"
+#include "api/api-doc/task_manager_test.json.hh"
+#include "tasks/test_module.hh"
+
+namespace api {
+
+namespace tmt = httpd::task_manager_test_json;
+using namespace json;
+
+void set_task_manager_test(http_context& ctx, routes& r, db::config& cfg) {
+    tmt::register_test_module.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        co_await ctx.tm.invoke_on_all([] (tasks::task_manager& tm) {
+            auto m = make_shared<tasks::test_module>(tm);
+            tm.register_module("test", m);
+        });
+        co_return json_void();
+    });
+
+    tmt::unregister_test_module.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        co_await ctx.tm.invoke_on_all([] (tasks::task_manager& tm) -> future<> {
+            auto module_name = "test";
+            auto module = tm.find_module(module_name);
+            co_await module->stop();
+        });
+        co_return json_void();
+    });
+
+    tmt::register_test_task.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        sharded<tasks::task_manager>& tms = ctx.tm;
+        auto it = req->query_parameters.find("task_id");
+        auto id = it != req->query_parameters.end() ? tasks::task_id{utils::UUID{it->second}} : tasks::task_id::create_null_id();
+        it = req->query_parameters.find("shard");
+        unsigned shard = it != req->query_parameters.end() ? boost::lexical_cast<unsigned>(it->second) : 0;
+        it = req->query_parameters.find("keyspace");
+        std::string keyspace = it != req->query_parameters.end() ? it->second : "";
+        it = req->query_parameters.find("table");
+        std::string table = it != req->query_parameters.end() ? it->second : "";
+        it = req->query_parameters.find("type");
+        std::string type = it != req->query_parameters.end() ? it->second : "";
+        it = req->query_parameters.find("entity");
+        std::string entity = it != req->query_parameters.end() ? it->second : "";
+        it = req->query_parameters.find("parent_id");
+        tasks::task_info data;
+        if (it != req->query_parameters.end()) {
+            data.id = tasks::task_id{utils::UUID{it->second}};
+            auto parent_ptr = co_await tasks::task_manager::lookup_task_on_all_shards(ctx.tm, data.id);
+            data.shard = parent_ptr->get_status().shard;
+        }
+
+        auto module = tms.local().find_module("test");
+        id = co_await module->make_task<tasks::test_task_impl>(shard, id, keyspace, table, type, entity, data);
+        co_await tms.invoke_on(shard, [id] (tasks::task_manager& tm) {
+            auto it = tm.get_all_tasks().find(id);
+            if (it != tm.get_all_tasks().end()) {
+                it->second->start();
+            }
+        });
+        co_return id.to_sstring();
+    });
+
+    tmt::unregister_test_task.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        auto id = tasks::task_id{utils::UUID{req->query_parameters["task_id"]}};
+        co_await tasks::task_manager::invoke_on_task(ctx.tm, id, [] (tasks::task_manager::task_ptr task) -> future<> {
+            tasks::test_task test_task{task};
+            co_await test_task.unregister_task();
+        });
+        co_return json_void();
+    });
+
+    tmt::finish_test_task.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto it = req->query_parameters.find("error");
+        bool fail = it != req->query_parameters.end();
+        std::string error = fail ? it->second : "";
+
+        co_await tasks::task_manager::invoke_on_task(ctx.tm, id, [fail, error = std::move(error)] (tasks::task_manager::task_ptr task) {
+            tasks::test_task test_task{task};
+            if (fail) {
+                test_task.finish_failed(std::make_exception_ptr(std::runtime_error(error)));
+            } else {
+                test_task.finish();
+            }
+            return make_ready_future<>();
+        });
+        co_return json_void();
+    });
+
+    tmt::get_and_update_ttl.set(r, [&ctx, &cfg] (std::unique_ptr<request> req) -> future<json::json_return_type> {
+        uint32_t ttl = cfg.task_ttl_seconds();
+        cfg.task_ttl_seconds.set(boost::lexical_cast<uint32_t>(req->query_parameters["ttl"]));
+        co_return json::json_return_type(ttl);
+    });
+}
+
+}
+
+#endif

api/task_manager_test.hh

@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#ifndef SCYLLA_BUILD_MODE_RELEASE
+
+#pragma once
+
+#include "api.hh"
+#include "db/config.hh"
+
+namespace api {
+
+void set_task_manager_test(http_context& ctx, routes& r, db::config& cfg);
+
+}
+
+#endif

build_mode.hh

@@ -0,0 +1,59 @@
+
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+#pragma once
+
+#ifndef SCYLLA_BUILD_MODE
+#error SCYLLA_BUILD_MODE must be defined
+#endif
+
+#ifndef STRINGIFY
+// We need to levels of indirection
+// to make a string out of the macro name.
+// The outer level expands the macro
+// and the inner level makes a string out of the expanded macro.
+#define STRINGIFY_VALUE(x) #x
+#define STRINGIFY_MACRO(x) STRINGIFY_VALUE(x)
+#endif
+
+#define SCYLLA_BUILD_MODE_STR STRINGIFY_MACRO(SCYLLA_BUILD_MODE)
+
+// We use plain macro definitions
+// so the preprocessor can expand them
+// inline in the #if directives below
+#define SCYLLA_BUILD_MODE_CODE_debug 0
+#define SCYLLA_BUILD_MODE_CODE_release 1
+#define SCYLLA_BUILD_MODE_CODE_dev 2
+#define SCYLLA_BUILD_MODE_CODE_sanitize 3
+#define SCYLLA_BUILD_MODE_CODE_coverage 4
+
+#define _SCYLLA_BUILD_MODE_CODE(sbm) SCYLLA_BUILD_MODE_CODE_ ## sbm
+#define SCYLLA_BUILD_MODE_CODE(sbm) _SCYLLA_BUILD_MODE_CODE(sbm)
+
+#if SCYLLA_BUILD_MODE_CODE(SCYLLA_BUILD_MODE) == SCYLLA_BUILD_MODE_CODE_debug
+#define SCYLLA_BUILD_MODE_DEBUG
+#elif SCYLLA_BUILD_MODE_CODE(SCYLLA_BUILD_MODE) == SCYLLA_BUILD_MODE_CODE_release
+#define SCYLLA_BUILD_MODE_RELEASE
+#elif SCYLLA_BUILD_MODE_CODE(SCYLLA_BUILD_MODE) == SCYLLA_BUILD_MODE_CODE_dev
+#define SCYLLA_BUILD_MODE_DEV
+#elif SCYLLA_BUILD_MODE_CODE(SCYLLA_BUILD_MODE) == SCYLLA_BUILD_MODE_CODE_sanitize
+#define SCYLLA_BUILD_MODE_SANITIZE
+#elif SCYLLA_BUILD_MODE_CODE(SCYLLA_BUILD_MODE) == SCYLLA_BUILD_MODE_CODE_coverage
+#define SCYLLA_BUILD_MODE_COVERAGE
+#else
+#error unrecognized SCYLLA_BUILD_MODE
+#endif
+
+#if (defined(SCYLLA_BUILD_MODE_RELEASE) || defined(SCYLLA_BUILD_MODE_DEV)) && defined(SEASTAR_DEBUG)
+#error SEASTAR_DEBUG is not expected to be defined when SCYLLA_BUILD_MODE is "release" or "dev"
+#endif
+
+#if (defined(SCYLLA_BUILD_MODE_DEBUG) || defined(SCYLLA_BUILD_MODE_SANITIZE)) && !defined(SEASTAR_DEBUG)
+#error SEASTAR_DEBUG is expected to be defined when SCYLLA_BUILD_MODE is "debug" or "sanitize"
+#endif

cache_flat_mutation_reader.hh

@@ -737,7 +737,9 @@ void cache_flat_mutation_reader::maybe_drop_last_entry() noexcept {
             && _snp->at_oldest_version()) {
 
         with_allocator(_snp->region().allocator(), [&] {
-            _last_row->on_evicted(_read_context.cache()._tracker);
+            cache_tracker& tracker = _read_context.cache()._tracker;
+            tracker.get_lru().remove(*_last_row);
+            _last_row->on_evicted(tracker);
         });
         _last_row = nullptr;
 

canonical_mutation.cc

@@ -15,14 +15,7 @@
 #include "converting_mutation_partition_applier.hh"
 #include "hashing_partition_visitor.hh"
 #include "utils/UUID.hh"
-#include "serializer.hh"
-#include "idl/uuid.dist.hh"
-#include "idl/keys.dist.hh"
 #include "idl/mutation.dist.hh"
-#include "serializer_impl.hh"
-#include "serialization_visitors.hh"
-#include "idl/uuid.dist.impl.hh"
-#include "idl/keys.dist.impl.hh"
 #include "idl/mutation.dist.impl.hh"
 #include <iostream>
 
@@ -44,7 +37,7 @@ canonical_mutation::canonical_mutation(const mutation& m)
                  }).end_canonical_mutation();
 }
 
-utils::UUID canonical_mutation::column_family_id() const {
+table_id canonical_mutation::column_family_id() const {
     auto in = ser::as_input_stream(_data);
     auto mv = ser::deserialize(in, boost::type<ser::canonical_mutation_view>());
     return mv.table_id();
@@ -120,17 +113,19 @@ std::ostream& operator<<(std::ostream& os, const canonical_mutation& cm) {
             auto&& entry = _cm.static_column_at(id);
             fmt::print(_os, "static column {} {}", bytes_to_text(entry.name()), collection_mutation_view::printer(*entry.type(), cmv));
         }
-        virtual void accept_row_tombstone(range_tombstone rt) override {
+        virtual stop_iteration accept_row_tombstone(range_tombstone rt) override {
             print_separator();
             fmt::print(_os, "row tombstone {}", rt);
+            return stop_iteration::no;
         }
-        virtual void accept_row(position_in_partition_view pipv, row_tombstone rt, row_marker rm, is_dummy, is_continuous) override {
+        virtual stop_iteration accept_row(position_in_partition_view pipv, row_tombstone rt, row_marker rm, is_dummy, is_continuous) override {
             if (_in_row) {
                 fmt::print(_os, "}}, ");
             }
             fmt::print(_os, "{{row {} tombstone {} marker {}", pipv, rt, rm);
             _in_row = true;
             _first = false;
+            return stop_iteration::no;
         }
         virtual void accept_row_cell(column_id id, atomic_cell ac) override {
             print_separator();

canonical_mutation.hh

@@ -14,10 +14,6 @@
 #include "bytes_ostream.hh"
 #include <iosfwd>
 
-namespace utils {
-    class UUID;
-} // namespace utils
-
 // Immutable mutation form which can be read using any schema version of the same table.
 // Safe to access from other shards via const&.
 // Safe to pass serialized across nodes.
@@ -39,7 +35,7 @@ public:
     // is not intended, user should sync the schema first.
     mutation to_mutation(schema_ptr) const;
 
-    utils::UUID column_family_id() const;
+    table_id column_family_id() const;
 
     const bytes_ostream& representation() const { return _data; }
 

cdc/generation.cc

@@ -455,7 +455,12 @@ static future<> update_streams_description(
                     noncopyable_function<unsigned()> get_num_token_owners,
                     abort_source& abort_src) -> future<> {
             while (true) {
-                co_await sleep_abortable(std::chrono::seconds(60), abort_src);
+                try {
+                    co_await sleep_abortable(std::chrono::seconds(60), abort_src);
+                } catch (seastar::sleep_aborted&) {
+                    cdc_log.warn( "Aborted update CDC description table with generation {}", gen_id);
+                    co_return;
+                }
                 try {
                     co_await do_update_streams_description(gen_id, *sys_dist_ks, { get_num_token_owners() });
                     co_return;
@@ -578,7 +583,7 @@ future<> generation_service::maybe_rewrite_streams_descriptions() {
         co_return;
     }
 
-    if (co_await db::system_keyspace::cdc_is_rewritten()) {
+    if (co_await _sys_ks.local().cdc_is_rewritten()) {
         co_return;
     }
 
@@ -602,13 +607,13 @@ future<> generation_service::maybe_rewrite_streams_descriptions() {
             continue;
         }
 
-        times_and_ttls.push_back(time_and_ttl{as_timepoint(s.id()), cdc_opts.ttl()});
+        times_and_ttls.push_back(time_and_ttl{as_timepoint(s.id().uuid()), cdc_opts.ttl()});
     }
 
     if (times_and_ttls.empty()) {
         // There's no point in rewriting old generations' streams (they don't contain any data).
         cdc_log.info("No CDC log tables present, not rewriting stream tables.");
-        co_return co_await db::system_keyspace::cdc_set_rewritten(std::nullopt);
+        co_return co_await _sys_ks.local().cdc_set_rewritten(std::nullopt);
     }
 
     auto get_num_token_owners = [tm = _token_metadata.get()] { return tm->count_normal_token_owners(); };
@@ -626,7 +631,7 @@ future<> generation_service::maybe_rewrite_streams_descriptions() {
             std::move(get_num_token_owners),
             _abort_src);
 
-    co_await db::system_keyspace::cdc_set_rewritten(last_rewritten);
+    co_await _sys_ks.local().cdc_set_rewritten(last_rewritten);
 }
 
 static void assert_shard_zero(const sstring& where) {
@@ -870,7 +875,7 @@ future<> generation_service::check_and_repair_cdc_streams() {
             { gms::application_state::CDC_GENERATION_ID, gms::versioned_value::cdc_generation_id(new_gen_id) },
             { gms::application_state::STATUS, *status }
     });
-    co_await db::system_keyspace::update_cdc_generation_id(new_gen_id);
+    co_await _sys_ks.local().update_cdc_generation_id(new_gen_id);
 }
 
 future<> generation_service::handle_cdc_generation(std::optional<cdc::generation_id> gen_id) {
@@ -1013,7 +1018,7 @@ future<bool> generation_service::do_handle_cdc_generation(cdc::generation_id gen
     // The assumption follows from the requirement of bootstrapping nodes sequentially.
     if (!_gen_id || get_ts(*_gen_id) < get_ts(gen_id)) {
         _gen_id = gen_id;
-        co_await db::system_keyspace::update_cdc_generation_id(gen_id);
+        co_await _sys_ks.local().update_cdc_generation_id(gen_id);
         co_await _gossiper.add_local_application_state(
                 gms::application_state::CDC_GENERATION_ID, gms::versioned_value::cdc_generation_id(gen_id));
     }

cdc/log.cc

@@ -59,7 +59,7 @@ using namespace std::chrono_literals;
 logging::logger cdc_log("cdc");
 
 namespace cdc {
-static schema_ptr create_log_schema(const schema&, std::optional<utils::UUID> = {}, schema_ptr = nullptr);
+static schema_ptr create_log_schema(const schema&, std::optional<table_id> = {}, schema_ptr = nullptr);
 }
 
 static constexpr auto cdc_group_name = "cdc";
@@ -485,7 +485,7 @@ bytes log_data_column_deleted_elements_name_bytes(const bytes& column_name) {
     return to_bytes(cdc_deleted_elements_column_prefix) + column_name;
 }
 
-static schema_ptr create_log_schema(const schema& s, std::optional<utils::UUID> uuid, schema_ptr old) {
+static schema_ptr create_log_schema(const schema& s, std::optional<table_id> uuid, schema_ptr old) {
     schema_builder b(s.ks_name(), log_name(s.cf_name()));
     b.with_partitioner(cdc::cdc_partitioner::classname);
     b.set_compaction_strategy(sstables::compaction_strategy_type::time_window);
@@ -1655,7 +1655,8 @@ public:
 
         auto partition_slice = query::partition_slice(std::move(bounds), std::move(static_columns), std::move(regular_columns), std::move(opts));
         const auto max_result_size = _ctx._proxy.get_max_result_size(partition_slice);
-        auto command = ::make_lw_shared<query::read_command>(_schema->id(), _schema->version(), partition_slice, query::max_result_size(max_result_size), query::row_limit(row_limit));
+        const auto tombstone_limit = query::tombstone_limit(_ctx._proxy.get_tombstone_limit());
+        auto command = ::make_lw_shared<query::read_command>(_schema->id(), _schema->version(), partition_slice, query::max_result_size(max_result_size), tombstone_limit, query::row_limit(row_limit));
 
         const auto select_cl = adjust_cl(write_cl);
 

column_computation.hh

@@ -13,6 +13,12 @@
 class schema;
 class partition_key;
 class clustering_row;
+struct atomic_cell_view;
+struct tombstone;
+
+namespace db::view {
+struct view_key_and_action;
+}
 
 class column_computation;
 using column_computation_ptr = std::unique_ptr<column_computation>;
@@ -22,7 +28,7 @@ using column_computation_ptr = std::unique_ptr<column_computation>;
  * Computed columns description is also available at docs/dev/system_schema_keyspace.md. They hold values
  * not provided directly by the user, but rather computed: from other column values and possibly other sources.
  * This class is able to serialize/deserialize column computations and perform the computation itself,
- * based on given schema, partition key and clustering row. Responsibility for providing enough data
+ * based on given schema, and partition key. Responsibility for providing enough data
  * in the clustering row in order for computation to succeed belongs to the caller. In particular,
  * generating a value might involve performing a read-before-write if the computation is performed
  * on more values than are present in the update request.
@@ -36,7 +42,19 @@ public:
     virtual column_computation_ptr clone() const = 0;
 
     virtual bytes serialize() const = 0;
-    virtual bytes_opt compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const = 0;
+    virtual bytes compute_value(const schema& schema, const partition_key& key) const = 0;
+    /*
+     * depends_on_non_primary_key_column for a column computation is needed to
+     * detect a case where the primary key of a materialized view depends on a
+     * non primary key column from the base table, but at the same time, the view
+     * itself doesn't have non-primary key columns. This is an issue, since as
+     * for now, it was assumed that no non-primary key columns in view schema
+     * meant that the update cannot change the primary key of the view, and
+     * therefore the update path can be simplified.
+     */
+    virtual bool depends_on_non_primary_key_column() const {
+        return false;
+    }
 };
 
 /*
@@ -54,7 +72,7 @@ public:
         return std::make_unique<legacy_token_column_computation>(*this);
     }
     virtual bytes serialize() const override;
-    virtual bytes_opt compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const override;
+    virtual bytes compute_value(const schema& schema, const partition_key& key) const override;
 };
 
 
@@ -75,5 +93,53 @@ public:
         return std::make_unique<token_column_computation>(*this);
     }
     virtual bytes serialize() const override;
-    virtual bytes_opt compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const override;
+    virtual bytes compute_value(const schema& schema, const partition_key& key) const override;
+};
+
+/*
+ * collection_column_computation is used for a secondary index on a collection
+ * column. In this case we don't have a single value to compute, but rather we
+ * want to return multiple values (e.g., all the keys in the collection).
+ * So this class does not implement the base class's compute_value() -
+ * instead it implements a new method compute_collection_values(), which
+ * can return multiple values. This new method is currently called only from
+ * the materialized-view code which uses collection_column_computation.
+ */
+class collection_column_computation final : public column_computation {
+    enum class kind {
+        keys,
+        values,
+        entries,
+    };
+    const bytes _collection_name;
+    const kind _kind;
+    collection_column_computation(const bytes& collection_name, kind kind) : _collection_name(collection_name), _kind(kind) {}
+
+    using collection_kv = std::pair<bytes_view, atomic_cell_view>;
+    void operate_on_collection_entries(
+            std::invocable<collection_kv*, collection_kv*, tombstone> auto&& old_and_new_row_func, const schema& schema,
+            const partition_key& key, const clustering_row& update, const std::optional<clustering_row>& existing) const;
+
+public:
+    static collection_column_computation for_keys(const bytes& collection_name) {
+        return {collection_name, kind::keys};
+    }
+    static collection_column_computation for_values(const bytes& collection_name) {
+        return {collection_name, kind::values};
+    }
+    static collection_column_computation for_entries(const bytes& collection_name) {
+        return {collection_name, kind::entries};
+    }
+    static column_computation_ptr for_target_type(std::string_view type, const bytes& collection_name);
+
+    virtual bytes serialize() const override;
+    virtual bytes compute_value(const schema& schema, const partition_key& key) const override;
+    virtual column_computation_ptr clone() const override {
+        return std::make_unique<collection_column_computation>(*this);
+    }
+    virtual bool depends_on_non_primary_key_column() const override {
+        return true;
+    }
+
+    std::vector<db::view::view_key_and_action> compute_values_with_action(const schema& schema, const partition_key& key, const clustering_row& row, const std::optional<clustering_row>& existing) const;
 };

compaction/compaction.cc

@@ -52,6 +52,7 @@
 #include "readers/filtering.hh"
 #include "readers/compacting.hh"
 #include "tombstone_gc.hh"
+#include "keys.hh"
 
 namespace sstables {
 
@@ -277,6 +278,18 @@ class compacted_fragments_writer {
     stop_func_t _stop_compaction_writer;
     std::optional<utils::observer<>> _stop_request_observer;
     bool _unclosed_partition = false;
+    struct partition_state {
+        dht::decorated_key_opt dk;
+        // Partition tombstone is saved for the purpose of replicating it to every fragment storing a partition pL.
+        // Then when reading from the SSTable run, we won't unnecessarily have to open >= 2 fragments, the one which
+        // contains the tombstone and another one(s) that has the partition slice being queried.
+        ::tombstone tombstone;
+        // Used to determine whether any active tombstones need closing at EOS.
+        ::tombstone current_emitted_tombstone;
+        // Track last emitted clustering row, which will be used to close active tombstone if splitting partition
+        position_in_partition last_pos = position_in_partition::before_all_clustered_rows();
+        bool is_splitting_partition = false;
+    } _current_partition;
 private:
     inline void maybe_abort_compaction();
 
@@ -286,6 +299,13 @@ private:
             consume_end_of_stream();
         });
     }
+
+    void stop_current_writer();
+    bool can_split_large_partition() const;
+    void track_last_position(position_in_partition_view pos);
+    void split_large_partition();
+    void do_consume_new_partition(const dht::decorated_key& dk);
+    stop_iteration do_consume_end_of_partition();
 public:
     explicit compacted_fragments_writer(compaction& c, creator_func_t cpw, stop_func_t scw)
             : _c(c)
@@ -304,7 +324,7 @@ public:
 
     void consume_new_partition(const dht::decorated_key& dk);
 
-    void consume(tombstone t) { _compaction_writer->writer.consume(t); }
+    void consume(tombstone t);
     stop_iteration consume(static_row&& sr, tombstone, bool) {
         maybe_abort_compaction();
         return _compaction_writer->writer.consume(std::move(sr));
@@ -312,17 +332,11 @@ public:
     stop_iteration consume(static_row&& sr) {
         return consume(std::move(sr), tombstone{}, bool{});
     }
-    stop_iteration consume(clustering_row&& cr, row_tombstone, bool) {
-        maybe_abort_compaction();
-        return _compaction_writer->writer.consume(std::move(cr));
-    }
+    stop_iteration consume(clustering_row&& cr, row_tombstone, bool);
     stop_iteration consume(clustering_row&& cr) {
         return consume(std::move(cr), row_tombstone{}, bool{});
     }
-    stop_iteration consume(range_tombstone_change&& rtc) {
-        maybe_abort_compaction();
-        return _compaction_writer->writer.consume(std::move(rtc));
-    }
+    stop_iteration consume(range_tombstone_change&& rtc);
 
     stop_iteration consume_end_of_partition();
     void consume_end_of_stream();
@@ -448,10 +462,11 @@ protected:
     uint64_t _estimated_partitions = 0;
     db::replay_position _rp;
     encoding_stats_collector _stats_collector;
+    bool _can_split_large_partition = false;
     bool _contains_multi_fragment_runs = false;
     mutation_source_metadata _ms_metadata = {};
     compaction_sstable_replacer_fn _replacer;
-    utils::UUID _run_identifier;
+    run_id _run_identifier;
     ::io_priority_class _io_priority;
     // optional clone of sstable set to be used for expiration purposes, so it will be set if expiration is enabled.
     std::optional<sstable_set> _sstable_set;
@@ -479,6 +494,7 @@ protected:
         , _type(descriptor.options.type())
         , _max_sstable_size(descriptor.max_sstable_bytes)
         , _sstable_level(descriptor.level)
+        , _can_split_large_partition(descriptor.can_split_large_partition)
         , _replacer(std::move(descriptor.replacer))
         , _run_identifier(descriptor.run_identifier)
         , _io_priority(descriptor.io_priority)
@@ -489,7 +505,7 @@ protected:
         for (auto& sst : _sstables) {
             _stats_collector.update(sst->get_encoding_stats_for_compaction());
         }
-        std::unordered_set<utils::UUID> ssts_run_ids;
+        std::unordered_set<run_id> ssts_run_ids;
         _contains_multi_fragment_runs = std::any_of(_sstables.begin(), _sstables.end(), [&ssts_run_ids] (shared_sstable& sst) {
             return !ssts_run_ids.insert(sst->run_identifier()).second;
         });
@@ -684,7 +700,8 @@ private:
                 reader.consume_in_thread(std::move(cfc));
             });
         });
-        return consumer(make_compacting_reader(make_sstable_reader(), compaction_time, max_purgeable_func()));
+        const auto& gc_state = _table_s.get_tombstone_gc_state();
+        return consumer(make_compacting_reader(make_sstable_reader(), compaction_time, max_purgeable_func(), gc_state));
     }
 
     future<> consume() {
@@ -704,6 +721,7 @@ private:
                     using compact_mutations = compact_for_compaction_v2<compacted_fragments_writer, compacted_fragments_writer>;
                     auto cfc = compact_mutations(*schema(), now,
                         max_purgeable_func(),
+                        _table_s.get_tombstone_gc_state(),
                         get_compacted_fragments_writer(),
                         get_gc_compacted_fragments_writer());
 
@@ -713,6 +731,7 @@ private:
                 using compact_mutations = compact_for_compaction_v2<compacted_fragments_writer, noop_compacted_fragments_consumer>;
                 auto cfc = compact_mutations(*schema(), now,
                     max_purgeable_func(),
+                    _table_s.get_tombstone_gc_state(),
                     get_compacted_fragments_writer(),
                     noop_compacted_fragments_consumer());
                 reader.consume_in_thread(std::move(cfc));
@@ -864,7 +883,51 @@ void compacted_fragments_writer::maybe_abort_compaction() {
     }
 }
 
-void compacted_fragments_writer::consume_new_partition(const dht::decorated_key& dk) {
+void compacted_fragments_writer::stop_current_writer() {
+    // stop sstable writer being currently used.
+    _stop_compaction_writer(&*_compaction_writer);
+    _compaction_writer = std::nullopt;
+}
+
+bool compacted_fragments_writer::can_split_large_partition() const {
+    return _c._can_split_large_partition;
+}
+
+void compacted_fragments_writer::track_last_position(position_in_partition_view pos) {
+    if (can_split_large_partition()) {
+        _current_partition.last_pos = pos;
+    }
+}
+
+void compacted_fragments_writer::split_large_partition() {
+    // Closes the active range tombstone if needed, before emitting partition end.
+    // after_key(last_pos) is used for both closing and re-opening the active tombstone, which
+    // will result in current fragment storing an inclusive end bound for last pos, and the
+    // next fragment storing an exclusive start bound for last pos. This is very important
+    // for not losing information on the range tombstone.
+    auto after_last_pos = position_in_partition::after_key(_current_partition.last_pos.key());
+    if (_current_partition.current_emitted_tombstone) {
+        auto rtc = range_tombstone_change(after_last_pos, tombstone{});
+        _c.log_debug("Closing active tombstone {} with {} for partition {}", _current_partition.current_emitted_tombstone, rtc, *_current_partition.dk);
+        _compaction_writer->writer.consume(std::move(rtc));
+    }
+    _c.log_debug("Splitting large partition {} in order to respect SSTable size limit of {}", *_current_partition.dk, pretty_printed_data_size(_c._max_sstable_size));
+    // Close partition in current writer, and open it again in a new writer.
+    do_consume_end_of_partition();
+    stop_current_writer();
+    do_consume_new_partition(*_current_partition.dk);
+    // Replicate partition tombstone to every fragment, allowing the SSTable run reader
+    // to open a single fragment during the read.
+    if (_current_partition.tombstone) {
+        consume(_current_partition.tombstone);
+    }
+    if (_current_partition.current_emitted_tombstone) {
+        _compaction_writer->writer.consume(range_tombstone_change(after_last_pos, _current_partition.current_emitted_tombstone));
+    }
+    _current_partition.is_splitting_partition = false;
+}
+
+void compacted_fragments_writer::do_consume_new_partition(const dht::decorated_key& dk) {
     maybe_abort_compaction();
     if (!_compaction_writer) {
         _compaction_writer = _create_compaction_writer(dk);
@@ -872,17 +935,55 @@ void compacted_fragments_writer::consume_new_partition(const dht::decorated_key&
 
     _c.on_new_partition();
     _compaction_writer->writer.consume_new_partition(dk);
-    _c._cdata.total_keys_written++;
     _unclosed_partition = true;
 }
 
-stop_iteration compacted_fragments_writer::consume_end_of_partition() {
-    auto ret = _compaction_writer->writer.consume_end_of_partition();
+stop_iteration compacted_fragments_writer::do_consume_end_of_partition() {
     _unclosed_partition = false;
+    return _compaction_writer->writer.consume_end_of_partition();
+}
+
+void compacted_fragments_writer::consume_new_partition(const dht::decorated_key& dk) {
+    _current_partition = {
+        .dk = dk,
+        .tombstone = tombstone(),
+        .current_emitted_tombstone = tombstone(),
+        .last_pos = position_in_partition(position_in_partition::partition_start_tag_t()),
+        .is_splitting_partition = false
+    };
+    do_consume_new_partition(dk);
+    _c._cdata.total_keys_written++;
+}
+
+void compacted_fragments_writer::consume(tombstone t) {
+    _current_partition.tombstone = t;
+    _compaction_writer->writer.consume(t);
+}
+
+stop_iteration compacted_fragments_writer::consume(clustering_row&& cr, row_tombstone, bool) {
+    maybe_abort_compaction();
+    if (_current_partition.is_splitting_partition) [[unlikely]] {
+        split_large_partition();
+    }
+    track_last_position(cr.position());
+    auto ret = _compaction_writer->writer.consume(std::move(cr));
+    if (can_split_large_partition() && ret == stop_iteration::yes) [[unlikely]] {
+        _current_partition.is_splitting_partition = true;
+    }
+    return stop_iteration::no;
+}
+
+stop_iteration compacted_fragments_writer::consume(range_tombstone_change&& rtc) {
+    maybe_abort_compaction();
+    _current_partition.current_emitted_tombstone = rtc.tombstone();
+    track_last_position(rtc.position());
+    return _compaction_writer->writer.consume(std::move(rtc));
+}
+
+stop_iteration compacted_fragments_writer::consume_end_of_partition() {
+    auto ret = do_consume_end_of_partition();
     if (ret == stop_iteration::yes) {
-        // stop sstable writer being currently used.
-        _stop_compaction_writer(&*_compaction_writer);
-        _compaction_writer = std::nullopt;
+        stop_current_writer();
     }
     return ret;
 }
@@ -1183,9 +1284,9 @@ public:
                 type,
                 schema.ks_name(),
                 schema.cf_name(),
-                partition_key_to_string(new_key.key(), schema),
+                new_key.key().with_schema(schema),
                 new_key,
-                partition_key_to_string(current_key.key(), schema),
+                current_key.key().with_schema(schema),
                 current_key,
                 action.empty() ? "" : "; ",
                 action);
@@ -1198,9 +1299,9 @@ public:
                 type,
                 schema.ks_name(),
                 schema.cf_name(),
-                partition_key_to_string(new_key.key(), schema),
+                new_key.key().with_schema(schema),
                 new_key,
-                partition_key_to_string(current_key.key(), schema),
+                current_key.key().with_schema(schema),
                 current_key,
                 action.empty() ? "" : "; ",
                 action);
@@ -1218,7 +1319,7 @@ public:
                 mf.mutation_fragment_kind(),
                 mf.has_key() ? format(" with key {}", mf.key().with_schema(schema)) : "",
                 mf.position(),
-                partition_key_to_string(key.key(), schema),
+                key.key().with_schema(schema),
                 key,
                 prev_pos.region(),
                 prev_pos.has_key() ? format(" with key {}", prev_pos.key().with_schema(schema)) : "",
@@ -1230,14 +1331,10 @@ public:
         const auto& schema = validator.schema();
         const auto& key = validator.previous_partition_key();
         clogger.error("[{} compaction {}.{}] Invalid end-of-stream, last partition {} ({}) didn't end with a partition-end fragment{}{}",
-                type, schema.ks_name(), schema.cf_name(), partition_key_to_string(key.key(), schema), key, action.empty() ? "" : "; ", action);
+                type, schema.ks_name(), schema.cf_name(), key.key().with_schema(schema), key, action.empty() ? "" : "; ", action);
     }
 
 private:
-    static sstring partition_key_to_string(const partition_key& key, const ::schema& s) {
-        sstring ret = format("{}", key.with_schema(s));
-        return utils::utf8::validate((const uint8_t*)ret.data(), ret.size()) ? ret : "<non-utf8-key>";
-    }
 
     class reader : public flat_mutation_reader_v2::impl {
         using skip = bool_class<class skip_tag>;
@@ -1249,17 +1346,20 @@ private:
         uint64_t& _validation_errors;
 
     private:
-        void maybe_abort_scrub() {
+        void maybe_abort_scrub(std::function<void()> report_error) {
             if (_scrub_mode == compaction_type_options::scrub::mode::abort) {
+                report_error();
                 throw compaction_aborted_exception(_schema->ks_name(), _schema->cf_name(), "scrub compaction found invalid data");
             }
             ++_validation_errors;
         }
 
         void on_unexpected_partition_start(const mutation_fragment_v2& ps) {
-            maybe_abort_scrub();
-            report_invalid_partition_start(compaction_type::Scrub, _validator, ps.as_partition_start().key(),
-                    "Rectifying by adding assumed missing partition-end");
+            auto report_fn = [this, &ps] (std::string_view action = "") {
+                report_invalid_partition_start(compaction_type::Scrub, _validator, ps.as_partition_start().key(), action);
+            };
+            maybe_abort_scrub(report_fn);
+            report_fn("Rectifying by adding assumed missing partition-end");
 
             auto pe = mutation_fragment_v2(*_schema, _permit, partition_end{});
             if (!_validator(pe)) {
@@ -1279,20 +1379,26 @@ private:
         }
 
         skip on_invalid_partition(const dht::decorated_key& new_key) {
-            maybe_abort_scrub();
+            auto report_fn = [this, &new_key] (std::string_view action = "") {
+                report_invalid_partition(compaction_type::Scrub, _validator, new_key, action);
+            };
+            maybe_abort_scrub(report_fn);
             if (_scrub_mode == compaction_type_options::scrub::mode::segregate) {
-                report_invalid_partition(compaction_type::Scrub, _validator, new_key, "Detected");
+                report_fn("Detected");
                 _validator.reset(new_key);
                 // Let the segregating interposer consumer handle this.
                 return skip::no;
             }
-            report_invalid_partition(compaction_type::Scrub, _validator, new_key, "Skipping");
+            report_fn("Skipping");
             _skip_to_next_partition = true;
             return skip::yes;
         }
 
         skip on_invalid_mutation_fragment(const mutation_fragment_v2& mf) {
-            maybe_abort_scrub();
+            auto report_fn = [this, &mf] (std::string_view action = "") {
+                report_invalid_mutation_fragment(compaction_type::Scrub, _validator, mf, "");
+            };
+            maybe_abort_scrub(report_fn);
 
             const auto& key = _validator.previous_partition_key();
 
@@ -1307,8 +1413,7 @@ private:
             // The only case a partition end is invalid is when it comes after
             // another partition end, and we can just drop it in that case.
             if (!mf.is_end_of_partition() && _scrub_mode == compaction_type_options::scrub::mode::segregate) {
-                report_invalid_mutation_fragment(compaction_type::Scrub, _validator, mf,
-                        "Injecting partition start/end to segregate out-of-order fragment");
+                report_fn("Injecting partition start/end to segregate out-of-order fragment");
                 push_mutation_fragment(*_schema, _permit, partition_end{});
 
                 // We loose the partition tombstone if any, but it will be
@@ -1321,16 +1426,19 @@ private:
                 return skip::no;
             }
 
-            report_invalid_mutation_fragment(compaction_type::Scrub, _validator, mf, "Skipping");
+            report_fn("Skipping");
 
             return skip::yes;
         }
 
         void on_invalid_end_of_stream() {
-            maybe_abort_scrub();
+            auto report_fn = [this] (std::string_view action = "") {
+                report_invalid_end_of_stream(compaction_type::Scrub, _validator, action);
+            };
+            maybe_abort_scrub(report_fn);
             // Handle missing partition_end
             push_mutation_fragment(mutation_fragment_v2(*_schema, _permit, partition_end{}));
-            report_invalid_end_of_stream(compaction_type::Scrub, _validator, "Rectifying by adding missing partition-end to the end of the stream");
+            report_fn("Rectifying by adding missing partition-end to the end of the stream");
         }
 
         void fill_buffer_from_underlying() {
@@ -1509,7 +1617,7 @@ class resharding_compaction final : public compaction {
         uint64_t estimated_partitions = 0;
     };
     std::vector<estimated_values> _estimation_per_shard;
-    std::vector<utils::UUID> _run_identifiers;
+    std::vector<run_id> _run_identifiers;
 private:
     // return estimated partitions per sstable for a given shard
     uint64_t partitions_per_sstable(shard_id s) const {
@@ -1533,7 +1641,7 @@ public:
             }
         }
         for (auto i : boost::irange(0u, smp::count)) {
-            _run_identifiers[i] = utils::make_random_uuid();
+            _run_identifiers[i] = run_id::create_random_id();
         }
     }
 
@@ -1766,7 +1874,7 @@ get_fully_expired_sstables(const table_state& table_s, const std::vector<sstable
     int64_t min_timestamp = std::numeric_limits<int64_t>::max();
 
     for (auto& sstable : overlapping) {
-        auto gc_before = sstable->get_gc_before_for_fully_expire(compaction_time);
+        auto gc_before = sstable->get_gc_before_for_fully_expire(compaction_time, table_s.get_tombstone_gc_state());
         if (sstable->get_max_local_deletion_time() >= gc_before) {
             min_timestamp = std::min(min_timestamp, sstable->get_stats_metadata().min_timestamp);
         }
@@ -1785,7 +1893,7 @@ get_fully_expired_sstables(const table_state& table_s, const std::vector<sstable
 
     // SStables that do not contain live data is added to list of possibly expired sstables.
     for (auto& candidate : compacting) {
-        auto gc_before = candidate->get_gc_before_for_fully_expire(compaction_time);
+        auto gc_before = candidate->get_gc_before_for_fully_expire(compaction_time, table_s.get_tombstone_gc_state());
         clogger.debug("Checking if candidate of generation {} and max_deletion_time {} is expired, gc_before is {}",
                     candidate->generation(), candidate->get_stats_metadata().max_local_deletion_time, gc_before);
         // A fully expired sstable which has an ancestor undeleted shouldn't be compacted because
@@ -1816,7 +1924,7 @@ get_fully_expired_sstables(const table_state& table_s, const std::vector<sstable
 }
 
 unsigned compaction_descriptor::fan_in() const {
-    return boost::copy_range<std::unordered_set<utils::UUID>>(sstables | boost::adaptors::transformed(std::mem_fn(&sstables::sstable::run_identifier))).size();
+    return boost::copy_range<std::unordered_set<run_id>>(sstables | boost::adaptors::transformed(std::mem_fn(&sstables::sstable::run_identifier))).size();
 }
 
 uint64_t compaction_descriptor::sstables_size() const {

compaction/compaction_descriptor.hh

@@ -14,7 +14,7 @@
 #include <variant>
 #include <seastar/core/smp.hh>
 #include <seastar/core/file.hh>
-#include "sstables/shared_sstable.hh"
+#include "sstables/types_fwd.hh"
 #include "sstables/sstable_set.hh"
 #include "utils/UUID.hh"
 #include "dht/i_partitioner.hh"
@@ -153,8 +153,10 @@ struct compaction_descriptor {
     int level;
     // Threshold size for sstable(s) to be created.
     uint64_t max_sstable_bytes;
+    // Can split large partitions at clustering boundary.
+    bool can_split_large_partition = false;
     // Run identifier of output sstables.
-    utils::UUID run_identifier;
+    sstables::run_id run_identifier;
     // The options passed down to the compaction code.
     // This also selects the kind of compaction to do.
     compaction_type_options options = compaction_type_options::make_regular();
@@ -176,7 +178,7 @@ struct compaction_descriptor {
                                    ::io_priority_class io_priority,
                                    int level = default_level,
                                    uint64_t max_sstable_bytes = default_max_sstable_bytes,
-                                   utils::UUID run_identifier = utils::make_random_uuid(),
+                                   run_id run_identifier = run_id::create_random_id(),
                                    compaction_type_options options = compaction_type_options::make_regular())
         : sstables(std::move(sstables))
         , level(level)
@@ -192,7 +194,7 @@ struct compaction_descriptor {
         : sstables(std::move(sstables))
         , level(default_level)
         , max_sstable_bytes(default_max_sstable_bytes)
-        , run_identifier(utils::make_random_uuid())
+        , run_identifier(run_id::create_random_id())
         , options(compaction_type_options::make_regular())
         , io_priority(io_priority)
         , has_only_fully_expired(has_only_fully_expired)

compaction/compaction_manager.cc

@@ -19,6 +19,7 @@
 #include "locator/abstract_replication_strategy.hh"
 #include "utils/fb_utilities.hh"
 #include "utils/UUID_gen.hh"
+#include "db/system_keyspace.hh"
 #include <cmath>
 #include <boost/algorithm/cxx11/any_of.hpp>
 #include <boost/range/algorithm/remove_if.hpp>
@@ -206,7 +207,7 @@ std::vector<sstables::shared_sstable> compaction_manager::get_candidates(compact
     candidates.reserve(t.main_sstable_set().all()->size());
     // prevents sstables that belongs to a partial run being generated by ongoing compaction from being
     // selected for compaction, which could potentially result in wrong behavior.
-    auto partial_run_identifiers = boost::copy_range<std::unordered_set<utils::UUID>>(_tasks
+    auto partial_run_identifiers = boost::copy_range<std::unordered_set<sstables::run_id>>(_tasks
             | boost::adaptors::filtered(std::mem_fn(&task::generating_output_run))
             | boost::adaptors::transformed(std::mem_fn(&task::output_run_id)));
     auto& cs = t.get_compaction_strategy();
@@ -347,8 +348,15 @@ future<sstables::compaction_result> compaction_manager::task::compact_sstables(s
 future<> compaction_manager::task::update_history(compaction::table_state& t, const sstables::compaction_result& res, const sstables::compaction_data& cdata) {
     auto ended_at = std::chrono::duration_cast<std::chrono::milliseconds>(res.stats.ended_at.time_since_epoch());
 
-    co_return co_await t.update_compaction_history(cdata.compaction_uuid, t.schema()->ks_name(), t.schema()->cf_name(), ended_at,
-                                                                    res.stats.start_size, res.stats.end_size);
+    if (_cm._sys_ks) {
+        // FIXME: add support to merged_rows. merged_rows is a histogram that
+        // shows how many sstables each row is merged from. This information
+        // cannot be accessed until we make combined_reader more generic,
+        // for example, by adding a reducer method.
+        auto sys_ks = _cm._sys_ks; // hold pointer on sys_ks
+        co_await sys_ks->update_compaction_history(cdata.compaction_uuid, t.schema()->ks_name(), t.schema()->cf_name(),
+                ended_at.count(), res.stats.start_size, res.stats.end_size, std::unordered_map<int32_t, int64_t>{});
+    }
 }
 
 class compaction_manager::major_compaction_task : public compaction_manager::task {
@@ -611,7 +619,7 @@ future<semaphore_units<named_semaphore_exception_factory>> compaction_manager::t
     });
 }
 
-void compaction_manager::task::setup_new_compaction(utils::UUID output_run_id) {
+void compaction_manager::task::setup_new_compaction(sstables::run_id output_run_id) {
     _compaction_data = create_compaction_data();
     _output_run_identifier = output_run_id;
     switch_state(state::active);
@@ -619,7 +627,7 @@ void compaction_manager::task::setup_new_compaction(utils::UUID output_run_id) {
 
 void compaction_manager::task::finish_compaction(state finish_state) noexcept {
     switch_state(finish_state);
-    _output_run_identifier = utils::null_uuid();
+    _output_run_identifier = sstables::run_id::create_null_id();
     if (finish_state != state::failed) {
         _compaction_retry.reset();
     }
@@ -657,6 +665,7 @@ compaction_manager::compaction_manager(config cfg, abort_source& as)
     , _update_compaction_static_shares_action([this] { return update_static_shares(static_shares()); })
     , _compaction_static_shares_observer(_cfg.static_shares.observe(_update_compaction_static_shares_action.make_observer()))
     , _strategy_control(std::make_unique<strategy_control>(*this))
+    , _tombstone_gc_state(&_repair_history_maps)
 {
     register_metrics();
     // Bandwidth throttling is node-wide, updater is needed on single shard
@@ -676,6 +685,7 @@ compaction_manager::compaction_manager()
     , _update_compaction_static_shares_action([] { return make_ready_future<>(); })
     , _compaction_static_shares_observer(_cfg.static_shares.observe(_update_compaction_static_shares_action.make_observer()))
     , _strategy_control(std::make_unique<strategy_control>(*this))
+    , _tombstone_gc_state(&_repair_history_maps)
 {
     // No metric registration because this constructor is supposed to be used only by the testing
     // infrastructure.
@@ -789,23 +799,27 @@ future<> compaction_manager::stop_tasks(std::vector<shared_ptr<task>> tasks, sst
     });
 }
 
-future<> compaction_manager::stop_ongoing_compactions(sstring reason, compaction::table_state* t, std::optional<sstables::compaction_type> type_opt) {
-    auto ongoing_compactions = get_compactions(t).size();
-    auto tasks = boost::copy_range<std::vector<shared_ptr<task>>>(_tasks | boost::adaptors::filtered([t, type_opt] (auto& task) {
-        return (!t || task->compacting_table() == t) && (!type_opt || task->type() == *type_opt);
-    }));
-    logging::log_level level = tasks.empty() ? log_level::debug : log_level::info;
-    if (cmlog.is_enabled(level)) {
-        std::string scope = "";
-        if (t) {
-            scope = fmt::format(" for table {}.{}", t->schema()->ks_name(), t->schema()->cf_name());
+future<> compaction_manager::stop_ongoing_compactions(sstring reason, compaction::table_state* t, std::optional<sstables::compaction_type> type_opt) noexcept {
+    try {
+        auto ongoing_compactions = get_compactions(t).size();
+        auto tasks = boost::copy_range<std::vector<shared_ptr<task>>>(_tasks | boost::adaptors::filtered([t, type_opt] (auto& task) {
+            return (!t || task->compacting_table() == t) && (!type_opt || task->type() == *type_opt);
+        }));
+        logging::log_level level = tasks.empty() ? log_level::debug : log_level::info;
+        if (cmlog.is_enabled(level)) {
+            std::string scope = "";
+            if (t) {
+                scope = fmt::format(" for table {}.{}", t->schema()->ks_name(), t->schema()->cf_name());
+            }
+            if (type_opt) {
+                scope += fmt::format(" {} type={}", scope.size() ? "and" : "for", *type_opt);
+            }
+            cmlog.log(level, "Stopping {} tasks for {} ongoing compactions{} due to {}", tasks.size(), ongoing_compactions, scope, reason);
         }
-        if (type_opt) {
-            scope += fmt::format(" {} type={}", scope.size() ? "and" : "for", *type_opt);
-        }
-        cmlog.log(level, "Stopping {} tasks for {} ongoing compactions{} due to {}", tasks.size(), ongoing_compactions, scope, reason);
+        return stop_tasks(std::move(tasks), std::move(reason));
+    } catch (...) {
+        return current_exception_as_future<>();
     }
-    return stop_tasks(std::move(tasks), std::move(reason));
 }
 
 future<> compaction_manager::drain() {
@@ -842,14 +856,31 @@ future<> compaction_manager::really_do_stop() {
     cmlog.info("Stopped");
 }
 
+template <typename Ex>
+requires std::is_base_of_v<std::exception, Ex> &&
+requires (const Ex& ex) {
+    { ex.code() } noexcept -> std::same_as<const std::error_code&>;
+}
+auto swallow_enospc(const Ex& ex) noexcept {
+    if (ex.code().value() != ENOSPC) {
+        return make_exception_future<>(std::make_exception_ptr(ex));
+    }
+
+    cmlog.warn("Got ENOSPC on stop, ignoring...");
+    return make_ready_future<>();
+}
+
 void compaction_manager::do_stop() noexcept {
-    if (_state == state::none || _state == state::stopped) {
+    if (_state == state::none || _stop_future) {
         return;
     }
 
     try {
         _state = state::stopped;
-        _stop_future = really_do_stop();
+        _stop_future = really_do_stop()
+            .handle_exception_type([] (const std::system_error& ex) { return swallow_enospc(ex); })
+            .handle_exception_type([] (const storage_io_error& ex) { return swallow_enospc(ex); })
+        ;
     } catch (...) {
         cmlog.error("Failed to stop the manager: {}", std::current_exception());
     }
@@ -1010,7 +1041,7 @@ future<> compaction_manager::maybe_wait_for_sstable_count_reduction(compaction::
     auto num_runs_for_compaction = [&, this] {
         auto& cs = t.get_compaction_strategy();
         auto desc = cs.get_sstables_for_compaction(t, get_strategy_control(), get_candidates(t));
-        return boost::copy_range<std::unordered_set<utils::UUID>>(
+        return boost::copy_range<std::unordered_set<sstables::run_id>>(
             desc.sstables
             | boost::adaptors::transformed(std::mem_fn(&sstables::sstable::run_identifier))).size();
     };
@@ -1050,7 +1081,7 @@ public:
     bool performed() const noexcept {
         return _performed;
     }
-
+private:
     future<> run_offstrategy_compaction(sstables::compaction_data& cdata) {
         // This procedure will reshape sstables in maintenance set until it's ready for
         // integration into main set.
@@ -1083,6 +1114,7 @@ public:
             return desc.sstables.size() ? std::make_optional(std::move(desc)) : std::nullopt;
         };
 
+        std::exception_ptr err;
         while (auto desc = get_next_job()) {
             desc->creator = [this, &new_unused_sstables, &t] (shard_id dummy) {
                 auto sst = t.make_sstable();
@@ -1091,7 +1123,16 @@ public:
             };
             auto input = boost::copy_range<std::unordered_set<sstables::shared_sstable>>(desc->sstables);
 
-            auto ret = co_await sstables::compact_sstables(std::move(*desc), cdata, t);
+            sstables::compaction_result ret;
+            try {
+                ret = co_await sstables::compact_sstables(std::move(*desc), cdata, t);
+            } catch (sstables::compaction_stopped_exception&) {
+                // If off-strategy compaction stopped on user request, let's not discard the partial work.
+                // Therefore, both un-reshaped and reshaped data will be integrated into main set, allowing
+                // regular compaction to continue from where off-strategy left off.
+                err = std::current_exception();
+                break;
+            }
             _performed = true;
 
             // update list of reshape candidates without input but with output added to it
@@ -1128,6 +1169,9 @@ public:
         for (auto& sst : sstables_to_remove) {
             sst->mark_for_deletion();
         }
+        if (err) {
+            co_await coroutine::return_exception_ptr(std::move(err));
+        }
     }
 protected:
     virtual future<compaction_stats_opt> do_run() override {
@@ -1537,7 +1581,7 @@ void compaction_manager::add(compaction::table_state& t) {
     }
 }
 
-future<> compaction_manager::remove(compaction::table_state& t) {
+future<> compaction_manager::remove(compaction::table_state& t) noexcept {
     auto handle = _compaction_state.extract(&t);
 
     if (!handle.empty()) {
@@ -1645,6 +1689,14 @@ compaction::strategy_control& compaction_manager::get_strategy_control() const n
     return *_strategy_control;
 }
 
+void compaction_manager::plug_system_keyspace(db::system_keyspace& sys_ks) noexcept {
+    _sys_ks = sys_ks.shared_from_this();
+}
+
+void compaction_manager::unplug_system_keyspace() noexcept {
+    _sys_ks = nullptr;
+}
+
 double compaction_backlog_tracker::backlog() const {
     return disabled() ? compaction_controller::disable_backlog : _impl->backlog(_ongoing_writes, _ongoing_compactions);
 }

compaction/compaction_manager.hh

@@ -8,6 +8,9 @@
 
 #pragma once
 
+#include <boost/icl/interval.hpp>
+#include <boost/icl/interval_map.hpp>
+
 #include <seastar/core/semaphore.hh>
 #include <seastar/core/sstring.hh>
 #include <seastar/core/shared_ptr.hh>
@@ -33,9 +36,19 @@
 #include "backlog_controller.hh"
 #include "seastarx.hh"
 #include "sstables/exceptions.hh"
+#include "tombstone_gc.hh"
+
+namespace db {
+class system_keyspace;
+}
 
 class compacting_sstable_registration;
 
+class repair_history_map {
+public:
+    boost::icl::interval_map<dht::token, gc_clock::time_point, boost::icl::partial_absorber, std::less, boost::icl::inplace_max> map;
+};
+
 // Compaction manager provides facilities to submit and track compaction jobs on
 // behalf of existing tables.
 class compaction_manager {
@@ -110,7 +123,7 @@ public:
         shared_future<compaction_stats_opt> _compaction_done = make_ready_future<compaction_stats_opt>();
         exponential_backoff_retry _compaction_retry = exponential_backoff_retry(std::chrono::seconds(5), std::chrono::seconds(300));
         sstables::compaction_type _type;
-        utils::UUID _output_run_identifier;
+        sstables::run_id _output_run_identifier;
         gate::holder _gate_holder;
         sstring _description;
 
@@ -134,7 +147,7 @@ public:
         // Return true if the task isn't stopped
         // and the compaction manager allows proceeding.
         inline bool can_proceed(throw_if_stopping do_throw_if_stopping = throw_if_stopping::no) const;
-        void setup_new_compaction(utils::UUID output_run_id = utils::null_uuid());
+        void setup_new_compaction(sstables::run_id output_run_id = sstables::run_id::create_null_id());
         void finish_compaction(state finish_state = state::done) noexcept;
 
         // Compaction manager stop itself if it finds an storage I/O error which results in
@@ -179,7 +192,7 @@ public:
         bool generating_output_run() const noexcept {
             return compaction_running() && _output_run_identifier;
         }
-        const utils::UUID& output_run_id() const noexcept {
+        const sstables::run_id& output_run_id() const noexcept {
             return _output_run_identifier;
         }
 
@@ -276,6 +289,8 @@ private:
     // being picked more than once.
     seastar::named_semaphore _off_strategy_sem = {1, named_semaphore_exception_factory{"off-strategy compaction"}};
 
+    seastar::shared_ptr<db::system_keyspace> _sys_ks;
+
     std::function<void()> compaction_submission_callback();
     // all registered tables are reevaluated at a constant interval.
     // Submission is a NO-OP when there's nothing to do, so it's fine to call it regularly.
@@ -294,6 +309,9 @@ private:
 
     class strategy_control;
     std::unique_ptr<strategy_control> _strategy_control;
+
+    per_table_history_maps _repair_history_maps;
+    tombstone_gc_state _tombstone_gc_state;
 private:
     future<compaction_stats_opt> perform_task(shared_ptr<task>);
 
@@ -470,6 +488,9 @@ public:
     // Run a function with compaction temporarily disabled for a table T.
     future<> run_with_compaction_disabled(compaction::table_state& t, std::function<future<> ()> func);
 
+    void plug_system_keyspace(db::system_keyspace& sys_ks) noexcept;
+    void unplug_system_keyspace() noexcept;
+
     // Adds a table to the compaction manager.
     // Creates a compaction_state structure that can be used for submitting
     // compaction jobs of all types.
@@ -477,7 +498,7 @@ public:
 
     // Remove a table from the compaction manager.
     // Cancel requests on table and wait for possible ongoing compactions.
-    future<> remove(compaction::table_state& t);
+    future<> remove(compaction::table_state& t) noexcept;
 
     const stats& get_stats() const {
         return _stats;
@@ -494,7 +515,7 @@ public:
     future<> stop_compaction(sstring type, compaction::table_state* table = nullptr);
 
     // Stops ongoing compaction of a given table and/or compaction_type.
-    future<> stop_ongoing_compactions(sstring reason, compaction::table_state* t = nullptr, std::optional<sstables::compaction_type> type_opt = {});
+    future<> stop_ongoing_compactions(sstring reason, compaction::table_state* t = nullptr, std::optional<sstables::compaction_type> type_opt = {}) noexcept;
 
     double backlog() {
         return _backlog_manager.backlog();
@@ -508,6 +529,14 @@ public:
 
     compaction::strategy_control& get_strategy_control() const noexcept;
 
+    tombstone_gc_state& get_tombstone_gc_state() noexcept {
+        return _tombstone_gc_state;
+    };
+
+    const tombstone_gc_state& get_tombstone_gc_state() const noexcept {
+        return _tombstone_gc_state;
+    };
+
     friend class compacting_sstable_registration;
     friend class compaction_weight_registration;
     friend class compaction_manager_test;

compaction/compaction_strategy.cc

@@ -50,7 +50,7 @@ std::vector<compaction_descriptor> compaction_strategy_impl::get_cleanup_compact
     }));
 }
 
-bool compaction_strategy_impl::worth_dropping_tombstones(const shared_sstable& sst, gc_clock::time_point compaction_time) {
+bool compaction_strategy_impl::worth_dropping_tombstones(const shared_sstable& sst, gc_clock::time_point compaction_time, const tombstone_gc_state& gc_state) {
     if (_disable_tombstone_compaction) {
         return false;
     }
@@ -61,7 +61,7 @@ bool compaction_strategy_impl::worth_dropping_tombstones(const shared_sstable& s
     if (db_clock::now()-_tombstone_compaction_interval < sst->data_file_write_time()) {
         return false;
     }
-    auto gc_before = sst->get_gc_before_for_drop_estimation(compaction_time);
+    auto gc_before = sst->get_gc_before_for_drop_estimation(compaction_time, gc_state);
     return sst->estimate_droppable_tombstone_ratio(gc_before) >= _tombstone_threshold;
 }
 
@@ -670,8 +670,8 @@ compaction_descriptor date_tiered_compaction_strategy::get_sstables_for_compacti
     }
 
     // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
-    auto e = boost::range::remove_if(candidates, [this, compaction_time] (const sstables::shared_sstable& sst) -> bool {
-        return !worth_dropping_tombstones(sst, compaction_time);
+    auto e = boost::range::remove_if(candidates, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
+        return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state());
     });
     candidates.erase(e, candidates.end());
     if (candidates.empty()) {

compaction/compaction_strategy_impl.hh

@@ -13,6 +13,7 @@
 #include "compaction_strategy.hh"
 #include "db_clock.hh"
 #include "compaction_descriptor.hh"
+#include "tombstone_gc.hh"
 
 namespace compaction {
 class table_state;
@@ -67,7 +68,7 @@ public:
 
     // Check if a given sstable is entitled for tombstone compaction based on its
     // droppable tombstone histogram and gc_before.
-    bool worth_dropping_tombstones(const shared_sstable& sst, gc_clock::time_point compaction_time);
+    bool worth_dropping_tombstones(const shared_sstable& sst, gc_clock::time_point compaction_time, const tombstone_gc_state& gc_state);
 
     virtual compaction_backlog_tracker& get_backlog_tracker() = 0;
 

compaction/leveled_compaction_strategy.cc

@@ -39,16 +39,16 @@ compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(t
     for (auto level = int(manifest.get_level_count()); level >= 0; level--) {
         auto& sstables = manifest.get_level(level);
         // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
-        auto e = boost::range::remove_if(sstables, [this, compaction_time] (const sstables::shared_sstable& sst) -> bool {
-            return !worth_dropping_tombstones(sst, compaction_time);
+        auto e = boost::range::remove_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
+            return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state());
         });
         sstables.erase(e, sstables.end());
         if (sstables.empty()) {
             continue;
         }
         auto& sst = *std::max_element(sstables.begin(), sstables.end(), [&] (auto& i, auto& j) {
-            auto gc_before1 = i->get_gc_before_for_drop_estimation(compaction_time);
-            auto gc_before2 = j->get_gc_before_for_drop_estimation(compaction_time);
+            auto gc_before1 = i->get_gc_before_for_drop_estimation(compaction_time, table_s.get_tombstone_gc_state());
+            auto gc_before2 = j->get_gc_before_for_drop_estimation(compaction_time, table_s.get_tombstone_gc_state());
             return i->estimate_droppable_tombstone_ratio(gc_before1) < j->estimate_droppable_tombstone_ratio(gc_before2);
         });
         return sstables::compaction_descriptor({ sst }, service::get_local_compaction_priority(), sst->get_sstable_level());

compaction/size_tiered_compaction_strategy.cc

@@ -169,8 +169,8 @@ size_tiered_compaction_strategy::get_sstables_for_compaction(table_state& table_
     // tombstone purge, i.e. less likely to shadow even older data.
     for (auto&& sstables : buckets | boost::adaptors::reversed) {
         // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
-        auto e = boost::range::remove_if(sstables, [this, compaction_time] (const sstables::shared_sstable& sst) -> bool {
-            return !worth_dropping_tombstones(sst, compaction_time);
+        auto e = boost::range::remove_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
+            return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state());
         });
         sstables.erase(e, sstables.end());
         if (sstables.empty()) {

compaction/table_state.hh

@@ -40,9 +40,9 @@ public:
     virtual sstables::shared_sstable make_sstable() const = 0;
     virtual sstables::sstable_writer_config configure_writer(sstring origin) const = 0;
     virtual api::timestamp_type min_memtable_timestamp() const = 0;
-    virtual future<> update_compaction_history(utils::UUID compaction_id, sstring ks_name, sstring cf_name, std::chrono::milliseconds ended_at, int64_t bytes_in, int64_t bytes_out) = 0;
     virtual future<> on_compaction_completion(sstables::compaction_completion_desc desc, sstables::offstrategy offstrategy) = 0;
     virtual bool is_auto_compaction_disabled_by_user() const noexcept = 0;
+    virtual const tombstone_gc_state& get_tombstone_gc_state() const noexcept = 0;
 };
 
 }

compaction/time_window_compaction_strategy.cc

@@ -43,6 +43,10 @@ time_window_compaction_strategy_options::time_window_compaction_strategy_options
         }
     }
 
+    if (window_size <= 0) {
+        throw exceptions::configuration_exception(fmt::format("{} must be greater than 1 for compaction_window_size", window_size));
+    }
+
     sstable_window_size = window_size * window_unit;
 
     it = options.find(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
@@ -270,8 +274,8 @@ time_window_compaction_strategy::get_next_non_expired_sstables(table_state& tabl
 
     // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
     // ratio is greater than threshold.
-    auto e = boost::range::remove_if(non_expiring_sstables, [this, compaction_time] (const shared_sstable& sst) -> bool {
-        return !worth_dropping_tombstones(sst, compaction_time);
+    auto e = boost::range::remove_if(non_expiring_sstables, [this, compaction_time, &table_s] (const shared_sstable& sst) -> bool {
+        return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state());
     });
     non_expiring_sstables.erase(e, non_expiring_sstables.end());
     if (non_expiring_sstables.empty()) {

compatible_ring_position.hh

@@ -10,88 +10,6 @@
 #pragma once
 
 #include "dht/i_partitioner.hh"
-#include <optional>
-#include <variant>
-
-// Wraps ring_position_view so it is compatible with old-style C++: default
-// constructor, stateless comparators, yada yada.
-class compatible_ring_position_view {
-    const ::schema* _schema = nullptr;
-    // Optional to supply a default constructor, no more.
-    std::optional<dht::ring_position_view> _rpv;
-public:
-    constexpr compatible_ring_position_view() = default;
-    compatible_ring_position_view(const schema& s, dht::ring_position_view rpv)
-        : _schema(&s), _rpv(rpv) {
-    }
-    const dht::ring_position_view& position() const {
-        return *_rpv;
-    }
-    const ::schema& schema() const {
-        return *_schema;
-    }
-    friend std::strong_ordering tri_compare(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return dht::ring_position_tri_compare(*x._schema, *x._rpv, *y._rpv);
-    }
-    friend bool operator<(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) < 0;
-    }
-    friend bool operator<=(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) <= 0;
-    }
-    friend bool operator>(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) > 0;
-    }
-    friend bool operator>=(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) >= 0;
-    }
-    friend bool operator==(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) == 0;
-    }
-    friend bool operator!=(const compatible_ring_position_view& x, const compatible_ring_position_view& y) {
-        return tri_compare(x, y) != 0;
-    }
-};
-
-// Wraps ring_position so it is compatible with old-style C++: default
-// constructor, stateless comparators, yada yada.
-class compatible_ring_position {
-    schema_ptr _schema;
-    // Optional to supply a default constructor, no more.
-    std::optional<dht::ring_position> _rp;
-public:
-    constexpr compatible_ring_position() = default;
-    compatible_ring_position(schema_ptr s, dht::ring_position rp)
-        : _schema(std::move(s)), _rp(std::move(rp)) {
-    }
-    dht::ring_position_view position() const {
-        return *_rp;
-    }
-    const ::schema& schema() const {
-        return *_schema;
-    }
-    friend std::strong_ordering tri_compare(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return dht::ring_position_tri_compare(*x._schema, *x._rp, *y._rp);
-    }
-    friend bool operator<(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) < 0;
-    }
-    friend bool operator<=(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) <= 0;
-    }
-    friend bool operator>(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) > 0;
-    }
-    friend bool operator>=(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) >= 0;
-    }
-    friend bool operator==(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) == 0;
-    }
-    friend bool operator!=(const compatible_ring_position& x, const compatible_ring_position& y) {
-        return tri_compare(x, y) != 0;
-    }
-};
 
 // Wraps ring_position or ring_position_view so either is compatible with old-style C++: default
 // constructor, stateless comparators, yada yada.
@@ -99,37 +17,22 @@ public:
 // on callers to keep ring position alive, allow lookup on containers that don't support different
 // key types, and also avoiding unnecessary copies.
 class compatible_ring_position_or_view {
-    // Optional to supply a default constructor, no more.
-    std::optional<std::variant<compatible_ring_position, compatible_ring_position_view>> _crp_or_view;
+    schema_ptr _schema;
+    lw_shared_ptr<dht::ring_position> _rp;
+    dht::ring_position_view_opt _rpv; // optional only for default ctor, nothing more
 public:
-    constexpr compatible_ring_position_or_view() = default;
+    compatible_ring_position_or_view() = default;
     explicit compatible_ring_position_or_view(schema_ptr s, dht::ring_position rp)
-        : _crp_or_view(compatible_ring_position(std::move(s), std::move(rp))) {
+        : _schema(std::move(s)), _rp(make_lw_shared<dht::ring_position>(std::move(rp))), _rpv(dht::ring_position_view(*_rp)) {
     }
     explicit compatible_ring_position_or_view(const schema& s, dht::ring_position_view rpv)
-        : _crp_or_view(compatible_ring_position_view(s, rpv)) {
+        : _schema(s.shared_from_this()), _rpv(rpv) {
     }
-    dht::ring_position_view position() const {
-        struct rpv_accessor {
-            dht::ring_position_view operator()(const compatible_ring_position& crp) {
-                return crp.position();
-            }
-            dht::ring_position_view operator()(const compatible_ring_position_view& crpv) {
-                return crpv.position();
-            }
-        };
-        return std::visit(rpv_accessor{}, *_crp_or_view);
+    const dht::ring_position_view& position() const {
+        return *_rpv;
     }
     friend std::strong_ordering tri_compare(const compatible_ring_position_or_view& x, const compatible_ring_position_or_view& y) {
-        struct schema_accessor {
-            const ::schema& operator()(const compatible_ring_position& crp) {
-                return crp.schema();
-            }
-            const ::schema& operator()(const compatible_ring_position_view& crpv) {
-                return crpv.schema();
-            }
-        };
-        return dht::ring_position_tri_compare(std::visit(schema_accessor{}, *x._crp_or_view), x.position(), y.position());
+        return dht::ring_position_tri_compare(*x._schema, x.position(), y.position());
     }
     friend bool operator<(const compatible_ring_position_or_view& x, const compatible_ring_position_or_view& y) {
         return tri_compare(x, y) < 0;

conf/scylla.yaml

@@ -410,6 +410,9 @@ commitlog_total_space_in_mb: -1
 # Log a warning when row number is larger than this value
 # compaction_rows_count_warning_threshold: 100000
 
+# Log a warning when writing a collection containing more elements than this value
+# compaction_collection_elements_count_warning_threshold: 10000
+
 # How long the coordinator should wait for seq or index scans to complete
 # range_request_timeout_in_ms: 10000
 # How long the coordinator should wait for writes to complete

configure.py

@@ -44,32 +44,18 @@ distro_extra_cflags = ''
 distro_extra_ldflags = ''
 distro_extra_cmake_args = []
 employ_ld_trickery = True
+has_wasmtime = False
+use_wasmtime_as_library = False
 
 # distro-specific setup
 def distro_setup_nix():
-    global os_ids, employ_ld_trickery
-    global distro_extra_ldflags, distro_extra_cflags, distro_extra_cmake_args
-
+    global os_ids, employ_ld_trickery, has_wasmtime, use_wasmtime_as_library
     os_ids = ['linux']
     employ_ld_trickery = False
+    has_wasmtime = True
+    use_wasmtime_as_library = True
 
-    libdirs = list(dict.fromkeys(os.environ.get('CMAKE_LIBRARY_PATH').split(':')))
-    incdirs = list(dict.fromkeys(os.environ.get('CMAKE_INCLUDE_PATH').split(':')))
-
-    # add nix {lib,inc}dirs to relevant flags, mimicing nix versions of cmake & autotools.
-    # also add rpaths to make sure that any built executables can run in place.
-    distro_extra_ldflags = ' '.join([ '-rpath ' + path + ' -L' + path for path in libdirs ]);
-    distro_extra_cflags = ' '.join([ '-isystem ' + path for path in incdirs ])
-
-    # indexers like clangd may or may not know which stdc++ or glibc
-    # the compiler was configured with, so make the relevant paths
-    # explicit on each compilation command line:
-    implicit_cflags = os.environ.get('IMPLICIT_CFLAGS').strip()
-    distro_extra_cflags += ' ' + implicit_cflags
-    # also propagate to cmake-built dependencies:
-    distro_extra_cmake_args = ['-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES:INTERNAL=' + implicit_cflags]
-
-if os.environ.get('NIX_BUILD_TOP'):
+if os.environ.get('NIX_CC'):
         distro_setup_nix()
 
 # distribution "internationalization", converting package names.
@@ -430,6 +416,7 @@ scylla_tests = set([
     'test/boost/estimated_histogram_test',
     'test/boost/summary_test',
     'test/boost/logalloc_test',
+    'test/boost/logalloc_standard_allocator_segment_pool_backend_test',
     'test/boost/managed_vector_test',
     'test/boost/managed_bytes_test',
     'test/boost/intrusive_array_test',
@@ -461,7 +448,7 @@ scylla_tests = set([
     'test/boost/schema_change_test',
     'test/boost/schema_registry_test',
     'test/boost/secondary_index_test',
-    'test/boost/tracing',
+    'test/boost/tracing_test',
     'test/boost/index_with_paging_test',
     'test/boost/serialization_test',
     'test/boost/serialized_action_test',
@@ -562,6 +549,7 @@ raft_tests = set([
     'test/raft/replication_test',
     'test/raft/randomized_nemesis_test',
     'test/raft/many_test',
+    'test/raft/raft_server_test',
     'test/raft/fsm_test',
     'test/raft/etcd_test',
     'test/raft/raft_sys_table_storage_test',
@@ -630,6 +618,8 @@ arg_parser.add_argument('--static-yaml-cpp', dest='staticyamlcpp', action='store
                         help='Link libyaml-cpp statically')
 arg_parser.add_argument('--tests-debuginfo', action='store', dest='tests_debuginfo', type=int, default=0,
                         help='Enable(1)/disable(0)compiler debug information generation for tests')
+arg_parser.add_argument('--perf-tests-debuginfo', action='store', dest='perf_tests_debuginfo', type=int, default=0,
+                        help='Enable(1)/disable(0)compiler debug information generation for perf tests')
 arg_parser.add_argument('--python', action='store', dest='python', default='python3',
                         help='Python3 path')
 arg_parser.add_argument('--split-dwarf', dest='split_dwarf', action='store_true', default=False,
@@ -652,6 +642,8 @@ arg_parser.add_argument('--clang-inline-threshold', action='store', type=int, de
                         help="LLVM-specific inline threshold compilation parameter")
 arg_parser.add_argument('--list-artifacts', dest='list_artifacts', action='store_true', default=False,
                         help='List all available build artifacts, that can be passed to --with')
+arg_parser.add_argument('--date-stamp', dest='date_stamp', type=str,
+                        help='Set datestamp for SCYLLA-VERSION-GEN')
 args = arg_parser.parse_args()
 
 if args.list_artifacts:
@@ -661,6 +653,7 @@ if args.list_artifacts:
 
 defines = ['XXH_PRIVATE_API',
            'SEASTAR_TESTING_MAIN',
+           'FMT_DEPRECATED_OSTREAM',
 ]
 
 scylla_raft_core = [
@@ -671,7 +664,8 @@ scylla_raft_core = [
     'raft/log.cc',
 ]
 
-scylla_core = (['replica/database.cc',
+scylla_core = (['message/messaging_service.cc',
+                'replica/database.cc',
                 'replica/table.cc',
                 'replica/distributed_loader.cc',
                 'replica/memtable.cc',
@@ -799,6 +793,8 @@ scylla_core = (['replica/database.cc',
                 'cql3/statements/raw/parsed_statement.cc',
                 'cql3/statements/property_definitions.cc',
                 'cql3/statements/update_statement.cc',
+                'cql3/statements/strongly_consistent_modification_statement.cc',
+                'cql3/statements/strongly_consistent_select_statement.cc',
                 'cql3/statements/delete_statement.cc',
                 'cql3/statements/prune_materialized_view_statement.cc',
                 'cql3/statements/batch_statement.cc',
@@ -947,7 +943,6 @@ scylla_core = (['replica/database.cc',
                 'locator/ec2_snitch.cc',
                 'locator/ec2_multi_region_snitch.cc',
                 'locator/gce_snitch.cc',
-                'message/messaging_service.cc',
                 'service/client_state.cc',
                 'service/storage_service.cc',
                 'service/misc_services.cc',
@@ -999,7 +994,6 @@ scylla_core = (['replica/database.cc',
                 'tracing/tracing.cc',
                 'tracing/trace_keyspace_helper.cc',
                 'tracing/trace_state.cc',
-                'tracing/tracing_backend_registry.cc',
                 'tracing/traced_file.cc',
                 'table_helper.cc',
                 'range_tombstone.cc',
@@ -1037,6 +1031,8 @@ scylla_core = (['replica/database.cc',
                 'service/raft/raft_group0.cc',
                 'direct_failure_detector/failure_detector.cc',
                 'service/raft/raft_group0_client.cc',
+                'service/broadcast_tables/experimental/lang.cc',
+                'tasks/task_manager.cc',
                 ] + [Antlr3Grammar('cql3/Cql.g')] + [Thrift('interface/cassandra.thrift', 'Cassandra')] \
                   + scylla_raft_core
                )
@@ -1073,6 +1069,10 @@ api = ['api/api.cc',
        'api/stream_manager.cc',
        Json2Code('api/api-doc/system.json'),
        'api/system.cc',
+       Json2Code('api/api-doc/task_manager.json'),
+       'api/task_manager.cc',
+       Json2Code('api/api-doc/task_manager_test.json'),
+       'api/task_manager_test.cc',
        'api/config.cc',
        Json2Code('api/api-doc/config.json'),
        'api/error_injection.cc',
@@ -1147,6 +1147,7 @@ idls = ['idl/gossip_digest.idl.hh',
         'idl/replica_exception.idl.hh',
         'idl/per_partition_rate_limit_info.idl.hh',
         'idl/position_in_partition.idl.hh',
+        'idl/experimental/broadcast_tables_lang.idl.hh',
         ]
 
 rusts = [
@@ -1174,7 +1175,7 @@ scylla_tests_dependencies = scylla_core + idls + scylla_tests_generic_dependenci
     'test/lib/random_schema.cc',
 ]
 
-scylla_raft_dependencies = scylla_raft_core + ['utils/uuid.cc']
+scylla_raft_dependencies = scylla_raft_core + ['utils/uuid.cc', 'utils/error_injection.cc']
 
 scylla_tools = ['tools/scylla-types.cc', 'tools/scylla-sstable.cc', 'tools/schema_loader.cc', 'tools/utils.cc']
 
@@ -1298,7 +1299,7 @@ deps['test/boost/linearizing_input_stream_test'] = [
     "test/boost/linearizing_input_stream_test.cc",
     "test/lib/log.cc",
 ]
-deps['test/boost/expr_test'] = ['test/boost/expr_test.cc'] + scylla_core
+deps['test/boost/expr_test'] = ['test/boost/expr_test.cc', 'test/lib/expr_test_utils.cc'] + scylla_core
 deps['test/boost/rate_limiter_test'] = ['test/boost/rate_limiter_test.cc', 'db/rate_limiter.cc']
 deps['test/boost/exceptions_optimized_test'] = ['test/boost/exceptions_optimized_test.cc', 'utils/exceptions.cc']
 deps['test/boost/exceptions_fallback_test'] = ['test/boost/exceptions_fallback_test.cc', 'utils/exceptions.cc']
@@ -1308,6 +1309,7 @@ deps['test/boost/schema_loader_test'] += ['tools/schema_loader.cc']
 deps['test/boost/rust_test'] += rusts
 
 deps['test/raft/replication_test'] = ['test/raft/replication_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
+deps['test/raft/raft_server_test'] = ['test/raft/raft_server_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
 deps['test/raft/randomized_nemesis_test'] = ['test/raft/randomized_nemesis_test.cc', 'direct_failure_detector/failure_detector.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
 deps['test/raft/failure_detector_test'] = ['test/raft/failure_detector_test.cc', 'direct_failure_detector/failure_detector.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
 deps['test/raft/many_test'] = ['test/raft/many_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
@@ -1407,7 +1409,8 @@ if flag_supported(flag='-Wstack-usage=4096', compiler=args.cxx):
     for mode in modes:
         modes[mode]['cxxflags'] += f' -Wstack-usage={modes[mode]["stack-usage-threshold"]} -Wno-error=stack-usage='
 
-has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')
+if not has_wasmtime:
+    has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')
 
 if has_wasmtime:
     if platform.machine() == 'aarch64':
@@ -1423,6 +1426,7 @@ linker_flags = linker_flags(compiler=args.cxx)
 
 dbgflag = '-g -gz' if args.debuginfo else ''
 tests_link_rule = 'link' if args.tests_debuginfo else 'link_stripped'
+perf_tests_link_rule = 'link' if args.perf_tests_debuginfo else 'link_stripped'
 
 # Strip if debuginfo is disabled, otherwise we end up with partial
 # debug info from the libraries we static link with
@@ -1536,7 +1540,8 @@ if args.artifacts:
 else:
     build_artifacts = all_artifacts
 
-status = subprocess.call("./SCYLLA-VERSION-GEN")
+date_stamp = f"--date-stamp {args.date_stamp}" if args.date_stamp else ""
+status = subprocess.call(f"./SCYLLA-VERSION-GEN {date_stamp}", shell=True)
 if status != 0:
     print('Version file generation failed')
     sys.exit(1)
@@ -1551,7 +1556,8 @@ scylla_product = file.read().strip()
 arch = platform.machine()
 
 for m, mode_config in modes.items():
-    cxxflags = "-DSCYLLA_VERSION=\"\\\"" + scylla_version + "\\\"\" -DSCYLLA_RELEASE=\"\\\"" + scylla_release + "\\\"\" -DSCYLLA_BUILD_MODE=\"\\\"" + m + "\\\"\""
+    mode_config['cxxflags'] += f" -DSCYLLA_BUILD_MODE={m}"
+    cxxflags = "-DSCYLLA_VERSION=\"\\\"" + scylla_version + "\\\"\" -DSCYLLA_RELEASE=\"\\\"" + scylla_release + "\\\"\""
     mode_config["per_src_extra_cxxflags"]["release.cc"] = cxxflags
     if mode_config["can_have_debug_info"]:
         mode_config['cxxflags'] += ' ' + dbgflag
@@ -1736,6 +1742,8 @@ libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-l
                 ])
 if has_wasmtime:
     print("Found wasmtime dependency, linking with libwasmtime")
+    if use_wasmtime_as_library:
+        libs += " -lwasmtime"
 
 if not args.staticboost:
     args.user_cflags += ' -DBOOST_TEST_DYN_LINK'
@@ -1754,7 +1762,7 @@ if any(filter(thrift_version.startswith, thrift_boost_versions)):
 for pkg in pkgs:
     args.user_cflags += ' ' + pkg_config(pkg, '--cflags')
     libs += ' ' + pkg_config(pkg, '--libs')
-args.user_cflags += ' -Iabseil'
+args.user_cflags += ' -isystem abseil'
 user_cflags = args.user_cflags + ' -fvisibility=hidden'
 user_ldflags = args.user_ldflags + ' -fvisibility=hidden'
 if args.staticcxx:
@@ -1817,8 +1825,14 @@ with open(buildfile, 'w') as f:
         rule copy
             command = cp --reflink=auto $in $out
             description = COPY $out
+        rule strip
+            command = scripts/strip.sh $in
         rule package
             command = scripts/create-relocatable-package.py --mode $mode $out
+        rule stripped_package
+            command = scripts/create-relocatable-package.py --stripped --mode $mode $out
+        rule debuginfo_package
+            command = dist/debuginfo/scripts/create-relocatable-package.py --mode $mode $out
         rule rpmbuild
             command = reloc/build_rpm.sh --reloc-pkg $in --builddir $out
         rule debbuild
@@ -1917,7 +1931,7 @@ with open(buildfile, 'w') as f:
                     for src in srcs
                     if src.endswith('.cc')]
             objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S')
-            if has_wasmtime:
+            if has_wasmtime and not use_wasmtime_as_library:
                 objs.append('/usr/lib64/libwasmtime.a')
             has_thrift = False
             for dep in deps[binary]:
@@ -1954,7 +1968,8 @@ with open(buildfile, 'w') as f:
                     # So we strip the tests by default; The user can very
                     # quickly re-link the test unstripped by adding a "_g"
                     # to the test name, e.g., "ninja build/release/testname_g"
-                    f.write('build $builddir/{}/{}: {}.{} {} | {} {}\n'.format(mode, binary, tests_link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
+                    link_rule = perf_tests_link_rule if binary.startswith('test/perf/') else tests_link_rule
+                    f.write('build $builddir/{}/{}: {}.{} {} | {} {}\n'.format(mode, binary, link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
                     f.write('   libs = {}\n'.format(local_libs))
                     f.write('build $builddir/{}/{}_g: {}.{} {} | {} {}\n'.format(mode, binary, regular_link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
                     f.write('   libs = {}\n'.format(local_libs))
@@ -1962,6 +1977,8 @@ with open(buildfile, 'w') as f:
                     f.write('build $builddir/{}/{}: {}.{} {} | {}\n'.format(mode, binary, regular_link_rule, mode, str.join(' ', objs), seastar_dep))
                     if has_thrift:
                         f.write('   libs =  {} {} $seastar_libs_{} $libs\n'.format(thrift_libs, maybe_static(args.staticboost, '-lboost_system'), mode))
+                    f.write(f'build $builddir/{mode}/{binary}.stripped: strip $builddir/{mode}/{binary}\n')
+                    f.write(f'build $builddir/{mode}/{binary}.debug: phony $builddir/{mode}/{binary}.stripped\n')
             for src in srcs:
                 if src.endswith('.cc'):
                     obj = '$builddir/' + mode + '/' + src.replace('.cc', '.o')
@@ -2096,22 +2113,36 @@ with open(buildfile, 'w') as f:
         f.write('  target = iotune\n'.format(**locals()))
         f.write(textwrap.dedent('''\
             build $builddir/{mode}/iotune: copy $builddir/{mode}/seastar/apps/iotune/iotune
+            build $builddir/{mode}/iotune.stripped: strip $builddir/{mode}/iotune
+            build $builddir/{mode}/iotune.debug: phony $builddir/{mode}/iotune.stripped
             ''').format(**locals()))
-        include_scylla_and_iotune = f'$builddir/{mode}/scylla $builddir/{mode}/iotune' if not args.dist_only else ''
-        f.write('build $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz: package {include_scylla_and_iotune} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter | always\n'.format(**locals()))
+        if args.dist_only:
+            include_scylla_and_iotune = ''
+            include_scylla_and_iotune_stripped = ''
+            include_scylla_and_iotune_debug = ''
+        else:
+            include_scylla_and_iotune = f'$builddir/{mode}/scylla $builddir/{mode}/iotune'
+            include_scylla_and_iotune_stripped = f'$builddir/{mode}/scylla.stripped $builddir/{mode}/iotune.stripped'
+            include_scylla_and_iotune_debug = f'$builddir/{mode}/scylla.debug $builddir/{mode}/iotune.debug'
+        f.write('build $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz: package {include_scylla_and_iotune} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter | always\n'.format(**locals()))
+        f.write('  mode = {mode}\n'.format(**locals()))
+        f.write('build $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz: stripped_package {include_scylla_and_iotune_stripped} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter.stripped | always\n'.format(**locals()))
+        f.write('  mode = {mode}\n'.format(**locals()))
+        f.write('build $builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz: debuginfo_package {include_scylla_and_iotune_debug} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter.debug | always\n'.format(**locals()))
         f.write('  mode = {mode}\n'.format(**locals()))
         f.write('build $builddir/{mode}/dist/tar/{scylla_product}-package.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n'.format(**locals()))
         f.write('  mode = {mode}\n'.format(**locals()))
         f.write('build $builddir/{mode}/dist/tar/{scylla_product}-{arch}-package.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n'.format(**locals()))
         f.write('  mode = {mode}\n'.format(**locals()))
 
-        f.write(f'build $builddir/dist/{mode}/redhat: rpmbuild $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
+        f.write(f'build $builddir/dist/{mode}/redhat: rpmbuild $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
         f.write(f'  mode = {mode}\n')
-        f.write(f'build $builddir/dist/{mode}/debian: debbuild $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
+        f.write(f'build $builddir/dist/{mode}/debian: debbuild $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
         f.write(f'  mode = {mode}\n')
         f.write(f'build dist-server-{mode}: phony $builddir/dist/{mode}/redhat $builddir/dist/{mode}/debian dist-server-compat-{mode} dist-server-compat-arch-{mode}\n')
         f.write(f'build dist-server-compat-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-package.tar.gz\n')
         f.write(f'build dist-server-compat-arch-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-{arch}-package.tar.gz\n')
+        f.write(f'build dist-server-debuginfo-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
         f.write(f'build dist-jmx-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz dist-jmx-rpm dist-jmx-deb dist-jmx-compat\n')
         f.write(f'build dist-tools-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz dist-tools-rpm dist-tools-deb dist-tools-compat\n')
         f.write(f'build dist-python3-{mode}: phony dist-python3-tar dist-python3-rpm dist-python3-deb dist-python3-compat dist-python3-compat-arch\n')
@@ -2155,9 +2186,10 @@ with open(buildfile, 'w') as f:
         build dist-server-deb: phony {' '.join(['$builddir/dist/{mode}/debian'.format(mode=mode) for mode in build_modes])}
         build dist-server-rpm: phony {' '.join(['$builddir/dist/{mode}/redhat'.format(mode=mode) for mode in build_modes])}
         build dist-server-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
+        build dist-server-debuginfo: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
         build dist-server-compat: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-package.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch) for mode in default_modes])}
         build dist-server-compat-arch: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-{arch}-package.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch) for mode in default_modes])}
-        build dist-server: phony dist-server-tar dist-server-compat dist-server-compat-arch dist-server-rpm dist-server-deb
+        build dist-server: phony dist-server-tar dist-server-debuginfo dist-server-compat dist-server-compat-arch dist-server-rpm dist-server-deb
 
         rule build-submodule-reloc
           command = cd $reloc_dir && ./reloc/build_reloc.sh --version $$(<../../build/SCYLLA-PRODUCT-FILE)-$$(sed 's/-/~/' <../../build/SCYLLA-VERSION-FILE)-$$(<../../build/SCYLLA-RELEASE-FILE) --nodeps $args
@@ -2227,7 +2259,7 @@ with open(buildfile, 'w') as f:
         build $builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz: copy tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
         build $builddir/{mode}/dist/tar/{scylla_product}-jmx-package.tar.gz: copy tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
 
-        build {mode}-dist: phony dist-server-{mode} dist-python3-{mode} dist-tools-{mode} dist-jmx-{mode} dist-unified-{mode}
+        build {mode}-dist: phony dist-server-{mode} dist-server-debuginfo-{mode} dist-python3-{mode} dist-tools-{mode} dist-jmx-{mode} dist-unified-{mode}
         build dist-{mode}: phony {mode}-dist
         build dist-check-{mode}: dist-check
           mode = {mode}
@@ -2270,7 +2302,9 @@ with open(buildfile, 'w') as f:
         build $builddir/debian/debian: debian_files_gen | always
         rule extract_node_exporter
             command = tar -C build -xvpf {node_exporter_filename} --no-same-owner && rm -rfv build/node_exporter && mv -v build/{node_exporter_dirname} build/node_exporter
-        build $builddir/node_exporter: extract_node_exporter | always
+        build $builddir/node_exporter/node_exporter: extract_node_exporter | always
+        build $builddir/node_exporter/node_exporter.stripped: strip $builddir/node_exporter/node_exporter
+        build $builddir/node_exporter/node_exporter.debug: phony $builddir/node_exporter/node_exporter.stripped
         rule print_help
              command = ./scripts/build-help.sh
         build help: print_help | always

counters.cc

@@ -12,10 +12,6 @@
 
 #include <boost/range/algorithm/sort.hpp>
 
-std::ostream& operator<<(std::ostream& os, const counter_id& id) {
-    return os << id.to_uuid();
-}
-
 std::ostream& operator<<(std::ostream& os, counter_shard_view csv) {
     return os << "{global_shard id: " << csv.id() << " value: " << csv.value()
               << " clock: " << csv.logical_clock() << "}";
@@ -174,9 +170,11 @@ std::optional<atomic_cell> counter_cell_view::difference(atomic_cell_view a, ato
 }
 
 
-void transform_counter_updates_to_shards(mutation& m, const mutation* current_state, uint64_t clock_offset, utils::UUID local_id) {
+void transform_counter_updates_to_shards(mutation& m, const mutation* current_state, uint64_t clock_offset, locator::host_id local_host_id) {
     // FIXME: allow current_state to be frozen_mutation
 
+    utils::UUID local_id = local_host_id.uuid();
+
     auto transform_new_row_to_shards = [&s = *m.schema(), clock_offset, local_id] (column_kind kind, auto& cells) {
         cells.for_each_cell([&] (column_id id, atomic_cell_or_collection& ac_o_c) {
             auto& cdef = s.column_at(kind, id);

counters.hh

@@ -14,51 +14,12 @@
 
 #include "atomic_cell.hh"
 #include "types.hh"
+#include "locator/host_id.hh"
 
 class mutation;
 class atomic_cell_or_collection;
 
-class counter_id {
-    int64_t _least_significant;
-    int64_t _most_significant;
-public:
-    static_assert(std::is_same<decltype(std::declval<utils::UUID>().get_least_significant_bits()), int64_t>::value
-            &&  std::is_same<decltype(std::declval<utils::UUID>().get_most_significant_bits()), int64_t>::value,
-        "utils::UUID is expected to work with two signed 64-bit integers");
-
-    counter_id() = default;
-    explicit counter_id(utils::UUID uuid) noexcept
-        : _least_significant(uuid.get_least_significant_bits())
-        , _most_significant(uuid.get_most_significant_bits())
-    { }
-
-    utils::UUID to_uuid() const {
-        return utils::UUID(_most_significant, _least_significant);
-    }
-
-    bool operator<(const counter_id& other) const {
-        return to_uuid() < other.to_uuid();
-    }
-    bool operator>(const counter_id& other) const {
-        return other.to_uuid() < to_uuid();
-    }
-    bool operator==(const counter_id& other) const {
-        return to_uuid() == other.to_uuid();
-    }
-    bool operator!=(const counter_id& other) const {
-        return !(*this == other);
-    }
-public:
-    // For tests.
-    static counter_id generate_random() {
-        return counter_id(utils::make_random_uuid());
-    }
-};
-static_assert(
-        std::is_standard_layout_v<counter_id> && std::is_trivial_v<counter_id>,
-        "counter_id should be a POD type");
-
-std::ostream& operator<<(std::ostream& os, const counter_id& id);
+using counter_id = utils::tagged_uuid<struct counter_id_tag>;
 
 template<mutable_view is_mutable>
 class basic_counter_shard_view {
@@ -406,13 +367,13 @@ struct counter_cell_mutable_view : basic_counter_cell_view<mutable_view::yes> {
 // Transforms mutation dst from counter updates to counter shards using state
 // stored in current_state.
 // If current_state is present it has to be in the same schema as dst.
-void transform_counter_updates_to_shards(mutation& dst, const mutation* current_state, uint64_t clock_offset, utils::UUID local_id);
+void transform_counter_updates_to_shards(mutation& dst, const mutation* current_state, uint64_t clock_offset, locator::host_id local_id);
 
 template<>
 struct appending_hash<counter_shard_view> {
     template<typename Hasher>
     void operator()(Hasher& h, const counter_shard_view& cshard) const {
-        ::feed_hash(h, cshard.id().to_uuid());
+        ::feed_hash(h, cshard.id());
         ::feed_hash(h, cshard.value());
         ::feed_hash(h, cshard.logical_clock());
     }

cql3/Cql.g

@@ -44,7 +44,7 @@ options {
 #include "cql3/statements/drop_aggregate_statement.hh"
 #include "cql3/statements/drop_service_level_statement.hh"
 #include "cql3/statements/detach_service_level_statement.hh"
-#include "cql3/statements/truncate_statement.hh"
+#include "cql3/statements/raw/truncate_statement.hh"
 #include "cql3/statements/raw/update_statement.hh"
 #include "cql3/statements/raw/insert_statement.hh"
 #include "cql3/statements/raw/delete_statement.hh"
@@ -371,7 +371,8 @@ useStatement returns [std::unique_ptr<raw::use_statement> stmt]
  * SELECT [JSON] <expression>
  * FROM <CF>
  * WHERE KEY = "key1" AND COL > 1 AND COL < 100
- * LIMIT <NUMBER>;
+ * LIMIT <NUMBER>
+ * [USING TIMEOUT <duration>];
  */
 selectStatement returns [std::unique_ptr<raw::select_statement> expr]
     @init {
@@ -398,7 +399,7 @@ selectStatement returns [std::unique_ptr<raw::select_statement> expr]
       ( K_LIMIT rows=intValue { limit = rows; } )?
       ( K_ALLOW K_FILTERING  { allow_filtering = true; } )?
       ( K_BYPASS K_CACHE { bypass_cache = true; })?
-      ( usingClause[attrs] )?
+      ( usingTimeoutClause[attrs] )?
       {
           auto params = make_lw_shared<raw::select_statement::parameters>(std::move(orderings), is_distinct, allow_filtering, statement_subtype, bypass_cache);
           $expr = std::make_unique<raw::select_statement>(std::move(cf), std::move(params),
@@ -518,6 +519,19 @@ usingClauseObjective[std::unique_ptr<cql3::attributes::raw>& attrs]
     | K_TIMEOUT to=term { attrs->timeout = to; }
     ;
 
+usingTimestampTimeoutClause[std::unique_ptr<cql3::attributes::raw>& attrs]
+    : K_USING usingTimestampTimeoutClauseObjective[attrs] ( K_AND usingTimestampTimeoutClauseObjective[attrs] )*
+    ;
+
+usingTimestampTimeoutClauseObjective[std::unique_ptr<cql3::attributes::raw>& attrs]
+    : K_TIMESTAMP ts=intValue { attrs->timestamp = ts; }
+    | K_TIMEOUT to=term { attrs->timeout = to; }
+    ;
+
+usingTimeoutClause[std::unique_ptr<cql3::attributes::raw>& attrs]
+    : K_USING K_TIMEOUT to=term { attrs->timeout = to; }
+    ;
+
 /**
  * UPDATE <CF>
  * USING TIMESTAMP <long>
@@ -552,7 +566,7 @@ updateConditions returns [conditions_type conditions]
 /**
  * DELETE name1, name2
  * FROM <CF>
- * USING TIMESTAMP <long>
+ * [USING (TIMESTAMP <long> | TIMEOUT <duration>) [AND ...]]
  * WHERE KEY = keyname
    [IF (EXISTS | name = value, ...)];
  */
@@ -564,7 +578,7 @@ deleteStatement returns [std::unique_ptr<raw::delete_statement> expr]
     }
     : K_DELETE ( dels=deleteSelection { column_deletions = std::move(dels); } )?
       K_FROM cf=columnFamilyName
-      ( usingClause[attrs] )?
+      ( usingTimestampTimeoutClause[attrs] )?
       K_WHERE wclause=whereClause
       ( K_IF ( K_EXISTS { if_exists = true; } | conditions=updateConditions ))?
       {
@@ -857,7 +871,8 @@ indexIdent returns [::shared_ptr<index_target::raw> id]
     @init {
         std::vector<::shared_ptr<cql3::column_identifier::raw>> columns;
     }
-    : c=cident                   { $id = index_target::raw::values_of(c); }
+    : c=cident                   { $id = index_target::raw::regular_values_of(c); }
+    | K_VALUES '(' c=cident ')'  { $id = index_target::raw::collection_values_of(c); }
     | K_KEYS '(' c=cident ')'    { $id = index_target::raw::keys_of(c); }
     | K_ENTRIES '(' c=cident ')' { $id = index_target::raw::keys_and_values_of(c); }
     | K_FULL '(' c=cident ')'    { $id = index_target::raw::full_collection(c); }
@@ -1057,10 +1072,18 @@ dropIndexStatement returns [std::unique_ptr<drop_index_statement> expr]
     ;
 
 /**
-  * TRUNCATE <CF>;
+  * TRUNCATE [TABLE] <CF>
+  * [USING TIMEOUT <duration>];
   */
-truncateStatement returns [std::unique_ptr<truncate_statement> stmt]
-    : K_TRUNCATE (K_COLUMNFAMILY)? cf=columnFamilyName { $stmt = std::make_unique<truncate_statement>(cf); }
+truncateStatement returns [std::unique_ptr<raw::truncate_statement> stmt]
+    @init {
+        auto attrs = std::make_unique<cql3::attributes::raw>();
+    }
+    : K_TRUNCATE (K_COLUMNFAMILY)? cf=columnFamilyName
+      ( usingTimeoutClause[attrs] )?
+      {
+        $stmt = std::make_unique<raw::truncate_statement>(std::move(cf), std::move(attrs));
+      }
     ;
 
 /**

cql3/column_condition.hh

@@ -73,6 +73,14 @@ public:
             std::move(in_values), nullptr, expr::oper_t::IN);
     }
 
+    const std::optional<expr::expression>& get_value() const {
+        return _value;
+    }
+
+    expr::oper_t get_operation() const {
+        return _op;
+    }
+
     class raw final {
     private:
         std::optional<cql3::expr::expression> _value;

cql3/constants.cc

@@ -10,6 +10,7 @@
 
 #include "cql3/constants.hh"
 #include "cql3/cql3_type.hh"
+#include "cql3/statements/strongly_consistent_modification_statement.hh"
 
 namespace cql3 {
 void constants::deleter::execute(mutation& m, const clustering_key_prefix& prefix, const update_parameters& params) {
@@ -22,4 +23,9 @@ void constants::deleter::execute(mutation& m, const clustering_key_prefix& prefi
         m.set_cell(prefix, column, params.make_dead_cell());
     }
 }
+
+void
+constants::setter::prepare_for_broadcast_tables(statements::broadcast_tables::prepared_update& query) const {
+    query.new_value = *_e;
+}
 }

cql3/constants.hh

@@ -11,12 +11,17 @@
 #pragma once
 
 #include "cql3/abstract_marker.hh"
+#include "cql3/query_options.hh"
 #include "cql3/update_parameters.hh"
 #include "cql3/operation.hh"
 #include "cql3/values.hh"
 #include "mutation.hh"
 #include <seastar/core/shared_ptr.hh>
 
+namespace service::broadcast_tables {
+    class update_query;
+}
+
 namespace cql3 {
 
 /**
@@ -44,6 +49,8 @@ public:
                 m.set_cell(prefix, column, params.make_cell(*column.type, value));
             }
         }
+
+        virtual void prepare_for_broadcast_tables(statements::broadcast_tables::prepared_update& query) const override;
     };
 
     struct adder final : operation {

cql3/expr/expression.cc

@@ -335,7 +335,8 @@ bool limits(const tuple_constructor& columns_tuple, const oper_t op, const expre
 /// True iff collection (list, set, or map) contains value.
 bool contains(const data_value& collection, const raw_value_view& value) {
     if (!value) {
-        return true; // Compatible with old code, which skips null terms in value comparisons.
+        // CONTAINS NULL should evaluate to NULL/false
+        return false;
     }
     auto col_type = static_pointer_cast<const collection_type_impl>(collection.type());
     auto&& element_type = col_type->is_set() ? col_type->name_comparator() : col_type->value_comparator();
@@ -373,7 +374,8 @@ bool contains(const column_value& col, const raw_value_view& value, const evalua
 /// True iff a column is a map containing \p key.
 bool contains_key(const column_value& col, cql3::raw_value_view key, const evaluation_inputs& inputs) {
     if (!key) {
-        return true; // Compatible with old code, which skips null terms in key comparisons.
+        // CONTAINS_KEY NULL should evaluate to NULL/false
+        return false;
     }
     auto type = col.col->type;
     const auto collection = get_value(col, inputs);
@@ -458,8 +460,9 @@ bool like(const column_value& cv, const raw_value_view& pattern, const evaluatio
 /// True iff the column value is in the set defined by rhs.
 bool is_one_of(const expression& col, const expression& rhs, const evaluation_inputs& inputs) {
     const cql3::raw_value in_list = evaluate(rhs, inputs);
-    statements::request_validations::check_false(
-            in_list.is_null(), "Invalid null value for column {}", col);
+    if (in_list.is_null()) {
+        return false;
+    }
 
     return boost::algorithm::any_of(get_list_elements(in_list), [&] (const managed_bytes_opt& b) {
         return equal(col, b, inputs);
@@ -695,7 +698,9 @@ value_list get_IN_values(
     if (in_list.is_unset_value()) {
         throw exceptions::invalid_request_exception(format("Invalid unset value for column {}", column_name));
     }
-    statements::request_validations::check_false(in_list.is_null(), "Invalid null value for column {}", column_name);
+    if (in_list.is_null()) {
+        return value_list();
+    }
     utils::chunked_vector<managed_bytes> list_elems = get_list_elements(in_list);
     return to_sorted_vector(std::move(list_elems) | non_null | deref, comparator);
 }
@@ -741,25 +746,25 @@ expression make_conjunction(expression a, expression b) {
     return conjunction{std::move(children)};
 }
 
-static
-void
-do_factorize(std::vector<expression>& factors, expression e) {
-    if (auto c = expr::as_if<conjunction>(&e)) {
-        for (auto&& element : c->children) {
-            do_factorize(factors, std::move(element));
-        }
-    } else {
-        factors.push_back(std::move(e));
-    }
-}
-
 std::vector<expression>
 boolean_factors(expression e) {
     std::vector<expression> ret;
-    do_factorize(ret, std::move(e));
+    for_each_boolean_factor(e, [&](const expression& boolean_factor) {
+        ret.push_back(boolean_factor);
+    });
     return ret;
 }
 
+void for_each_boolean_factor(const expression& e, const noncopyable_function<void (const expression&)>& for_each_func) {
+    if (auto conj = as_if<conjunction>(&e)) {
+        for (const expression& child : conj->children) {
+            for_each_boolean_factor(child, for_each_func);
+        }
+    } else {
+        for_each_func(e);
+    }
+}
+
 template<typename T>
 nonwrapping_range<std::remove_cvref_t<T>> to_range(oper_t op, T&& val) {
     using U = std::remove_cvref_t<T>;
@@ -818,11 +823,37 @@ value_set possible_lhs_values(const column_definition* cdef, const expression& e
                                         : to_range(oper.op, std::move(*val));
                             } else if (oper.op == oper_t::IN) {
                                 return get_IN_values(oper.rhs, options, type->as_less_comparator(), cdef->name_as_text());
+                            } else if (oper.op == oper_t::CONTAINS || oper.op == oper_t::CONTAINS_KEY) {
+                                managed_bytes_opt val = evaluate(oper.rhs, options).to_managed_bytes_opt();
+                                if (!val) {
+                                    return empty_value_set; // All NULL comparisons fail; no column values match.
+                                }
+                                return value_set(value_list{*val});
                             }
                             throw std::logic_error(format("possible_lhs_values: unhandled operator {}", oper));
                         },
                         [&] (const subscript& s) -> value_set {
-                            on_internal_error(expr_logger, "possible_lhs_values: subscripts are not supported as the LHS of a binary expression");
+                            const column_value& col = get_subscripted_column(s);
+
+                            if (!cdef || cdef != col.col) {
+                                return unbounded_value_set;
+                            }
+
+                            managed_bytes_opt sval = evaluate(s.sub, options).to_managed_bytes_opt();
+                            if (!sval) {
+                                return empty_value_set; // NULL can't be a map key
+                            }
+
+                            if (oper.op == oper_t::EQ) {
+                                managed_bytes_opt rval = evaluate(oper.rhs, options).to_managed_bytes_opt();
+                                if (!rval) {
+                                    return empty_value_set; // All NULL comparisons fail; no column values match.
+                                }
+                                managed_bytes_opt elements[] = {sval, rval};
+                                managed_bytes val = tuple_type_impl::build_value_fragmented(elements);
+                                return value_set(value_list{val});
+                            }
+                            throw std::logic_error(format("possible_lhs_values: unhandled operator {}", oper));
                         },
                         [&] (const tuple_constructor& tuple) -> value_set {
                             if (!cdef) {
@@ -979,11 +1010,26 @@ nonwrapping_range<managed_bytes> to_range(const value_set& s) {
         }, s);
 }
 
-bool is_supported_by(const expression& expr, const secondary_index::index& idx) {
-    using std::placeholders::_1;
+namespace {
+constexpr inline secondary_index::index::supports_expression_v operator&&(secondary_index::index::supports_expression_v v1, secondary_index::index::supports_expression_v v2) {
+    using namespace secondary_index;
+    auto True = index::supports_expression_v::from_bool(true);
+    return v1 == True && v2 == True ? True : index::supports_expression_v::from_bool(false);
+}
+
+secondary_index::index::supports_expression_v is_supported_by_helper(const expression& expr, const secondary_index::index& idx) {
+    using ret_t = secondary_index::index::supports_expression_v;
+    using namespace secondary_index;
     return expr::visit(overloaded_functor{
-            [&] (const conjunction& conj) {
-                return boost::algorithm::all_of(conj.children, std::bind(is_supported_by, _1, idx));
+            [&] (const conjunction& conj) -> ret_t {
+                if (conj.children.empty()) {
+                    return index::supports_expression_v::from_bool(true);
+                }
+                auto init = is_supported_by_helper(conj.children[0], idx);
+                return std::accumulate(std::begin(conj.children) + 1, std::end(conj.children), init, 
+                        [&] (ret_t acc, const expression& child) -> ret_t {
+                            return acc && is_supported_by_helper(child, idx);
+                });
             },
             [&] (const binary_operator& oper) {
                 return expr::visit(overloaded_functor{
@@ -997,57 +1043,64 @@ bool is_supported_by(const expression& expr, const secondary_index::index& idx)
                                 }
                             }
                             // We don't use index table for multi-column restrictions, as it cannot avoid filtering.
-                            return false;
+                            return index::supports_expression_v::from_bool(false);
                         },
-                        [&] (const token&) { return false; },
-                        [&] (const subscript& s) -> bool {
-                            // We don't support indexes on map entries yet.
-                            return false;
+                        [&] (const token&) { return index::supports_expression_v::from_bool(false); },
+                        [&] (const subscript& s) -> ret_t {
+                            const column_value& col = get_subscripted_column(s);
+                            return idx.supports_subscript_expression(*col.col, oper.op);
                         },
-                        [&] (const binary_operator&) -> bool {
+                        [&] (const binary_operator&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: nested binary operators are not supported");
                         },
-                        [&] (const conjunction&) -> bool {
+                        [&] (const conjunction&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: conjunctions are not supported as the LHS of a binary expression");
                         },
-                        [] (const constant&) -> bool {
+                        [] (const constant&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: constants are not supported as the LHS of a binary expression");
                         },
-                        [] (const unresolved_identifier&) -> bool {
+                        [] (const unresolved_identifier&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: an unresolved identifier is not supported as the LHS of a binary expression");
                         },
-                        [&] (const column_mutation_attribute&) -> bool {
+                        [&] (const column_mutation_attribute&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: writetime/ttl are not supported as the LHS of a binary expression");
                         },
-                        [&] (const function_call&) -> bool {
+                        [&] (const function_call&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: function calls are not supported as the LHS of a binary expression");
                         },
-                        [&] (const cast&) -> bool {
+                        [&] (const cast&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: typecasts are not supported as the LHS of a binary expression");
                         },
-                        [&] (const field_selection&) -> bool {
+                        [&] (const field_selection&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: field selections are not supported as the LHS of a binary expression");
                         },
-                        [&] (const null&) -> bool {
+                        [&] (const null&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: nulls are not supported as the LHS of a binary expression");
                         },
-                        [&] (const bind_variable&) -> bool {
+                        [&] (const bind_variable&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: bind variables are not supported as the LHS of a binary expression");
                         },
-                        [&] (const untyped_constant&) -> bool {
+                        [&] (const untyped_constant&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: untyped constants are not supported as the LHS of a binary expression");
                         },
-                        [&] (const collection_constructor&) -> bool {
+                        [&] (const collection_constructor&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: collection constructors are not supported as the LHS of a binary expression");
                         },
-                        [&] (const usertype_constructor&) -> bool {
+                        [&] (const usertype_constructor&) -> ret_t {
                             on_internal_error(expr_logger, "is_supported_by: user type constructors are not supported as the LHS of a binary expression");
                         },
                     }, oper.lhs);
             },
-            [] (const auto& default_case) { return false; }
+            [] (const auto& default_case) { return index::supports_expression_v::from_bool(false); }
         }, expr);
 }
+}
+
+bool is_supported_by(const expression& expr, const secondary_index::index& idx) {
+    auto s = is_supported_by_helper(expr, idx);
+    return s != secondary_index::index::supports_expression_v::from_bool(false);
+}
+
 
 bool has_supporting_index(
         const expression& expr,

cql3/expr/expression.hh

@@ -666,6 +666,9 @@ inline auto find_clustering_order(const expression& e) {
 /// empty conjunction).
 std::vector<expression> boolean_factors(expression e);
 
+/// Run the given function for each element in the top level conjunction.
+void for_each_boolean_factor(const expression& e, const noncopyable_function<void (const expression&)>& for_each_func);
+
 /// True iff binary_operator involves a collection.
 extern bool is_on_collection(const binary_operator&);
 

cql3/operation.cc

@@ -18,6 +18,7 @@
 #include "types/list.hh"
 #include "types/set.hh"
 #include "types/user.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
 
 namespace cql3 {
 
@@ -297,8 +298,8 @@ operation::set_counter_value_from_tuple_list::prepare(data_dictionary::database
                 if (id <= last) {
                     throw marshal_exception(
                                     format("invalid counter id order, {} <= {}",
-                                                    id.to_uuid().to_sstring(),
-                                                    last.to_uuid().to_sstring()));
+                                                    id.uuid().to_sstring(),
+                                                    last.uuid().to_sstring()));
                 }
                 last = id;
                 // TODO: maybe allow more than global values to propagate,
@@ -357,4 +358,10 @@ operation::element_deletion::prepare(data_dictionary::database db, const sstring
     abort();
 }
 
+void
+operation::prepare_for_broadcast_tables(statements::broadcast_tables::prepared_update&) const {
+    // FIXME: implement for every type of `operation`.
+    throw service::broadcast_tables::unsupported_operation_error{};
+}
+
 }

cql3/operation.hh

@@ -11,6 +11,7 @@
 #pragma once
 
 #include <seastar/core/shared_ptr.hh>
+#include "cql3/cql3_type.hh"
 #include "exceptions/exceptions.hh"
 #include "data_dictionary/data_dictionary.hh"
 #include "update_parameters.hh"
@@ -21,6 +22,10 @@
 
 namespace cql3 {
 
+namespace statements::broadcast_tables {
+    struct prepared_update;
+}
+
 class update_parameters;
 
 /**
@@ -85,6 +90,8 @@ public:
      * Execute the operation.
      */
     virtual void execute(mutation& m, const clustering_key_prefix& prefix, const update_parameters& params) = 0;
+    
+    virtual void prepare_for_broadcast_tables(statements::broadcast_tables::prepared_update&) const;
 
     /**
      * A parsed raw UPDATE operation.

cql3/query_processor.cc

@@ -55,7 +55,7 @@ public:
     }
 };
 
-query_processor::query_processor(service::storage_proxy& proxy, service::forward_service& forwarder, data_dictionary::database db, service::migration_notifier& mn, service::migration_manager& mm, query_processor::memory_config mcfg, cql_config& cql_cfg, utils::loading_cache_config auth_prep_cache_cfg)
+query_processor::query_processor(service::storage_proxy& proxy, service::forward_service& forwarder, data_dictionary::database db, service::migration_notifier& mn, service::migration_manager& mm, query_processor::memory_config mcfg, cql_config& cql_cfg, utils::loading_cache_config auth_prep_cache_cfg, service::raft_group0_client& group0_client)
         : _migration_subscriber{std::make_unique<migration_subscriber>(this)}
         , _proxy(proxy)
         , _forwarder(forwarder)
@@ -64,6 +64,7 @@ query_processor::query_processor(service::storage_proxy& proxy, service::forward
         , _mm(mm)
         , _mcfg(mcfg)
         , _cql_config(cql_cfg)
+        , _group0_client(group0_client)
         , _internal_state(new internal_state())
         , _prepared_cache(prep_cache_log, _mcfg.prepared_statment_cache_size)
         , _authorized_prepared_cache(std::move(auth_prep_cache_cfg), authorized_prepared_statements_cache_log)

cql3/query_processor.hh

@@ -23,6 +23,7 @@
 #include "exceptions/exceptions.hh"
 #include "lang/wasm_instance_cache.hh"
 #include "service/migration_listener.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "transport/messages/result_message.hh"
 #include "service/qos/service_level_controller.hh"
 #include "service/client_state.hh"
@@ -98,6 +99,7 @@ private:
     service::migration_manager& _mm;
     memory_config _mcfg;
     const cql_config& _cql_config;
+    service::raft_group0_client& _group0_client;
 
     struct stats {
         uint64_t prepare_invocations = 0;
@@ -138,7 +140,7 @@ public:
     static std::unique_ptr<statements::raw::parsed_statement> parse_statement(const std::string_view& query);
     static std::vector<std::unique_ptr<statements::raw::parsed_statement>> parse_statements(std::string_view queries);
 
-    query_processor(service::storage_proxy& proxy, service::forward_service& forwarder, data_dictionary::database db, service::migration_notifier& mn, service::migration_manager& mm, memory_config mcfg, cql_config& cql_cfg, utils::loading_cache_config auth_prep_cache_cfg);
+    query_processor(service::storage_proxy& proxy, service::forward_service& forwarder, data_dictionary::database db, service::migration_notifier& mn, service::migration_manager& mm, memory_config mcfg, cql_config& cql_cfg, utils::loading_cache_config auth_prep_cache_cfg, service::raft_group0_client& group0_client);
 
     ~query_processor();
 
@@ -202,6 +204,10 @@ public:
         return _prepared_cache.find(key);
     }
 
+    service::raft_group0_client& get_group0_client() {
+        return _group0_client;
+    }
+
     inline
     future<::shared_ptr<cql_transport::messages::result_message>>
     execute_prepared(

cql3/restrictions/statement_restrictions.cc

@@ -1169,8 +1169,11 @@ struct multi_column_range_accumulator {
             intersect_all(to_range(binop.op, clustering_key_prefix(std::move(values))));
         } else if (binop.op == oper_t::IN) {
             const cql3::raw_value tup = expr::evaluate(binop.rhs, options);
-            statements::request_validations::check_false(tup.is_null(), "Invalid null value for IN restriction");
-            process_in_values(expr::get_list_of_tuples_elements(tup, *type_of(binop.rhs)));
+            utils::chunked_vector<std::vector<managed_bytes_opt>> tuple_elems;
+            if (tup.is_value()) {
+                tuple_elems = expr::get_list_of_tuples_elements(tup, *type_of(binop.rhs));
+            }
+            process_in_values(std::move(tuple_elems));
         } else {
             on_internal_error(rlogger, format("multi_column_range_accumulator: unexpected atom {}", binop));
         }
@@ -1879,17 +1882,12 @@ std::vector<query::clustering_range> statement_restrictions::get_global_index_cl
     std::transform(pk_value.cbegin(), pk_value.cend(), pkv_linearized.begin(),
                    [] (const managed_bytes& mb) { return to_bytes(mb); });
     auto& token_column = idx_tbl_schema.clustering_column_at(0);
-    bytes_opt token_bytes = token_column.get_computation().compute_value(
-            *_schema, pkv_linearized, clustering_row(clustering_key_prefix::make_empty()));
-    if (!token_bytes) {
-        on_internal_error(rlogger,
-                          format("null value for token column in indexing table {}",
-                                 token_column.name_as_text()));
-    }
+    bytes token_bytes = token_column.get_computation().compute_value(*_schema, pkv_linearized);
+
     // WARNING: We must not yield to another fiber from here until the function's end, lest this RHS be
     // overwritten.
     const_cast<expr::expression&>(expr::as<binary_operator>((*_idx_tbl_ck_prefix)[0]).rhs) =
-            expr::constant(raw_value::make_value(*token_bytes), token_column.type);
+            expr::constant(raw_value::make_value(token_bytes), token_column.type);
 
     // Multi column restrictions are not added to _idx_tbl_ck_prefix, they are handled later by filtering.
     return get_single_column_clustering_bounds(options, idx_tbl_schema, *_idx_tbl_ck_prefix);

cql3/statements/alter_table_statement.cc

@@ -409,7 +409,8 @@ cql3::statements::alter_table_statement::prepare(data_dictionary::database db, c
 
 future<::shared_ptr<messages::result_message>>
 alter_table_statement::execute(query_processor& qp, service::query_state& state, const query_options& options) const {
-    std::optional<sstring> warning = check_restricted_table_properties(qp, keyspace(), column_family(), *_properties);
+    auto s = validation::validate_column_family(qp.db(), keyspace(), column_family());
+    std::optional<sstring> warning = check_restricted_table_properties(qp, s, keyspace(), column_family(), *_properties);
     return schema_altering_statement::execute(qp, state, options).then([this, warning = std::move(warning)] (::shared_ptr<messages::result_message> msg) {
         if (warning) {
             msg->add_warning(*warning);

cql3/statements/cas_request.cc

@@ -91,7 +91,8 @@ lw_shared_ptr<query::read_command> cas_request::read_command(query_processor& qp
     options.set(query::partition_slice::option::always_return_static_content);
     query::partition_slice ps(std::move(ranges), *_schema, columns_to_read, options);
     ps.set_partition_row_limit(max_rows);
-    return make_lw_shared<query::read_command>(_schema->id(), _schema->version(), std::move(ps), qp.proxy().get_max_result_size(ps));
+    return make_lw_shared<query::read_command>(_schema->id(), _schema->version(), std::move(ps), qp.proxy().get_max_result_size(ps),
+            query::tombstone_limit(qp.proxy().get_tombstone_limit()));
 }
 
 bool cas_request::applies_to() const {

cql3/statements/cf_prop_defs.cc

@@ -20,6 +20,7 @@
 #include "tombstone_gc.hh"
 #include "db/per_partition_rate_limit_extension.hh"
 #include "db/per_partition_rate_limit_options.hh"
+#include "utils/bloom_calculations.hh"
 
 #include <boost/algorithm/string/predicate.hpp>
 
@@ -152,6 +153,16 @@ void cf_prop_defs::validate(const data_dictionary::database db, sstring ks_name,
         throw exceptions::configuration_exception(KW_MAX_INDEX_INTERVAL + " must be greater than " + KW_MIN_INDEX_INTERVAL);
     }
 
+    if (get_simple(KW_BF_FP_CHANCE)) {
+        double bloom_filter_fp_chance = get_double(KW_BF_FP_CHANCE, 0/*not used*/);
+        double min_bloom_filter_fp_chance = utils::bloom_calculations::min_supported_bloom_filter_fp_chance();
+        if (bloom_filter_fp_chance <= min_bloom_filter_fp_chance || bloom_filter_fp_chance > 1.0) {
+            throw exceptions::configuration_exception(format(
+                "{} must be larger than {} and less than or equal to 1.0 (got {})",
+                KW_BF_FP_CHANCE, min_bloom_filter_fp_chance, bloom_filter_fp_chance));
+        }
+    }
+
     speculative_retry::from_sstring(get_string(KW_SPECULATIVE_RETRY, speculative_retry(speculative_retry::type::NONE, 0).to_sstring()));
 }
 
@@ -189,10 +200,10 @@ int32_t cf_prop_defs::get_paxos_grace_seconds() const {
     return get_int(KW_PAXOSGRACESECONDS, DEFAULT_GC_GRACE_SECONDS);
 }
 
-std::optional<utils::UUID> cf_prop_defs::get_id() const {
+std::optional<table_id> cf_prop_defs::get_id() const {
     auto id = get_simple(KW_ID);
     if (id) {
-        return utils::UUID(*id);
+        return std::make_optional<table_id>(utils::UUID(*id));
     }
 
     return std::nullopt;

cql3/statements/cf_prop_defs.hh

@@ -100,7 +100,7 @@ public:
     int32_t get_default_time_to_live() const;
     int32_t get_gc_grace_seconds() const;
     int32_t get_paxos_grace_seconds() const;
-    std::optional<utils::UUID> get_id() const;
+    std::optional<table_id> get_id() const;
     bool get_synchronous_updates_flag() const;
 
     void apply_to_builder(schema_builder& builder, schema::extensions_map schema_extensions) const;

cql3/statements/create_aggregate_statement.cc

@@ -66,7 +66,7 @@ shared_ptr<db::functions::function> create_aggregate_statement::create(query_pro
         auto dummy_ident = ::make_shared<column_identifier>("", true);
         auto column_spec = make_lw_shared<column_specification>("", "", dummy_ident, state_type);
         auto initcond_term = expr::evaluate(prepare_expression(_ival.value(), db, _name.keyspace, nullptr, {column_spec}), query_options::DEFAULT);
-        initcond = std::move(initcond_term).to_bytes();
+        initcond = std::move(initcond_term).to_bytes_opt();
     }
 
     return ::make_shared<functions::user_aggregate>(_name, initcond, std::move(state_func), std::move(reduce_func), std::move(final_func));

cql3/statements/create_index_statement.cc

@@ -10,6 +10,7 @@
 
 #include <seastar/core/coroutine.hh>
 #include "create_index_statement.hh"
+#include "exceptions/exceptions.hh"
 #include "prepared_statement.hh"
 #include "validation.hh"
 #include "service/storage_proxy.hh"
@@ -28,6 +29,7 @@
 
 #include <boost/range/adaptor/transformed.hpp>
 #include <boost/algorithm/string/join.hpp>
+#include <stdexcept>
 
 namespace cql3 {
 
@@ -51,6 +53,16 @@ create_index_statement::check_access(query_processor& qp, const service::client_
     return state.has_column_family_access(qp.db(), keyspace(), column_family(), auth::permission::ALTER);
 }
 
+static sstring target_type_name(index_target::target_type type) {
+    switch (type) {
+        case index_target::target_type::keys: return "keys()";
+        case index_target::target_type::keys_and_values: return "entries()";
+        case index_target::target_type::collection_values: return "values()";
+        case index_target::target_type::regular_values: return "value";
+        default: throw std::invalid_argument("should not reach");
+    }
+}
+
 void
 create_index_statement::validate(query_processor& qp, const service::client_state& state) const
 {
@@ -89,6 +101,7 @@ std::vector<::shared_ptr<index_target>> create_index_statement::validate_while_e
         validate_targets_for_multi_column_index(targets);
     }
 
+    const bool is_local_index = targets.size() > 0 && std::holds_alternative<index_target::multiple_columns>(targets.front()->value);
     for (auto& target : targets) {
         auto* ident = std::get_if<::shared_ptr<column_identifier>>(&target->value);
         if (!ident) {
@@ -98,7 +111,7 @@ std::vector<::shared_ptr<index_target>> create_index_statement::validate_while_e
 
         if (cd == nullptr) {
             throw exceptions::invalid_request_exception(
-                    format("No column definition found for column {}", target->as_string()));
+                    format("No column definition found for column {}", target->column_name()));
         }
 
         //NOTICE(sarna): Should be lifted after resolving issue #2963
@@ -127,14 +140,25 @@ std::vector<::shared_ptr<index_target>> create_index_statement::validate_while_e
         if (cd->kind == column_kind::partition_key && cd->is_on_all_components()) {
             throw exceptions::invalid_request_exception(
                     format("Cannot create secondary index on partition key column {}",
-                            target->as_string()));
+                            target->column_name()));
         }
 
         if (cd->type->is_multi_cell()) {
-            // NOTICE(sarna): should be lifted after #2962 (indexes on non-frozen collections) is implemented
-            // NOTICE(kbraun): don't forget about non-frozen user defined types
-            throw exceptions::invalid_request_exception(
-                    format("Cannot create secondary index on non-frozen collection or UDT column {}", cd->name_as_text()));
+            if (cd->type->is_collection()) {
+                if (!db.features().collection_indexing) {
+                    throw exceptions::invalid_request_exception(
+                        "Indexing of collection columns not supported by some older nodes in this cluster. Please upgrade them.");
+                }
+                if (is_local_index) {
+                    throw exceptions::invalid_request_exception(
+                            format("Local secondary index on collection column {} is not implemented yet.", target->column_name()));
+                }
+                validate_not_full_index(*target);
+                validate_for_collection(*target, *cd);
+                rewrite_target_for_collection(*target, *cd);
+            } else {
+                throw exceptions::invalid_request_exception(format("Cannot create secondary index on UDT column {}", cd->name_as_text()));
+            }
         } else if (cd->type->is_collection()) {
             validate_for_frozen_collection(*target);
         } else {
@@ -194,8 +218,8 @@ void create_index_statement::validate_for_frozen_collection(const index_target&
     if (target.type != index_target::target_type::full) {
         throw exceptions::invalid_request_exception(
                 format("Cannot create index on {} of frozen collection column {}",
-                        index_target::index_option(target.type),
-                        target.as_string()));
+                        target_type_name(target.type),
+                        target.column_name()));
     }
 }
 
@@ -206,16 +230,70 @@ void create_index_statement::validate_not_full_index(const index_target& target)
     }
 }
 
+void create_index_statement::validate_for_collection(const index_target& target, const column_definition& cd) const
+{
+    auto throw_exception = [&] {
+        const char* msg_format = "Cannot create secondary index on {} of non-frozen collection column {}";
+        throw exceptions::invalid_request_exception(format(msg_format, to_sstring(target.type), cd.name_as_text()));
+    };
+    switch (target.type) {
+        case index_target::target_type::full:
+            throw std::logic_error("invalid target type(full) in validate_for_collection");
+        case index_target::target_type::regular_values:
+            break;
+        case index_target::target_type::collection_values:
+            break;
+        case index_target::target_type::keys:
+            [[fallthrough]];
+        case index_target::target_type::keys_and_values:
+            if (!cd.type->is_map()) {
+                const char* msg_format = "Cannot create secondary index on {} of column {} with non-map type";
+                throw exceptions::invalid_request_exception(format(msg_format, to_sstring(target.type), cd.name_as_text()));
+            }
+            break;
+    }
+}
+
+void create_index_statement::rewrite_target_for_collection(index_target& target, const column_definition& cd) const
+{
+    // In Cassandra, `CREATE INDEX ON table(collection)` works the same as `CREATE INDEX ON table(VALUES(collection))`,
+    // and index on VALUES(collection) indexes values, if the collection was a map or a list, but it indexes the keys, if it
+    // was a set. Rewrite it to clean the mess.
+    switch (target.type) {
+        case index_target::target_type::full:
+            throw std::logic_error("invalid target type(full) in rewrite_target_for_collection");
+        case index_target::target_type::keys:
+            // If it was keys, then it must have been a map.
+            break;
+        case index_target::target_type::keys_and_values:
+            // If it was entries, then it must have been a map.
+            break;
+        case index_target::target_type::regular_values:
+            // Regular values for collections means the same as collection values.
+            [[fallthrough]];
+        case index_target::target_type::collection_values:
+            if (cd.type->is_map() || cd.type->is_list()) {
+                target.type = index_target::target_type::collection_values;
+            } else if (cd.type->is_set()) {
+                target.type = index_target::target_type::keys;
+            } else {
+                throw std::logic_error(format("rewrite_target_for_collection: unknown collection type {}", cd.type->cql3_type_name()));
+            }
+            break;
+    }
+}
+
+
 void create_index_statement::validate_is_values_index_if_target_column_not_collection(
         const column_definition* cd, const index_target& target) const
 {
     if (!cd->type->is_collection()
-            && target.type != index_target::target_type::values) {
+            && target.type != index_target::target_type::regular_values) {
         throw exceptions::invalid_request_exception(
                 format("Cannot create index on {} of column {}; only non-frozen collections support {} indexes",
-                       index_target::index_option(target.type),
-                       target.as_string(),
-                       index_target::index_option(target.type)));
+                       target_type_name(target.type),
+                       target.column_name(),
+                       target_type_name(target.type)));
     }
 }
 
@@ -226,7 +304,7 @@ void create_index_statement::validate_target_column_is_map_if_index_involves_key
         if (!is_map) {
             throw exceptions::invalid_request_exception(
                     format("Cannot create index on {} of column {} with non-map type",
-                           index_target::index_option(target.type), target.as_string()));
+                           target_type_name(target.type), target.column_name()));
         }
     }
 }
@@ -240,10 +318,10 @@ void create_index_statement::validate_targets_for_multi_column_index(std::vector
     }
     std::unordered_set<sstring> columns;
     for (auto& target : targets) {
-        if (columns.contains(target->as_string())) {
-            throw exceptions::invalid_request_exception(format("Duplicate column {} in index target list", target->as_string()));
+        if (columns.contains(target->column_name())) {
+            throw exceptions::invalid_request_exception(format("Duplicate column {} in index target list", target->column_name()));
         }
-        columns.emplace(target->as_string());
+        columns.emplace(target->column_name());
     }
 }
 
@@ -257,9 +335,9 @@ schema_ptr create_index_statement::build_index_schema(query_processor& qp) const
     if (accepted_name.empty()) {
         std::optional<sstring> index_name_root;
         if (targets.size() == 1) {
-           index_name_root = targets[0]->as_string();
+           index_name_root = targets[0]->column_name();
         } else if ((targets.size() == 2 && std::holds_alternative<index_target::multiple_columns>(targets.front()->value))) {
-            index_name_root = targets[1]->as_string();
+            index_name_root = targets[1]->column_name();
         }
         accepted_name = db.get_available_index_name(keyspace(), column_family(), index_name_root);
     }

cql3/statements/create_index_statement.hh

@@ -55,6 +55,8 @@ private:
     void validate_for_local_index(const schema& schema) const;
     void validate_for_frozen_collection(const index_target& target) const;
     void validate_not_full_index(const index_target& target) const;
+    void validate_for_collection(const index_target& target, const column_definition&) const;
+    void rewrite_target_for_collection(index_target& target, const column_definition&) const;
     void validate_is_values_index_if_target_column_not_collection(const column_definition* cd,
                                                                   const index_target& target) const;
     void validate_target_column_is_map_if_index_involves_keys(bool is_map, const index_target& target) const;

cql3/statements/create_table_statement.cc

@@ -30,6 +30,7 @@
 #include "service/migration_manager.hh"
 #include "service/storage_proxy.hh"
 #include "db/config.hh"
+#include "compaction/time_window_compaction_strategy.hh"
 
 namespace cql3 {
 
@@ -41,7 +42,7 @@ create_table_statement::create_table_statement(cf_name name,
                                                ::shared_ptr<cf_prop_defs> properties,
                                                bool if_not_exists,
                                                column_set_type static_columns,
-                                               const std::optional<utils::UUID>& id)
+                                               const std::optional<table_id>& id)
     : schema_altering_statement{name}
     , _use_compact_storage(false)
     , _static_columns{static_columns}
@@ -426,6 +427,7 @@ void create_table_statement::raw_statement::add_column_alias(::shared_ptr<column
 // in the table's options are done elsewhere.
 std::optional<sstring> check_restricted_table_properties(
     query_processor& qp,
+    std::optional<schema_ptr> schema,
     const sstring& keyspace, const sstring& table,
     const cf_prop_defs& cfprops)
 {
@@ -434,6 +436,19 @@ std::optional<sstring> check_restricted_table_properties(
     // function before cfprops.validate() (there, validate() is only called
     // in prepare_schema_mutations(), in the middle of execute).
     auto strategy = cfprops.get_compaction_strategy_class();
+    sstables::compaction_strategy_type current_strategy = sstables::compaction_strategy_type::null;
+    gc_clock::duration current_ttl = gc_clock::duration::zero();
+    // cfprops doesn't return any of the table attributes unless the attribute
+    // has been specified in the CQL statement. If a schema is defined, then
+    // this was an ALTER TABLE statement.
+    if (schema) {
+        current_strategy = (*schema)->compaction_strategy();
+        current_ttl = (*schema)->default_time_to_live();
+    }
+
+    // Evaluate whether the strategy to evaluate was explicitly passed
+    auto cs = (strategy) ? strategy : current_strategy;
+
     if (strategy && *strategy == sstables::compaction_strategy_type::date_tiered) {
         switch(qp.db().get_config().restrict_dtcs()) {
         case db::tri_mode_restriction_t::mode::TRUE:
@@ -452,12 +467,52 @@ std::optional<sstring> check_restricted_table_properties(
             break;
         }
     }
+    if (cs == sstables::compaction_strategy_type::time_window) {
+        std::map<sstring, sstring> options = (strategy) ? cfprops.get_compaction_type_options() : (*schema)->compaction_strategy_options();
+        sstables::time_window_compaction_strategy_options twcs_options(options);
+        long ttl = (cfprops.has_property(cf_prop_defs::KW_DEFAULT_TIME_TO_LIVE)) ? cfprops.get_default_time_to_live() : current_ttl.count();
+        auto max_windows = qp.db().get_config().twcs_max_window_count();
+
+        // It may happen that an user tries to update an unrelated table property. Allow the request through.
+        if (!cfprops.has_property(cf_prop_defs::KW_DEFAULT_TIME_TO_LIVE) && !strategy) {
+            return std::nullopt;
+        }
+
+        if (ttl > 0) {
+            // Ideally we should not need the window_size check below. However, given #2336 it may happen that some incorrectly
+            // table created with a window_size=0 may exist, which would cause a division by zero (eg: in an ALTER statement).
+            // Given that, an invalid window size is treated as 1 minute, which is the smaller "supported" window size for TWCS.
+            auto window_size = twcs_options.get_sstable_window_size() > std::chrono::seconds::zero() ? twcs_options.get_sstable_window_size() : std::chrono::seconds(60);
+            auto window_count = std::chrono::seconds(ttl) / window_size;
+            if (max_windows > 0 && window_count > max_windows) {
+                throw exceptions::configuration_exception(fmt::format("The setting of default_time_to_live={} and compaction window={}(s) "
+                                                   "can lead to {} windows, which is larger than the allowed number of windows specified "
+                                                   "by the twcs_max_window_count ({}) parameter. Note that default_time_to_live=0 is also "
+                                                   "highly discouraged.", ttl, twcs_options.get_sstable_window_size().count(), window_count, max_windows));
+            }
+        } else {
+              switch (qp.db().get_config().restrict_twcs_without_default_ttl()) {
+              case db::tri_mode_restriction_t::mode::TRUE:
+                  throw exceptions::configuration_exception(
+                      "TimeWindowCompactionStrategy tables without a strict default_time_to_live setting "
+                      "are forbidden. You may override this restriction by setting restrict_twcs_without_default_ttl "
+                      "configuration option to false.");
+              case db::tri_mode_restriction_t::mode::WARN:
+                  return format("TimeWindowCompactionStrategy tables without a default_time_to_live "
+                      "may potentially introduce too many windows. Ensure that insert statements specify a "
+                      "TTL (via USING TTL), when inserting data to this table. The restrict_twcs_without_default_ttl "
+                      "configuration option can be changed to silence this warning or make it into an error");
+              case db::tri_mode_restriction_t::mode::FALSE:
+                  break;
+              }
+        }
+   }
     return std::nullopt;
 }
 
 future<::shared_ptr<messages::result_message>>
 create_table_statement::execute(query_processor& qp, service::query_state& state, const query_options& options) const {
-    std::optional<sstring> warning = check_restricted_table_properties(qp, keyspace(), column_family(), *_properties);
+    std::optional<sstring> warning = check_restricted_table_properties(qp, std::nullopt, keyspace(), column_family(), *_properties);
     return schema_altering_statement::execute(qp, state, options).then([this, warning = std::move(warning)] (::shared_ptr<messages::result_message> msg) {
         if (warning) {
             msg->add_warning(*warning);

cql3/statements/create_table_statement.hh

@@ -60,13 +60,13 @@ class create_table_statement : public schema_altering_statement {
     column_set_type _static_columns;
     const ::shared_ptr<cf_prop_defs> _properties;
     const bool _if_not_exists;
-    std::optional<utils::UUID> _id;
+    std::optional<table_id> _id;
 public:
     create_table_statement(cf_name name,
                            ::shared_ptr<cf_prop_defs> properties,
                            bool if_not_exists,
                            column_set_type static_columns,
-                           const std::optional<utils::UUID>& id);
+                           const std::optional<table_id>& id);
 
     virtual future<> check_access(query_processor& qp, const service::client_state& state) const override;
 
@@ -130,6 +130,7 @@ public:
 
 std::optional<sstring> check_restricted_table_properties(
     query_processor& qp,
+    std::optional<schema_ptr> schema,
     const sstring& keyspace, const sstring& table,
     const cf_prop_defs& cfprops);
 

cql3/statements/delete_statement.cc

@@ -99,9 +99,7 @@ delete_statement::delete_statement(cf_name name,
     , _deletions(std::move(deletions))
     , _where_clause(std::move(where_clause))
 {
-    if (_attrs->time_to_live) {
-        throw exceptions::invalid_request_exception("TTL attribute is not allowed for deletes");
-    }
+    assert(!_attrs->time_to_live.has_value());
 }
 
 }

cql3/statements/index_target.cc

@@ -8,6 +8,7 @@
  * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
  */
 
+#include <regex>
 #include <stdexcept>
 #include "index_target.hh"
 #include "index/secondary_index.hh"
@@ -21,9 +22,11 @@ using db::index::secondary_index;
 
 const sstring index_target::target_option_name = "target";
 const sstring index_target::custom_index_option_name = "class_name";
+const std::regex index_target::target_regex("^(keys|entries|values|full)\\((.+)\\)$");
 
-sstring index_target::as_string() const {
+sstring index_target::column_name() const {
     struct as_string_visitor {
+        const index_target* target;
         sstring operator()(const std::vector<::shared_ptr<column_identifier>>& columns) const {
             return "(" + boost::algorithm::join(columns | boost::adaptors::transformed(
                     [](const ::shared_ptr<cql3::column_identifier>& ident) -> sstring {
@@ -36,7 +39,7 @@ sstring index_target::as_string() const {
         }
     };
 
-    return std::visit(as_string_visitor(), value);
+    return std::visit(as_string_visitor {this}, value);
 }
 
 index_target::target_type index_target::from_sstring(const sstring& s)
@@ -45,26 +48,66 @@ index_target::target_type index_target::from_sstring(const sstring& s)
         return index_target::target_type::keys;
     } else if (s == "entries") {
         return index_target::target_type::keys_and_values;
+    } else if (s == "regular_values") {
+        return index_target::target_type::regular_values;
     } else if (s == "values") {
-        return index_target::target_type::values;
+        return index_target::target_type::collection_values;
     } else if (s == "full") {
         return index_target::target_type::full;
     }
     throw std::runtime_error(format("Unknown target type: {}", s));
 }
 
-sstring index_target::index_option(target_type type) {
-    switch (type) {
-        case target_type::keys: return secondary_index::index_keys_option_name;
-        case target_type::keys_and_values: return secondary_index::index_entries_option_name;
-        case target_type::values: return secondary_index::index_values_option_name;
-        default: throw std::invalid_argument("should not reach");
+index_target::target_type index_target::from_target_string(const sstring& target) {
+    std::cmatch match;
+    if (std::regex_match(target.data(), match, target_regex)) {
+        return index_target::from_sstring(match[1].str());
     }
+    return target_type::regular_values;
+}
+
+// A CQL column's name may contain any characters. If we use this string as-is
+// inside a target string, it may confuse us when we later try to parse the
+// resulting string (e.g., see issue #10707). We should therefore use the
+// function escape_target_column() to "escape" the target column name, and use
+// the reverse function unescape_target_column() to undo this.
+// Cassandra uses for this escaping the CQL syntax of this column name
+// (basically, column_identifier::as_cql_name()). This is an overkill,
+// but since we already have such code, we might as well use it.
+sstring index_target::escape_target_column(const cql3::column_identifier& col) {
+    return col.to_cql_string();
+}
+sstring index_target::unescape_target_column(std::string_view str) {
+    // We don't have a reverse version of util::maybe_quote(), so
+    // we need to open-code it here. Cassandra has this too - in
+    // index/TargetParser.java
+    if (str.size() >= 2 && str.starts_with('"') && str.ends_with('"')) {
+        str.remove_prefix(1);
+        str.remove_suffix(1);
+        // remove doubled quotes in the middle of the string, which to_cql_string()
+        // adds. This code is inefficient but rarely called so it's fine.
+        static const std::regex double_quote_re("\"\"");
+        return std::regex_replace(std::string(str), double_quote_re, "\"");
+    }
+    return sstring(str);
+}
+
+sstring index_target::column_name_from_target_string(const sstring& target) {
+    std::cmatch match;
+    if (std::regex_match(target.data(), match, target_regex)) {
+        return unescape_target_column(match[2].str());
+    }
+    return unescape_target_column(target);
 }
 
 ::shared_ptr<index_target::raw>
-index_target::raw::values_of(::shared_ptr<column_identifier::raw> c) {
-    return ::make_shared<raw>(c, target_type::values);
+index_target::raw::regular_values_of(::shared_ptr<column_identifier::raw> c) {
+    return ::make_shared<raw>(c, target_type::regular_values);
+}
+
+::shared_ptr<index_target::raw>
+index_target::raw::collection_values_of(::shared_ptr<column_identifier::raw> c) {
+    return ::make_shared<raw>(c, target_type::collection_values);
 }
 
 ::shared_ptr<index_target::raw>
@@ -84,7 +127,7 @@ index_target::raw::full_collection(::shared_ptr<column_identifier::raw> c) {
 
 ::shared_ptr<index_target::raw>
 index_target::raw::columns(std::vector<::shared_ptr<column_identifier::raw>> c) {
-    return ::make_shared<raw>(std::move(c), target_type::values);
+    return ::make_shared<raw>(std::move(c), target_type::regular_values);
 }
 
 ::shared_ptr<index_target>
@@ -115,10 +158,11 @@ sstring to_sstring(index_target::target_type type)
     switch (type) {
     case index_target::target_type::keys: return "keys";
     case index_target::target_type::keys_and_values: return "entries";
-    case index_target::target_type::values: return "values";
+    case index_target::target_type::regular_values: return "regular_values";
+    case index_target::target_type::collection_values: return "values";
     case index_target::target_type::full: return "full";
     }
-    return "";
+    throw std::runtime_error("to_sstring(index_target::target_type): should not reach");
 }
 
 }

cql3/statements/index_target.hh

@@ -13,6 +13,7 @@
 #include <seastar/core/shared_ptr.hh>
 #include "cql3/column_identifier.hh"
 #include <variant>
+#include <regex>
 
 namespace cql3 {
 
@@ -21,26 +22,42 @@ namespace statements {
 struct index_target {
     static const sstring target_option_name;
     static const sstring custom_index_option_name;
+    static const std::regex target_regex;
 
-    using single_column =::shared_ptr<column_identifier>;
+    enum class target_type {
+        regular_values, collection_values, keys, keys_and_values, full
+    };
+
+    using single_column = ::shared_ptr<column_identifier>;
     using multiple_columns = std::vector<::shared_ptr<column_identifier>>;
     using value_type = std::variant<single_column, multiple_columns>;
 
-    enum class target_type {
-        values, keys, keys_and_values, full
-    };
-
     const value_type value;
-    const target_type type;
+    target_type type;
 
     index_target(::shared_ptr<column_identifier> c, target_type t) : value(c) , type(t) {}
     index_target(std::vector<::shared_ptr<column_identifier>> c, target_type t) : value(std::move(c)), type(t) {}
 
-    sstring as_string() const;
+    sstring column_name() const;
 
-    static sstring index_option(target_type type);
     static target_type from_column_definition(const column_definition& cd);
+    // Parses index_target::target_type from it's textual form.
+    // e.g. from_sstring("keys") == index_target::target_type::keys
     static index_target::target_type from_sstring(const sstring& s);
+    // Parses index_target::target_type from index target string form.
+    // e.g. from_target_string("keys(some_column)") == index_target::target_type::keys
+    static index_target::target_type from_target_string(const sstring& s);
+    // Parses column name from index target string form
+    // e.g. column_name_from_target_string("keys(some_column)") == "some_column"
+    static sstring column_name_from_target_string(const sstring& s);
+
+    // A CQL column's name may contain any characters. If we use this string
+    // as-is inside a target string, it may confuse us when we later try to
+    // parse the resulting string (e.g., see issue #10707). We should
+    // therefore use the function escape_target_column() to "escape" the
+    // target column name, and the reverse function unescape_target_column().
+    static sstring escape_target_column(const cql3::column_identifier& col);
+    static sstring unescape_target_column(std::string_view str);
 
     class raw {
     public:
@@ -54,7 +71,8 @@ struct index_target {
         raw(::shared_ptr<column_identifier::raw> c, target_type t) : value(c), type(t) {}
         raw(std::vector<::shared_ptr<column_identifier::raw>> pk_columns, target_type t) : value(pk_columns), type(t) {}
 
-        static ::shared_ptr<raw> values_of(::shared_ptr<column_identifier::raw> c);
+        static ::shared_ptr<raw> regular_values_of(::shared_ptr<column_identifier::raw> c);
+        static ::shared_ptr<raw> collection_values_of(::shared_ptr<column_identifier::raw> c);
         static ::shared_ptr<raw> keys_of(::shared_ptr<column_identifier::raw> c);
         static ::shared_ptr<raw> keys_and_values_of(::shared_ptr<column_identifier::raw> c);
         static ::shared_ptr<raw> full_collection(::shared_ptr<column_identifier::raw> c);

cql3/statements/modification_statement.cc

@@ -8,8 +8,10 @@
  * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
  */
 
+#include "cql3/cql_statement.hh"
 #include "types/map.hh"
 #include "cql3/statements/modification_statement.hh"
+#include "cql3/statements/strongly_consistent_modification_statement.hh"
 #include "cql3/statements/raw/modification_statement.hh"
 #include "cql3/statements/prepared_statement.hh"
 #include "cql3/util.hh"
@@ -28,6 +30,7 @@
 #include "cas_request.hh"
 #include "cql3/query_processor.hh"
 #include "service/storage_proxy.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
 
 template<typename T = void>
 using coordinator_result = exceptions::coordinator_result<T>;
@@ -211,7 +214,7 @@ modification_statement::read_command(query_processor& qp, query::clustering_row_
     }
     query::partition_slice ps(std::move(ranges), *s, columns_to_read(), update_parameters::options);
     const auto max_result_size = qp.proxy().get_max_result_size(ps);
-    return make_lw_shared<query::read_command>(s->id(), s->version(), std::move(ps), query::max_result_size(max_result_size));
+    return make_lw_shared<query::read_command>(s->id(), s->version(), std::move(ps), query::max_result_size(max_result_size), query::tombstone_limit::max);
 }
 
 std::vector<query::clustering_range>
@@ -439,13 +442,30 @@ modification_statement::process_where_clause(data_dictionary::database db, expr:
     }
 }
 
+::shared_ptr<strongly_consistent_modification_statement>
+modification_statement::prepare_for_broadcast_tables() const {
+    // FIXME: implement for every type of `modification_statement`.
+    throw service::broadcast_tables::unsupported_operation_error{};
+}
+
 namespace raw {
 
+::shared_ptr<cql_statement_opt_metadata>
+modification_statement::prepare_statement(data_dictionary::database db, prepare_context& ctx, cql_stats& stats) {
+    ::shared_ptr<cql3::statements::modification_statement> statement = prepare(db, ctx, stats);
+
+    if (service::broadcast_tables::is_broadcast_table_statement(keyspace(), column_family())) {
+        return statement->prepare_for_broadcast_tables();
+    } else {
+        return statement;
+    }
+}
+
 std::unique_ptr<prepared_statement>
 modification_statement::prepare(data_dictionary::database db, cql_stats& stats) {
     schema_ptr schema = validation::validate_column_family(db, keyspace(), column_family());
     auto meta = get_prepare_context();
-    auto statement = prepare(db, meta, stats);
+    auto statement = prepare_statement(db, meta, stats);
     auto partition_key_bind_indices = meta.get_partition_key_bind_indexes(*schema);
     return std::make_unique<prepared_statement>(std::move(statement), meta, std::move(partition_key_bind_indices));
 }

cql3/statements/modification_statement.hh

@@ -33,6 +33,7 @@ class operation;
 
 namespace statements {
 
+class strongly_consistent_modification_statement;
 
 namespace raw { class modification_statement; }
 
@@ -69,10 +70,11 @@ public:
 protected:
     std::vector<::shared_ptr<operation>> _column_operations;
     cql_stats& _stats;
-private:
+
     // Separating normal and static conditions makes things somewhat easier
     std::vector<lw_shared_ptr<column_condition>> _regular_conditions;
     std::vector<lw_shared_ptr<column_condition>> _static_conditions;
+private:
     const ks_selector _ks_sel;
 
     // True if this statement has _if_exists or _if_not_exists or other
@@ -256,6 +258,9 @@ public:
     future<std::vector<mutation>> get_mutations(query_processor& qp, const query_options& options, db::timeout_clock::time_point timeout, bool local, int64_t now, service::query_state& qs) const;
 
     virtual json_cache_opt maybe_prepare_json_cache(const query_options& options) const;
+
+    virtual ::shared_ptr<strongly_consistent_modification_statement> prepare_for_broadcast_tables() const;
+
 protected:
     /**
      * If there are conditions on the statement, this is called after the where clause and conditions have been

cql3/statements/prune_materialized_view_statement.cc

@@ -32,7 +32,8 @@ static future<> delete_ghost_rows(dht::partition_range_vector partition_ranges,
     auto selection = cql3::selection::selection::for_columns(view, key_columns);
 
     query::partition_slice partition_slice(std::move(clustering_bounds), {},  {}, selection->get_query_options());
-    auto command = ::make_lw_shared<query::read_command>(view->id(), view->version(), partition_slice, proxy.get_max_result_size(partition_slice));
+    auto command = ::make_lw_shared<query::read_command>(view->id(), view->version(), partition_slice, proxy.get_max_result_size(partition_slice),
+            query::tombstone_limit(proxy.get_tombstone_limit()));
 
     tracing::trace(state.get_trace_state(), "Deleting ghost rows from partition ranges {}", partition_ranges);
 

cql3/statements/raw/modification_statement.hh

@@ -14,6 +14,7 @@
 #include "cql3/column_identifier.hh"
 #include "cql3/column_condition.hh"
 #include "cql3/attributes.hh"
+#include "cql3/cql_statement.hh"
 
 #include <seastar/core/shared_ptr.hh>
 
@@ -38,10 +39,11 @@ private:
     const bool _if_not_exists;
     const bool _if_exists;
 protected:
-    modification_statement(cf_name name, std::unique_ptr<attributes::raw> attrs, conditions_vector conditions, bool if_not_exists, bool if_exists);
+    modification_statement(cf_name name, std::unique_ptr<attributes::raw> attrs, conditions_vector conditions = {}, bool if_not_exists = false, bool if_exists = false);
 
 public:
     virtual std::unique_ptr<prepared_statement> prepare(data_dictionary::database db, cql_stats& stats) override;
+    ::shared_ptr<cql_statement_opt_metadata> prepare_statement(data_dictionary::database db, prepare_context& ctx, cql_stats& stats);
     ::shared_ptr<cql3::statements::modification_statement> prepare(data_dictionary::database db, prepare_context& ctx, cql_stats& stats) const;
 protected:
     virtual ::shared_ptr<cql3::statements::modification_statement> prepare_internal(data_dictionary::database db, schema_ptr schema,

cql3/statements/raw/truncate_statement.hh

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ *
+ * Modified by ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
+ */
+
+#pragma once
+
+#include "cql3/statements/raw/cf_statement.hh"
+#include "cql3/attributes.hh"
+
+namespace cql3 {
+
+namespace statements {
+
+namespace raw {
+
+class truncate_statement : public raw::cf_statement {
+private:
+    std::unique_ptr<attributes::raw> _attrs;
+public:
+    /**
+     * Creates a new truncate_statement from a column family name, and attributes.
+     *
+     * @param name column family being operated on
+     * @param attrs additional attributes for statement (timeout)
+     */
+    truncate_statement(cf_name name, std::unique_ptr<attributes::raw> attrs);
+
+    virtual std::unique_ptr<prepared_statement> prepare(data_dictionary::database db, cql_stats& stats) override;
+};
+
+} // namespace raw
+
+} // namespace statements
+
+} // namespace cql3

cql3/statements/select_statement.cc

@@ -10,10 +10,13 @@
 
 #include "cql3/statements/select_statement.hh"
 #include "cql3/expr/expression.hh"
+#include "cql3/statements/index_target.hh"
 #include "cql3/statements/raw/select_statement.hh"
 #include "cql3/query_processor.hh"
 #include "cql3/statements/prune_materialized_view_statement.hh"
+#include "cql3/statements/strongly_consistent_select_statement.hh"
 
+#include "service/broadcast_tables/experimental/lang.hh"
 #include "transport/messages/result_message.hh"
 #include "cql3/functions/as_json_function.hh"
 #include "cql3/selection/selection.hh"
@@ -353,11 +356,12 @@ select_statement::do_execute(query_processor& qp,
             _schema->version(),
             std::move(slice),
             max_result_size,
+            query::tombstone_limit(qp.proxy().get_tombstone_limit()),
             query::row_limit(limit),
             query::partition_limit(query::max_partitions),
             now,
             tracing::make_trace_info(state.get_trace_state()),
-            utils::UUID(),
+            query_id::create_null_id(),
             query::is_first_page::no,
             options.get_timestamp(state));
     command->allow_limit = db::allow_per_partition_rate_limit::yes;
@@ -524,11 +528,12 @@ indexed_table_select_statement::prepare_command_for_base_query(query_processor&
             _schema->version(),
             std::move(slice),
             qp.proxy().get_max_result_size(slice),
+            query::tombstone_limit(qp.proxy().get_tombstone_limit()),
             query::row_limit(get_limit(options)),
             query::partition_limit(query::max_partitions),
             now,
             tracing::make_trace_info(state.get_trace_state()),
-            utils::UUID(),
+            query_id::create_null_id(),
             query::is_first_page::no,
             options.get_timestamp(state));
     cmd->allow_limit = db::allow_per_partition_rate_limit::yes;
@@ -564,12 +569,14 @@ indexed_table_select_statement::do_execute_base_query(
 
     const bool is_paged = bool(paging_state);
     base_query_state query_state{cmd->get_row_limit() * queried_ranges_count, std::move(ranges_to_vnodes)};
-    return do_with(std::move(query_state), [this, is_paged, &qp, &state, &options, cmd, timeout] (auto&& query_state) {
+    {
         auto& merger = query_state.merger;
         auto& ranges_to_vnodes = query_state.ranges_to_vnodes;
         auto& concurrency = query_state.concurrency;
         auto& previous_result_size = query_state.previous_result_size;
-        return utils::result_repeat([this, is_paged, &previous_result_size, &ranges_to_vnodes, &merger, &qp, &state, &options, &concurrency, cmd, timeout]() {
+        query::short_read is_short_read = query::short_read::no;
+        bool page_limit_reached = false;
+        while (!is_short_read && !ranges_to_vnodes.empty() && !page_limit_reached) {
             // Starting with 1 range, we check if the result was a short read, and if not,
             // we continue exponentially, asking for 2x more ranges than before
             dht::partition_range_vector prange = ranges_to_vnodes(concurrency);
@@ -606,19 +613,19 @@ indexed_table_select_statement::do_execute_base_query(
             if (previous_result_size < query::result_memory_limiter::maximum_result_size && concurrency < max_base_table_query_concurrency) {
                 concurrency *= 2;
             }
-            return qp.proxy().query_result(_schema, command, std::move(prange), options.get_consistency(), {timeout, state.get_permit(), state.get_client_state(), state.get_trace_state()})
-            .then(utils::result_wrap([is_paged, &previous_result_size, &ranges_to_vnodes, &merger] (service::storage_proxy::coordinator_query_result qr) -> coordinator_result<stop_iteration> {
-                auto is_short_read = qr.query_result->is_short_read();
-                // Results larger than 1MB should be shipped to the client immediately
-                const bool page_limit_reached = is_paged && qr.query_result->buf().size() >= query::result_memory_limiter::maximum_result_size;
-                previous_result_size = qr.query_result->buf().size();
-                merger(std::move(qr.query_result));
-                return stop_iteration(is_short_read || ranges_to_vnodes.empty() || page_limit_reached);
-            }));
-        }).then(utils::result_wrap([&merger, cmd]() mutable {
-            return make_ready_future<coordinator_result<value_type>>(value_type(merger.get(), std::move(cmd)));
-        }));
-    });
+            coordinator_result<service::storage_proxy::coordinator_query_result> rqr = co_await qp.proxy().query_result(_schema, command, std::move(prange), options.get_consistency(), {timeout, state.get_permit(), state.get_client_state(), state.get_trace_state()});
+            if (!rqr.has_value()) {
+                co_return std::move(rqr).as_failure();
+            }
+            auto& qr = rqr.value();
+            is_short_read = qr.query_result->is_short_read();
+            // Results larger than 1MB should be shipped to the client immediately
+            page_limit_reached = is_paged && qr.query_result->buf().size() >= query::result_memory_limiter::maximum_result_size;
+            previous_result_size = qr.query_result->buf().size();
+            merger(std::move(qr.query_result));
+        }
+        co_return coordinator_result<value_type>(value_type(merger.get(), std::move(cmd)));
+    }
 }
 
 future<shared_ptr<cql_transport::messages::result_message>>
@@ -963,18 +970,15 @@ static void append_base_key_to_index_ck(std::vector<managed_bytes_view>& explode
 bytes indexed_table_select_statement::compute_idx_token(const partition_key& key) const {
     const column_definition& cdef = *_view_schema->clustering_key_columns().begin();
     clustering_row empty_row(clustering_key_prefix::make_empty());
-    bytes_opt computed_value;
+    bytes computed_value;
     if (!cdef.is_computed()) {
         // FIXME(pgrabowski): this legacy code is here for backward compatibility and should be removed
         // once "computed_columns feature" is supported by every node
-        computed_value = legacy_token_column_computation().compute_value(*_schema, key, empty_row);
+        computed_value = legacy_token_column_computation().compute_value(*_schema, key);
     } else {
-        computed_value = cdef.get_computation().compute_value(*_schema, key, empty_row);
+        computed_value = cdef.get_computation().compute_value(*_schema, key);
     }
-    if (!computed_value) {
-        throw std::logic_error(format("No value computed for idx_token column {}", cdef.name()));
-    }
-    return *computed_value;
+    return computed_value;
 }
 
 lw_shared_ptr<const service::pager::paging_state> indexed_table_select_statement::generate_view_paging_state_from_base_query_results(lw_shared_ptr<const service::pager::paging_state> paging_state,
@@ -1237,11 +1241,12 @@ indexed_table_select_statement::read_posting_list(query_processor& qp,
             _view_schema->version(),
             partition_slice,
             qp.proxy().get_max_result_size(partition_slice),
+            query::tombstone_limit(qp.proxy().get_tombstone_limit()),
             query::row_limit(limit),
             query::partition_limit(query::max_partitions),
             now,
             tracing::make_trace_info(state.get_trace_state()),
-            utils::UUID(),
+            query_id::create_null_id(),
             query::is_first_page::no,
             options.get_timestamp(state));
 
@@ -1299,6 +1304,12 @@ indexed_table_select_statement::find_index_partition_ranges(query_processor& qp,
         // to avoid outputting the same partition key twice, but luckily in
         // the sorted order, these will be adjacent.
         std::optional<dht::decorated_key> last_dk;
+        if (options.get_paging_state()) {
+            auto paging_state = options.get_paging_state();
+            auto base_pk = generate_base_key_from_index_pk<partition_key>(paging_state->get_partition_key(),
+                paging_state->get_clustering_key(), *_schema, *_view_schema);
+            last_dk = dht::decorate_key(*_schema, base_pk);
+        }
         for (size_t i = 0; i < rs.size(); i++) {
             const auto& row = rs.at(i);
             std::vector<bytes> pk_columns;
@@ -1335,6 +1346,24 @@ indexed_table_select_statement::find_index_clustering_rows(query_processor& qp,
         auto rs = cql3::untyped_result_set(rows);
         std::vector<primary_key> primary_keys;
         primary_keys.reserve(rs.size());
+
+        std::optional<std::reference_wrapper<primary_key>> last_primary_key;
+        // Set last_primary_key if indexing map values and not in the first
+        // query page. See comment below why last_primary_key is needed for
+        // indexing map values. We have a test for this with paging:
+        // test_secondary_index.py::test_index_map_values_paging.
+        std::optional<primary_key> page_start_primary_key;
+        if (_index.target_type() == cql3::statements::index_target::target_type::collection_values &&
+            options.get_paging_state()) {
+            auto paging_state = options.get_paging_state();
+            auto base_pk = generate_base_key_from_index_pk<partition_key>(paging_state->get_partition_key(),
+                paging_state->get_clustering_key(), *_schema, *_view_schema);
+            auto base_dk = dht::decorate_key(*_schema, base_pk);
+            auto base_ck = generate_base_key_from_index_pk<clustering_key>(paging_state->get_partition_key(),
+                paging_state->get_clustering_key(), *_schema, *_view_schema);
+            page_start_primary_key = primary_key{std::move(base_dk), std::move(base_ck)};
+            last_primary_key = *page_start_primary_key;
+        }
         for (size_t i = 0; i < rs.size(); i++) {
             const auto& row = rs.at(i);
             auto pk_columns = _schema->partition_key_columns() | boost::adaptors::transformed([&] (auto& cdef) {
@@ -1346,7 +1375,24 @@ indexed_table_select_statement::find_index_clustering_rows(query_processor& qp,
                 return row.get_blob(cdef.name_as_text());
             });
             auto ck = clustering_key::from_range(ck_columns);
+
+            if (_index.target_type() == cql3::statements::index_target::target_type::collection_values) {
+                // The index on collection values is special in a way, as its' clustering key contains not only the
+                // base primary key, but also a column that holds the keys of the cells in the collection, which
+                // allows to distinguish cells with different keys but the same value.
+                // This has an unwanted consequence, that it's possible to receive two identical base table primary
+                // keys. Thankfully, they are guaranteed to occur consequently.
+                if (last_primary_key) {
+                    const auto& [last_dk, last_ck] = last_primary_key->get();
+                    if (last_dk.equal(*_schema, dk) && last_ck.equal(*_schema, ck)) {
+                        continue;
+                    }
+                }
+            }
+
             primary_keys.emplace_back(primary_key{std::move(dk), std::move(ck)});
+            // Last use of this reference will be before next .emplace_back to the vector.
+            last_primary_key = primary_keys.back();
         }
         auto paging_state = rows->rs().get_metadata().paging_state();
         return make_ready_future<coordinator_result<value_type>>(value_type(std::move(primary_keys), std::move(paging_state)));
@@ -1481,11 +1527,12 @@ parallelized_select_statement::do_execute(
         _schema->version(),
         std::move(slice),
         qp.proxy().get_max_result_size(slice),
+        query::tombstone_limit(qp.proxy().get_tombstone_limit()),
         query::row_limit(query::max_rows),
         query::partition_limit(query::max_partitions),
         now,
         tracing::make_trace_info(state.get_trace_state()),
-        utils::UUID(),
+        query_id::create_null_id(),
         query::is_first_page::no,
         options.get_timestamp(state)
     );
@@ -1526,12 +1573,8 @@ parallelized_select_statement::do_execute(
 namespace raw {
 
 static void validate_attrs(const cql3::attributes::raw& attrs) {
-    if (attrs.timestamp) {
-        throw exceptions::invalid_request_exception("Specifying TIMESTAMP is not legal for SELECT statement");
-    }
-    if (attrs.time_to_live) {
-        throw exceptions::invalid_request_exception("Specifying TTL is not legal for SELECT statement");
-    }
+    assert(!attrs.timestamp.has_value());
+    assert(!attrs.time_to_live.has_value());
 }
 
 select_statement::select_statement(cf_name cf_name,
@@ -1689,6 +1732,20 @@ std::unique_ptr<prepared_statement> select_statement::prepare(data_dictionary::d
             stats,
             std::move(prepared_attrs)
         );
+    } else if (service::broadcast_tables::is_broadcast_table_statement(keyspace(), column_family())) {
+        stmt = ::make_shared<cql3::statements::strongly_consistent_select_statement>(
+                schema,
+                ctx.bound_variables_size(),
+                _parameters,
+                std::move(selection),
+                std::move(restrictions),
+                std::move(group_by_cell_indices),
+                is_reversed_,
+                std::move(ordering_comparator),
+                prepare_limit(db, ctx, _limit),
+                prepare_limit(db, ctx, _per_partition_limit),
+                stats,
+                std::move(prepared_attrs));
     } else {
         stmt = ::make_shared<cql3::statements::primary_key_select_statement>(
                 schema,
@@ -2126,7 +2183,10 @@ std::unique_ptr<cql3::statements::raw::select_statement> build_select_statement(
         out << join(", ", cols);
     }
     // Note that cf_name may need to be quoted, just like column names above.
-    out << " FROM " << util::maybe_quote(sstring(cf_name)) << " WHERE " << where_clause << " ALLOW FILTERING";
+    out << " FROM " << util::maybe_quote(sstring(cf_name));
+    if (!where_clause.empty()) {
+        out << " WHERE " << where_clause << " ALLOW FILTERING";
+    }
     return do_with_parser(out.str(), std::mem_fn(&cql3_parser::CqlParser::selectStatement));
 }
 

cql3/statements/strongly_consistent_modification_statement.cc

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ *
+ * Modified by ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
+ */
+
+
+#include "cql3/statements/strongly_consistent_modification_statement.hh"
+
+#include <boost/range/adaptors.hpp>
+#include <optional>
+
+#include <seastar/core/future.hh>
+#include <seastar/util/variant_utils.hh>
+
+#include "bytes.hh"
+#include "cql3/attributes.hh"
+#include "cql3/expr/expression.hh"
+#include "cql3/operation.hh"
+#include "cql3/query_processor.hh"
+#include "cql3/values.hh"
+#include "timeout_config.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
+
+namespace cql3 {
+
+static logging::logger logger("strongly_consistent_modification_statement");
+
+namespace statements {
+
+strongly_consistent_modification_statement::strongly_consistent_modification_statement(
+    uint32_t bound_terms,
+    schema_ptr schema,
+    broadcast_tables::prepared_update query)
+    : cql_statement_opt_metadata{&timeout_config::write_timeout}
+    , _bound_terms{bound_terms}
+    , _schema{schema}
+    , _query{std::move(query)}
+{ }
+
+future<::shared_ptr<cql_transport::messages::result_message>>
+strongly_consistent_modification_statement::execute(query_processor& qp, service::query_state& qs, const query_options& options) const {
+    return execute_without_checking_exception_message(qp, qs, options)
+            .then(cql_transport::messages::propagate_exception_as_future<shared_ptr<cql_transport::messages::result_message>>);
+}
+
+static
+service::broadcast_tables::update_query
+evaluate_prepared(
+    const broadcast_tables::prepared_update& query,
+    const query_options& options) {
+    return service::broadcast_tables::update_query{
+        .key = expr::evaluate(query.key, options).to_bytes(),
+        .new_value = expr::evaluate(query.new_value, options).to_bytes(),
+        .value_condition = query.value_condition
+            ? std::optional<bytes_opt>{expr::evaluate(*query.value_condition, options).to_bytes_opt()}
+            : std::nullopt
+    };
+}
+    
+future<::shared_ptr<cql_transport::messages::result_message>>
+strongly_consistent_modification_statement::execute_without_checking_exception_message(query_processor& qp, service::query_state& qs, const query_options& options) const {
+        if (this_shard_id() != 0) {
+        co_return ::make_shared<cql_transport::messages::result_message::bounce_to_shard>(0, cql3::computed_function_values{});
+    }
+
+    auto result = co_await service::broadcast_tables::execute(
+        qp.get_group0_client(),
+        { evaluate_prepared(_query, options) }
+    );
+    
+    co_return co_await std::visit(make_visitor(
+        [] (service::broadcast_tables::query_result_conditional_update& qr) -> future<::shared_ptr<cql_transport::messages::result_message>> {
+            auto result_set = std::make_unique<cql3::result_set>(std::vector{
+                make_lw_shared<cql3::column_specification>(
+                    db::system_keyspace::NAME,
+                    db::system_keyspace::BROADCAST_KV_STORE,
+                    ::make_shared<cql3::column_identifier>("[applied]", false),
+                    boolean_type
+                ),
+                make_lw_shared<cql3::column_specification>(
+                    db::system_keyspace::NAME,
+                    db::system_keyspace::BROADCAST_KV_STORE,
+                    ::make_shared<cql3::column_identifier>("value", true),
+                    utf8_type
+                )
+            });
+
+            result_set->add_row({ boolean_type->decompose(qr.is_applied), qr.previous_value });
+
+            return make_ready_future<::shared_ptr<cql_transport::messages::result_message>>(
+                ::make_shared<cql_transport::messages::result_message::rows>(cql3::result{std::move(result_set)}));
+        },
+        [] (service::broadcast_tables::query_result_none&) -> future<::shared_ptr<cql_transport::messages::result_message>> {
+            return make_ready_future<::shared_ptr<cql_transport::messages::result_message>>();
+        },
+        [] (service::broadcast_tables::query_result_select&) -> future<::shared_ptr<cql_transport::messages::result_message>> {
+            on_internal_error(logger, "incorrect query result ");
+        }
+    ), result);
+}
+
+uint32_t strongly_consistent_modification_statement::get_bound_terms() const {
+    return _bound_terms;
+}
+
+future<> strongly_consistent_modification_statement::check_access(query_processor& qp, const service::client_state& state) const {
+    const data_dictionary::database db = qp.db();
+    auto f = state.has_column_family_access(db, _schema->ks_name(), _schema->cf_name(), auth::permission::MODIFY);
+    if (_query.value_condition.has_value()) {
+        f = f.then([this, &state, db] {
+           return state.has_column_family_access(db, _schema->ks_name(), _schema->cf_name(), auth::permission::SELECT);
+        });
+    }
+    return f;
+}
+
+void strongly_consistent_modification_statement::validate(query_processor&, const service::client_state& state) const {
+    // Nothing to do, all validation has been done by prepare().
+}
+
+bool strongly_consistent_modification_statement::depends_on(std::string_view ks_name, std::optional<std::string_view> cf_name) const {
+    return _schema->ks_name() == ks_name && (!cf_name || _schema->cf_name() == *cf_name);
+}
+
+}
+
+}

cql3/statements/strongly_consistent_modification_statement.hh

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ *
+ * Modified by ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
+ */
+
+#pragma once
+
+#include "cql3/cql_statement.hh"
+#include "cql3/statements/modification_statement.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
+
+namespace cql3 {
+
+namespace statements {
+
+namespace broadcast_tables {
+
+struct prepared_update {
+    expr::expression key;
+    expr::expression new_value;
+    std::optional<expr::expression> value_condition;
+};
+
+}
+
+class strongly_consistent_modification_statement : public cql_statement_opt_metadata {
+    const uint32_t _bound_terms;
+    const schema_ptr _schema;
+    const broadcast_tables::prepared_update _query;
+
+public:
+    strongly_consistent_modification_statement(uint32_t bound_terms, schema_ptr schema, broadcast_tables::prepared_update query);
+
+    virtual future<::shared_ptr<cql_transport::messages::result_message>>
+    execute(query_processor& qp, service::query_state& qs, const query_options& options) const override;
+
+    virtual future<::shared_ptr<cql_transport::messages::result_message>>
+    execute_without_checking_exception_message(query_processor& qp, service::query_state& qs, const query_options& options) const override;
+
+    virtual uint32_t get_bound_terms() const override;
+
+    virtual future<> check_access(query_processor& qp, const service::client_state& state) const override;
+
+    // Validate before execute, using client state and current schema
+    void validate(query_processor&, const service::client_state& state) const override;
+
+    virtual bool depends_on(std::string_view ks_name, std::optional<std::string_view> cf_name) const override;
+};
+
+
+}
+
+}

cql3/statements/strongly_consistent_select_statement.cc

@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ *
+ * Modified by ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
+ */
+
+
+#include "cql3/statements/strongly_consistent_select_statement.hh"
+
+#include <seastar/core/future.hh>
+#include <seastar/core/on_internal_error.hh>
+
+#include "cql3/restrictions/statement_restrictions.hh"
+#include "cql3/query_processor.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
+
+namespace cql3 {
+
+namespace statements {
+
+static logging::logger logger("strongly_consistent_select_statement");
+
+static
+expr::expression get_key(const cql3::expr::expression& partition_key_restrictions) {
+    const auto* conjunction = cql3::expr::as_if<cql3::expr::conjunction>(&partition_key_restrictions);
+
+    if (!conjunction || conjunction->children.size() != 1) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format(
+            "partition key restriction: {}", partition_key_restrictions));
+    }
+
+    const auto* key_restriction = cql3::expr::as_if<cql3::expr::binary_operator>(&conjunction->children[0]);
+
+    if (!key_restriction) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format("partition key restriction: {}", *conjunction));
+    }
+
+    const auto* column = cql3::expr::as_if<cql3::expr::column_value>(&key_restriction->lhs);
+
+    if (!column || column->col->kind != column_kind::partition_key ||
+        key_restriction->op != cql3::expr::oper_t::EQ) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format("key restriction: {}", *key_restriction));
+    }
+
+    return key_restriction->rhs;
+}
+
+static
+bool is_selecting_only_value(const cql3::selection::selection& selection) {
+    return selection.is_trivial() &&
+           selection.get_column_count() == 1 &&
+           selection.get_columns()[0]->name() == "value";
+}
+
+strongly_consistent_select_statement::strongly_consistent_select_statement(schema_ptr schema, uint32_t bound_terms,
+                                                                           lw_shared_ptr<const parameters> parameters,
+                                                                           ::shared_ptr<selection::selection> selection,
+                                                                           ::shared_ptr<const restrictions::statement_restrictions> restrictions,
+                                                                           ::shared_ptr<std::vector<size_t>> group_by_cell_indices,
+                                                                           bool is_reversed,
+                                                                           ordering_comparator_type ordering_comparator,
+                                                                           std::optional<expr::expression> limit,
+                                                                           std::optional<expr::expression> per_partition_limit,
+                                                                           cql_stats &stats,
+                                                                           std::unique_ptr<attributes> attrs)
+    : select_statement{schema, bound_terms, parameters, selection, restrictions, group_by_cell_indices, is_reversed, ordering_comparator, std::move(limit), std::move(per_partition_limit), stats, std::move(attrs)},
+      _query{prepare_query()}
+{ }
+
+broadcast_tables::prepared_select strongly_consistent_select_statement::prepare_query() const {
+    if (!is_selecting_only_value(*_selection)) {
+        throw service::broadcast_tables::unsupported_operation_error("only 'value' selector is allowed");
+    }
+
+    return {
+        .key = get_key(_restrictions->get_partition_key_restrictions())
+    };
+}
+
+static
+service::broadcast_tables::select_query
+evaluate_prepared(
+    const broadcast_tables::prepared_select& query,
+    const query_options& options) {
+    return service::broadcast_tables::select_query{
+        .key = expr::evaluate(query.key, options).to_bytes()
+    };
+}
+
+future<::shared_ptr<cql_transport::messages::result_message>>
+strongly_consistent_select_statement::execute_without_checking_exception_message(query_processor& qp, service::query_state& qs, const query_options& options) const {
+    if (this_shard_id() != 0) {
+        co_return ::make_shared<cql_transport::messages::result_message::bounce_to_shard>(0, cql3::computed_function_values{});
+    }
+
+    auto result = co_await service::broadcast_tables::execute(
+        qp.get_group0_client(),
+        { evaluate_prepared(_query, options) }
+    );
+    
+    auto query_result = std::get_if<service::broadcast_tables::query_result_select>(&result);
+
+    if (!query_result) {
+        on_internal_error(logger, "incorrect query result ");
+    }
+
+    auto result_set = std::make_unique<cql3::result_set>(std::vector{
+        make_lw_shared<cql3::column_specification>(
+            db::system_keyspace::NAME,
+            db::system_keyspace::BROADCAST_KV_STORE,
+            ::make_shared<cql3::column_identifier>("value", true),
+            utf8_type
+        )
+    });
+
+    if (query_result->value) {
+        result_set->add_row({ query_result->value });
+    }
+
+    co_return ::make_shared<cql_transport::messages::result_message::rows>(cql3::result{std::move(result_set)});
+}
+
+}
+
+}

cql3/statements/strongly_consistent_select_statement.hh

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ *
+ * Modified by ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
+ */
+
+#pragma once
+
+#include "cql3/expr/expression.hh"
+#include "cql3/statements/select_statement.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
+
+namespace cql3 {
+
+namespace statements {
+
+namespace broadcast_tables {
+
+struct prepared_select {
+    expr::expression key;
+};
+
+}
+
+class strongly_consistent_select_statement : public select_statement {
+    const broadcast_tables::prepared_select _query;
+
+    broadcast_tables::prepared_select prepare_query() const;
+public:
+    strongly_consistent_select_statement(schema_ptr schema,
+                     uint32_t bound_terms,
+                     lw_shared_ptr<const parameters> parameters,
+                     ::shared_ptr<selection::selection> selection,
+                     ::shared_ptr<const restrictions::statement_restrictions> restrictions,
+                     ::shared_ptr<std::vector<size_t>> group_by_cell_indices,
+                     bool is_reversed,
+                     ordering_comparator_type ordering_comparator,
+                     std::optional<expr::expression> limit,
+                     std::optional<expr::expression> per_partition_limit,
+                     cql_stats &stats,
+                     std::unique_ptr<cql3::attributes> attrs);
+
+    virtual future<::shared_ptr<cql_transport::messages::result_message>>
+        execute_without_checking_exception_message(query_processor& qp, service::query_state& qs, const query_options& options) const override;
+};
+
+}
+
+}

cql3/statements/truncate_statement.cc

@@ -8,6 +8,7 @@
  * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
  */
 
+#include "cql3/statements/raw/truncate_statement.hh"
 #include "cql3/statements/truncate_statement.hh"
 #include "cql3/statements/prepared_statement.hh"
 #include "cql3/cql_statement.hh"
@@ -15,15 +16,54 @@
 #include "cql3/query_processor.hh"
 #include "service/storage_proxy.hh"
 #include <optional>
+#include "validation.hh"
 
 namespace cql3 {
 
 namespace statements {
 
-truncate_statement::truncate_statement(cf_name name)
-    : cf_statement{std::move(name)}
-    , cql_statement_no_metadata(&timeout_config::truncate_timeout)
+namespace raw {
+
+truncate_statement::truncate_statement(cf_name name, std::unique_ptr<attributes::raw> attrs)
+        : cf_statement(std::move(name))
+        , _attrs(std::move(attrs))
 {
+    // Validate the attributes.
+    // Currently, TRUNCATE supports only USING TIMEOUT
+    assert(!_attrs->timestamp.has_value());
+    assert(!_attrs->time_to_live.has_value());
+}
+
+std::unique_ptr<prepared_statement> truncate_statement::prepare(data_dictionary::database db, cql_stats& stats) {
+    schema_ptr schema = validation::validate_column_family(db, keyspace(), column_family());
+    auto prepared_attributes = _attrs->prepare(db, keyspace(), column_family());
+    auto ctx = get_prepare_context();
+    prepared_attributes->fill_prepare_context(ctx);
+    auto stmt = ::make_shared<cql3::statements::truncate_statement>(std::move(schema), std::move(prepared_attributes));
+    return std::make_unique<prepared_statement>(std::move(stmt));
+}
+
+} // namespace raw
+
+truncate_statement::truncate_statement(schema_ptr schema, std::unique_ptr<attributes> prepared_attrs)
+    : cql_statement_no_metadata(&timeout_config::truncate_timeout)
+    , _schema{std::move(schema)}
+    , _attrs(std::move(prepared_attrs))
+{
+}
+
+truncate_statement::truncate_statement(const truncate_statement& ts)
+    : cql_statement_no_metadata(ts)
+    , _schema(ts._schema)
+    , _attrs(std::make_unique<attributes>(*ts._attrs))
+{ }
+
+const sstring& truncate_statement::keyspace() const {
+    return _schema->ks_name();
+}
+
+const sstring& truncate_statement::column_family() const {
+    return _schema->cf_name();
 }
 
 uint32_t truncate_statement::get_bound_terms() const
@@ -31,11 +71,6 @@ uint32_t truncate_statement::get_bound_terms() const
     return 0;
 }
 
-std::unique_ptr<prepared_statement> truncate_statement::prepare(data_dictionary::database db,cql_stats& stats)
-{
-    return std::make_unique<prepared_statement>(::make_shared<truncate_statement>(*this));
-}
-
 bool truncate_statement::depends_on(std::string_view ks_name, std::optional<std::string_view> cf_name) const
 {
     return false;
@@ -60,13 +95,18 @@ truncate_statement::execute(query_processor& qp, service::query_state& state, co
     if (qp.db().find_schema(keyspace(), column_family())->is_view()) {
         throw exceptions::invalid_request_exception("Cannot TRUNCATE materialized view directly; must truncate base table instead");
     }
-    return qp.proxy().truncate_blocking(keyspace(), column_family()).handle_exception([](auto ep) {
+    auto timeout_in_ms = std::chrono::duration_cast<std::chrono::milliseconds>(get_timeout(state.get_client_state(), options));
+    return qp.proxy().truncate_blocking(keyspace(), column_family(), timeout_in_ms).handle_exception([](auto ep) {
         throw exceptions::truncate_exception(ep);
     }).then([] {
         return ::shared_ptr<cql_transport::messages::result_message>{};
     });
 }
 
+db::timeout_clock::duration truncate_statement::get_timeout(const service::client_state& state, const query_options& options) const {
+    return _attrs->is_timeout_set() ? _attrs->get_timeout(options) : state.get_timeout_config().truncate_timeout;
+}
+
 }
 
 }

cql3/statements/truncate_statement.hh

@@ -12,6 +12,7 @@
 
 #include "cql3/statements/raw/cf_statement.hh"
 #include "cql3/cql_statement.hh"
+#include "cql3/attributes.hh"
 
 namespace cql3 {
 
@@ -19,14 +20,20 @@ class query_processor;
 
 namespace statements {
 
-class truncate_statement : public raw::cf_statement, public cql_statement_no_metadata {
+class truncate_statement : public cql_statement_no_metadata {
+    schema_ptr _schema;
+    const std::unique_ptr<attributes> _attrs;
 public:
-    truncate_statement(cf_name name);
+    truncate_statement(schema_ptr schema, std::unique_ptr<attributes> prepared_attrs);
+    truncate_statement(const truncate_statement&);
+    truncate_statement(truncate_statement&&) = default;
+
+    const sstring& keyspace() const;
+
+    const sstring& column_family() const;
 
     virtual uint32_t get_bound_terms() const override;
 
-    virtual std::unique_ptr<prepared_statement> prepare(data_dictionary::database db, cql_stats& stats) override;
-
     virtual bool depends_on(std::string_view ks_name, std::optional<std::string_view> cf_name) const override;
 
     virtual future<> check_access(query_processor& qp, const service::client_state& state) const override;
@@ -35,6 +42,8 @@ public:
 
     virtual future<::shared_ptr<cql_transport::messages::result_message>>
     execute(query_processor& qp, service::query_state& state, const query_options& options) const override;
+private:
+    db::timeout_clock::duration get_timeout(const service::client_state& state, const query_options& options) const;
 };
 
 }

cql3/statements/update_statement.cc

@@ -9,6 +9,9 @@
  */
 
 #include "update_statement.hh"
+#include "cql3/expr/expression.hh"
+#include "cql3/statements/strongly_consistent_modification_statement.hh"
+#include "service/broadcast_tables/experimental/lang.hh"
 #include "raw/update_statement.hh"
 
 #include "raw/insert_statement.hh"
@@ -22,6 +25,7 @@
 #include "types/user.hh"
 #include "concrete_types.hh"
 #include "validation.hh"
+#include <optional>
 
 namespace cql3 {
 
@@ -254,6 +258,94 @@ void insert_prepared_json_statement::execute_operations_for_key(mutation& m, con
     }
 }
 
+
+static
+expr::expression get_key(const cql3::expr::expression& partition_key_restrictions) {
+    const auto* conjunction = cql3::expr::as_if<cql3::expr::conjunction>(&partition_key_restrictions);
+
+    if (!conjunction || conjunction->children.size() != 1) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format(
+            "partition key restriction: {}", partition_key_restrictions));
+    }
+
+    const auto* key_restriction = cql3::expr::as_if<cql3::expr::binary_operator>(&conjunction->children[0]);
+
+    if (!key_restriction) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format("partition key restriction: {}", *conjunction));
+    }
+
+    const auto* column = cql3::expr::as_if<cql3::expr::column_value>(&key_restriction->lhs);
+
+    if (!column || column->col->kind != column_kind::partition_key ||
+        key_restriction->op != cql3::expr::oper_t::EQ) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format("key restriction: {}", *key_restriction));
+    }
+
+    return key_restriction->rhs;
+}
+
+static
+void prepare_new_value(broadcast_tables::prepared_update& query, const std::vector<::shared_ptr<operation>>& operations) {
+    if (operations.size() != 1) {
+        throw service::broadcast_tables::unsupported_operation_error("only one operation is allowed and must be of the form \"value = X\"");
+    }
+
+    operations[0]->prepare_for_broadcast_tables(query);
+}
+
+static
+std::optional<expr::expression> get_value_condition(const std::vector<lw_shared_ptr<column_condition>>& conditions) {
+    if (conditions.size() == 0) {
+        return std::nullopt;
+    }
+
+    if (conditions.size() > 1) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format("conditions: {}", fmt::join(
+            conditions | boost::adaptors::transformed([] (const auto& cond) {
+                return fmt::format("{}{}", cond->get_operation(), cond->get_value());
+            }), ", ")));
+    }
+
+    const auto& condition = conditions[0];
+
+    if (!condition->get_value() || condition->get_operation() != cql3::expr::oper_t::EQ) {
+        throw service::broadcast_tables::unsupported_operation_error(fmt::format(
+            "condition: {}{}", condition->get_operation(), condition->get_value()));
+    }
+
+    return condition->get_value();
+}
+
+::shared_ptr<strongly_consistent_modification_statement>
+update_statement::prepare_for_broadcast_tables() const {
+    if (attrs) {
+        if (attrs->is_time_to_live_set()) {
+            throw service::broadcast_tables::unsupported_operation_error{"USING TTL"};
+        }
+
+        if (attrs->is_timestamp_set()) {
+            throw service::broadcast_tables::unsupported_operation_error{"USING TIMESTAMP"};
+        }
+
+        if (attrs->is_timeout_set()) {
+            throw service::broadcast_tables::unsupported_operation_error{"USING TIMEOUT"};
+        }
+    }
+
+    broadcast_tables::prepared_update query = {
+        .key = get_key(restrictions().get_partition_key_restrictions()),
+        .value_condition = get_value_condition(_regular_conditions),
+    };
+
+    prepare_new_value(query, _column_operations);
+
+    return ::make_shared<strongly_consistent_modification_statement>(
+        get_bound_terms(),
+        s,
+        query
+    );
+}
+
 namespace raw {
 
 insert_statement::insert_statement(cf_name name,

cql3/statements/update_statement.hh

@@ -42,6 +42,9 @@ private:
     virtual void add_update_for_key(mutation& m, const query::clustering_range& range, const update_parameters& params, const json_cache_opt& json_cache) const override;
 
     virtual void execute_operations_for_key(mutation& m, const clustering_key_prefix& prefix, const update_parameters& params, const json_cache_opt& json_cache) const;
+
+public:
+    virtual ::shared_ptr<strongly_consistent_modification_statement> prepare_for_broadcast_tables() const override;
 };
 
 /*

cql3/values.cc

@@ -78,4 +78,8 @@ bool operator==(const raw_value& v1, const raw_value& v2) {
     return false;
 }
 
+std::ostream& operator<<(std::ostream& os, const raw_value& value) {
+    return os << value.view();
+}
+
 }

cql3/values.hh

@@ -85,6 +85,14 @@ public:
     bool is_unset_value() const {
         return std::holds_alternative<unset_value>(_data);
     }
+    // An empty value is not null or unset, but it has 0 bytes of data.
+    // An empty int value can be created in CQL using blobasint(0x).
+    bool is_empty_value() const {
+        if (is_null() || is_unset_value()) {
+            return false;
+        }
+        return size_bytes() == 0;
+    }
     bool is_value() const {
         return _data.index() <= 1;
     }
@@ -233,6 +241,14 @@ public:
     bool is_null_or_unset() const {
         return !is_value();
     }
+    // An empty value is not null or unset, but it has 0 bytes of data.
+    // An empty int value can be created in CQL using blobasint(0x).
+    bool is_empty_value() const {
+        if (is_null_or_unset()) {
+            return false;
+        }
+        return view().size_bytes() == 0;
+    }
     bool is_value() const {
         return _data.index() <= 1;
     }
@@ -240,10 +256,28 @@ public:
         return is_value();
     }
     bytes to_bytes() && {
-        switch (_data.index()) {
-        case 0:  return std::move(std::get<bytes>(_data));
-        default: return ::to_bytes(std::get<managed_bytes>(_data));
-        }
+        return std::visit(overloaded_functor{
+            [](bytes&& bytes_val) { return std::move(bytes_val); },
+            [](managed_bytes&& managed_bytes_val) { return ::to_bytes(managed_bytes_val); },
+            [](null_value&&) -> bytes {
+                throw std::runtime_error("to_bytes() called on raw value that is null");
+            },
+            [](unset_value&&) -> bytes {
+                throw std::runtime_error("to_bytes() called on raw value that is unset_value");
+            }
+        }, std::move(_data));
+    }
+    bytes_opt to_bytes_opt() && {
+        return std::visit(overloaded_functor{
+            [](bytes&& bytes_val) { return bytes_opt(bytes_val); },
+            [](managed_bytes&& managed_bytes_val) { return bytes_opt(::to_bytes(managed_bytes_val)); },
+            [](null_value&&) -> bytes_opt {
+                return std::nullopt;
+            },
+            [](unset_value&&) -> bytes_opt {
+                return std::nullopt;
+            }
+        }, std::move(_data));
     }
     managed_bytes to_managed_bytes() && {
         return std::visit(overloaded_functor{
@@ -273,6 +307,7 @@ public:
     friend class raw_value_view;
 
     friend bool operator==(const raw_value& v1, const raw_value& v2);
+    friend std::ostream& operator<<(std::ostream& os, const raw_value& value);
 };
 
 }

data_dictionary/data_dictionary.cc

@@ -100,12 +100,12 @@ database::find_table(std::string_view ks, std::string_view table) const {
 }
 
 std::optional<table>
-database::try_find_table(utils::UUID id) const {
+database::try_find_table(table_id id) const {
     return _ops->try_find_table(*this, id);
 }
 
 table
-database::find_column_family(utils::UUID uuid) const {
+database::find_column_family(table_id uuid) const {
     auto t = try_find_table(uuid);
     if (!t) {
         throw no_such_column_family(uuid);
@@ -119,7 +119,7 @@ database::find_schema(std::string_view ks, std::string_view table) const {
 }
 
 schema_ptr
-database::find_schema(utils::UUID uuid) const {
+database::find_schema(table_id uuid) const {
     return find_column_family(uuid).schema();
 }
 
@@ -326,7 +326,7 @@ no_such_keyspace::no_such_keyspace(std::string_view ks_name)
 {
 }
 
-no_such_column_family::no_such_column_family(const utils::UUID& uuid)
+no_such_column_family::no_such_column_family(const table_id& uuid)
     : runtime_error{format("Can't find a column family with UUID {}", uuid)}
 {
 }
@@ -336,7 +336,7 @@ no_such_column_family::no_such_column_family(std::string_view ks_name, std::stri
 {
 }
 
-no_such_column_family::no_such_column_family(std::string_view ks_name, const utils::UUID& uuid)
+no_such_column_family::no_such_column_family(std::string_view ks_name, const table_id& uuid)
     : runtime_error{format("Can't find a column family with UUID {} in keyspace {}", uuid, ks_name)}
 {
 }

data_dictionary/data_dictionary.hh

@@ -14,7 +14,7 @@
 #include <vector>
 #include <seastar/core/shared_ptr.hh>
 #include "seastarx.hh"
-#include "utils/UUID.hh"
+#include "schema_fwd.hh"
 
 namespace replica {
 class database; // For transition; remove
@@ -62,9 +62,9 @@ public:
 
 class no_such_column_family : public std::runtime_error {
 public:
-    no_such_column_family(const utils::UUID& uuid);
+    no_such_column_family(const table_id& uuid);
     no_such_column_family(std::string_view ks_name, std::string_view cf_name);
-    no_such_column_family(std::string_view ks_name, const utils::UUID& uuid);
+    no_such_column_family(std::string_view ks_name, const table_id& uuid);
 };
 
 class table {
@@ -105,13 +105,13 @@ public:
     std::vector<keyspace> get_keyspaces() const;
     std::vector<table> get_tables() const;
     table find_table(std::string_view ks, std::string_view table) const;  // throws no_such_column_family
-    table find_column_family(utils::UUID uuid) const;  // throws no_such_column_family
+    table find_column_family(table_id uuid) const;  // throws no_such_column_family
     schema_ptr find_schema(std::string_view ks, std::string_view table) const;  // throws no_such_column_family
-    schema_ptr find_schema(utils::UUID uuid) const;  // throws no_such_column_family
+    schema_ptr find_schema(table_id uuid) const;  // throws no_such_column_family
     table find_column_family(schema_ptr s) const;
     bool has_schema(std::string_view ks_name, std::string_view cf_name) const;
     std::optional<table> try_find_table(std::string_view ks, std::string_view table) const;
-    std::optional<table> try_find_table(utils::UUID id) const;
+    std::optional<table> try_find_table(table_id id) const;
     const db::config& get_config() const;
     std::set<sstring> existing_index_names(std::string_view ks_name, std::string_view cf_to_exclude = sstring()) const;
     schema_ptr find_indexed_table(std::string_view ks_name, std::string_view index_name) const;

data_dictionary/impl.hh

@@ -21,7 +21,7 @@ public:
     virtual std::vector<keyspace> get_keyspaces(database db) const = 0;
     virtual std::vector<table> get_tables(database db) const = 0;
     virtual std::optional<table> try_find_table(database db, std::string_view ks, std::string_view tab) const = 0;
-    virtual std::optional<table> try_find_table(database db, utils::UUID id) const = 0;
+    virtual std::optional<table> try_find_table(database db, table_id id) const = 0;
     virtual const secondary_index::secondary_index_manager& get_index_manager(table t) const = 0;
     virtual schema_ptr get_table_schema(table t) const = 0;
     virtual lw_shared_ptr<keyspace_metadata> get_keyspace_metadata(keyspace ks) const = 0;

db/batchlog_manager.cc

@@ -25,19 +25,14 @@
 #include "system_keyspace.hh"
 #include "utils/rate_limiter.hh"
 #include "log.hh"
-#include "serializer.hh"
 #include "db_clock.hh"
 #include "unimplemented.hh"
 #include "gms/failure_detector.hh"
 #include "gms/gossiper.hh"
 #include "schema_registry.hh"
-#include "idl/uuid.dist.hh"
 #include "idl/frozen_schema.dist.hh"
-#include "serializer_impl.hh"
-#include "serialization_visitors.hh"
-#include "db/schema_tables.hh"
-#include "idl/uuid.dist.impl.hh"
 #include "idl/frozen_schema.dist.impl.hh"
+#include "db/schema_tables.hh"
 #include "message/messaging_service.hh"
 #include "cql3/untyped_result_set.hh"
 #include "service_permit.hh"

db/cache_tracker.hh

@@ -92,7 +92,7 @@ public:
     void insert(partition_entry&) noexcept;
     void insert(partition_version&) noexcept;
     void insert(rows_entry&) noexcept;
-    void on_remove() noexcept;
+    void remove(rows_entry&) noexcept;
     void clear_continuity(cache_entry& ce) noexcept;
     void on_partition_erase() noexcept;
     void on_partition_merge() noexcept;
@@ -123,9 +123,12 @@ public:
 };
 
 inline
-void cache_tracker::on_remove() noexcept {
+void cache_tracker::remove(rows_entry& entry) noexcept {
     --_stats.rows;
     ++_stats.row_removals;
+    if (entry.is_linked()) {
+        _lru.remove(entry);
+    }
 }
 
 inline

db/commitlog/commitlog.cc

@@ -225,7 +225,6 @@ const std::string db::commitlog::descriptor::FILENAME_EXTENSION(".log");
 class db::commitlog::segment_manager : public ::enable_shared_from_this<segment_manager> {
 public:
     config cfg;
-    std::vector<sstring> _segments_to_replay;
     const uint64_t max_size;
     const uint64_t max_mutation_size;
     // Divide the size-on-disk threshold by #cpus used, since we assume
@@ -479,7 +478,8 @@ public:
     buffer_type acquire_buffer(size_t s, size_t align);
     temporary_buffer<char> allocate_single_buffer(size_t, size_t);
 
-    future<std::vector<descriptor>> list_descriptors(sstring dir);
+    future<std::vector<descriptor>> list_descriptors(sstring dir) const;
+    future<std::vector<sstring>> get_segments_to_replay() const;
 
     flush_handler_id add_flush_handler(flush_handler h) {
         auto id = ++_flush_ids;
@@ -499,7 +499,7 @@ private:
     void abort_recycled_list(std::exception_ptr);
 
     size_t max_request_controller_units() const;
-    segment_id_type _ids = 0;
+    segment_id_type _ids = 0, _low_id = 0;
     std::vector<sseg_ptr> _segments;
     queue<sseg_ptr> _reserve_segments;
     queue<named_file> _recycled_segments;
@@ -1436,7 +1436,7 @@ future<> db::commitlog::segment_manager::replenish_reserve() {
 }
 
 future<std::vector<db::commitlog::descriptor>>
-db::commitlog::segment_manager::list_descriptors(sstring dirname) {
+db::commitlog::segment_manager::list_descriptors(sstring dirname) const {
     auto dir = co_await open_checked_directory(commit_error_handler, dirname);
     std::vector<db::commitlog::descriptor> result;
 
@@ -1466,6 +1466,26 @@ db::commitlog::segment_manager::list_descriptors(sstring dirname) {
     co_return result;
 }
 
+// #11237 - make get_segments_to_replay on-demand. Since we base the time-part of
+// descriptor ids on hightest of wall-clock and segments found on disk on init,
+// we can just scan files now and include only those representing generations before
+// the creation of this commitlog instance.
+// This _could_ give weird results iff we had a truly sharded commitlog folder
+// where init of the instances was not very synced _and_ we allowed more than
+// one shard to do replay. But allowed usage always either is one dir - one shard (hints)
+// or does shard 0 init first (main/database), then replays on 0 as well.
+future<std::vector<sstring>> db::commitlog::segment_manager::get_segments_to_replay() const {
+    std::vector<sstring> segments_to_replay;
+    auto descs = co_await list_descriptors(cfg.commit_log_location);
+    for (auto& d : descs) {
+        auto id = replay_position(d.id).base_id();
+        if (id <= _low_id) {
+            segments_to_replay.push_back(cfg.commit_log_location + "/" + d.filename());
+        }
+    }
+    co_return segments_to_replay;
+}
+
 future<> db::commitlog::segment_manager::init() {
     auto descs = co_await list_descriptors(cfg.commit_log_location);
 
@@ -1473,11 +1493,12 @@ future<> db::commitlog::segment_manager::init() {
     segment_id_type id = *cfg.base_segment_id;
     for (auto& d : descs) {
         id = std::max(id, replay_position(d.id).base_id());
-        _segments_to_replay.push_back(cfg.commit_log_location + "/" + d.filename());
     }
 
     // base id counter is [ <shard> | <base> ]
     _ids = replay_position(this_shard_id(), id).id;
+    _low_id = id;
+
     // always run the timer now, since we need to handle segment pre-alloc etc as well.
     _timer.set_callback(std::bind(&segment_manager::on_timer, this));
     auto delay = this_shard_id() * std::ceil(double(cfg.commitlog_sync_period_in_ms) / smp::count);
@@ -2031,7 +2052,7 @@ future<> db::commitlog::segment_manager::shutdown() {
         }
     }
     co_await _shutdown_promise->get_shared_future();
-    clogger.info("Commitlog shutdown complete");
+    clogger.debug("Commitlog shutdown complete");
 }
 
 void db::commitlog::segment_manager::add_file_to_dispose(named_file f, dispose_mode mode) {
@@ -2043,9 +2064,16 @@ future<> db::commitlog::segment_manager::delete_segments(std::vector<sstring> fi
         // Note: this is only for replay files. We can decide to
         // recycle these, but they don't count into footprint,
         // thus unopened named_files are what we want (known_size == 0)
+
+        // #11184 - include replay footprint so we make sure to delete any segments
+        // pushing us over limits in "delete_segments" (assumed called after replay)
+        // #11237 - cannot do this already in "init()" - we need to be able to create
+        // segments before replay+delete, and more to the point: shards that _don't_ replay
+        // must not add this to their footprint (a.) shared, b.) not there after this call)
+        totals.total_size_on_disk += co_await file_size(s);
         _files_to_dispose.emplace_back(s, dispose_mode::Delete);
     }
-    return do_pending_deletes();
+    co_return co_await do_pending_deletes();
 }
 
 void db::commitlog::segment_manager::abort_recycled_list(std::exception_ptr ep) {
@@ -2088,7 +2116,6 @@ future<> db::commitlog::segment_manager::do_pending_deletes() {
     }
 
     std::exception_ptr recycle_error;
-    size_t num_deleted = 0;
     auto exts = cfg.extensions;
 
     clogger.debug("Discarding segments {}", ftd);
@@ -2136,7 +2163,6 @@ future<> db::commitlog::segment_manager::do_pending_deletes() {
 
             // last resort.
             co_await f.remove_file();
-            ++num_deleted;
         } catch (...) {
             clogger.error("Could not delete segment {}: {}", f.name(), std::current_exception());
         }
@@ -2943,8 +2969,8 @@ future<std::vector<sstring>> db::commitlog::list_existing_segments(const sstring
     });
 }
 
-std::vector<sstring> db::commitlog::get_segments_to_replay() const {
-    return std::move(_segment_manager->_segments_to_replay);
+future<std::vector<sstring>> db::commitlog::get_segments_to_replay() const {
+    return _segment_manager->get_segments_to_replay();
 }
 
 future<> db::commitlog::delete_segments(std::vector<sstring> files) const {

db/commitlog/commitlog.hh

@@ -270,10 +270,8 @@ public:
     /**
      * Returns a vector of segment paths which were
      * preexisting when this instance of commitlog was created.
-     *
-     * The list will be empty when called for the second time.
      */
-    std::vector<sstring> get_segments_to_replay() const;
+    future<std::vector<sstring>> get_segments_to_replay() const;
 
     /**
      * Delete aforementioned segments, and possible metadata