token_metadata: make it a template with NodeId=inet_address/host_id

NodeId is used in all internal token_metadata data structures, that
previously used inet_address. We choose topology::key_kind based
on the value of the template parameter.

generic_token_metadata::update_topology overload with host_id
parameter is added to make update_topology_change_info work,
it now uses NodeId as a parameter type.

topology::remove_endpoint(host_id) is added to make
generic_token_metadata::remove_endpoint(NodeId) work.

pending_endpoints_for and endpoints_for_reading are just removed - they
are not used and not implemented. The declarations were left by mistake
from a refactoring in which these methods were moved to erm.

generic_token_metadata_base is extracted to contain declarations, common
to both token_metadata versions.

Templates are explicitly instantiated inside token_metadata.cc, since
implementation part is also a template and it's not exposed to the header.

There are no other behavioral changes in this commit, just syntax
fixes to make token_metadata a template.

api/api_init.hh

@@ -10,6 +10,7 @@
 #include <seastar/http/httpd.hh>
 #include <seastar/core/future.hh>
 
+#include "locator/host_id.hh"
 #include "replica/database_fwd.hh"
 #include "tasks/task_manager.hh"
 #include "seastarx.hh"
@@ -32,9 +33,16 @@ namespace streaming {
 class stream_manager;
 }
 
+namespace gms {
+    class inet_address;
+}
+
 namespace locator {
 
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<host_id>;
 class shared_token_metadata;
 class snitch_ptr;
 

cdc/log.hh

@@ -29,13 +29,17 @@
 #include "timestamp.hh"
 #include "tracing/trace_state.hh"
 #include "utils/UUID.hh"
+#include "locator/host_id.hh"
 
 class schema;
 using schema_ptr = seastar::lw_shared_ptr<const schema>;
 
 namespace locator {
 
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<host_id>;
 
 } // namespace locator
 

cql3/statements/create_keyspace_statement.hh

@@ -17,7 +17,10 @@
 
 namespace locator {
 
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<host_id>;
 
 };
 

cql3/statements/ks_prop_defs.hh

@@ -12,6 +12,7 @@
 
 #include "cql3/statements/property_definitions.hh"
 #include "data_dictionary/storage_options.hh"
+#include "locator/host_id.hh"
 
 #include <seastar/core/shared_ptr.hh>
 #include <seastar/core/sstring.hh>
@@ -20,9 +21,15 @@
 namespace data_dictionary {
 class keyspace_metadata;
 }
+namespace gms {
+    class inet_address;
+}
 
 namespace locator {
-    class token_metadata;
+    template <typename NodeId>
+    class generic_token_metadata;
+    using token_metadata = generic_token_metadata<gms::inet_address>;
+    using token_metadata2 = generic_token_metadata<host_id>;
     class shared_token_metadata;
     struct snitch_ptr;
     class abstract_replication_strategy;

db/view/view_update_checks.hh

@@ -10,6 +10,7 @@
 
 #include <seastar/core/future.hh>
 #include "streaming/stream_reason.hh"
+#include "locator/host_id.hh"
 #include "seastarx.hh"
 
 namespace replica {
@@ -23,7 +24,10 @@ class system_distributed_keyspace;
 }
 
 namespace locator {
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<host_id>;
 }
 
 namespace db::view {

locator/token_metadata.hh

@@ -43,7 +43,10 @@ class abstract_replication_strategy;
 
 using token = dht::token;
 
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<locator::host_id>;
 class tablet_metadata;
 
 struct host_id_or_endpoint {
@@ -68,14 +71,16 @@ struct host_id_or_endpoint {
 
     // Map the host_id to endpoint based on whichever of them is set,
     // using the token_metadata
-    void resolve(const token_metadata& tm);
+    template <typename NodeId>
+    void resolve(const generic_token_metadata<NodeId>& tm);
 };
 
+template <typename NodeId>
 class token_metadata_impl;
+template <typename NodeId = gms::inet_address>
 struct topology_change_info;
 
-class token_metadata final {
-    std::unique_ptr<token_metadata_impl> _impl;
+class generic_token_metadata_base {
 public:
     struct config {
         topology::config topo_cfg;
@@ -83,6 +88,11 @@ public:
     using inet_address = gms::inet_address;
     using version_t = service::topology::version_t;
     using version_tracker_t = utils::phased_barrier::operation;
+};
+
+template <typename NodeId = gms::inet_address>
+class generic_token_metadata final: public generic_token_metadata_base {
+    std::unique_ptr<token_metadata_impl<NodeId>> _impl;
 private:
     friend class token_metadata_ring_splitter;
     class tokens_iterator {
@@ -94,24 +104,24 @@ private:
         using reference = token&;
     public:
         tokens_iterator() = default;
-        tokens_iterator(const token& start, const token_metadata_impl* token_metadata);
+        tokens_iterator(const token& start, const token_metadata_impl<NodeId>* token_metadata);
         bool operator==(const tokens_iterator& it) const;
         const token& operator*() const;
         tokens_iterator& operator++();
     private:
         std::vector<token>::const_iterator _cur_it;
         size_t _remaining = 0;
-        const token_metadata_impl* _token_metadata = nullptr;
+        const token_metadata_impl<NodeId>* _token_metadata = nullptr;
 
-        friend class token_metadata_impl;
+        friend class token_metadata_impl<NodeId>;
     };
 
 public:
-    token_metadata(config cfg);
-    explicit token_metadata(std::unique_ptr<token_metadata_impl> impl);
-    token_metadata(token_metadata&&) noexcept; // Can't use "= default;" - hits some static_assert in unique_ptr
-    token_metadata& operator=(token_metadata&&) noexcept;
-    ~token_metadata();
+    generic_token_metadata(config cfg);
+    explicit generic_token_metadata(std::unique_ptr<token_metadata_impl<NodeId>> impl);
+    generic_token_metadata(generic_token_metadata&&) noexcept; // Can't use "= default;" - hits some static_assert in unique_ptr
+    generic_token_metadata& operator=(generic_token_metadata&&) noexcept;
+    ~generic_token_metadata();
     const std::vector<token>& sorted_tokens() const;
     const tablet_metadata& tablets() const;
     tablet_metadata& tablets();
@@ -121,19 +131,19 @@ public:
     //
     // Note: the function is not exception safe!
     // It must be called only on a temporary copy of the token_metadata
-    future<> update_normal_tokens(std::unordered_set<token> tokens, inet_address endpoint);
+    future<> update_normal_tokens(std::unordered_set<token> tokens, NodeId endpoint);
     const token& first_token(const token& start) const;
     size_t first_token_index(const token& start) const;
-    std::optional<inet_address> get_endpoint(const token& token) const;
-    std::vector<token> get_tokens(const inet_address& addr) const;
-    const std::unordered_map<token, inet_address>& get_token_to_endpoint() const;
-    const std::unordered_set<inet_address>& get_leaving_endpoints() const;
-    const std::unordered_map<token, inet_address>& get_bootstrap_tokens() const;
+    std::optional<NodeId> get_endpoint(const token& token) const;
+    std::vector<token> get_tokens(const NodeId& addr) const;
+    const std::unordered_map<token, NodeId>& get_token_to_endpoint() const;
+    const std::unordered_set<NodeId>& get_leaving_endpoints() const;
+    const std::unordered_map<token, NodeId>& get_bootstrap_tokens() const;
 
     /**
      * Update or add endpoint given its inet_address and endpoint_dc_rack.
      */
-    void update_topology(inet_address ep, std::optional<endpoint_dc_rack> opt_dr, std::optional<node::state> opt_st = std::nullopt,
+    void update_topology(NodeId ep, std::optional<endpoint_dc_rack> opt_dr, std::optional<node::state> opt_st = std::nullopt,
                          std::optional<shard_id> shard_count = std::nullopt);
     /**
      * Creates an iterable range of the sorted tokens starting at the token t
@@ -182,39 +192,39 @@ public:
     /// Returns host_id of the local node.
     host_id get_my_id() const;
 
-    void add_bootstrap_token(token t, inet_address endpoint);
+    void add_bootstrap_token(token t, NodeId endpoint);
 
-    void add_bootstrap_tokens(std::unordered_set<token> tokens, inet_address endpoint);
+    void add_bootstrap_tokens(std::unordered_set<token> tokens, NodeId endpoint);
 
     void remove_bootstrap_tokens(std::unordered_set<token> tokens);
 
-    void add_leaving_endpoint(inet_address endpoint);
-    void del_leaving_endpoint(inet_address endpoint);
+    void add_leaving_endpoint(NodeId endpoint);
+    void del_leaving_endpoint(NodeId endpoint);
 
-    void remove_endpoint(inet_address endpoint);
+    void remove_endpoint(NodeId endpoint);
 
     // Checks if the node is part of the token ring. If yes, the node is one of
     // the nodes that owns the tokens and inside the set _normal_token_owners.
-    bool is_normal_token_owner(inet_address endpoint) const;
+    bool is_normal_token_owner(NodeId endpoint) const;
 
-    bool is_leaving(inet_address endpoint) const;
+    bool is_leaving(NodeId endpoint) const;
 
     // Is this node being replaced by another node
-    bool is_being_replaced(inet_address endpoint) const;
+    bool is_being_replaced(NodeId endpoint) const;
 
     // Is any node being replaced by another node
     bool is_any_node_being_replaced() const;
 
-    void add_replacing_endpoint(inet_address existing_node, inet_address replacing_node);
+    void add_replacing_endpoint(NodeId existing_node, NodeId replacing_node);
 
-    void del_replacing_endpoint(inet_address existing_node);
+    void del_replacing_endpoint(NodeId existing_node);
 
     /**
      * Create a full copy of token_metadata using asynchronous continuations.
      * The caller must ensure that the cloned object will not change if
      * the function yields.
      */
-    future<token_metadata> clone_async() const noexcept;
+    future<generic_token_metadata> clone_async() const noexcept;
 
     /**
      * Create a copy of TokenMetadata with only tokenToEndpointMap. That is, pending ranges,
@@ -222,7 +232,7 @@ public:
      * The caller must ensure that the cloned object will not change if
      * the function yields.
      */
-    future<token_metadata> clone_only_token_map() const noexcept;
+    future<generic_token_metadata> clone_only_token_map() const noexcept;
     /**
      * Create a copy of TokenMetadata with tokenToEndpointMap reflecting situation after all
      * current leave operations have finished.
@@ -231,7 +241,7 @@ public:
      *
      * @return a future holding a new token metadata
      */
-    future<token_metadata> clone_after_all_left() const noexcept;
+    future<generic_token_metadata> clone_after_all_left() const noexcept;
 
     /**
      * Gently clear the token_metadata members.
@@ -251,13 +261,13 @@ public:
     static boost::icl::interval<token>::interval_type range_to_interval(range<dht::token> r);
     static range<dht::token> interval_to_range(boost::icl::interval<token>::interval_type i);
 
-    future<> update_topology_change_info(dc_rack_fn<gms::inet_address>& get_dc_rack);
+    future<> update_topology_change_info(dc_rack_fn<NodeId>& get_dc_rack);
 
-    const std::optional<topology_change_info>& get_topology_change_info() const;
+    const std::optional<topology_change_info<NodeId>>& get_topology_change_info() const;
 
     token get_predecessor(token t) const;
 
-    const std::unordered_set<inet_address>& get_all_endpoints() const;
+    const std::unordered_set<NodeId>& get_all_endpoints() const;
 
     /* Returns the number of different endpoints that own tokens in the ring.
      * Bootstrapping tokens are not taken into account. */
@@ -275,7 +285,7 @@ public:
      * @return a (stable copy, won't be modified) Token to Endpoint map for all the normal and bootstrapping nodes
      *         in the cluster.
      */
-    std::map<token, inet_address> get_normal_and_bootstrapping_token_to_endpoint_map() const;
+    std::map<token, NodeId> get_normal_and_bootstrapping_token_to_endpoint_map() const;
 
     long get_ring_version() const;
     void invalidate_cached_rings();
@@ -283,20 +293,26 @@ public:
     version_t get_version() const;
     void set_version(version_t version);
 
-    friend class token_metadata_impl;
+    friend class token_metadata_impl<NodeId>;
     friend class shared_token_metadata;
 private:
     void set_version_tracker(version_tracker_t tracker);
 };
 
+extern template class generic_token_metadata<locator::host_id>;
+extern template class generic_token_metadata<gms::inet_address>;
+extern template void host_id_or_endpoint::resolve(const token_metadata& tm);
+extern template void host_id_or_endpoint::resolve(const token_metadata2& tm);
+
+template <typename NodeId>
 struct topology_change_info {
-    token_metadata_ptr target_token_metadata;
-    token_metadata_ptr base_token_metadata;
+    lw_shared_ptr<generic_token_metadata<NodeId>> target_token_metadata;
+    lw_shared_ptr<generic_token_metadata<NodeId>> base_token_metadata;
     std::vector<dht::token> all_tokens;
     token_metadata::read_new_t read_new;
 
-    topology_change_info(token_metadata_ptr target_token_metadata_,
-        token_metadata_ptr base_token_metadata_,
+    topology_change_info(lw_shared_ptr<generic_token_metadata<NodeId>> target_token_metadata_,
+        lw_shared_ptr<generic_token_metadata<NodeId>> base_token_metadata_,
         std::vector<dht::token> all_tokens_,
         token_metadata::read_new_t read_new_);
     future<> clear_gently();
@@ -310,6 +326,11 @@ mutable_token_metadata_ptr make_token_metadata_ptr(Args... args) {
     return make_lw_shared<token_metadata>(std::forward<Args>(args)...);
 }
 
+template <typename... Args>
+mutable_token_metadata2_ptr make_token_metadata2_ptr(Args... args) {
+    return make_lw_shared<token_metadata2>(std::forward<Args>(args)...);
+}
+
 class shared_token_metadata {
     mutable_token_metadata_ptr _shared;
     token_metadata_lock_func _lock_func;

locator/token_metadata_fwd.hh

@@ -11,9 +11,13 @@
 
 namespace locator {
 
-class token_metadata;
-
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
 using token_metadata_ptr = lw_shared_ptr<const token_metadata>;
 using mutable_token_metadata_ptr = lw_shared_ptr<token_metadata>;
+using token_metadata2 = generic_token_metadata<host_id>;
+using token_metadata2_ptr = lw_shared_ptr<const token_metadata2>;
+using mutable_token_metadata2_ptr = lw_shared_ptr<token_metadata2>;
 
 } // namespace locator

locator/topology.cc

@@ -491,6 +491,17 @@ bool topology::remove_endpoint(inet_address ep)
     return false;
 }
 
+bool topology::remove_endpoint(locator::host_id host_id)
+{
+    auto node = find_node(host_id);
+    tlogger.debug("topology[{}]: remove_endpoint: host_id={}: {}", fmt::ptr(this), host_id, debug_format(node));
+    if (node) {
+        remove_node(node);
+        return true;
+    }
+    return false;
+}
+
 bool topology::has_node(host_id id) const noexcept {
     auto node = find_node(id);
     tlogger.trace("topology[{}]: has_node: host_id={}: {}", fmt::ptr(this), id, debug_format(node));

locator/topology.hh

@@ -258,6 +258,8 @@ public:
      */
     bool remove_endpoint(inet_address ep);
 
+    bool remove_endpoint(locator::host_id ep);
+
     /**
      * Returns true iff contains given endpoint.
      */
@@ -422,6 +424,7 @@ private:
         return _nodes_by_endpoint;
     };
 
+    template <typename NodeId>
     friend class token_metadata_impl;
 public:
     void test_compare_endpoints(const inet_address& address, const inet_address& a1, const inet_address& a2) const;

node_ops/node_ops_ctl.hh

@@ -13,6 +13,7 @@
 #include "locator/host_id.hh"
 #include "node_ops/id.hh"
 #include "schema/schema_fwd.hh"
+#include "locator/host_id.hh"
 
 #include <seastar/core/abort_source.hh>
 
@@ -24,7 +25,10 @@ class storage_service;
 }
 
 namespace locator {
-class token_metadata;
+template <typename NodeId>
+class generic_token_metadata;
+using token_metadata = generic_token_metadata<gms::inet_address>;
+using token_metadata2 = generic_token_metadata<host_id>;
 }
 
 class node_ops_info {