/*
 * Copyright (C) 2021-present ScyllaDB
 */

/*
 * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
 */
#pragma once

#include "service/raft/join_node.hh"
#include "service/raft/raft_group_registry.hh"
#include "service/raft/discovery.hh"
#include "service/raft/group0_fwd.hh"
#include "gms/feature.hh"
#include "raft/raft.hh"
#include "utils/updateable_value.hh"
#include <seastar/core/gate.hh>

namespace cql3 { class query_processor; }

namespace db { class system_keyspace; }

namespace gms { class gossiper; class feature_service; }

namespace service {

extern const char* const raft_upgrade_doc;

class migration_manager;
class raft_group0_client;
class storage_service;

// Wrapper for `discovery` which persists the learned peers on disk.
class persistent_discovery {
    discovery _discovery;
    cql3::query_processor& _qp;
    seastar::named_gate _gate;

public:
    using peer_list = discovery::peer_list;
    using tick_output = discovery::tick_output;

    // See `discovery::discovery`.
    // The provided seed list will be extended with already known persisted peers.
    // `my_addr` must be the same across restarts.
    static future<persistent_discovery> make(discovery_peer my_addr, peer_list seeds, cql3::query_processor&);

    // Run the discovery algorithm to find information about group 0.
    future<group0_info> run(
        netw::messaging_service&,
        gate::holder pause_shutdown,
        abort_source&,
        discovery_peer my_addr);

    // Must be called and waited for before destroying the object.
    // Must not be called concurrently with `run`.
    // Can be called concurrently with `request`.
    future<> stop();

    // See `discovery::request`.
    future<std::optional<peer_list>> request(peer_list peers);

private:
    // See `discovery::response`.
    void response(discovery_peer from, const peer_list& peers);

    // See `discovery::tick`.
    future<tick_output> tick();

    persistent_discovery(discovery_peer my_addr, const peer_list&, cql3::query_processor&);
};

// Abstracts away the details of a handshake used to join an existing group 0.
// None of its methods are called if the node creates a new group 0.
class group0_handshaker {
public:
    virtual ~group0_handshaker() {}

    // Called after initial discovery run is finished, but before starting
    // the raft server. This is only called once and is not retried in case
    // it or `post_server_start` fails.
    virtual future<> pre_server_start(const group0_info& info) = 0;

    // Performs a handshake that should result in this node being added to group 0.
    // Called after the raft server is started.
    //
    // - If the function returns true, the node has been added to group 0
    //   successfully.
    // - If the function returns false, this step has failed but should be
    //   retried.
    // - An exception is interpreted as an irrecoverable error and causes
    //   setup_group0 to fail.
    //
    // The implementation can use the abort_source parameter to abort the
    // function if needed.
    virtual future<bool> post_server_start(const group0_info& info, abort_source& as) = 0;
};

class raft_group0 {
    seastar::named_gate _shutdown_gate;
    seastar::abort_source& _abort_source;
    raft_group_registry& _raft_gr;
    sharded<netw::messaging_service>& _ms;
    gms::gossiper& _gossiper;
    gms::feature_service& _feat;
    raft_group0_client& _client;
    seastar::scheduling_group _sg;

    // Status of leader discovery. Initially there is no group 0,
    // and the variant contains no state. During initial cluster
    // bootstrap a discovery object is created, which is then
    // substituted by group0 id when a leader is discovered or
    // created.
    std::variant<std::monostate, service::persistent_discovery, raft::group_id> _group0;

    gms::feature::listener_registration _raft_support_listener;

    seastar::metrics::metric_groups _metrics;
    void register_metrics();

    // Status of the raft group0 for monitoring.
    enum class status_for_monitoring : uint8_t {
        unused = 0,
        normal = 1,
        aborted = 2
    } _status_for_monitoring;

    // Fiber monitoring and logging gaining/losing leadership events in group 0.
    future<> leadership_monitor_fiber();
    future<> _leadership_monitor = make_ready_future<>();
    abort_source _leadership_monitor_as;
    utils::updateable_value_source<bool> _leadership_observable;
    std::optional<shared_future<>> _aborted;

public:
    // Passed to `setup_group0` when replacing a node.
    struct replace_info {
        raft::server_id raft_id;
    };

    // Assumes that the provided services are fully started.
    raft_group0(seastar::abort_source& abort_source,
        service::raft_group_registry& raft_gr,
        sharded<netw::messaging_service>& ms,
        gms::gossiper& gs,
        gms::feature_service& feat,
        raft_group0_client& client,
        seastar::scheduling_group sg);

    // Initialises RPC verbs on all shards.
    // Call after construction but before using the object.
    future<> start();

    // Deinitializes the group0 Raft server and stops all background activity.
    // The server object remains valid, but new requests will get raft::stopped_error.
    // This function is idempotent: it can be called multiple times; subsequent calls
    // will wait for the abort initiated by the first call to complete.
    future<> abort_and_drain();

    // Destroys the Raft server instance registered in raft_group_registry.
    // It must be ensured that no one accesses it from this point on, including via raft_group0_client.
    // Must be called before destroying the raft_group0 object.
    void destroy();

    // Run the discovery algorithm.
    //
    // Discovers an existing group 0 cluster or elects a server (called a 'leader')
    // responsible for creating a new group 0 cluster if one doesn't exist
    // (in particular, we may become that leader).
    //
    // See 'raft-in-scylla.md', 'Establishing group 0 in a fresh cluster'.
    future<group0_info> discover_group0(const std::vector<gms::inet_address>& seeds, cql3::query_processor& qp);

    // Call during the startup procedure, after gossiping has started.
    //
    // If we're performing the replace operation, pass the IP and Raft ID of the replaced node
    // obtained using the shadow round through the `replace_info` parameter.
    //
    // If the local RAFT feature is enabled, does one of the following:
    // - join group 0 (if we're bootstrapping),
    // - prepare us for the upgrade procedure, which will create group 0 later (if we're upgrading).
    //
    // Cannot be called twice.
    //
    // Also make sure to call `finish_setup_after_join` after the node has joined the cluster and entered NORMAL state.
    future<> setup_group0(db::system_keyspace&, const std::unordered_set<gms::inet_address>& initial_contact_nodes, shared_ptr<group0_handshaker> handshaker,
                          service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm,
                          const join_node_request_params& params);

    // Call during the startup procedure before networking is enabled.
    //
    // If the local RAFT feature is enabled and we're not in RECOVERY mode:
    // - start group 0 server if it exists,
    // - disable migration_manager schema pulls if we're bootstrapping
    //   or Raft is already fully functioning.
    //
    // Cannot be called twice.
    //
    future<> setup_group0_if_exist(db::system_keyspace&, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm);

    // Call at the end of the startup procedure, after the node entered NORMAL state.
    // `setup_group0()` must have finished earlier.
    //
    // If the node has just bootstrapped, causes the group 0 server to become a voter.
    //
    future<> finish_setup_after_join(service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm);

    // Check whether the given Raft server is a member of group 0 configuration
    // according to our current knowledge.
    //
    // If `include_voters_only` is `true`, returns `true` only if the server is a voting member.
    //
    // Precondition: joined_group0(). In particular, this can be called safely
    // if wait_for_raft() was called earlier and returned `true`.
    bool is_member(raft::server_id, bool include_voters_only);

    // Modify the voter status of the given servers in group 0.
    //
    // The `voters_add` are changed to voters, the `voters_del` are changed to non-voters.
    //
    // The `voters_add` and `voters_del` sets must not intersect.
    //
    // Assumes we've finished the startup procedure (`setup_group0()` finished earlier).
    // `wait_for_raft` must've also been called earlier and returned `true`.
    future<> modify_voters(const std::unordered_set<raft::server_id>& voters_add, const std::unordered_set<raft::server_id>& voters_del, abort_source& as,
            std::optional<raft_timeout> timeout = std::nullopt);

    // Assumes that this node's Raft server ID is already initialized and returns it.
    // It's a fatal error if the id is missing.
    //
    // The returned ID is not empty.
    const raft::server_id& load_my_id();

    // Remove the node from raft config, retries raft::commit_status_unknown.
    // The function can only be called after wait_for_raft() successfully
    // completes and current state of group0 is not RECOVERY.
    future<> remove_from_raft_config(raft::server_id id);

    // Creates a group0 adder which uses the legacy GROUP0_MODIFY_CONFIG RPC.
    // It is meant to be used as a fallback when a proper handshake procedure
    // cannot be used (e.g. when completing the upgrade or group0 procedures
    // or when joining an old cluster which does not support JOIN_NODE RPC).
    shared_ptr<group0_handshaker> make_legacy_handshaker(raft::is_voter can_vote);

    raft_group0_client& client() {
        return _client;
    }

    // Return an instance of group 0. Valid only on shard 0,
    // after boot/upgrade is complete
    raft::server& group0_server() {
        return _raft_gr.group0();
    }

    // Returns a wrapper for the group0_server() with timeouts support.
    raft_server_with_timeouts group0_server_with_timeouts() {
        return _raft_gr.group0_with_timeouts();
    }

    // Hold shutdown gate to be waited during shutdown
    gate::holder hold_group0_gate() {
        return _shutdown_gate.hold();
    }

    // Returns true after the group 0 server has been started.
    bool joined_group0() const;

    utils::observer<bool> observe_leadership(std::function<void(bool)>);

    // Returns scheduling group group0 is configured to run with
    seastar::scheduling_group get_scheduling_group() {
        return _sg;
    }
private:
    static void init_rpc_verbs(raft_group0& shard0_this);
    static future<> uninit_rpc_verbs(netw::messaging_service& ms);

    future<> do_abort_and_drain();

    // Handle peer_exchange RPC
    future<group0_peer_exchange> peer_exchange(discovery::peer_list peers);

    raft_server_for_group create_server_for_group0(raft::group_id id, raft::server_id my_id, service::storage_service& ss, cql3::query_processor& qp,
        service::migration_manager& mm);

    // Start a Raft server for the cluster-wide group 0 and join it to the group.
    // Called during bootstrap or upgrade.
    //
    // Uses `seeds` as contact points to discover other servers which will be part of group 0.
    //
    // If the server becomes the discovery leader, meaning that is it elected as the server
    // which creates group 0, then it will become a voter. Otherwise it will use the `handshaker`
    // to ask the discovery leader to join the node to the group, and the voting status
    // depends on how exactly this is implemented.
    // `as_voter` determines whether the server joins as a voter. If `false`, it will join
    // as a non-voter with one exception: if it becomes the 'discovery leader', meaning that
    // it is elected as the server which creates group 0, it will become a voter.
    //
    // The Raft server may already exist on disk (e.g. because we initialized it earlier but then crashed),
    // but it doesn't have to. It may also already be part of group 0, but if not, we will attempt
    // to discover and join it.
    //
    // Persists group 0 ID on disk at the end so subsequent restarts of Scylla process can detect that group 0
    // has already been joined and the server initialized.
    //
    // When joining an existing cluster, the provided handshaker should result in the node being added
    // as a non-voter. When the cluster is being upgraded and group 0 is being created, the node
    // should be joined as a voter.
    //
    // Preconditions: Raft local feature enabled
    // and we haven't initialized group 0 yet after last Scylla start (`joined_group0()` is false).
    // Postcondition: `joined_group0()` is true.
    future<> join_group0(std::vector<gms::inet_address> seeds, shared_ptr<group0_handshaker> handshaker, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm, db::system_keyspace& sys_ks,
            const join_node_request_params& params);

    // Start an existing Raft server for the cluster-wide group 0.
    // Assumes the server was already added to the group earlier so we don't attempt to join it again.
    //
    // Preconditions: `group0_id` must be non-null and equal to the ID of group 0 that we joined earlier.
    // The existing group 0 server must not have been started yet after the last restart of Scylla
    // (`joined_group0()` is false).
    // Postcondition: `joined_group0()` is true.
    //
    // XXX: perhaps it would be good to make this function callable multiple times,
    // if we want to handle crashes of the group 0 server without crashing the entire Scylla process
    // (we could then try restarting the server internally).
    future<> start_server_for_group0(raft::group_id group0_id, service::storage_service& ss, cql3::query_processor& qp, service::migration_manager& mm);

    // Modify the given server voter status in Raft group 0 configuration.
    // Retries on raft::commit_status_unknown.
    //
    // Does not affect non-members. This behavior is only guaranteed if no concurrent membership changes are happening.
    future<> modify_raft_voter_status(const std::unordered_set<raft::server_id>& voters_add, const std::unordered_set<raft::server_id>& voters_del,
            abort_source& as, std::optional<raft_timeout> timeout = std::nullopt);

    // Returns true if in maintenance mode
    bool maintenance_mode();
};

} // end of namespace service
