/* * * Modified by ScyllaDB * Copyright (C) 2015-present ScyllaDB */ /* * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) */ #pragma once #include #include #include #include "utils/bounded_stats_deque.hh" #include #include #include #include #include #include "gms/inet_address.hh" namespace gms { class i_failure_detection_event_listener; class endpoint_state; class arrival_window { public: using clk = seastar::lowres_system_clock; private: clk::time_point _tlast{clk::time_point::min()}; utils::bounded_stats_deque _arrival_intervals; std::chrono::milliseconds _initial; std::chrono::milliseconds _max_interval; std::chrono::milliseconds _min_interval; // this is useless except to provide backwards compatibility in phi_convict_threshold, // because everyone seems pretty accustomed to the default of 8, and users who have // already tuned their phi_convict_threshold for their own environments won't need to // change. static constexpr double PHI_FACTOR{M_LOG10El}; public: arrival_window(int size, std::chrono::milliseconds initial, std::chrono::milliseconds max_interval, std::chrono::milliseconds min_interval) : _arrival_intervals(size) , _initial(initial) , _max_interval(max_interval) , _min_interval(min_interval) { } void add(clk::time_point value, const gms::inet_address& ep); double mean() const; // see CASSANDRA-2597 for an explanation of the math at work here. double phi(clk::time_point tnow); size_t size() { return _arrival_intervals.size(); } clk::time_point last_update() const { return _tlast; } friend std::ostream& operator<<(std::ostream& os, const arrival_window& w); }; /** * This FailureDetector is an implementation of the paper titled * "The Phi Accrual Failure Detector" by Hayashibara. * Check the paper and the IFailureDetector interface for details. */ class failure_detector { private: static constexpr int SAMPLE_SIZE = 1000; // this is useless except to provide backwards compatibility in phi_convict_threshold, // because everyone seems pretty accustomed to the default of 8, and users who have // already tuned their phi_convict_threshold for their own environments won't need to // change. static constexpr double PHI_FACTOR{M_LOG10El}; std::map _arrival_samples; std::list _fd_evnt_listeners; double _phi = 8; std::chrono::milliseconds _initial; std::chrono::milliseconds _max_interval; static constexpr std::chrono::milliseconds DEFAULT_MAX_PAUSE{5000}; std::chrono::milliseconds get_max_local_pause() { // FIXME: cassandra.max_local_pause_in_ms #if 0 if (System.getProperty("cassandra.max_local_pause_in_ms") != null) { long pause = Long.parseLong(System.getProperty("cassandra.max_local_pause_in_ms")); logger.warn("Overriding max local pause time to {}ms", pause); return pause * 1000000L; } else { return DEFAULT_MAX_PAUSE; } #endif return DEFAULT_MAX_PAUSE; } std::optional _last_interpret; arrival_window::clk::time_point _last_paused; public: failure_detector(double phi, std::chrono::milliseconds initial, std::chrono::milliseconds max_interval) : _phi(phi), _initial(initial), _max_interval(max_interval) { } const std::map& arrival_samples() const { return _arrival_samples; } public: /** * Dump the inter arrival times for examination if necessary. */ #if 0 void dumpInterArrivalTimes() { File file = FileUtils.createTempFile("failuredetector-", ".dat"); OutputStream os = null; try { os = new BufferedOutputStream(new FileOutputStream(file, true)); os.write(toString().getBytes()); } catch (IOException e) { throw new FSWriteError(e, file); } finally { FileUtils.closeQuietly(os); } } #endif void set_phi_convict_threshold(double phi); double get_phi_convict_threshold(); void report(inet_address ep); void interpret(inet_address ep); void force_conviction(inet_address ep); void remove(inet_address ep); void register_failure_detection_event_listener(i_failure_detection_event_listener* listener); void unregister_failure_detection_event_listener(i_failure_detection_event_listener* listener); friend std::ostream& operator<<(std::ostream& os, const failure_detector& x); }; } // namespace gms