/* * Copyright (C) 2015-present ScyllaDB * */ /* * SPDX-License-Identifier: AGPL-3.0-or-later */ #pragma once #include #include #include #include #include #include #include "locator/abstract_replication_strategy.hh" #include "seastarx.hh" #include "utils/config_file.hh" #include "utils/enum_option.hh" #include "locator/host_id.hh" #include "gms/inet_address.hh" #include "db/hints/host_filter.hh" #include "utils/updateable_value.hh" #include "utils/s3/creds.hh" #include "utils/error_injection.hh" namespace seastar { class file; struct logging_settings; namespace tls { class credentials_builder; } namespace log_cli { class options; } } namespace db { namespace fs = std::filesystem; class extensions; /* * This type is not use, and probably never will be. * So it makes sense to jump through hoops just to ensure * it is in fact handled properly... */ struct seed_provider_type { seed_provider_type() = default; seed_provider_type(sstring n, std::initializer_list opts = { }) : class_name(std::move(n)), parameters(std::move(opts)) { } sstring class_name; std::unordered_map parameters; bool operator==(const seed_provider_type& other) const { return class_name == other.class_name && parameters == other.parameters; } }; inline std::istream& operator>>(std::istream& is, seed_provider_type&); // Describes a single error injection that should be enabled at startup. struct error_injection_at_startup { sstring name; bool one_shot = false; utils::error_injection_parameters parameters; bool operator==(const error_injection_at_startup& other) const { return name == other.name && one_shot == other.one_shot && parameters == other.parameters; } }; std::istream& operator>>(std::istream& is, error_injection_at_startup&); } template<> struct fmt::formatter { constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } auto format(const db::error_injection_at_startup&, fmt::format_context& ctx) const -> decltype(ctx.out()); }; namespace utils { sstring config_value_as_json(const db::seed_provider_type& v); sstring config_value_as_json(const log_level& v); sstring config_value_as_json(const std::unordered_map& v); } namespace db { /// Enumeration of all valid values for the `experimental` config entry. struct experimental_features_t { enum class feature { UNUSED, UDF, ALTERNATOR_STREAMS, BROADCAST_TABLES, KEYSPACE_STORAGE_OPTIONS, }; static std::map map(); // See enum_option. static std::vector> all(); }; /// A restriction that can be in three modes: true (the operation is disabled), /// false (the operation is allowed), or warn (the operation is allowed but /// produces a warning in the log). struct tri_mode_restriction_t { enum class mode { FALSE, TRUE, WARN }; static std::unordered_map map(); // for enum_option<> }; using tri_mode_restriction = enum_option; struct replication_strategy_restriction_t { static std::unordered_map map(); // for enum_option<> }; constexpr unsigned default_murmur3_partitioner_ignore_msb_bits = 12; class config final : public utils::config_file { public: config(); config(std::shared_ptr); ~config(); // For testing only void add_cdc_extension(); void add_per_partition_rate_limit_extension(); void add_tags_extension(); void add_tombstone_gc_extension(); /// True iff the feature is enabled. bool check_experimental(experimental_features_t::feature f) const; void setup_directories(); /** * Scans the environment variables for configuration files directory * definition. It's either $SCYLLA_CONF, $SCYLLA_HOME/conf or "conf" if none * of SCYLLA_CONF and SCYLLA_HOME is defined. * * @return path of the directory where configuration files are located * according the environment variables definitions. */ static fs::path get_conf_dir(); static fs::path get_conf_sub(fs::path); using string_map = std::unordered_map; //program_options::string_map; using string_list = std::vector; using seed_provider_type = db::seed_provider_type; using hinted_handoff_enabled_type = db::hints::host_filter; using error_injection_at_startup = db::error_injection_at_startup; /* * All values and documentation taken from * http://docs.datastax.com/en/cassandra/2.1/cassandra/configuration/configCassandra_yaml_r.html */ named_value background_writer_scheduling_quota; named_value auto_adjust_flush_quota; named_value memtable_flush_static_shares; named_value compaction_static_shares; named_value compaction_enforce_min_threshold; named_value compaction_flush_all_tables_before_major_seconds; named_value cluster_name; named_value listen_address; named_value listen_interface; named_value listen_interface_prefer_ipv6; named_value work_directory; named_value commitlog_directory; named_value schema_commitlog_directory; named_value data_file_directories; named_value hints_directory; named_value view_hints_directory; named_value saved_caches_directory; named_value commit_failure_policy; named_value disk_failure_policy; named_value endpoint_snitch; named_value rpc_address; named_value rpc_interface; named_value rpc_interface_prefer_ipv6; named_value seed_provider; named_value compaction_throughput_mb_per_sec; named_value compaction_large_partition_warning_threshold_mb; named_value compaction_large_row_warning_threshold_mb; named_value compaction_large_cell_warning_threshold_mb; named_value compaction_rows_count_warning_threshold; named_value compaction_collection_elements_count_warning_threshold; named_value memtable_total_space_in_mb; named_value concurrent_reads; named_value concurrent_writes; named_value concurrent_counter_writes; named_value incremental_backups; named_value snapshot_before_compaction; named_value phi_convict_threshold; named_value failure_detector_timeout_in_ms; named_value direct_failure_detector_ping_timeout_in_ms; named_value commitlog_sync; named_value commitlog_segment_size_in_mb; named_value schema_commitlog_segment_size_in_mb; named_value commitlog_sync_period_in_ms; named_value commitlog_sync_batch_window_in_ms; named_value commitlog_total_space_in_mb; named_value commitlog_reuse_segments; // unused. retained for upgrade compat named_value commitlog_flush_threshold_in_mb; named_value commitlog_use_o_dsync; named_value commitlog_use_hard_size_limit; named_value compaction_preheat_key_cache; named_value concurrent_compactors; named_value in_memory_compaction_limit_in_mb; named_value preheat_kernel_page_cache; named_value sstable_preemptive_open_interval_in_mb; named_value defragment_memory_on_idle; named_value memtable_allocation_type; named_value memtable_cleanup_threshold; named_value file_cache_size_in_mb; named_value memtable_flush_queue_size; named_value memtable_flush_writers; named_value memtable_heap_space_in_mb; named_value memtable_offheap_space_in_mb; named_value column_index_size_in_kb; named_value column_index_auto_scale_threshold_in_kb; named_value index_summary_capacity_in_mb; named_value index_summary_resize_interval_in_minutes; named_value reduce_cache_capacity_to; named_value reduce_cache_sizes_at; named_value stream_throughput_outbound_megabits_per_sec; named_value inter_dc_stream_throughput_outbound_megabits_per_sec; named_value stream_io_throughput_mb_per_sec; named_value stream_plan_ranges_fraction; named_value trickle_fsync; named_value trickle_fsync_interval_in_kb; named_value auto_bootstrap; named_value batch_size_warn_threshold_in_kb; named_value batch_size_fail_threshold_in_kb; named_value broadcast_address; named_value listen_on_broadcast_address; named_value initial_token; named_value num_tokens; named_value partitioner; named_value storage_port; named_value auto_snapshot; named_value key_cache_keys_to_save; named_value key_cache_save_period; named_value key_cache_size_in_mb; named_value row_cache_keys_to_save; named_value row_cache_size_in_mb; named_value row_cache_save_period; named_value memory_allocator; named_value counter_cache_size_in_mb; named_value counter_cache_save_period; named_value counter_cache_keys_to_save; named_value tombstone_warn_threshold; named_value tombstone_failure_threshold; named_value query_tombstone_page_limit; named_value query_page_size_in_bytes; named_value range_request_timeout_in_ms; named_value read_request_timeout_in_ms; named_value counter_write_request_timeout_in_ms; named_value cas_contention_timeout_in_ms; named_value truncate_request_timeout_in_ms; named_value write_request_timeout_in_ms; named_value request_timeout_in_ms; named_value cross_node_timeout; named_value internode_send_buff_size_in_bytes; named_value internode_recv_buff_size_in_bytes; named_value internode_compression; named_value inter_dc_tcp_nodelay; named_value streaming_socket_timeout_in_ms; named_value start_native_transport; named_value native_transport_port; named_value maintenance_socket; named_value maintenance_socket_group; named_value maintenance_mode; named_value native_transport_port_ssl; named_value native_shard_aware_transport_port; named_value native_shard_aware_transport_port_ssl; named_value native_transport_max_threads; named_value native_transport_max_frame_size_in_mb; named_value broadcast_rpc_address; named_value rpc_port; named_value start_rpc; named_value rpc_keepalive; named_value rpc_max_threads; named_value rpc_min_threads; named_value rpc_recv_buff_size_in_bytes; named_value rpc_send_buff_size_in_bytes; named_value rpc_server_type; named_value cache_hit_rate_read_balancing; named_value dynamic_snitch_badness_threshold; named_value dynamic_snitch_reset_interval_in_ms; named_value dynamic_snitch_update_interval_in_ms; named_value hinted_handoff_enabled; named_value max_hinted_handoff_concurrency; named_value hinted_handoff_throttle_in_kb; named_value max_hint_window_in_ms; named_value max_hints_delivery_threads; named_value batchlog_replay_throttle_in_kb; named_value request_scheduler; named_value request_scheduler_id; named_value request_scheduler_options; named_value thrift_framed_transport_size_in_mb; named_value thrift_max_message_length_in_mb; named_value authenticator; named_value internode_authenticator; named_value authorizer; named_value role_manager; named_value permissions_validity_in_ms; named_value permissions_update_interval_in_ms; named_value permissions_cache_max_entries; named_value server_encryption_options; named_value client_encryption_options; named_value alternator_encryption_options; named_value ssl_storage_port; named_value enable_in_memory_data_store; named_value enable_cache; named_value enable_commitlog; named_value volatile_system_keyspace_for_testing; named_value api_port; named_value api_address; named_value api_ui_dir; named_value api_doc_dir; named_value load_balance; named_value consistent_rangemovement; named_value join_ring; named_value load_ring_state; named_value replace_node_first_boot; named_value replace_address; named_value replace_address_first_boot; named_value ignore_dead_nodes_for_replace; named_value override_decommission; named_value enable_repair_based_node_ops; named_value allowed_repair_based_node_ops; named_value enable_compacting_data_for_streaming_and_repair; named_value repair_partition_count_estimation_ratio; named_value ring_delay_ms; named_value shadow_round_ms; named_value fd_max_interval_ms; named_value fd_initial_value_ms; named_value shutdown_announce_in_ms; named_value developer_mode; named_value skip_wait_for_gossip_to_settle; named_value force_gossip_generation; named_value>> experimental_features; named_value lsa_reclamation_step; named_value prometheus_port; named_value prometheus_address; named_value prometheus_prefix; named_value prometheus_allow_protobuf; named_value abort_on_lsa_bad_alloc; named_value murmur3_partitioner_ignore_msb_bits; named_value unspooled_dirty_soft_limit; named_value sstable_summary_ratio; named_value components_memory_reclaim_threshold; named_value large_memory_allocation_warning_threshold; named_value enable_deprecated_partitioners; named_value enable_keyspace_column_family_metrics; named_value enable_node_aggregated_table_metrics; named_value enable_sstable_data_integrity_check; named_value enable_sstable_key_validation; named_value cpu_scheduler; named_value view_building; named_value enable_sstables_mc_format; named_value enable_sstables_md_format; named_value sstable_format; named_value uuid_sstable_identifiers_enabled; named_value table_digest_insensitive_to_expiry; named_value enable_dangerous_direct_import_of_cassandra_counters; named_value enable_shard_aware_drivers; named_value enable_ipv6_dns_lookup; named_value abort_on_internal_error; named_value max_partition_key_restrictions_per_query; named_value max_clustering_key_restrictions_per_query; named_value max_memory_for_unlimited_query_soft_limit; named_value max_memory_for_unlimited_query_hard_limit; named_value reader_concurrency_semaphore_serialize_limit_multiplier; named_value reader_concurrency_semaphore_kill_limit_multiplier; named_value twcs_max_window_count; named_value initial_sstable_loading_concurrency; named_value enable_3_1_0_compatibility_mode; named_value enable_user_defined_functions; named_value user_defined_function_time_limit_ms; named_value user_defined_function_allocation_limit_bytes; named_value user_defined_function_contiguous_allocation_limit_bytes; named_value schema_registry_grace_period; named_value max_concurrent_requests_per_shard; named_value cdc_dont_rewrite_streams; named_value strict_allow_filtering; named_value strict_is_not_null_in_views; named_value reversed_reads_auto_bypass_cache; named_value enable_optimized_reversed_reads; named_value enable_cql_config_updates; named_value enable_parallelized_aggregation; named_value alternator_port; named_value alternator_https_port; named_value alternator_address; named_value alternator_enforce_authorization; named_value alternator_write_isolation; named_value alternator_streams_time_window_s; named_value alternator_timeout_in_ms; named_value alternator_ttl_period_in_seconds; named_value alternator_describe_endpoints; named_value abort_on_ebadf; named_value redis_port; named_value redis_ssl_port; named_value redis_read_consistency_level; named_value redis_write_consistency_level; named_value redis_database_count; named_value redis_keyspace_replication_strategy_options; named_value sanitizer_report_backtrace; named_value flush_schema_tables_after_modification; // Options to restrict (forbid, warn or somehow limit) certain operations // or options which non-expert users are more likely to regret than to // enjoy: named_value restrict_replication_simplestrategy; named_value restrict_dtcs; named_value restrict_twcs_without_default_ttl; named_value restrict_future_timestamp; named_value ignore_truncation_record; named_value force_schema_commit_log; named_value task_ttl_seconds; named_value nodeops_watchdog_timeout_seconds; named_value nodeops_heartbeat_interval_seconds; named_value cache_index_pages; named_value index_cache_fraction; named_value consistent_cluster_management; named_value force_gossip_topology_changes; named_value wasm_cache_memory_fraction; named_value wasm_cache_timeout_in_ms; named_value wasm_cache_instance_size_limit; named_value wasm_udf_yield_fuel; named_value wasm_udf_total_fuel; named_value wasm_udf_memory_limit; named_value relabel_config_file; named_value object_storage_config_file; // wasm_udf_reserved_memory is static because the options in db::config // are parsed using seastar::app_template, while this option is used for // configuring the Seastar memory subsystem. static constexpr size_t wasm_udf_reserved_memory = 50 * 1024 * 1024; named_value live_updatable_config_params_changeable_via_cql; bool are_live_updatable_config_params_changeable_via_cql() const override { return live_updatable_config_params_changeable_via_cql(); } // authenticator options named_value auth_superuser_name; named_value auth_superuser_salted_password; named_value>> auth_certificate_role_queries; named_value minimum_replication_factor_fail_threshold; named_value minimum_replication_factor_warn_threshold; named_value maximum_replication_factor_warn_threshold; named_value maximum_replication_factor_fail_threshold; named_value tablets_initial_scale_factor; named_value target_tablet_size_in_bytes; named_value>> replication_strategy_warn_list; named_value>> replication_strategy_fail_list; named_value service_levels_interval; seastar::logging_settings logging_settings(const log_cli::options&) const; const db::extensions& extensions() const; utils::updateable_value_source> object_storage_config; named_value> error_injections_at_startup; named_value topology_barrier_stall_detector_threshold_seconds; named_value enable_tablets; static const sstring default_tls_priority; private: template struct log_legacy_value : public named_value { using MyBase = named_value; using MyBase::MyBase; T value_or(T&& t) const { return this->is_set() ? (*this)() : t; } // do not add to boost::options. We only care about yaml config void add_command_line_option(boost::program_options::options_description_easy_init&) override {} }; log_legacy_value default_log_level; log_legacy_value> logger_log_level; log_legacy_value log_to_stdout, log_to_syslog; void maybe_in_workdir(named_value&, const char*); void maybe_in_workdir(named_value&, const char*); std::shared_ptr _extensions; }; } namespace utils { template V get_or_default(const std::unordered_map& ss, const K2& key, const V2& def = V()) { const auto iter = ss.find(key); if (iter != ss.end()) { return iter->second; } return def; } inline bool is_true(sstring val) { std::transform(val.begin(), val.end(), val.begin(), ::tolower); return val == "true" || val == "1"; } future<> configure_tls_creds_builder(seastar::tls::credentials_builder& creds, db::config::string_map options); future resolve(const config_file::named_value&, gms::inet_address::opt_family family = {}, gms::inet_address::opt_family preferred = {}); /*! * \brief read the the relabel config from a file * * Will throw an exception if there is a conflict with the metrics names */ future<> update_relabel_config_from_file(const std::string& name); std::vector split_comma_separated_list(sstring comma_separated_list); }