mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 'Extract config file mechanism + allow additional' from Calle

Browse Source 
"Extracts the yaml/boost-po aspects of the "self-describing" db::config
into an abstract type.

db::config is then reimplemented in said type, removing some of the
slightly cumbersome entanglement with seastar opts (log).

Adds a main hook for additional configuration files (options + file)"

* 'calle/config' of github.com:scylladb/seastar-dev:
  main/init: Add registerable configuration objects
  db::config: Re-implement on utils/config_file.
  utils::config_file: Abstract out config file to external type
This commit is contained in:
Duarte Nunes
2017-10-18 09:50:53 +01:00
parent ae92055b52 12a54805ea
commit c468e59817
8 changed files with 870 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
configure.py Executable file → Normal file
View File

@@ -460,6 +460,7 @@ scylla_core = (['database.cc',
'utils/dynamic_bitset.cc',
'utils/managed_bytes.cc',
'utils/exceptions.cc',
'utils/config_file.cc',
'gms/version_generator.cc',
'gms/versioned_value.cc',
'gms/gossiper.cc',

436
db/config.cc
View File

@@ -20,104 +20,38 @@
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#include <yaml-cpp/yaml.h>
#include <boost/program_options.hpp>
#include <unordered_map>
#include <regex>
#include "config.hh"
#include "core/file.hh"
#include "core/reactor.hh"
#include "core/shared_ptr.hh"
#include "core/fstream.hh"
#include "core/do_with.hh"
#include "core/print.hh"
#include "log.hh"
#include <boost/any.hpp>
#include <boost/program_options.hpp>
#include <yaml-cpp/yaml.h>
static logging::logger clogger("config");
#include <seastar/core/print.hh>
#include <seastar/util/log.hh>
db::config::config()
:
// Initialize members to defaults.
#define _mk_init(name, type, deflt, status, desc, ...) \
name(deflt),
_make_config_values(_mk_init)
_dummy(0)
{}
namespace bpo = boost::program_options;
#include "config.hh"
#include "log.hh"
#include "utils/config_file_impl.hh"
namespace YAML {
/*
* Add converters as needed here...
*/
// yaml-cpp conversion would do well to have some enable_if-stuff to make it possible
// to do more broad spectrum converters.
template<>
struct convert<sstring> {
static Node encode(sstring rhs) {
auto p = rhs.c_str();
return convert<const char *>::encode(p);
}
static bool decode(const Node& node, sstring& rhs) {
struct convert<seastar::log_level> {
static bool decode(const Node& node, seastar::log_level& rhs) {
std::string tmp;
if (!convert<std::string>::decode(node, tmp)) {
return false;
}
rhs = tmp;
return true;
}
};
template <>
struct convert<db::config::string_list> {
static Node encode(const db::config::string_list& rhs) {
Node node(NodeType::Sequence);
for (auto& s : rhs) {
node.push_back(convert<sstring>::encode(s));
}
return node;
}
static bool decode(const Node& node, db::config::string_list& rhs) {
if (!node.IsSequence()) {
return false;
}
rhs.clear();
for (auto& n : node) {
sstring tmp;
if (!convert<sstring>::decode(n,tmp)) {
return false;
}
rhs.push_back(tmp);
}
return true;
}
};
template<>
struct convert<seastar::program_options::string_map>{
static Node encode(const seastar::program_options::string_map& rhs) {
Node node(NodeType::Map);
for (auto& p : rhs) {
node.force_insert(p.first, p.second);
}
return node;
}
static bool decode(const Node& node, seastar::program_options::string_map& rhs) {
if (!node.IsMap()) {
return false;
}
rhs.clear();
for (auto& n : node) {
rhs[n.first.as<sstring>()] = n.second.as<sstring>();
}
rhs = boost::lexical_cast<seastar::log_level>(tmp);
return true;
}
};
template<>
struct convert<db::config::seed_provider_type> {
static Node encode(const db::config::seed_provider_type&) {
throw std::runtime_error("should not reach");
}
static bool decode(const Node& node, db::config::seed_provider_type& rhs) {
if (!node.IsSequence()) {
return false;
@@ -143,292 +77,39 @@ struct convert<db::config::seed_provider_type> {
}
};
}
/*
* Helper type to do compile time exclusion of Unused/invalid options from
* command line.
*
* Only opts marked "used" should get a boost::opt
*
*/
template<typename T, db::config::value_status S>
struct do_value_opt;
#define _mk_name(name, ...) name,
#define str(x) #x
#define _mk_init(name, type, deflt, status, desc, ...) , name(str(name), type(deflt), desc)
template<typename T>
struct do_value_opt<T, db::config::value_status::Used> {
template<typename Func>
void operator()(Func&& func, const char* name, const T& dflt, T * dst, db::config::config_source & src, const char* desc) const {
func(name, db::config::value_status::Used, dflt, dst, src, desc);
}
};
db::config::config()
: utils::config_file({ _make_config_values(_mk_name)
default_log_level, logger_log_level, log_to_stdout, log_to_syslog })
_make_config_values(_mk_init)
, default_log_level("default_log_level")
, logger_log_level("logger_log_level")
, log_to_stdout("log_to_stdout")
, log_to_syslog("log_to_syslog")
{}
template <typename T>
struct do_value_opt<T, db::config::value_status::UsedFromSeastar> {
template <typename Func>
void operator()(Func&& func, const char* name, const T& dflt, T* dst, db::config::config_source& src, const char* desc) const {
func(name, db::config::value_status::UsedFromSeastar, dflt, dst, src, desc);
}
};
namespace utils {
template<>
struct do_value_opt<db::config::seed_provider_type, db::config::value_status::Used> {
using seed_provider_type = db::config::seed_provider_type;
template<typename Func>
void operator()(Func&& func, const char* name, const seed_provider_type& dflt, seed_provider_type * dst, db::config::config_source & src, const char* desc) const {
func((sstring(name) + "_class_name").c_str(), db::config::value_status::Used, dflt.class_name, &dst->class_name, src, desc);
func((sstring(name) + "_parameters").c_str(), db::config::value_status::Used, dflt.parameters, &dst->parameters, src, desc);
}
};
template<typename T>
struct do_value_opt<T, db::config::value_status::Unused> {
template<typename... Args> void operator()(Args&&...) const {}
};
template<typename T>
struct do_value_opt<T, db::config::value_status::Invalid> {
template<typename... Args> void operator()(Args&&...) const {}
};
/*
* Our own bpo::typed_valye.
* Only difference is that we _don't_ apply defaults (they are already applied) and therefore, the notifier doesn't
* execute for default values (only for explicitly provided ones).
* Needed to make aliases and usage-checking work properly.
*/
template<class T, class charT = char>
class typed_value_ex : public bpo::typed_value<T, charT> {
public:
typedef bpo::typed_value<T, charT> _Super;
typed_value_ex(T* store_to)
: _Super(store_to)
{}
bool apply_default(boost::any&) const override {
return false;
}
};
template<class T>
inline typed_value_ex<T>* value_ex(T* v) {
typed_value_ex<T>* r = new typed_value_ex<T>(v);
return r;
void config_file::named_value<db::config::seed_provider_type,
db::config::value_status::Used>::add_command_line_option(
boost::program_options::options_description_easy_init& init,
const stdx::string_view& name, const stdx::string_view& desc) {
init((hyphenate(name) + "-class-name").data(),
value_ex(&_value.class_name)->notifier(
[this](auto&&) {_source = config_source::CommandLine;}),
desc.data());
init((hyphenate(name) + "-parameters").data(),
value_ex(&_value.parameters)->notifier(
[this](auto&&) {_source = config_source::CommandLine;}),
desc.data());
}
template<class T>
inline typed_value_ex<std::vector<T>>* value_ex(std::vector<T>* v) {
auto r = new typed_value_ex<std::vector<T>>(v);
r->multitoken();
return r;
}
static sstring replace_underscores_with_hyphens(const char* v) {
sstring result(v);
std::replace(result.begin(), result.end(), '_', '-');
return result;
}
bpo::options_description_easy_init db::config::add_options(bpo::options_description& opts) {
bpo::options_description_easy_init init = opts.add_options();
auto opt_add =
[&init, &opts](const char* name, const value_status& status, const auto& dflt, auto* dst, auto& src, const char* desc) {
const auto hyphenated_name = replace_underscores_with_hyphens(name);
switch (status) {
case value_status::Used: {
init(hyphenated_name.c_str(),
value_ex(dst)->default_value(dflt)->notifier([&src](auto) {
src = config_source::CommandLine;
}),
desc);
break;
}
case value_status::UsedFromSeastar: {
auto* seastar_opt = opts.find_nothrow(hyphenated_name, false);
if (!seastar_opt) {
throw std::runtime_error(seastar::sprint("Expected Seastar to define a command-line option '%s'",
hyphenated_name));
}
//
// Just a type-check.
//
using value_type = typename std::remove_pointer<decltype(dst)>::type;
auto value = boost::dynamic_pointer_cast<const bpo::typed_value<value_type>>(seastar_opt->semantic());
if (value == nullptr) {
throw std::runtime_error(
sprint(
"The type of the Seastar-defined option '%s' does not match its declared type "
"in Scylla's configuration",
hyphenated_name));
}
break;
}
case value_status::Invalid:
case value_status::Unused:
// Nothing.
break;
}
};
// Add all used opts as command line opts
#define _add_boost_opt(name, type, deflt, status, desc, ...) \
do_value_opt<type, value_status::status>()(opt_add, #name, type( deflt ), &name._value, name._source, desc);
_make_config_values(_add_boost_opt)
auto alias_add =
[&init](const char* name, auto& dst, const char* desc) {
init(name,
value_ex(&dst._value)->notifier([&dst](auto&) {
dst._source = config_source::CommandLine;
}),
desc);
};
// Handle "old" syntax with "aliases"
alias_add("datadir", data_file_directories, "alias for 'data-file-directories'");
alias_add("thrift-port", rpc_port, "alias for 'rpc-port'");
alias_add("cql-port", native_transport_port, "alias for 'native-transport-port'");
return opts.add_options();
}
void db::config::apply_seastar_options(boost::program_options::variables_map& vars) {
const auto do_apply = [&vars](const char* name, const db::config::value_status& status, auto* dst, auto& src) {
// This could be done at compile-time, but this is more clear and it's not a bottleneck.
if (status == db::config::value_status::UsedFromSeastar) {
const auto hyphenated_name = replace_underscores_with_hyphens(name);
using value_type = typename std::remove_pointer<decltype(dst)>::type;
const auto& variable_value = vars[hyphenated_name];
*dst = variable_value.as<value_type>();
if (!variable_value.defaulted()) {
src = db::config::config_source::CommandLine;
}
}
};
#define _add_seastar_opt(name, type, deflt, status, desc, ...) \
do_apply(#name, db::config::value_status::status, &(name._value), name._source);
_make_config_values(_add_seastar_opt)
}
// Virtual dispatch to convert yaml->data type.
struct handle_yaml {
virtual ~handle_yaml() = default;
virtual void operator()(const YAML::Node&) = 0;
virtual db::config::value_status status() const = 0;
virtual db::config::config_source source() const = 0;
};
template<typename T, db::config::value_status S>
struct handle_yaml_impl : public handle_yaml {
typedef db::config::value<T, S> value_type;
handle_yaml_impl(value_type& v, db::config::config_source& src) : _dst(v), _src(src) {}
void operator()(const YAML::Node& node) override {
_dst(node.as<T>());
_src = db::config::config_source::SettingsFile;
}
db::config::value_status status() const override {
return _dst.status();
}
db::config::config_source source() const override {
return _src;
}
value_type& _dst;
db::config::config_source&
_src;
};
void db::config::read_from_yaml(const sstring& yaml) {
read_from_yaml(yaml.c_str());
}
void db::config::read_from_yaml(const char* yaml) {
std::unordered_map<sstring, std::unique_ptr<handle_yaml>> values;
#define _add_yaml_opt(name, type, deflt, status, desc, ...) \
values.emplace(#name, std::make_unique<handle_yaml_impl<type, value_status::status>>(name, name._source));
_make_config_values(_add_yaml_opt)
/*
* Note: this is not very "half-fault" tolerant. I.e. there could be
* yaml syntax errors that origin handles and still sets the options
* where as we don't...
* There are no exhaustive attempts at converting, we rely on syntax of
* file mapping to the data type...
*/
auto doc = YAML::Load(yaml);
for (auto node : doc) {
auto label = node.first.as<sstring>();
auto i = values.find(label);
if (i == values.end()) {
clogger.warn("Unknown option {} ignored.", label);
continue;
}
if (i->second->source() > config_source::SettingsFile) {
clogger.debug("Option {} already set by commandline. ignored.", label);
continue;
}
switch (i->second->status()) {
case value_status::Invalid:
clogger.warn("Option {} is not applicable. Ignoring.", label);
continue;
case value_status::Unused:
clogger.warn("Option {} is not (yet) used.", label);
break;
default:
break;
}
if (node.second.IsNull()) {
clogger.debug("Option {}, empty value. Skipping.", label);
continue;
}
// Still, a syntax error is an error warning, not a fail
try {
(*i->second)(node.second);
} catch (...) {
clogger.error("Option {}, exception while converting value.", label);
}
}
for (auto& p : values) {
if (p.second->status() > value_status::Used) {
continue;
}
if (p.second->source() > config_source::None) {
continue;
}
clogger.debug("Option {} not set", p.first);
}
}
future<> db::config::read_from_file(file f) {
return f.size().then([this, f](size_t s) {
return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
read_from_yaml(sstring(buf.begin(), buf.end()));
});
});
});
}
future<> db::config::read_from_file(const sstring& filename) {
return open_file_dma(filename, open_flags::ro).then([this](file f) {
return read_from_file(std::move(f));
});
}
boost::filesystem::path db::config::get_conf_dir() {
@@ -454,3 +135,42 @@ void db::config::check_experimental(const sstring& what) const {
throw std::runtime_error(sprint("%s is currently disabled. Start Scylla with --experimental=on to enable.", what));
}
}
namespace bpo = boost::program_options;
logging::settings db::config::logging_settings(const bpo::variables_map& map) const {
struct convert {
std::unordered_map<sstring, seastar::log_level> operator()(const seastar::program_options::string_map& map) const {
std::unordered_map<sstring, seastar::log_level> res;
for (auto& p : map) {
res.emplace(p.first, (*this)(p.second));
};
return res;
}
seastar::log_level operator()(const sstring& s) const {
return boost::lexical_cast<seastar::log_level>(s);
}
bool operator()(bool b) const {
return b;
}
};
auto value = [&map](auto v, auto dummy) {
auto name = utils::hyphenate(v.name());
const bpo::variable_value& opt = map[name];
if (opt.defaulted() && v.is_set()) {
return v();
}
using expected = std::decay_t<decltype(dummy)>;
return convert()(opt.as<expected>());
};
return logging::settings{ value(logger_log_level, seastar::program_options::string_map())
, value(default_log_level, sstring())
, value(log_to_stdout, bool())
, value(log_to_syslog, bool())
};
}

118
db/config.hh
View File

@@ -25,12 +25,16 @@
#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <unordered_map>
#include "core/sstring.hh"
#include "core/future.hh"
#include "util/program-options.hh"
#include "seastarx.hh"
namespace seastar { class file; }
#include <seastar/core/sstring.hh>
#include <seastar/core/future.hh>
#include <seastar/util/program-options.hh>
#include <seastar/util/log.hh>
#include "seastarx.hh"
#include "utils/config_file.hh"
namespace seastar { class file; struct logging_settings; }
namespace db {
@@ -47,79 +51,16 @@ struct seed_provider_type {
: class_name(std::move(n)), parameters(std::move(opts)) {
}
sstring class_name;
program_options::string_map parameters;
std::unordered_map<sstring, sstring> parameters;
};
class config {
class config : public utils::config_file {
public:
enum class value_status {
UsedFromSeastar,
Used,
Unused,
Invalid,
};
enum class config_source : uint8_t {
None,
SettingsFile,
CommandLine
};
template<typename T, value_status S>
struct value {
typedef T type;
typedef value<T, S> MyType;
value(const T& t = T()) : _value(t)
{}
value_status status() const {
return S;
}
config_source source() const {
return _source;
}
bool is_set() const {
return _source > config_source::None;
}
MyType & operator()(const T& t) {
_value = t;
return *this;
}
MyType & operator()(T&& t, config_source src = config_source::None) {
_value = std::move(t);
if (src > config_source::None) {
_source = src;
}
return *this;
}
const T& operator()() const {
return _value;
}
T& operator()() {
return _value;
}
private:
friend class config;
T _value = T();
config_source _source = config_source::None;
};
config();
// Throws exception if experimental feature is disabled.
void check_experimental(const sstring& what) const;
boost::program_options::options_description_easy_init
add_options(boost::program_options::options_description&);
/* Apply the value of Seastar options to the configuration. These are marked `UsedFromSeastar`. */
void apply_seastar_options(boost::program_options::variables_map& vars);
void read_from_yaml(const sstring&);
void read_from_yaml(const char *);
future<> read_from_file(const sstring&);
future<> read_from_file(file);
/**
* Scans the environment variables for configuration files directory
* definition. It's either $SCYLLA_CONF, $SCYLLA_HOME/conf or "conf" if none
@@ -129,8 +70,10 @@ public:
* according the environment variables definitions.
*/
static boost::filesystem::path get_conf_dir();
using string_map = program_options::string_map;
typedef std::vector<sstring> string_list;
using string_map = std::unordered_map<sstring, sstring>;
//program_options::string_map;
using string_list = std::vector<sstring>;
using seed_provider_type = db::seed_provider_type;
/*
@@ -735,10 +678,6 @@ public:
val(ssl_storage_port, uint32_t, 7001, Used, \
"The SSL port for encrypted communication. Unused unless enabled in encryption_options." \
) \
val(default_log_level, sstring, /* none */, UsedFromSeastar, "") \
val(logger_log_level, string_map, /* none */, UsedFromSeastar, "") \
val(log_to_stdout, bool, /* none */, UsedFromSeastar, "") \
val(log_to_syslog, bool, /* none */, UsedFromSeastar, "") \
val(enable_in_memory_data_store, bool, false, Used, "Enable in memory mode (system tables are always persisted)") \
val(enable_cache, bool, true, Used, "Enable cache") \
val(enable_commitlog, bool, true, Used, "Enable commitlog") \
@@ -781,17 +720,30 @@ public:
/* done! */
#define _make_value_member(name, type, deflt, status, desc, ...) \
value<type, value_status::status> name;
named_value<type, value_status::status> name;
_make_config_values(_make_value_member)
private:
struct bound_value;
struct bound_values;
struct cmdline_args;
struct yaml_config;
seastar::logging_settings logging_settings(const boost::program_options::variables_map&) const;
int _dummy;
private:
template<typename T>
struct log_legacy_value : public named_value<T, value_status::Used> {
using MyBase = named_value<T, value_status::Used>;
using MyBase::MyBase;
T value_or(T&& t) const {
return this->is_set() ? (*this)() : t;
}
// do not add to boost::options. We only care about yaml config
void add_command_line_option(boost::program_options::options_description_easy_init&,
const stdx::string_view&, const stdx::string_view&) override {}
};
log_legacy_value<seastar::log_level> default_log_level;
log_legacy_value<std::unordered_map<sstring, seastar::log_level>> logger_log_level;
log_legacy_value<bool> log_to_stdout, log_to_syslog;
};
}

26
init.hh
View File

@@ -47,3 +47,29 @@ void init_ms_fd_gossiper(sstring listen_address
, sstring cluster_name = "Test Cluster"
, double phi = 8
, bool sltba = false);
/**
* Very simplistic config registry. Allows hooking in a config object
* to the "main" sequence.
*/
class configurable {
public:
configurable() {
// We auto register. Not that like cycle is assumed to be forever
// and scope should be managed elsewhere.
register_configurable(*this);
}
virtual ~configurable()
{}
// Hook to add command line options
virtual void append_options(boost::program_options::options_description_easy_init&)
{};
// Called after command line is parsed and db/config populated.
// Hooked config can for example take this oppurtunity to load any file(s).
virtual future<> initialize(const boost::program_options::variables_map&) {
return make_ready_future();
}
private:
static void register_configurable(configurable &);
};

44
main.cc
View File

@@ -67,6 +67,24 @@ using namespace std::chrono_literals;
namespace bpo = boost::program_options;
static std::vector<std::reference_wrapper<configurable>>& configurables() {
static std::vector<std::reference_wrapper<configurable>> configurables;
return configurables;
}
void configurable::register_configurable(configurable & c) {
configurables().emplace_back(std::ref(c));
}
template<typename K, typename V, typename... Args, typename K2, typename V2 = V>
V get_or_default(const std::unordered_map<K, V, Args...>& ss, const K2& key, const V2& def = V()) {
const auto iter = ss.find(key);
if (iter != ss.end()) {
return iter->second;
}
return def;
}
static boost::filesystem::path relative_conf_dir(boost::filesystem::path path) {
static auto conf_dir = db::config::get_conf_dir(); // this is not gonna change in our life time
return conf_dir / path;
@@ -77,8 +95,6 @@ read_config(bpo::variables_map& opts, db::config& cfg) {
using namespace boost::filesystem;
sstring file;
cfg.apply_seastar_options(opts);
if (opts.count("options-file") > 0) {
file = opts["options-file"].as<sstring>();
} else {
@@ -255,8 +271,14 @@ int main(int ac, char** av) {
auto cfg = make_lw_shared<db::config>();
bool help_version = false;
cfg->add_options(app.get_options_description())
// TODO : default, always read?
auto init = app.get_options_description().add_options();
cfg->add_options(init);
for (configurable& c : configurables()) {
c.append_options(init);
}
init // TODO : default, always read?
("options-file", bpo::value<sstring>(), "configuration file (i.e. <SCYLLA_HOME>/conf/scylla.yaml)")
("version", bpo::bool_switch(&help_version), "print version number and exit")
;
@@ -297,18 +319,12 @@ int main(int ac, char** av) {
return seastar::async([cfg, &db, &qp, &proxy, &mm, &ctx, &opts, &dirs, &pctx, &prometheus_server, &return_value, &cf_cache_hitrate_calculator] {
read_config(opts, *cfg).get();
{
std::unordered_map<sstring, logging::log_level> logger_levels;
log_cli::parse_logger_levels(cfg->logger_log_level(), std::inserter(logger_levels, logger_levels.end()));
logging::apply_settings(logging::settings{
std::move(logger_levels),
log_cli::parse_log_level(cfg->default_log_level()),
cfg->log_to_stdout(),
cfg->log_to_syslog()});
for (configurable& c : configurables()) {
c.initialize(opts).get();
}
logging::apply_settings(cfg->logging_settings(opts));
verify_rlimit(cfg->developer_mode());
verify_adequate_memory_per_shard(cfg->developer_mode());
if (cfg->partitioner() != "org.apache.cassandra.dht.Murmur3Partitioner") {

325
utils/config_file.cc Normal file
View File

@@ -0,0 +1,325 @@
/*
* Copyright (C) 2017 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#include <unordered_map>
#include <regex>
#include <yaml-cpp/yaml.h>
#include <boost/program_options.hpp>
#include <boost/any.hpp>
#include <boost/range/adaptor/filtered.hpp>
#include <seastar/core/file.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/shared_ptr.hh>
#include <seastar/core/fstream.hh>
#include <seastar/core/do_with.hh>
#include <seastar/core/print.hh>
#include "config_file.hh"
#include "config_file_impl.hh"
namespace bpo = boost::program_options;
template<>
std::istream& std::operator>>(std::istream& is, std::unordered_map<seastar::sstring, seastar::sstring>& map) {
std::istreambuf_iterator<char> i(is), e;
int level = 0;
bool sq = false, dq = false, qq = false;
sstring key, val;
sstring* ps = &key;
auto add = [&] {
if (!key.empty()) {
map[key] = std::move(val);
}
key = {};
val = {};
ps = &key;
};
while (i != e && level >= 0) {
auto c = *i++;
switch (c) {
case '\\':
qq = !qq;
if (qq) {
continue;
}
break;
case '\'':
if (!qq) {
sq = !sq;
}
break;
case '"':
if (!qq) {
dq = !dq;
}
break;
case '=':
if (level <= 1 && !sq && !dq && !qq) {
ps = &val;
continue;
}
break;
case '{': case '[':
if (!sq && !dq && !qq) {
++level;
continue;
}
break;
case ']': case '}':
if (!sq && !dq && !qq && level > 0) {
--level;
continue;
}
break;
case ',':
if (level == 1 && !sq && !dq && !qq) {
add();
continue;
}
break;
case ' ': case '\t': case '\n':
if (!sq && !dq && !qq) {
continue;
}
break;
default:
break;
}
if (level == 0) {
++level;
}
qq = false;
ps->append(&c, 1);
}
add();
return is;
}
template<>
std::istream& std::operator>>(std::istream& is, std::vector<seastar::sstring>& res) {
std::istreambuf_iterator<char> i(is), e;
int level = 0;
bool sq = false, dq = false, qq = false;
sstring val;
auto add = [&] {
if (!val.empty()) {
res.emplace_back(std::exchange(val, {}));
}
val = {};
};
while (i != e && level >= 0) {
auto c = *i++;
switch (c) {
case '\\':
qq = !qq;
if (qq) {
continue;
}
break;
case '\'':
if (!qq) {
sq = !sq;
}
break;
case '"':
if (!qq) {
dq = !dq;
}
break;
case '{': case '[':
if (!sq && !dq && !qq) {
++level;
continue;
}
break;
case '}': case ']':
if (!sq && !dq && !qq && level > 0) {
--level;
continue;
}
break;
case ',':
if (level == 1 && !sq && !dq && !qq) {
add();
continue;
}
break;
case ' ': case '\t': case '\n':
if (!sq && !dq && !qq) {
continue;
}
break;
default:
break;
}
if (level == 0) {
++level;
}
qq = false;
val.append(&c, 1);
}
add();
return is;
}
template std::istream& std::operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
sstring utils::hyphenate(const stdx::string_view& v) {
sstring result(v.begin(), v.end());
std::replace(result.begin(), result.end(), '_', '-');
return result;
}
utils::config_file::config_file(std::initializer_list<cfg_ref> cfgs)
: _cfgs(cfgs)
{}
void utils::config_file::add(cfg_ref cfg) {
_cfgs.emplace_back(cfg);
}
void utils::config_file::add(std::initializer_list<cfg_ref> cfgs) {
_cfgs.insert(_cfgs.end(), cfgs.begin(), cfgs.end());
}
bpo::options_description utils::config_file::get_options_description() {
bpo::options_description opts("");
return get_options_description(opts);
}
bpo::options_description utils::config_file::get_options_description(boost::program_options::options_description opts) {
auto init = opts.add_options();
add_options(init);
return std::move(opts);
}
bpo::options_description_easy_init&
utils::config_file::add_options(bpo::options_description_easy_init& init) {
for (config_src& src : _cfgs) {
if (src.status() == value_status::Used) {
auto&& name = src.name();
sstring tmp(name.begin(), name.end());
std::replace(tmp.begin(), tmp.end(), '_', '-');
src.add_command_line_option(init, tmp, src.desc());
}
}
return init;
}
void utils::config_file::read_from_yaml(const sstring& yaml) {
read_from_yaml(yaml.c_str());
}
void utils::config_file::read_from_yaml(const char* yaml) {
std::unordered_map<sstring, cfg_ref> values;
/*
* Note: this is not very "half-fault" tolerant. I.e. there could be
* yaml syntax errors that origin handles and still sets the options
* where as we don't...
* There are no exhaustive attempts at converting, we rely on syntax of
* file mapping to the data type...
*/
auto doc = YAML::Load(yaml);
for (auto node : doc) {
auto label = node.first.as<sstring>();
auto i = std::find_if(_cfgs.begin(), _cfgs.end(), [&label](const config_src& cfg) { return cfg.name() == label; });
if (i == _cfgs.end()) {
throw std::invalid_argument("Unknown option " + label);
}
config_src& cfg = *i;
if (cfg.source() > config_source::SettingsFile) {
// already set
continue;
}
switch (cfg.status()) {
case value_status::Invalid:
throw std::invalid_argument("Option " + label + "is not applicable");
case value_status::Unused:
default:
break;
}
if (node.second.IsNull()) {
continue;
}
// Still, a syntax error is an error warning, not a fail
cfg.set_value(node.second);
}
}
utils::config_file::configs utils::config_file::set_values() const {
return boost::copy_range<configs>(_cfgs | boost::adaptors::filtered([] (const config_src& cfg) {
return cfg.status() > value_status::Used || cfg.source() > config_source::None;
}));
}
utils::config_file::configs utils::config_file::unset_values() const {
configs res;
for (config_src& cfg : _cfgs) {
if (cfg.status() > value_status::Used) {
continue;
}
if (cfg.source() > config_source::None) {
continue;
}
res.emplace_back(cfg);
}
return res;
}
future<> utils::config_file::read_from_file(file f) {
return f.size().then([this, f](size_t s) {
return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
read_from_yaml(sstring(buf.begin(), buf.end()));
});
});
});
}
future<> utils::config_file::read_from_file(const sstring& filename) {
return open_file_dma(filename, open_flags::ro).then([this](file f) {
return read_from_file(std::move(f));
});
}

161
utils/config_file.hh Normal file
View File

@@ -0,0 +1,161 @@
/*
* Copyright (C) 2017 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <unordered_map>
#include <iosfwd>
#include <experimental/string_view>
#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <seastar/core/sstring.hh>
#include <seastar/core/future.hh>
#include "seastarx.hh"
#include "stdx.hh"
namespace seastar { class file; }
namespace YAML { class Node; }
namespace utils {
namespace bpo = boost::program_options;
class config_file {
public:
typedef std::unordered_map<sstring, sstring> string_map;
typedef std::vector<sstring> string_list;
enum class value_status {
Used,
Unused,
Invalid,
};
enum class config_source : uint8_t {
None,
SettingsFile,
CommandLine
};
struct config_src {
stdx::string_view _name, _desc;
public:
config_src(stdx::string_view name, stdx::string_view desc)
: _name(name)
, _desc(desc)
{}
virtual ~config_src() {}
const stdx::string_view & name() const {
return _name;
}
const stdx::string_view & desc() const {
return _desc;
}
virtual void add_command_line_option(
bpo::options_description_easy_init&, const stdx::string_view&,
const stdx::string_view&) = 0;
virtual void set_value(const YAML::Node&) = 0;
virtual value_status status() const = 0;
virtual config_source source() const = 0;
};
template<typename T, value_status S = value_status::Used>
struct named_value : public config_src {
private:
friend class config;
stdx::string_view _name, _desc;
T _value = T();
config_source _source = config_source::None;
public:
typedef T type;
typedef named_value<T, S> MyType;
named_value(stdx::string_view name, const T& t = T(), stdx::string_view desc = {})
: config_src(name, desc)
, _value(t)
{}
value_status status() const override {
return S;
}
config_source source() const override {
return _source;
}
bool is_set() const {
return _source > config_source::None;
}
MyType & operator()(const T& t) {
_value = t;
return *this;
}
MyType & operator()(T&& t, config_source src = config_source::None) {
_value = std::move(t);
if (src > config_source::None) {
_source = src;
}
return *this;
}
const T& operator()() const {
return _value;
}
T& operator()() {
return _value;
}
void add_command_line_option(bpo::options_description_easy_init&,
const stdx::string_view&, const stdx::string_view&) override;
void set_value(const YAML::Node&) override;
};
typedef std::reference_wrapper<config_src> cfg_ref;
config_file(std::initializer_list<cfg_ref> = {});
void add(cfg_ref);
void add(std::initializer_list<cfg_ref>);
boost::program_options::options_description get_options_description();
boost::program_options::options_description get_options_description(boost::program_options::options_description);
boost::program_options::options_description_easy_init&
add_options(boost::program_options::options_description_easy_init&);
void read_from_yaml(const sstring&);
void read_from_yaml(const char *);
future<> read_from_file(const sstring&);
future<> read_from_file(file);
using configs = std::vector<cfg_ref>;
configs set_values() const;
configs unset_values() const;
private:
configs
_cfgs;
};
}

214
utils/config_file_impl.hh Normal file
View File

@@ -0,0 +1,214 @@
/*
* Copyright (C) 2015 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <iterator>
#include <regex>
#include <yaml-cpp/yaml.h>
#include <boost/any.hpp>
#include "config_file.hh"
namespace YAML {
/*
* Add converters as needed here...
*
* TODO: Maybe we should just define all node conversionas as "lexical_cast".
* However, vanilla yamp-cpp does some special treatment of scalar types,
* mainly inf handling etc. Hm.
*/
template<>
struct convert<seastar::sstring> {
static bool decode(const Node& node, sstring& rhs) {
std::string tmp;
if (!convert<std::string>::decode(node, tmp)) {
return false;
}
rhs = tmp;
return true;
}
};
template<typename K, typename V, typename... Rest>
struct convert<std::unordered_map<K, V, Rest...>> {
using map_type = std::unordered_map<K, V, Rest...>;
static bool decode(const Node& node, map_type& rhs) {
if (!node.IsMap()) {
return false;
}
rhs.clear();
for (auto& n : node) {
rhs[n.first.as<K>()] = n.second.as<V>();
}
return true;
}
};
}
namespace std {
// these are also defined in sstring.hh. redefine to get different syntax
template<typename K, typename V>
std::ostream& operator<<(std::ostream& os, const std::pair<K, V>& p) {
return os << p.first << '=' << p.second;
}
template<typename K, typename V, typename... Args>
std::ostream& operator<<(std::ostream& os, const std::unordered_map<K, V, Args...>& map) {
using value_type = typename std::unordered_map<K, V, Args...>::value_type;
os << '{';
std::copy(map.begin(), map.end(), std::ostream_iterator<value_type>(os, ","));
return os << '}';
}
template<typename K, typename V, typename... Args>
std::istream& operator>>(std::istream&, std::unordered_map<K, V, Args...>&);
template<>
std::istream& operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
extern template
std::istream& operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
template<typename V, typename... Args>
std::istream& operator>>(std::istream&, std::vector<V, Args...>&);
template<>
std::istream& operator>>(std::istream&, std::vector<seastar::sstring>&);
extern template
std::istream& operator>>(std::istream&, std::vector<seastar::sstring>&);
template<typename K, typename V, typename... Args>
std::istream& operator>>(std::istream& is, std::unordered_map<K, V, Args...>& map) {
std::unordered_map<sstring, sstring> tmp;
is >> tmp;
for (auto& p : tmp) {
map[boost::lexical_cast<K>(p.first)] = boost::lexical_cast<V>(p.second);
}
return is;
}
template<typename V, typename... Args>
std::istream& operator>>(std::istream& is, std::vector<V, Args...>& dst) {
std::vector<seastar::sstring> tmp;
is >> tmp;
for (auto& v : tmp) {
dst.emplace_back(boost::lexical_cast<V>(v));
}
return is;
}
template<typename K, typename V, typename... Args>
void validate(boost::any& out, const std::vector<std::string>& in, std::unordered_map<K, V, Args...>*, int utf8) {
using map_type = std::unordered_map<K, V, Args...>;
if (out.empty()) {
out = boost::any(map_type());
}
static const std::regex key(R"foo((?:^|\:)([^=:]+)=)foo");
auto* p = boost::any_cast<map_type>(&out);
for (const auto& s : in) {
std::sregex_iterator i(s.begin(), s.end(), key), e;
if (i == e) {
throw boost::program_options::invalid_option_value(s);
}
while (i != e) {
auto k = (*i)[1].str();
auto vs = s.begin() + i->position() + i->length();
auto ve = s.end();
if (++i != e) {
ve = s.begin() + i->position();
}
(*p)[boost::lexical_cast<K>(k)] = boost::lexical_cast<V>(sstring(vs, ve));
}
}
}
}
namespace utils {
namespace {
/*
* Our own bpo::typed_valye.
* Only difference is that we _don't_ apply defaults (they are already applied)
* Needed to make aliases work properly.
*/
template<class T, class charT = char>
class typed_value_ex : public bpo::typed_value<T, charT> {
public:
typedef bpo::typed_value<T, charT> _Super;
typed_value_ex(T* store_to)
: _Super(store_to)
{}
bool apply_default(boost::any& value_store) const override {
return false;
}
};
template<class T>
inline typed_value_ex<T>* value_ex(T* v) {
typed_value_ex<T>* r = new typed_value_ex<T>(v);
return r;
}
template<class T>
inline typed_value_ex<std::vector<T>>* value_ex(std::vector<T>* v) {
auto r = new typed_value_ex<std::vector<T>>(v);
r->multitoken();
return r;
}
}
sstring hyphenate(const stdx::string_view&);
}
template<typename T, utils::config_file::value_status S>
void utils::config_file::named_value<T, S>::add_command_line_option(
boost::program_options::options_description_easy_init& init,
const stdx::string_view& name, const stdx::string_view& desc) {
init(hyphenate(name).data(), value_ex(&_value)->notifier([this](auto&&) { _source = config_source::CommandLine; }), desc.data());
}
template<typename T, utils::config_file::value_status S>
void utils::config_file::named_value<T, S>::set_value(const YAML::Node& node) {
(*this)(node.as<T>());
_source = config_source::SettingsFile;
}