utils::config_file: Abstract out config file to external type

Handling all the boost::commandline + YAML stuff.
This patch only provides an external version of these functions,
it does not modify the db::config object. That is for a follow-up
patch.
This commit is contained in:
Calle Wilund
2017-08-22 08:36:09 +00:00
parent baeec0935f
commit 05db87e068
4 changed files with 701 additions and 0 deletions

1
configure.py Executable file → Normal file
View File

@@ -460,6 +460,7 @@ scylla_core = (['database.cc',
'utils/dynamic_bitset.cc',
'utils/managed_bytes.cc',
'utils/exceptions.cc',
'utils/config_file.cc',
'gms/version_generator.cc',
'gms/versioned_value.cc',
'gms/gossiper.cc',

325
utils/config_file.cc Normal file
View File

@@ -0,0 +1,325 @@
/*
* Copyright (C) 2017 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#include <unordered_map>
#include <regex>
#include <yaml-cpp/yaml.h>
#include <boost/program_options.hpp>
#include <boost/any.hpp>
#include <boost/range/adaptor/filtered.hpp>
#include <seastar/core/file.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/shared_ptr.hh>
#include <seastar/core/fstream.hh>
#include <seastar/core/do_with.hh>
#include <seastar/core/print.hh>
#include "config_file.hh"
#include "config_file_impl.hh"
namespace bpo = boost::program_options;
template<>
std::istream& std::operator>>(std::istream& is, std::unordered_map<seastar::sstring, seastar::sstring>& map) {
std::istreambuf_iterator<char> i(is), e;
int level = 0;
bool sq = false, dq = false, qq = false;
sstring key, val;
sstring* ps = &key;
auto add = [&] {
if (!key.empty()) {
map[key] = std::move(val);
}
key = {};
val = {};
ps = &key;
};
while (i != e && level >= 0) {
auto c = *i++;
switch (c) {
case '\\':
qq = !qq;
if (qq) {
continue;
}
break;
case '\'':
if (!qq) {
sq = !sq;
}
break;
case '"':
if (!qq) {
dq = !dq;
}
break;
case '=':
if (level <= 1 && !sq && !dq && !qq) {
ps = &val;
continue;
}
break;
case '{': case '[':
if (!sq && !dq && !qq) {
++level;
continue;
}
break;
case ']': case '}':
if (!sq && !dq && !qq && level > 0) {
--level;
continue;
}
break;
case ',':
if (level == 1 && !sq && !dq && !qq) {
add();
continue;
}
break;
case ' ': case '\t': case '\n':
if (!sq && !dq && !qq) {
continue;
}
break;
default:
break;
}
if (level == 0) {
++level;
}
qq = false;
ps->append(&c, 1);
}
add();
return is;
}
template<>
std::istream& std::operator>>(std::istream& is, std::vector<seastar::sstring>& res) {
std::istreambuf_iterator<char> i(is), e;
int level = 0;
bool sq = false, dq = false, qq = false;
sstring val;
auto add = [&] {
if (!val.empty()) {
res.emplace_back(std::exchange(val, {}));
}
val = {};
};
while (i != e && level >= 0) {
auto c = *i++;
switch (c) {
case '\\':
qq = !qq;
if (qq) {
continue;
}
break;
case '\'':
if (!qq) {
sq = !sq;
}
break;
case '"':
if (!qq) {
dq = !dq;
}
break;
case '{': case '[':
if (!sq && !dq && !qq) {
++level;
continue;
}
break;
case '}': case ']':
if (!sq && !dq && !qq && level > 0) {
--level;
continue;
}
break;
case ',':
if (level == 1 && !sq && !dq && !qq) {
add();
continue;
}
break;
case ' ': case '\t': case '\n':
if (!sq && !dq && !qq) {
continue;
}
break;
default:
break;
}
if (level == 0) {
++level;
}
qq = false;
val.append(&c, 1);
}
add();
return is;
}
template std::istream& std::operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
sstring utils::hyphenate(const stdx::string_view& v) {
sstring result(v.begin(), v.end());
std::replace(result.begin(), result.end(), '_', '-');
return result;
}
utils::config_file::config_file(std::initializer_list<cfg_ref> cfgs)
: _cfgs(cfgs)
{}
void utils::config_file::add(cfg_ref cfg) {
_cfgs.emplace_back(cfg);
}
void utils::config_file::add(std::initializer_list<cfg_ref> cfgs) {
_cfgs.insert(_cfgs.end(), cfgs.begin(), cfgs.end());
}
bpo::options_description utils::config_file::get_options_description() {
bpo::options_description opts("");
return get_options_description(opts);
}
bpo::options_description utils::config_file::get_options_description(boost::program_options::options_description opts) {
auto init = opts.add_options();
add_options(init);
return std::move(opts);
}
bpo::options_description_easy_init&
utils::config_file::add_options(bpo::options_description_easy_init& init) {
for (config_src& src : _cfgs) {
if (src.status() == value_status::Used) {
auto&& name = src.name();
sstring tmp(name.begin(), name.end());
std::replace(tmp.begin(), tmp.end(), '_', '-');
src.add_command_line_option(init, tmp, src.desc());
}
}
return init;
}
void utils::config_file::read_from_yaml(const sstring& yaml) {
read_from_yaml(yaml.c_str());
}
void utils::config_file::read_from_yaml(const char* yaml) {
std::unordered_map<sstring, cfg_ref> values;
/*
* Note: this is not very "half-fault" tolerant. I.e. there could be
* yaml syntax errors that origin handles and still sets the options
* where as we don't...
* There are no exhaustive attempts at converting, we rely on syntax of
* file mapping to the data type...
*/
auto doc = YAML::Load(yaml);
for (auto node : doc) {
auto label = node.first.as<sstring>();
auto i = std::find_if(_cfgs.begin(), _cfgs.end(), [&label](const config_src& cfg) { return cfg.name() == label; });
if (i == _cfgs.end()) {
throw std::invalid_argument("Unknown option " + label);
}
config_src& cfg = *i;
if (cfg.source() > config_source::SettingsFile) {
// already set
continue;
}
switch (cfg.status()) {
case value_status::Invalid:
throw std::invalid_argument("Option " + label + "is not applicable");
case value_status::Unused:
default:
break;
}
if (node.second.IsNull()) {
continue;
}
// Still, a syntax error is an error warning, not a fail
cfg.set_value(node.second);
}
}
utils::config_file::configs utils::config_file::set_values() const {
return boost::copy_range<configs>(_cfgs | boost::adaptors::filtered([] (const config_src& cfg) {
return cfg.status() > value_status::Used || cfg.source() > config_source::None;
}));
}
utils::config_file::configs utils::config_file::unset_values() const {
configs res;
for (config_src& cfg : _cfgs) {
if (cfg.status() > value_status::Used) {
continue;
}
if (cfg.source() > config_source::None) {
continue;
}
res.emplace_back(cfg);
}
return res;
}
future<> utils::config_file::read_from_file(file f) {
return f.size().then([this, f](size_t s) {
return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
read_from_yaml(sstring(buf.begin(), buf.end()));
});
});
});
}
future<> utils::config_file::read_from_file(const sstring& filename) {
return open_file_dma(filename, open_flags::ro).then([this](file f) {
return read_from_file(std::move(f));
});
}

161
utils/config_file.hh Normal file
View File

@@ -0,0 +1,161 @@
/*
* Copyright (C) 2017 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <unordered_map>
#include <iosfwd>
#include <experimental/string_view>
#include <boost/program_options.hpp>
#include <boost/filesystem.hpp>
#include <seastar/core/sstring.hh>
#include <seastar/core/future.hh>
#include "seastarx.hh"
#include "stdx.hh"
namespace seastar { class file; }
namespace YAML { class Node; }
namespace utils {
namespace bpo = boost::program_options;
class config_file {
public:
typedef std::unordered_map<sstring, sstring> string_map;
typedef std::vector<sstring> string_list;
enum class value_status {
Used,
Unused,
Invalid,
};
enum class config_source : uint8_t {
None,
SettingsFile,
CommandLine
};
struct config_src {
stdx::string_view _name, _desc;
public:
config_src(stdx::string_view name, stdx::string_view desc)
: _name(name)
, _desc(desc)
{}
virtual ~config_src() {}
const stdx::string_view & name() const {
return _name;
}
const stdx::string_view & desc() const {
return _desc;
}
virtual void add_command_line_option(
bpo::options_description_easy_init&, const stdx::string_view&,
const stdx::string_view&) = 0;
virtual void set_value(const YAML::Node&) = 0;
virtual value_status status() const = 0;
virtual config_source source() const = 0;
};
template<typename T, value_status S = value_status::Used>
struct named_value : public config_src {
private:
friend class config;
stdx::string_view _name, _desc;
T _value = T();
config_source _source = config_source::None;
public:
typedef T type;
typedef named_value<T, S> MyType;
named_value(stdx::string_view name, const T& t = T(), stdx::string_view desc = {})
: config_src(name, desc)
, _value(t)
{}
value_status status() const override {
return S;
}
config_source source() const override {
return _source;
}
bool is_set() const {
return _source > config_source::None;
}
MyType & operator()(const T& t) {
_value = t;
return *this;
}
MyType & operator()(T&& t, config_source src = config_source::None) {
_value = std::move(t);
if (src > config_source::None) {
_source = src;
}
return *this;
}
const T& operator()() const {
return _value;
}
T& operator()() {
return _value;
}
void add_command_line_option(bpo::options_description_easy_init&,
const stdx::string_view&, const stdx::string_view&) override;
void set_value(const YAML::Node&) override;
};
typedef std::reference_wrapper<config_src> cfg_ref;
config_file(std::initializer_list<cfg_ref> = {});
void add(cfg_ref);
void add(std::initializer_list<cfg_ref>);
boost::program_options::options_description get_options_description();
boost::program_options::options_description get_options_description(boost::program_options::options_description);
boost::program_options::options_description_easy_init&
add_options(boost::program_options::options_description_easy_init&);
void read_from_yaml(const sstring&);
void read_from_yaml(const char *);
future<> read_from_file(const sstring&);
future<> read_from_file(file);
using configs = std::vector<cfg_ref>;
configs set_values() const;
configs unset_values() const;
private:
configs
_cfgs;
};
}

214
utils/config_file_impl.hh Normal file
View File

@@ -0,0 +1,214 @@
/*
* Copyright (C) 2015 ScyllaDB
*
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <iterator>
#include <regex>
#include <yaml-cpp/yaml.h>
#include <boost/any.hpp>
#include "config_file.hh"
namespace YAML {
/*
* Add converters as needed here...
*
* TODO: Maybe we should just define all node conversionas as "lexical_cast".
* However, vanilla yamp-cpp does some special treatment of scalar types,
* mainly inf handling etc. Hm.
*/
template<>
struct convert<seastar::sstring> {
static bool decode(const Node& node, sstring& rhs) {
std::string tmp;
if (!convert<std::string>::decode(node, tmp)) {
return false;
}
rhs = tmp;
return true;
}
};
template<typename K, typename V, typename... Rest>
struct convert<std::unordered_map<K, V, Rest...>> {
using map_type = std::unordered_map<K, V, Rest...>;
static bool decode(const Node& node, map_type& rhs) {
if (!node.IsMap()) {
return false;
}
rhs.clear();
for (auto& n : node) {
rhs[n.first.as<K>()] = n.second.as<V>();
}
return true;
}
};
}
namespace std {
// these are also defined in sstring.hh. redefine to get different syntax
template<typename K, typename V>
std::ostream& operator<<(std::ostream& os, const std::pair<K, V>& p) {
return os << p.first << '=' << p.second;
}
template<typename K, typename V, typename... Args>
std::ostream& operator<<(std::ostream& os, const std::unordered_map<K, V, Args...>& map) {
using value_type = typename std::unordered_map<K, V, Args...>::value_type;
os << '{';
std::copy(map.begin(), map.end(), std::ostream_iterator<value_type>(os, ","));
return os << '}';
}
template<typename K, typename V, typename... Args>
std::istream& operator>>(std::istream&, std::unordered_map<K, V, Args...>&);
template<>
std::istream& operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
extern template
std::istream& operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);
template<typename V, typename... Args>
std::istream& operator>>(std::istream&, std::vector<V, Args...>&);
template<>
std::istream& operator>>(std::istream&, std::vector<seastar::sstring>&);
extern template
std::istream& operator>>(std::istream&, std::vector<seastar::sstring>&);
template<typename K, typename V, typename... Args>
std::istream& operator>>(std::istream& is, std::unordered_map<K, V, Args...>& map) {
std::unordered_map<sstring, sstring> tmp;
is >> tmp;
for (auto& p : tmp) {
map[boost::lexical_cast<K>(p.first)] = boost::lexical_cast<V>(p.second);
}
return is;
}
template<typename V, typename... Args>
std::istream& operator>>(std::istream& is, std::vector<V, Args...>& dst) {
std::vector<seastar::sstring> tmp;
is >> tmp;
for (auto& v : tmp) {
dst.emplace_back(boost::lexical_cast<V>(v));
}
return is;
}
template<typename K, typename V, typename... Args>
void validate(boost::any& out, const std::vector<std::string>& in, std::unordered_map<K, V, Args...>*, int utf8) {
using map_type = std::unordered_map<K, V, Args...>;
if (out.empty()) {
out = boost::any(map_type());
}
static const std::regex key(R"foo((?:^|\:)([^=:]+)=)foo");
auto* p = boost::any_cast<map_type>(&out);
for (const auto& s : in) {
std::sregex_iterator i(s.begin(), s.end(), key), e;
if (i == e) {
throw boost::program_options::invalid_option_value(s);
}
while (i != e) {
auto k = (*i)[1].str();
auto vs = s.begin() + i->position() + i->length();
auto ve = s.end();
if (++i != e) {
ve = s.begin() + i->position();
}
(*p)[boost::lexical_cast<K>(k)] = boost::lexical_cast<V>(sstring(vs, ve));
}
}
}
}
namespace utils {
namespace {
/*
* Our own bpo::typed_valye.
* Only difference is that we _don't_ apply defaults (they are already applied)
* Needed to make aliases work properly.
*/
template<class T, class charT = char>
class typed_value_ex : public bpo::typed_value<T, charT> {
public:
typedef bpo::typed_value<T, charT> _Super;
typed_value_ex(T* store_to)
: _Super(store_to)
{}
bool apply_default(boost::any& value_store) const override {
return false;
}
};
template<class T>
inline typed_value_ex<T>* value_ex(T* v) {
typed_value_ex<T>* r = new typed_value_ex<T>(v);
return r;
}
template<class T>
inline typed_value_ex<std::vector<T>>* value_ex(std::vector<T>* v) {
auto r = new typed_value_ex<std::vector<T>>(v);
r->multitoken();
return r;
}
}
sstring hyphenate(const stdx::string_view&);
}
template<typename T, utils::config_file::value_status S>
void utils::config_file::named_value<T, S>::add_command_line_option(
boost::program_options::options_description_easy_init& init,
const stdx::string_view& name, const stdx::string_view& desc) {
init(hyphenate(name).data(), value_ex(&_value)->notifier([this](auto&&) { _source = config_source::CommandLine; }), desc.data());
}
template<typename T, utils::config_file::value_status S>
void utils::config_file::named_value<T, S>::set_value(const YAML::Node& node) {
(*this)(node.as<T>());
_source = config_source::SettingsFile;
}