scylladb/utils/config_file.cc

/*
 * Copyright (C) 2017 ScyllaDB
 *
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <unordered_map>
#include <regex>

#include <yaml-cpp/yaml.h>

#include <boost/program_options.hpp>
#include <boost/any.hpp>
#include <boost/range/adaptor/filtered.hpp>

#include <seastar/core/file.hh>
#include <seastar/core/reactor.hh>
#include <seastar/core/shared_ptr.hh>
#include <seastar/core/fstream.hh>
#include <seastar/core/do_with.hh>
#include <seastar/core/print.hh>
#include <seastar/core/thread.hh>

#include <seastar/json/json_elements.hh>

#include "config_file.hh"
#include "config_file_impl.hh"

namespace bpo = boost::program_options;

template<>
std::istream& std::operator>>(std::istream& is, std::unordered_map<seastar::sstring, seastar::sstring>& map) {
   std::istreambuf_iterator<char> i(is), e;

   int level = 0;
   bool sq = false, dq = false, qq = false;
   sstring key, val;
   sstring* ps = &key;

   auto add = [&] {
      if (!key.empty()) {
         map[key] = std::move(val);
      }
      key = {};
      val = {};
      ps = &key;
   };

   while (i != e && level >= 0) {
      auto c = *i++;

      switch (c) {
      case '\\':
         qq = !qq;
         if (qq) {
            continue;
         }
         break;
      case '\'':
         if (!qq) {
            sq = !sq;
         }
         break;
      case '"':
         if (!qq) {
            dq = !dq;
         }
         break;
      case '=':
         if (level <= 1 && !sq && !dq && !qq) {
            ps = &val;
            continue;
         }
         break;
      case '{': case '[':
         if (!sq && !dq && !qq) {
            ++level;
            continue;
         }
         break;
      case ']': case '}':
         if (!sq && !dq && !qq && level > 0) {
            --level;
            continue;
         }
         break;
      case ',':
         if (level == 1 && !sq && !dq && !qq) {
            add();
            continue;
         }
         break;
      case ' ': case '\t': case '\n':
         if (!sq && !dq && !qq) {
            continue;
         }
         break;
      default:
         break;
      }

      if (level == 0) {
         ++level;
      }
      qq = false;
      ps->append(&c, 1);
   }

   add();

   return is;
}

template<>
std::istream& std::operator>>(std::istream& is, std::vector<seastar::sstring>& res) {
   std::istreambuf_iterator<char> i(is), e;

   int level = 0;
   bool sq = false, dq = false, qq = false;
   sstring val;

   auto add = [&] {
      if (!val.empty()) {
         res.emplace_back(std::exchange(val, {}));
      }
      val = {};
   };

   while (i != e && level >= 0) {
      auto c = *i++;

      switch (c) {
      case '\\':
         qq = !qq;
         if (qq) {
            continue;
         }
         break;
      case '\'':
         if (!qq) {
            sq = !sq;
         }
         break;
      case '"':
         if (!qq) {
            dq = !dq;
         }
         break;
      case '{': case '[':
         if (!sq && !dq && !qq) {
            ++level;
            continue;
         }
         break;
      case '}': case ']':
         if (!sq && !dq && !qq && level > 0) {
            --level;
            continue;
         }
         break;
      case ',':
         if (level == 1 && !sq && !dq && !qq) {
            add();
            continue;
         }
         break;
      case ' ': case '\t': case '\n':
         if (!sq && !dq && !qq) {
            continue;
         }
         break;
      default:
         break;
      }

      if (level == 0) {
         ++level;
      }
      qq = false;
      val.append(&c, 1);
   }

   add();

   return is;
}
template std::istream& std::operator>>(std::istream&, std::unordered_map<seastar::sstring, seastar::sstring>&);

thread_local unsigned utils::config_file::s_shard_id = 0;

json::json_return_type
utils::config_type::to_json(const void* value) const {
    return _to_json(value);
}

json::json_return_type
utils::config_file::config_src::value_as_json() const {
    return _type->to_json(current_value());
}

sstring utils::hyphenate(const std::string_view& v) {
    sstring result(v.begin(), v.end());
    std::replace(result.begin(), result.end(), '_', '-');
    return result;
}

utils::config_file::config_file(std::initializer_list<cfg_ref> cfgs)
    : _cfgs(cfgs)
{}

void utils::config_file::add(cfg_ref cfg, std::unique_ptr<any_value> value) {
    if (_per_shard_values.size() != 1) {
        throw std::runtime_error("Can only add config_src to config_file during initialization");
    }
    _cfgs.emplace_back(cfg);
    auto undo = defer([&] { _cfgs.pop_back(); });
    cfg.get()._per_shard_values_offset = _per_shard_values[0].size();
    _per_shard_values[0].emplace_back(std::move(value));
    undo.cancel();
}

void utils::config_file::add(std::initializer_list<cfg_ref> cfgs) {
    _cfgs.insert(_cfgs.end(), cfgs.begin(), cfgs.end());
}

void utils::config_file::add(const std::vector<cfg_ref> & cfgs) {
    _cfgs.insert(_cfgs.end(), cfgs.begin(), cfgs.end());
}

bpo::options_description utils::config_file::get_options_description() {
    bpo::options_description opts("");
    return get_options_description(opts);
}

bpo::options_description utils::config_file::get_options_description(boost::program_options::options_description opts) {
    auto init = opts.add_options();
    add_options(init);
    return std::move(opts);
}

bpo::options_description_easy_init&
utils::config_file::add_options(bpo::options_description_easy_init& init) {
    for (config_src& src : _cfgs) {
        if (src.status() == value_status::Used) {
            auto&& name = src.name();
            sstring tmp(name.begin(), name.end());
            std::replace(tmp.begin(), tmp.end(), '_', '-');
            src.add_command_line_option(init, tmp, src.desc());
        }
    }
    return init;
}

void utils::config_file::read_from_yaml(const sstring& yaml, error_handler h) {
    read_from_yaml(yaml.c_str(), std::move(h));
}

void utils::config_file::read_from_yaml(const char* yaml, error_handler h) {
    std::unordered_map<sstring, cfg_ref> values;

    if (!h) {
        h = [](auto & opt, auto & msg, auto) {
            throw std::invalid_argument(msg + " : " + opt);
        };
    }
    /*
     * Note: this is not very "half-fault" tolerant. I.e. there could be
     * yaml syntax errors that origin handles and still sets the options
     * where as we don't...
     * There are no exhaustive attempts at converting, we rely on syntax of
     * file mapping to the data type...
     */
    auto doc = YAML::Load(yaml);
    for (auto node : doc) {
        auto label = node.first.as<sstring>();

        auto i = std::find_if(_cfgs.begin(), _cfgs.end(), [&label](const config_src& cfg) { return cfg.name() == label; });
        if (i == _cfgs.end()) {
            h(label, "Unknown option", std::nullopt);
            continue;
        }

        config_src& cfg = *i;

        if (cfg.source() > config_source::SettingsFile) {
            // already set
            continue;
        }
        switch (cfg.status()) {
        case value_status::Invalid:
            h(label, "Option is not applicable", cfg.status());
            continue;
        case value_status::Unused:
        default:
            break;
        }
        if (node.second.IsNull()) {
            continue;
        }
        // Still, a syntax error is an error warning, not a fail
        try {
            cfg.set_value(node.second);
        } catch (std::exception& e) {
            h(label, e.what(), cfg.status());
        } catch (...) {
            h(label, "Could not convert value", cfg.status());
        }
    }
}

utils::config_file::configs utils::config_file::set_values() const {
    return boost::copy_range<configs>(_cfgs | boost::adaptors::filtered([] (const config_src& cfg) {
        return cfg.status() > value_status::Used || cfg.source() > config_source::None;
    }));
}

utils::config_file::configs utils::config_file::unset_values() const {
    configs res;
    for (config_src& cfg : _cfgs) {
        if (cfg.status() > value_status::Used) {
            continue;
        }
        if (cfg.source() > config_source::None) {
            continue;
        }
        res.emplace_back(cfg);
    }
    return res;
}

future<> utils::config_file::read_from_file(file f, error_handler h) {
    return f.size().then([this, f, h](size_t s) {
        return do_with(make_file_input_stream(f), [this, s, h](input_stream<char>& in) {
            return in.read_exactly(s).then([this, h](temporary_buffer<char> buf) {
               read_from_yaml(sstring(buf.begin(), buf.end()), h);
            });
        });
    });
}

future<> utils::config_file::read_from_file(const sstring& filename, error_handler h) {
    return open_file_dma(filename, open_flags::ro).then([this, h](file f) {
       return read_from_file(std::move(f), h);
    });
}

future<> utils::config_file::broadcast_to_all_shards() {
    return async([this] {
        if (_per_shard_values.size() != smp::count) {
            _per_shard_values.resize(smp::count);
            smp::invoke_on_all([this] {
                auto cpu = engine().cpu_id();
                if (cpu != 0) {
                    s_shard_id = cpu;
                    auto& shard_0_values = _per_shard_values[0];
                    auto nr_values = shard_0_values.size();
                    auto& this_shard_values = _per_shard_values[cpu];
                    this_shard_values.resize(nr_values);
                    for (size_t i = 0; i != nr_values; ++i) {
                        this_shard_values[i] = shard_0_values[i]->clone();
                    }
                }
            }).get();
        } else {
            smp::invoke_on_all([this] {
                if (s_shard_id != 0) {
                    auto& shard_0_values = _per_shard_values[0];
                    auto nr_values = shard_0_values.size();
                    auto& this_shard_values = _per_shard_values[s_shard_id];
                    for (size_t i = 0; i != nr_values; ++i) {
                        this_shard_values[i]->update_from(shard_0_values[i].get());
                    }
                }
            }).get();
        }

        // #4713
        // We can have values retained that are not pointing to
        // our storage (extension config). Need to broadcast
        // these configs as well.
        std::set<config_file *> files;
        for (config_src& v : _cfgs) {
            auto f = v.get_config_file();
            if (f != this) {
                files.insert(f);
            }
        }
        for (auto* f : files) {
            f->broadcast_to_all_shards().get();
        }
    });
}