diff --git a/configure.py b/configure.py index 1f907c419e..16148e9d8a 100755 --- a/configure.py +++ b/configure.py @@ -495,6 +495,7 @@ scylla_core = (['database.cc', 'utils/buffer_input_stream.cc', 'utils/limiting_data_source.cc', 'utils/updateable_value.cc', + 'utils/directories.cc', 'mutation_partition.cc', 'mutation_partition_view.cc', 'mutation_partition_serializer.cc', diff --git a/main.cc b/main.cc index 063ccf08b4..82fc5e041e 100644 --- a/main.cc +++ b/main.cc @@ -42,8 +42,8 @@ #include "db/commitlog/commitlog_replayer.hh" #include "db/view/view_builder.hh" #include "utils/runtime.hh" -#include "utils/file_lock.hh" #include "log.hh" +#include "utils/directories.hh" #include "debug.hh" #include "init.hh" #include "release.hh" @@ -214,57 +214,6 @@ public: } }; -static future<> disk_sanity(sstring path, bool developer_mode) { - return check_direct_io_support(path).then([] { - return make_ready_future<>(); - }).handle_exception([path](auto ep) { - startlog.error("Could not access {}: {}", path, ep); - return make_exception_future<>(ep); - }); -}; - -class directories { -public: - future<> touch_and_lock(sstring path) { - return io_check([path] { return recursive_touch_directory(path); }).then_wrapped([this, path] (future<> f) { - try { - f.get(); - return utils::file_lock::acquire(path + "/.lock").then([this](utils::file_lock lock) { - _locks.emplace_back(std::move(lock)); - }).handle_exception([path](auto ep) { - // only do this because "normal" unhandled exception exit in seastar - // _drops_ system_error message ("what()") and thus does not quite deliver - // the relevant info to the user - try { - std::rethrow_exception(ep); - } catch (std::exception& e) { - startlog.error("Could not initialize {}: {}", path, e.what()); - throw; - } catch (...) { - throw; - } - }); - } catch (...) { - startlog.error("Directory '{}' cannot be initialized. Tried to do it but failed with: {}", path, std::current_exception()); - throw; - } - }); - } - template - future<> touch_and_lock(_Iter i, _Iter e) { - return parallel_for_each(i, e, [this](sstring path) { - return touch_and_lock(std::move(path)); - }); - } - template - future<> touch_and_lock(_Range&& r) { - return touch_and_lock(std::begin(r), std::end(r)); - } -private: - std::vector - _locks; -}; - static void verify_rlimit(bool developer_mode) { @@ -504,7 +453,7 @@ int main(int ac, char** av) { auto& mm = service::get_migration_manager(); api::http_context ctx(db, proxy); httpd::http_server_control prometheus_server; - directories dirs; + utils::directories dirs; sharded feature_service; return app.run(ac, av, [&] () -> future { @@ -750,45 +699,8 @@ int main(int ac, char** av) { }); verify_seastar_io_scheduler(opts.count("max-io-requests"), opts.count("io-properties") || opts.count("io-properties-file"), cfg->developer_mode()).get(); - supervisor::notify("creating data directories"); - dirs.touch_and_lock(db.local().get_config().data_file_directories()).get(); - supervisor::notify("creating commitlog directory"); - dirs.touch_and_lock(db.local().get_config().commitlog_directory()).get(); - std::unordered_set directories; - directories.insert(db.local().get_config().data_file_directories().cbegin(), - db.local().get_config().data_file_directories().cend()); - directories.insert(db.local().get_config().commitlog_directory()); - supervisor::notify("creating hints directories"); - if (hinted_handoff_enabled) { - fs::path hints_base_dir(db.local().get_config().hints_directory()); - dirs.touch_and_lock(db.local().get_config().hints_directory()).get(); - directories.insert(db.local().get_config().hints_directory()); - for (unsigned i = 0; i < smp::count; ++i) { - sstring shard_dir((hints_base_dir / seastar::to_sstring(i).c_str()).native()); - dirs.touch_and_lock(shard_dir).get(); - directories.insert(std::move(shard_dir)); - } - } - fs::path view_pending_updates_base_dir = fs::path(db.local().get_config().view_hints_directory()); - sstring view_pending_updates_base_dir_str = view_pending_updates_base_dir.native(); - dirs.touch_and_lock(view_pending_updates_base_dir_str).get(); - directories.insert(view_pending_updates_base_dir_str); - for (unsigned i = 0; i < smp::count; ++i) { - sstring shard_dir((view_pending_updates_base_dir / seastar::to_sstring(i).c_str()).native()); - dirs.touch_and_lock(shard_dir).get(); - directories.insert(std::move(shard_dir)); - } - - supervisor::notify("verifying directories"); - parallel_for_each(directories, [&db] (sstring pathname) { - return disk_sanity(pathname, db.local().get_config().developer_mode()).then([dir = std::move(pathname)] { - return distributed_loader::verify_owner_and_mode(fs::path(dir)).handle_exception([](auto ep) { - startlog.error("Failed owner and mode verification: {}", ep); - return make_exception_future<>(ep); - }); - }); - }).get(); + dirs.init(*cfg, bool(hinted_handoff_enabled)).get(); // Initialization of a keyspace is done by shard 0 only. For system // keyspace, the procedure will go through the hardcoded column diff --git a/utils/directories.cc b/utils/directories.cc new file mode 100644 index 0000000000..925a66b5b5 --- /dev/null +++ b/utils/directories.cc @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#include "init.hh" +#include "supervisor.hh" +#include "directories.hh" +#include "distributed_loader.hh" +#include "disk-error-handler.hh" + +namespace utils { + +static future<> disk_sanity(fs::path path, bool developer_mode) { + return check_direct_io_support(path.native()).then([] { + return make_ready_future<>(); + }).handle_exception([path](auto ep) { + startlog.error("Could not access {}: {}", path, ep); + return make_exception_future<>(ep); + }); +}; + +future<> directories::touch_and_lock(fs::path path) { + return io_check([path] { return recursive_touch_directory(path.native()); }).then_wrapped([this, path] (future<> f) { + try { + f.get(); + return file_lock::acquire(path / ".lock").then([this](file_lock lock) { + _locks.emplace_back(std::move(lock)); + }).handle_exception([path](auto ep) { + // only do this because "normal" unhandled exception exit in seastar + // _drops_ system_error message ("what()") and thus does not quite deliver + // the relevant info to the user + try { + std::rethrow_exception(ep); + } catch (std::exception& e) { + startlog.error("Could not initialize {}: {}", path, e.what()); + throw; + } catch (...) { + throw; + } + }); + } catch (...) { + startlog.error("Directory '{}' cannot be initialized. Tried to do it but failed with: {}", path, std::current_exception()); + throw; + } + }); +} + +static void add(fs::path path, std::vector& to) { + to.push_back(path); +} + +static void add(sstring path, std::vector& to) { + add(fs::path(path), to); +} + +static void add(std::vector paths, std::vector& to) { + for (auto& path : paths) { + add(path, to); + } +} + +static void add_sharded(sstring p, std::vector& to) { + fs::path path(p); + + add(path, to); + for (unsigned i = 0; i < smp::count; i++) { + add(path / seastar::to_sstring(i).c_str(), to); + } +} + +future<> directories::init(db::config& cfg, bool hinted_handoff_enabled) { + std::vector paths; + + add(cfg.data_file_directories(), paths); + add(cfg.commitlog_directory(), paths); + if (hinted_handoff_enabled) { + add_sharded(cfg.hints_directory(), paths); + } + add_sharded(cfg.view_hints_directory(), paths); + + supervisor::notify("creating and verifying directories"); + return parallel_for_each(paths, [this, &cfg] (fs::path path) { + return touch_and_lock(path).then([path = std::move(path), &cfg] { + return disk_sanity(path, cfg.developer_mode()).then([path = std::move(path)] { + return distributed_loader::verify_owner_and_mode(path).handle_exception([](auto ep) { + startlog.error("Failed owner and mode verification: {}", ep); + return make_exception_future<>(ep); + }); + }); + }); + }); +} + +} // namespace utils diff --git a/utils/directories.hh b/utils/directories.hh new file mode 100644 index 0000000000..dc4b690e2d --- /dev/null +++ b/utils/directories.hh @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include +#include +#include +#include "utils/file_lock.hh" + +using namespace seastar; + +namespace utils { + +class directories { +public: + future<> init(db::config& cfg, bool hinted_handoff_enabled); +private: + future<> touch_and_lock(fs::path path); + std::vector _locks; +}; + +} // namespace utils diff --git a/utils/file_lock.cc b/utils/file_lock.cc index b8df3654a0..b8102e3071 100644 --- a/utils/file_lock.cc +++ b/utils/file_lock.cc @@ -29,12 +29,13 @@ class utils::file_lock::impl { public: - impl(sstring path) + impl(fs::path path) : _path(std::move(path)), _fd( - file_desc::open(_path, O_RDWR | O_CREAT | O_CLOEXEC, + file_desc::open(_path.native(), O_RDWR | O_CREAT | O_CLOEXEC, S_IRWXU)) { if (::lockf(_fd.get(), F_TLOCK, 0) != 0) { - throw std::system_error(errno, std::system_category(), "Could not acquire lock: " + _path); + throw std::system_error(errno, std::system_category(), + "Could not acquire lock: " + _path.native()); } } impl(impl&&) = default; @@ -46,13 +47,11 @@ public: auto r = ::lockf(_fd.get(), F_ULOCK, 0); assert(r == 0); } - sstring - _path; - file_desc - _fd; + fs::path _path; + file_desc _fd; }; -utils::file_lock::file_lock(sstring path) +utils::file_lock::file_lock(fs::path path) : _impl(std::make_unique(std::move(path))) {} @@ -63,11 +62,11 @@ utils::file_lock::file_lock(file_lock&& f) noexcept utils::file_lock::~file_lock() {} -sstring utils::file_lock::path() const { +fs::path utils::file_lock::path() const { return _impl ? _impl->_path : ""; } -future utils::file_lock::acquire(sstring path) { +future utils::file_lock::acquire(fs::path path) { // meh. not really any future stuff here. but pretend, for the // day when a future version of lock etc is added. try { diff --git a/utils/file_lock.hh b/utils/file_lock.hh index 73b612796a..ec885d7da8 100644 --- a/utils/file_lock.hh +++ b/utils/file_lock.hh @@ -23,11 +23,14 @@ #include #include +#include #include #include #include "seastarx.hh" +namespace fs = std::filesystem; + namespace utils { class file_lock { public: @@ -38,15 +41,15 @@ namespace utils { file_lock& operator=(file_lock&&) = default; - static future acquire(sstring); + static future acquire(fs::path); - sstring path() const; + fs::path path() const; sstring to_string() const { - return path(); + return path().native(); } private: class impl; - file_lock(sstring); + file_lock(fs::path); std::unique_ptr _impl; };