mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-06-03 21:47:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge "Sanitize and speed-up (a bit) directories set up" from Pavel

Browse Source 
"
On start there are two things that scylla does on data/commitlog/etc.
dirs: locks and verifies permissions. Right now these two actions are
managed by different approaches, it's convenient to merge them.

Also the introduced in this set directories class makes a ground for
better --workdir option handling. In particular, right now the db::config
entries are modified after options parse to update directories with
the workdir prefix. With the directories class at hands will be able
to stop doing this.
"

* 'br-directories-cleanup' of https://github.com/xemul/scylla:
  directories: Make internals work on fs::path
  directories: Cleanup adding dirs to the vector to work on
  directories: Drop seastar::async usage
  directories: Do touch_and_lock and verify sequentially
  directories: Do touch_and_lock in parallel
  directories: Move the whole stuff into own .cc file
  directories: Move all the dirs code into .init method
  file_lock: Work with fs::path, not sstring
This commit is contained in:
Avi Kivity
2019-12-15 16:02:46 +02:00
parent 5e502ccea9 23a8d32920
commit 00ae4af94c
6 changed files with 172 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
configure.py
View File

@@ -495,6 +495,7 @@ scylla_core = (['database.cc',
'utils/buffer_input_stream.cc',
'utils/limiting_data_source.cc',
'utils/updateable_value.cc',
'utils/directories.cc',
'mutation_partition.cc',
'mutation_partition_view.cc',
'mutation_partition_serializer.cc',

94
main.cc
View File

@@ -42,8 +42,8 @@
#include "db/commitlog/commitlog_replayer.hh"
#include "db/view/view_builder.hh"
#include "utils/runtime.hh"
#include "utils/file_lock.hh"
#include "log.hh"
#include "utils/directories.hh"
#include "debug.hh"
#include "init.hh"
#include "release.hh"
@@ -214,57 +214,6 @@ public:
}
};
static future<> disk_sanity(sstring path, bool developer_mode) {
return check_direct_io_support(path).then([] {
return make_ready_future<>();
}).handle_exception([path](auto ep) {
startlog.error("Could not access {}: {}", path, ep);
return make_exception_future<>(ep);
});
};
class directories {
public:
future<> touch_and_lock(sstring path) {
return io_check([path] { return recursive_touch_directory(path); }).then_wrapped([this, path] (future<> f) {
try {
f.get();
return utils::file_lock::acquire(path + "/.lock").then([this](utils::file_lock lock) {
_locks.emplace_back(std::move(lock));
}).handle_exception([path](auto ep) {
// only do this because "normal" unhandled exception exit in seastar
// _drops_ system_error message ("what()") and thus does not quite deliver
// the relevant info to the user
try {
std::rethrow_exception(ep);
} catch (std::exception& e) {
startlog.error("Could not initialize {}: {}", path, e.what());
throw;
} catch (...) {
throw;
}
});
} catch (...) {
startlog.error("Directory '{}' cannot be initialized. Tried to do it but failed with: {}", path, std::current_exception());
throw;
}
});
}
template<typename _Iter>
future<> touch_and_lock(_Iter i, _Iter e) {
return parallel_for_each(i, e, [this](sstring path) {
return touch_and_lock(std::move(path));
});
}
template<typename _Range>
future<> touch_and_lock(_Range&& r) {
return touch_and_lock(std::begin(r), std::end(r));
}
private:
std::vector<utils::file_lock>
_locks;
};
static
void
verify_rlimit(bool developer_mode) {
@@ -504,7 +453,7 @@ int main(int ac, char** av) {
auto& mm = service::get_migration_manager();
api::http_context ctx(db, proxy);
httpd::http_server_control prometheus_server;
directories dirs;
utils::directories dirs;
sharded<gms::feature_service> feature_service;
return app.run(ac, av, [&] () -> future<int> {
@@ -750,45 +699,8 @@ int main(int ac, char** av) {
});
verify_seastar_io_scheduler(opts.count("max-io-requests"), opts.count("io-properties") || opts.count("io-properties-file"),
cfg->developer_mode()).get();
supervisor::notify("creating data directories");
dirs.touch_and_lock(db.local().get_config().data_file_directories()).get();
supervisor::notify("creating commitlog directory");
dirs.touch_and_lock(db.local().get_config().commitlog_directory()).get();
std::unordered_set<sstring> directories;
directories.insert(db.local().get_config().data_file_directories().cbegin(),
db.local().get_config().data_file_directories().cend());
directories.insert(db.local().get_config().commitlog_directory());
supervisor::notify("creating hints directories");
if (hinted_handoff_enabled) {
fs::path hints_base_dir(db.local().get_config().hints_directory());
dirs.touch_and_lock(db.local().get_config().hints_directory()).get();
directories.insert(db.local().get_config().hints_directory());
for (unsigned i = 0; i < smp::count; ++i) {
sstring shard_dir((hints_base_dir / seastar::to_sstring(i).c_str()).native());
dirs.touch_and_lock(shard_dir).get();
directories.insert(std::move(shard_dir));
}
}
fs::path view_pending_updates_base_dir = fs::path(db.local().get_config().view_hints_directory());
sstring view_pending_updates_base_dir_str = view_pending_updates_base_dir.native();
dirs.touch_and_lock(view_pending_updates_base_dir_str).get();
directories.insert(view_pending_updates_base_dir_str);
for (unsigned i = 0; i < smp::count; ++i) {
sstring shard_dir((view_pending_updates_base_dir / seastar::to_sstring(i).c_str()).native());
dirs.touch_and_lock(shard_dir).get();
directories.insert(std::move(shard_dir));
}
supervisor::notify("verifying directories");
parallel_for_each(directories, [&db] (sstring pathname) {
return disk_sanity(pathname, db.local().get_config().developer_mode()).then([dir = std::move(pathname)] {
return distributed_loader::verify_owner_and_mode(fs::path(dir)).handle_exception([](auto ep) {
startlog.error("Failed owner and mode verification: {}", ep);
return make_exception_future<>(ep);
});
});
}).get();
dirs.init(*cfg, bool(hinted_handoff_enabled)).get();
// Initialization of a keyspace is done by shard 0 only. For system
// keyspace, the procedure will go through the hardcoded column

111
utils/directories.cc Normal file
View File

@@ -0,0 +1,111 @@
/*
* Copyright (C) 2019 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#include "init.hh"
#include "supervisor.hh"
#include "directories.hh"
#include "distributed_loader.hh"
#include "disk-error-handler.hh"
namespace utils {
static future<> disk_sanity(fs::path path, bool developer_mode) {
return check_direct_io_support(path.native()).then([] {
return make_ready_future<>();
}).handle_exception([path](auto ep) {
startlog.error("Could not access {}: {}", path, ep);
return make_exception_future<>(ep);
});
};
future<> directories::touch_and_lock(fs::path path) {
return io_check([path] { return recursive_touch_directory(path.native()); }).then_wrapped([this, path] (future<> f) {
try {
f.get();
return file_lock::acquire(path / ".lock").then([this](file_lock lock) {
_locks.emplace_back(std::move(lock));
}).handle_exception([path](auto ep) {
// only do this because "normal" unhandled exception exit in seastar
// _drops_ system_error message ("what()") and thus does not quite deliver
// the relevant info to the user
try {
std::rethrow_exception(ep);
} catch (std::exception& e) {
startlog.error("Could not initialize {}: {}", path, e.what());
throw;
} catch (...) {
throw;
}
});
} catch (...) {
startlog.error("Directory '{}' cannot be initialized. Tried to do it but failed with: {}", path, std::current_exception());
throw;
}
});
}
static void add(fs::path path, std::vector<fs::path>& to) {
to.push_back(path);
}
static void add(sstring path, std::vector<fs::path>& to) {
add(fs::path(path), to);
}
static void add(std::vector<sstring> paths, std::vector<fs::path>& to) {
for (auto& path : paths) {
add(path, to);
}
}
static void add_sharded(sstring p, std::vector<fs::path>& to) {
fs::path path(p);
add(path, to);
for (unsigned i = 0; i < smp::count; i++) {
add(path / seastar::to_sstring(i).c_str(), to);
}
}
future<> directories::init(db::config& cfg, bool hinted_handoff_enabled) {
std::vector<fs::path> paths;
add(cfg.data_file_directories(), paths);
add(cfg.commitlog_directory(), paths);
if (hinted_handoff_enabled) {
add_sharded(cfg.hints_directory(), paths);
}
add_sharded(cfg.view_hints_directory(), paths);
supervisor::notify("creating and verifying directories");
return parallel_for_each(paths, [this, &cfg] (fs::path path) {
return touch_and_lock(path).then([path = std::move(path), &cfg] {
return disk_sanity(path, cfg.developer_mode()).then([path = std::move(path)] {
return distributed_loader::verify_owner_and_mode(path).handle_exception([](auto ep) {
startlog.error("Failed owner and mode verification: {}", ep);
return make_exception_future<>(ep);
});
});
});
});
}
} // namespace utils

41
utils/directories.hh Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (C) 2019 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <unordered_set>
#include <seastar/core/future.hh>
#include <seastar/core/smp.hh>
#include "utils/file_lock.hh"
using namespace seastar;
namespace utils {
class directories {
public:
future<> init(db::config& cfg, bool hinted_handoff_enabled);
private:
future<> touch_and_lock(fs::path path);
std::vector<file_lock> _locks;
};
} // namespace utils

19
utils/file_lock.cc
View File

@@ -29,12 +29,13 @@
class utils::file_lock::impl {
public:
impl(sstring path)
impl(fs::path path)
: _path(std::move(path)), _fd(
file_desc::open(_path, O_RDWR | O_CREAT | O_CLOEXEC,
file_desc::open(_path.native(), O_RDWR | O_CREAT | O_CLOEXEC,
S_IRWXU)) {
if (::lockf(_fd.get(), F_TLOCK, 0) != 0) {
throw std::system_error(errno, std::system_category(), "Could not acquire lock: " + _path);
throw std::system_error(errno, std::system_category(),
"Could not acquire lock: " + _path.native());
}
}
impl(impl&&) = default;
@@ -46,13 +47,11 @@ public:
auto r = ::lockf(_fd.get(), F_ULOCK, 0);
assert(r == 0);
}
sstring
_path;
file_desc
_fd;
fs::path _path;
file_desc _fd;
};
utils::file_lock::file_lock(sstring path)
utils::file_lock::file_lock(fs::path path)
: _impl(std::make_unique<impl>(std::move(path)))
{}
@@ -63,11 +62,11 @@ utils::file_lock::file_lock(file_lock&& f) noexcept
utils::file_lock::~file_lock()
{}
sstring utils::file_lock::path() const {
fs::path utils::file_lock::path() const {
return _impl ? _impl->_path : "";
}
future<utils::file_lock> utils::file_lock::acquire(sstring path) {
future<utils::file_lock> utils::file_lock::acquire(fs::path path) {
// meh. not really any future stuff here. but pretend, for the
// day when a future version of lock etc is added.
try {

11
utils/file_lock.hh
View File

@@ -23,11 +23,14 @@
#include <memory>
#include <ostream>
#include <filesystem>
#include <seastar/core/sstring.hh>
#include <seastar/core/future.hh>
#include "seastarx.hh"
namespace fs = std::filesystem;
namespace utils {
class file_lock {
public:
@@ -38,15 +41,15 @@ namespace utils {
file_lock& operator=(file_lock&&) = default;
static future<file_lock> acquire(sstring);
static future<file_lock> acquire(fs::path);
sstring path() const;
fs::path path() const;
sstring to_string() const {
return path();
return path().native();
}
private:
class impl;
file_lock(sstring);
file_lock(fs::path);
std::unique_ptr<impl> _impl;
};