fec57c83e6
`maybe_rehash` is complimentary and is not strictly require to succeed. If it fails, it will retry on the next call, but there's no reason to throw an exception that will fail its caller, since `maybe_rehash` is called as the final step after the caller has already succeeded with its action. Minor enhancement for the error path, no backport required. Closes scylladb/scylladb#19910 * github.com:scylladb/scylladb: cell_locker: maybe_rehash: reindent cell_locker: maybe_rehash: ignore allocation failures
560 lines
19 KiB
C++
560 lines
19 KiB
C++
/*
|
|
* Copyright (C) 2017-present ScyllaDB
|
|
*/
|
|
|
|
/*
|
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <exception>
|
|
|
|
#include <boost/intrusive/unordered_set.hpp>
|
|
|
|
#include "utils/assert.hh"
|
|
#include "utils/small_vector.hh"
|
|
#include "mutation/mutation_partition.hh"
|
|
#include "utils/xx_hasher.hh"
|
|
|
|
#include "db/timeout_clock.hh"
|
|
#include "log.hh"
|
|
|
|
extern logging::logger cell_locker_log;
|
|
|
|
class cells_range {
|
|
using ids_vector_type = utils::small_vector<column_id, 5>;
|
|
|
|
position_in_partition_view _position;
|
|
ids_vector_type _ids;
|
|
public:
|
|
using iterator = ids_vector_type::iterator;
|
|
using const_iterator = ids_vector_type::const_iterator;
|
|
|
|
cells_range()
|
|
: _position(position_in_partition_view(position_in_partition_view::static_row_tag_t())) { }
|
|
|
|
explicit cells_range(position_in_partition_view pos, const row& cells)
|
|
: _position(pos)
|
|
{
|
|
_ids.reserve(cells.size());
|
|
cells.for_each_cell([this] (auto id, auto&&) {
|
|
_ids.emplace_back(id);
|
|
});
|
|
}
|
|
|
|
position_in_partition_view position() const { return _position; }
|
|
bool empty() const { return _ids.empty(); }
|
|
|
|
auto begin() const { return _ids.begin(); }
|
|
auto end() const { return _ids.end(); }
|
|
};
|
|
|
|
class partition_cells_range {
|
|
const mutation_partition& _mp;
|
|
public:
|
|
class iterator {
|
|
const mutation_partition& _mp;
|
|
std::optional<mutation_partition::rows_type::const_iterator> _position;
|
|
cells_range _current;
|
|
public:
|
|
explicit iterator(const mutation_partition& mp)
|
|
: _mp(mp)
|
|
, _current(position_in_partition_view(position_in_partition_view::static_row_tag_t()), mp.static_row().get())
|
|
{ }
|
|
|
|
iterator(const mutation_partition& mp, mutation_partition::rows_type::const_iterator it)
|
|
: _mp(mp)
|
|
, _position(it)
|
|
{ }
|
|
|
|
iterator& operator++() {
|
|
if (!_position) {
|
|
_position = _mp.clustered_rows().begin();
|
|
} else {
|
|
++(*_position);
|
|
}
|
|
if (_position != _mp.clustered_rows().end()) {
|
|
auto it = *_position;
|
|
_current = cells_range(position_in_partition_view(position_in_partition_view::clustering_row_tag_t(), it->key()),
|
|
it->row().cells());
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
iterator operator++(int) {
|
|
iterator it(*this);
|
|
operator++();
|
|
return it;
|
|
}
|
|
|
|
cells_range& operator*() {
|
|
return _current;
|
|
}
|
|
|
|
cells_range* operator->() {
|
|
return &_current;
|
|
}
|
|
|
|
bool operator==(const iterator& other) const {
|
|
return _position == other._position;
|
|
}
|
|
};
|
|
public:
|
|
explicit partition_cells_range(const mutation_partition& mp) : _mp(mp) { }
|
|
|
|
iterator begin() const {
|
|
return iterator(_mp);
|
|
}
|
|
iterator end() const {
|
|
return iterator(_mp, _mp.clustered_rows().end());
|
|
}
|
|
};
|
|
|
|
class locked_cell;
|
|
|
|
struct cell_locker_stats {
|
|
uint64_t lock_acquisitions = 0;
|
|
uint64_t operations_waiting_for_lock = 0;
|
|
};
|
|
|
|
class cell_locker {
|
|
private:
|
|
class partition_entry;
|
|
|
|
struct cell_address {
|
|
position_in_partition position;
|
|
column_id id;
|
|
};
|
|
|
|
class cell_entry : public bi::unordered_set_base_hook<bi::link_mode<bi::auto_unlink>>,
|
|
public enable_lw_shared_from_this<cell_entry> {
|
|
partition_entry& _parent;
|
|
cell_address _address;
|
|
db::timeout_semaphore _semaphore { 0 };
|
|
|
|
friend class cell_locker;
|
|
public:
|
|
cell_entry(partition_entry& parent, position_in_partition position, column_id id)
|
|
: _parent(parent)
|
|
, _address { std::move(position), id }
|
|
{ }
|
|
|
|
// Upgrades cell_entry to another schema.
|
|
// Changes the value of cell_address, so cell_entry has to be
|
|
// temporarily removed from its parent partition_entry.
|
|
// Returns true if the cell_entry still exist in the new schema and
|
|
// should be reinserted.
|
|
bool upgrade(const schema& from, const schema& to, column_kind kind) noexcept {
|
|
auto& old_column_mapping = from.get_column_mapping();
|
|
auto& column = old_column_mapping.column_at(kind, _address.id);
|
|
auto cdef = to.get_column_definition(column.name());
|
|
if (!cdef) {
|
|
return false;
|
|
}
|
|
_address.id = cdef->id;
|
|
return true;
|
|
}
|
|
|
|
const position_in_partition& position() const {
|
|
return _address.position;
|
|
}
|
|
|
|
future<> lock(db::timeout_clock::time_point _timeout) {
|
|
return _semaphore.wait(_timeout);
|
|
}
|
|
void unlock() {
|
|
_semaphore.signal();
|
|
}
|
|
|
|
~cell_entry() {
|
|
if (!is_linked()) {
|
|
return;
|
|
}
|
|
unlink();
|
|
if (!--_parent._cell_count) {
|
|
delete &_parent;
|
|
}
|
|
}
|
|
|
|
class hasher {
|
|
const ::schema* _schema; // pointer instead of reference for default assignment
|
|
public:
|
|
explicit hasher(const schema& s) : _schema(&s) { }
|
|
|
|
size_t operator()(const cell_address& ca) const {
|
|
xx_hasher hasher;
|
|
ca.position.feed_hash(hasher, *_schema);
|
|
::feed_hash(hasher, ca.id);
|
|
return static_cast<size_t>(hasher.finalize_uint64());
|
|
}
|
|
size_t operator()(const cell_entry& ce) const {
|
|
return operator()(ce._address);
|
|
}
|
|
};
|
|
|
|
class equal_compare {
|
|
position_in_partition::equal_compare _cmp;
|
|
private:
|
|
bool do_compare(const cell_address& a, const cell_address& b) const {
|
|
return a.id == b.id && _cmp(a.position, b.position);
|
|
}
|
|
public:
|
|
explicit equal_compare(const schema& s) : _cmp(s) { }
|
|
bool operator()(const cell_address& ca, const cell_entry& ce) const {
|
|
return do_compare(ca, ce._address);
|
|
}
|
|
bool operator()(const cell_entry& ce, const cell_address& ca) const {
|
|
return do_compare(ca, ce._address);
|
|
}
|
|
bool operator()(const cell_entry& a, const cell_entry& b) const {
|
|
return do_compare(a._address, b._address);
|
|
}
|
|
};
|
|
};
|
|
|
|
class partition_entry : public bi::unordered_set_base_hook<bi::link_mode<bi::auto_unlink>> {
|
|
using cells_type = bi::unordered_set<cell_entry,
|
|
bi::equal<cell_entry::equal_compare>,
|
|
bi::hash<cell_entry::hasher>,
|
|
bi::constant_time_size<false>>;
|
|
|
|
static constexpr size_t initial_bucket_count = 16;
|
|
using max_load_factor = std::ratio<3, 4>;
|
|
dht::decorated_key _key;
|
|
cell_locker& _parent;
|
|
size_t _rehash_at_size = compute_rehash_at_size(initial_bucket_count);
|
|
std::unique_ptr<cells_type::bucket_type[]> _buckets; // TODO: start with internal storage?
|
|
size_t _cell_count = 0; // cells_type::empty() is not O(1) if the hook is auto-unlink
|
|
cells_type::bucket_type _internal_buckets[initial_bucket_count];
|
|
cells_type _cells;
|
|
schema_ptr _schema;
|
|
|
|
friend class cell_entry;
|
|
private:
|
|
static constexpr size_t compute_rehash_at_size(size_t bucket_count) {
|
|
return bucket_count * max_load_factor::num / max_load_factor::den;
|
|
}
|
|
|
|
// Try to rehash the set, if needed.
|
|
// The function may fail silently on bad_alloc (logging a warning).
|
|
// Rehashing would be retried at a later time on failure.
|
|
void maybe_rehash() {
|
|
if (_cell_count >= _rehash_at_size) {
|
|
auto new_bucket_count = std::min(_cells.bucket_count() * 2, _cells.bucket_count() + 1024);
|
|
try {
|
|
auto buckets = std::make_unique<cells_type::bucket_type[]>(new_bucket_count);
|
|
|
|
_cells.rehash(cells_type::bucket_traits(buckets.get(), new_bucket_count));
|
|
_buckets = std::move(buckets);
|
|
} catch (const std::bad_alloc&) {
|
|
cell_locker_log.warn("Could not rehash cell_locker partition cells set: bucket_count={} new_bucket_count={}: {}", _cells.bucket_count(), new_bucket_count, std::current_exception());
|
|
}
|
|
// Attempt rehash at the new size in both success and failure paths.
|
|
// On failure, we don't want to retry too soon since it may take some time
|
|
// for memory to free up.
|
|
_rehash_at_size = compute_rehash_at_size(new_bucket_count);
|
|
}
|
|
}
|
|
public:
|
|
partition_entry(schema_ptr s, cell_locker& parent, const dht::decorated_key& dk)
|
|
: _key(dk)
|
|
, _parent(parent)
|
|
, _cells(cells_type::bucket_traits(_internal_buckets, initial_bucket_count),
|
|
cell_entry::hasher(*s), cell_entry::equal_compare(*s))
|
|
, _schema(s)
|
|
{ }
|
|
|
|
~partition_entry() {
|
|
if (is_linked()) {
|
|
_parent._partition_count--;
|
|
}
|
|
}
|
|
|
|
// Upgrades partition entry to new schema. Returns false if all
|
|
// cell_entries has been removed during the upgrade.
|
|
bool upgrade(schema_ptr new_schema);
|
|
|
|
void insert(lw_shared_ptr<cell_entry> cell) {
|
|
_cells.insert(*cell);
|
|
_cell_count++;
|
|
maybe_rehash();
|
|
}
|
|
|
|
cells_type& cells() {
|
|
return _cells;
|
|
}
|
|
|
|
struct hasher {
|
|
size_t operator()(const dht::decorated_key& dk) const {
|
|
return std::hash<dht::decorated_key>()(dk);
|
|
}
|
|
size_t operator()(const partition_entry& pe) const {
|
|
return operator()(pe._key);
|
|
}
|
|
};
|
|
|
|
class equal_compare {
|
|
dht::decorated_key_equals_comparator _cmp;
|
|
public:
|
|
explicit equal_compare(const schema& s) : _cmp(s) { }
|
|
bool operator()(const dht::decorated_key& dk, const partition_entry& pe) {
|
|
return _cmp(dk, pe._key);
|
|
}
|
|
bool operator()(const partition_entry& pe, const dht::decorated_key& dk) {
|
|
return _cmp(dk, pe._key);
|
|
}
|
|
bool operator()(const partition_entry& a, const partition_entry& b) {
|
|
return _cmp(a._key, b._key);
|
|
}
|
|
};
|
|
};
|
|
|
|
using partitions_type = bi::unordered_set<partition_entry,
|
|
bi::equal<partition_entry::equal_compare>,
|
|
bi::hash<partition_entry::hasher>,
|
|
bi::constant_time_size<false>>;
|
|
|
|
static constexpr size_t initial_bucket_count = 4 * 1024;
|
|
using max_load_factor = std::ratio<3, 4>;
|
|
|
|
std::unique_ptr<partitions_type::bucket_type[]> _buckets;
|
|
partitions_type _partitions;
|
|
size_t _partition_count = 0;
|
|
size_t _rehash_at_size = compute_rehash_at_size(initial_bucket_count);
|
|
schema_ptr _schema;
|
|
|
|
// partitions_type uses equality comparator which keeps a reference to the
|
|
// original schema, we must ensure that it doesn't die.
|
|
schema_ptr _original_schema;
|
|
cell_locker_stats& _stats;
|
|
|
|
friend class locked_cell;
|
|
private:
|
|
struct locker;
|
|
|
|
static constexpr size_t compute_rehash_at_size(size_t bucket_count) {
|
|
return bucket_count * max_load_factor::num / max_load_factor::den;
|
|
}
|
|
|
|
// Try to rehash the set, if needed.
|
|
// The function may fail silently on bad_alloc (logging a warning).
|
|
// Rehashing would be retried at a later time on failure.
|
|
void maybe_rehash() {
|
|
if (_partition_count >= _rehash_at_size) {
|
|
auto new_bucket_count = std::min(_partitions.bucket_count() * 2, _partitions.bucket_count() + 64 * 1024);
|
|
try {
|
|
auto buckets = std::make_unique<partitions_type::bucket_type[]>(new_bucket_count);
|
|
|
|
_partitions.rehash(partitions_type::bucket_traits(buckets.get(), new_bucket_count));
|
|
_buckets = std::move(buckets);
|
|
} catch (const std::bad_alloc&) {
|
|
cell_locker_log.warn("Could not rehash cell_locker partitions set: bucket_count={} new_bucket_count={}: {}", _partitions.bucket_count(), new_bucket_count, std::current_exception());
|
|
}
|
|
// Attempt rehash at the new size in both success and failure paths.
|
|
// On failure, we don't want to retry too soon since it may take some time
|
|
// for memory to free up.
|
|
_rehash_at_size = compute_rehash_at_size(new_bucket_count);
|
|
}
|
|
}
|
|
public:
|
|
explicit cell_locker(schema_ptr s, cell_locker_stats& stats)
|
|
: _buckets(std::make_unique<partitions_type::bucket_type[]>(initial_bucket_count))
|
|
, _partitions(partitions_type::bucket_traits(_buckets.get(), initial_bucket_count),
|
|
partition_entry::hasher(), partition_entry::equal_compare(*s))
|
|
, _schema(s)
|
|
, _original_schema(std::move(s))
|
|
, _stats(stats)
|
|
{ }
|
|
|
|
~cell_locker() {
|
|
SCYLLA_ASSERT(_partitions.empty());
|
|
}
|
|
|
|
void set_schema(schema_ptr s) {
|
|
_schema = s;
|
|
}
|
|
schema_ptr schema() const {
|
|
return _schema;
|
|
}
|
|
|
|
// partition_cells_range is required to be in cell_locker::schema()
|
|
future<std::vector<locked_cell>> lock_cells(const dht::decorated_key& dk, partition_cells_range&& range,
|
|
db::timeout_clock::time_point timeout);
|
|
};
|
|
|
|
|
|
class locked_cell {
|
|
lw_shared_ptr<cell_locker::cell_entry> _entry;
|
|
public:
|
|
explicit locked_cell(lw_shared_ptr<cell_locker::cell_entry> entry)
|
|
: _entry(std::move(entry)) { }
|
|
|
|
locked_cell(const locked_cell&) = delete;
|
|
locked_cell(locked_cell&&) = default;
|
|
|
|
~locked_cell() {
|
|
if (_entry) {
|
|
_entry->unlock();
|
|
}
|
|
}
|
|
};
|
|
|
|
struct cell_locker::locker {
|
|
cell_entry::hasher _hasher;
|
|
cell_entry::equal_compare _eq_cmp;
|
|
partition_entry& _partition_entry;
|
|
|
|
partition_cells_range _range;
|
|
partition_cells_range::iterator _current_ck;
|
|
cells_range::const_iterator _current_cell;
|
|
|
|
db::timeout_clock::time_point _timeout;
|
|
std::vector<locked_cell> _locks;
|
|
cell_locker_stats& _stats;
|
|
private:
|
|
void update_ck() {
|
|
if (!is_done()) {
|
|
_current_cell = _current_ck->begin();
|
|
}
|
|
}
|
|
|
|
future<> lock_next();
|
|
|
|
bool is_done() const { return _current_ck == _range.end(); }
|
|
public:
|
|
explicit locker(const ::schema& s, cell_locker_stats& st, partition_entry& pe, partition_cells_range&& range, db::timeout_clock::time_point timeout)
|
|
: _hasher(s)
|
|
, _eq_cmp(s)
|
|
, _partition_entry(pe)
|
|
, _range(std::move(range))
|
|
, _current_ck(_range.begin())
|
|
, _timeout(timeout)
|
|
, _stats(st)
|
|
{
|
|
update_ck();
|
|
}
|
|
|
|
locker(const locker&) = delete;
|
|
locker(locker&&) = delete;
|
|
|
|
future<> lock_all() {
|
|
// Cannot defer before first call to lock_next().
|
|
return lock_next().then([this] {
|
|
return do_until([this] { return is_done(); }, [this] {
|
|
return lock_next();
|
|
});
|
|
});
|
|
}
|
|
|
|
std::vector<locked_cell> get() && { return std::move(_locks); }
|
|
};
|
|
|
|
inline
|
|
future<std::vector<locked_cell>> cell_locker::lock_cells(const dht::decorated_key& dk, partition_cells_range&& range, db::timeout_clock::time_point timeout) {
|
|
partition_entry::hasher pe_hash;
|
|
partition_entry::equal_compare pe_eq(*_schema);
|
|
|
|
auto it = _partitions.find(dk, pe_hash, pe_eq);
|
|
std::unique_ptr<partition_entry> partition;
|
|
if (it == _partitions.end()) {
|
|
partition = std::make_unique<partition_entry>(_schema, *this, dk);
|
|
} else if (!it->upgrade(_schema)) {
|
|
partition = std::unique_ptr<partition_entry>(&*it);
|
|
_partition_count--;
|
|
_partitions.erase(it);
|
|
}
|
|
|
|
if (partition) {
|
|
std::vector<locked_cell> locks;
|
|
for (auto&& r : range) {
|
|
if (r.empty()) {
|
|
continue;
|
|
}
|
|
for (auto&& c : r) {
|
|
auto cell = make_lw_shared<cell_entry>(*partition, position_in_partition(r.position()), c);
|
|
_stats.lock_acquisitions++;
|
|
partition->insert(cell);
|
|
locks.emplace_back(std::move(cell));
|
|
}
|
|
}
|
|
|
|
if (!locks.empty()) {
|
|
_partitions.insert(*partition.release());
|
|
_partition_count++;
|
|
maybe_rehash();
|
|
}
|
|
return make_ready_future<std::vector<locked_cell>>(std::move(locks));
|
|
}
|
|
|
|
auto l = std::make_unique<locker>(*_schema, _stats, *it, std::move(range), timeout);
|
|
auto f = l->lock_all();
|
|
return f.then([l = std::move(l)] {
|
|
return std::move(*l).get();
|
|
});
|
|
}
|
|
|
|
inline
|
|
future<> cell_locker::locker::lock_next() {
|
|
while (!is_done()) {
|
|
if (_current_cell == _current_ck->end()) {
|
|
++_current_ck;
|
|
update_ck();
|
|
continue;
|
|
}
|
|
|
|
auto cid = *_current_cell++;
|
|
|
|
cell_address ca { position_in_partition(_current_ck->position()), cid };
|
|
auto it = _partition_entry.cells().find(ca, _hasher, _eq_cmp);
|
|
if (it != _partition_entry.cells().end()) {
|
|
_stats.operations_waiting_for_lock++;
|
|
return it->lock(_timeout).then([this, ce = it->shared_from_this()] () mutable {
|
|
_stats.operations_waiting_for_lock--;
|
|
_stats.lock_acquisitions++;
|
|
_locks.emplace_back(std::move(ce));
|
|
});
|
|
}
|
|
|
|
auto cell = make_lw_shared<cell_entry>(_partition_entry, position_in_partition(_current_ck->position()), cid);
|
|
_stats.lock_acquisitions++;
|
|
_partition_entry.insert(cell);
|
|
_locks.emplace_back(std::move(cell));
|
|
}
|
|
return make_ready_future<>();
|
|
}
|
|
|
|
inline
|
|
bool cell_locker::partition_entry::upgrade(schema_ptr new_schema) {
|
|
if (_schema == new_schema) {
|
|
return true;
|
|
}
|
|
|
|
auto buckets = std::make_unique<cells_type::bucket_type[]>(_cells.bucket_count());
|
|
auto cells = cells_type(cells_type::bucket_traits(buckets.get(), _cells.bucket_count()),
|
|
cell_entry::hasher(*new_schema), cell_entry::equal_compare(*new_schema));
|
|
|
|
_cells.clear_and_dispose([&] (cell_entry* cell_ptr) noexcept {
|
|
auto& cell = *cell_ptr;
|
|
auto kind = cell.position().is_static_row() ? column_kind::static_column
|
|
: column_kind::regular_column;
|
|
auto reinsert = cell.upgrade(*_schema, *new_schema, kind);
|
|
if (reinsert) {
|
|
cells.insert(cell);
|
|
} else {
|
|
_cell_count--;
|
|
}
|
|
});
|
|
|
|
// bi::unordered_set move assignment is actually a swap.
|
|
// Original _buckets cannot be destroyed before the container using them is
|
|
// so we need to explicitly make sure that the original _cells is no more.
|
|
_cells = std::move(cells);
|
|
auto destroy = [] (auto) { };
|
|
destroy(std::move(cells));
|
|
|
|
_buckets = std::move(buckets);
|
|
_schema = new_schema;
|
|
return _cell_count;
|
|
}
|