5348d9f596
Because of the small size optimization, not all copies will call the allocator, so allocation failure injection may miss this site if the value is not large enough. Make the testing more effective by marking this place explicitly as an allocation point.
439 lines
12 KiB
C++
439 lines
12 KiB
C++
|
|
/*
|
|
* Copyright (C) 2015 ScyllaDB
|
|
*/
|
|
|
|
/*
|
|
* This file is part of Scylla.
|
|
*
|
|
* Scylla is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* Scylla is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <stdint.h>
|
|
#include <memory>
|
|
#include "bytes.hh"
|
|
#include "utils/allocation_strategy.hh"
|
|
#include <seastar/core/unaligned.hh>
|
|
#include <seastar/util/alloc_failure_injector.hh>
|
|
#include <unordered_map>
|
|
#include <type_traits>
|
|
|
|
struct blob_storage {
|
|
struct [[gnu::packed]] ref_type {
|
|
blob_storage* ptr;
|
|
|
|
ref_type() {}
|
|
ref_type(blob_storage* ptr) : ptr(ptr) {}
|
|
operator blob_storage*() const { return ptr; }
|
|
blob_storage* operator->() const { return ptr; }
|
|
blob_storage& operator*() const { return *ptr; }
|
|
};
|
|
using size_type = uint32_t;
|
|
using char_type = bytes_view::value_type;
|
|
|
|
ref_type* backref;
|
|
size_type size;
|
|
size_type frag_size;
|
|
ref_type next;
|
|
char_type data[];
|
|
|
|
blob_storage(ref_type* backref, size_type size, size_type frag_size) noexcept
|
|
: backref(backref)
|
|
, size(size)
|
|
, frag_size(frag_size)
|
|
, next(nullptr)
|
|
{
|
|
*backref = this;
|
|
}
|
|
|
|
blob_storage(blob_storage&& o) noexcept
|
|
: backref(o.backref)
|
|
, size(o.size)
|
|
, frag_size(o.frag_size)
|
|
, next(o.next)
|
|
{
|
|
*backref = this;
|
|
o.next = nullptr;
|
|
if (next) {
|
|
next->backref = &next;
|
|
}
|
|
memcpy(data, o.data, frag_size);
|
|
}
|
|
} __attribute__((packed));
|
|
|
|
// A managed version of "bytes" (can be used with LSA).
|
|
class managed_bytes {
|
|
struct linearization_context {
|
|
unsigned _nesting = 0;
|
|
// Map from first blob_storage address to linearized version
|
|
// We use the blob_storage address to be insentive to moving
|
|
// a managed_bytes object.
|
|
std::unordered_map<const blob_storage*, std::unique_ptr<bytes_view::value_type[]>> _state;
|
|
void enter() {
|
|
++_nesting;
|
|
}
|
|
void leave() {
|
|
if (!--_nesting) {
|
|
_state.clear();
|
|
}
|
|
}
|
|
void forget(const blob_storage* p) noexcept;
|
|
};
|
|
static thread_local linearization_context _linearization_context;
|
|
public:
|
|
struct linearization_context_guard {
|
|
linearization_context_guard() {
|
|
_linearization_context.enter();
|
|
}
|
|
~linearization_context_guard() {
|
|
_linearization_context.leave();
|
|
}
|
|
};
|
|
private:
|
|
static constexpr size_t max_inline_size = 15;
|
|
struct small_blob {
|
|
bytes_view::value_type data[max_inline_size];
|
|
int8_t size; // -1 -> use blob_storage
|
|
};
|
|
union u {
|
|
u() {}
|
|
~u() {}
|
|
blob_storage::ref_type ptr;
|
|
small_blob small;
|
|
} _u;
|
|
static_assert(sizeof(small_blob) > sizeof(blob_storage*), "inline size too small");
|
|
private:
|
|
bool external() const {
|
|
return _u.small.size < 0;
|
|
}
|
|
size_t max_seg(allocation_strategy& alctr) {
|
|
return alctr.preferred_max_contiguous_allocation() - sizeof(blob_storage);
|
|
}
|
|
void free_chain(blob_storage* p) noexcept {
|
|
if (p->next && _linearization_context._nesting) {
|
|
_linearization_context.forget(p);
|
|
}
|
|
auto& alctr = current_allocator();
|
|
while (p) {
|
|
auto n = p->next;
|
|
alctr.destroy(p);
|
|
p = n;
|
|
}
|
|
}
|
|
const bytes_view::value_type* read_linearize() const {
|
|
if (!external()) {
|
|
return _u.small.data;
|
|
} else if (!_u.ptr->next) {
|
|
return _u.ptr->data;
|
|
} else {
|
|
return do_linearize();
|
|
}
|
|
}
|
|
bytes_view::value_type& value_at_index(blob_storage::size_type index) {
|
|
if (!external()) {
|
|
return _u.small.data[index];
|
|
}
|
|
blob_storage* a = _u.ptr;
|
|
while (index >= a->frag_size) {
|
|
index -= a->frag_size;
|
|
a = a->next;
|
|
}
|
|
return a->data[index];
|
|
}
|
|
const bytes_view::value_type* do_linearize() const;
|
|
public:
|
|
using size_type = blob_storage::size_type;
|
|
struct initialized_later {};
|
|
|
|
managed_bytes() {
|
|
_u.small.size = 0;
|
|
}
|
|
|
|
managed_bytes(const blob_storage::char_type* ptr, size_type size)
|
|
: managed_bytes(bytes_view(ptr, size)) {}
|
|
|
|
managed_bytes(const bytes& b) : managed_bytes(static_cast<bytes_view>(b)) {}
|
|
|
|
managed_bytes(initialized_later, size_type size) {
|
|
memory::on_alloc_point();
|
|
if (size <= max_inline_size) {
|
|
_u.small.size = size;
|
|
} else {
|
|
_u.small.size = -1;
|
|
auto& alctr = current_allocator();
|
|
auto maxseg = max_seg(alctr);
|
|
auto now = std::min(size_t(size), maxseg);
|
|
void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
|
|
sizeof(blob_storage) + now, alignof(blob_storage));
|
|
auto first = new (p) blob_storage(&_u.ptr, size, now);
|
|
auto last = first;
|
|
size -= now;
|
|
try {
|
|
while (size) {
|
|
auto now = std::min(size_t(size), maxseg);
|
|
void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
|
|
sizeof(blob_storage) + now, alignof(blob_storage));
|
|
last = new (p) blob_storage(&last->next, 0, now);
|
|
size -= now;
|
|
}
|
|
} catch (...) {
|
|
free_chain(first);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
managed_bytes(bytes_view v) : managed_bytes(initialized_later(), v.size()) {
|
|
if (!external()) {
|
|
std::copy(v.begin(), v.end(), _u.small.data);
|
|
return;
|
|
}
|
|
auto p = v.data();
|
|
auto s = v.size();
|
|
auto b = _u.ptr;
|
|
while (s) {
|
|
memcpy(b->data, p, b->frag_size);
|
|
p += b->frag_size;
|
|
s -= b->frag_size;
|
|
b = b->next;
|
|
}
|
|
assert(!b);
|
|
}
|
|
|
|
managed_bytes(std::initializer_list<bytes::value_type> b) : managed_bytes(b.begin(), b.size()) {}
|
|
|
|
~managed_bytes() noexcept {
|
|
if (external()) {
|
|
free_chain(_u.ptr);
|
|
}
|
|
}
|
|
|
|
managed_bytes(const managed_bytes& o) : managed_bytes(initialized_later(), o.size()) {
|
|
if (!external()) {
|
|
memcpy(data(), o.data(), size());
|
|
return;
|
|
}
|
|
auto s = size();
|
|
const blob_storage::ref_type* next_src = &o._u.ptr;
|
|
blob_storage* blob_src = nullptr;
|
|
size_type size_src = 0;
|
|
size_type offs_src = 0;
|
|
blob_storage::ref_type* next_dst = &_u.ptr;
|
|
blob_storage* blob_dst = nullptr;
|
|
size_type size_dst = 0;
|
|
size_type offs_dst = 0;
|
|
while (s) {
|
|
if (!size_src) {
|
|
blob_src = *next_src;
|
|
next_src = &blob_src->next;
|
|
size_src = blob_src->frag_size;
|
|
offs_src = 0;
|
|
}
|
|
if (!size_dst) {
|
|
blob_dst = *next_dst;
|
|
next_dst = &blob_dst->next;
|
|
size_dst = blob_dst->frag_size;
|
|
offs_dst = 0;
|
|
}
|
|
auto now = std::min(size_src, size_dst);
|
|
memcpy(blob_dst->data + offs_dst, blob_src->data + offs_src, now);
|
|
s -= now;
|
|
offs_src += now; size_src -= now;
|
|
offs_dst += now; size_dst -= now;
|
|
}
|
|
assert(size_src == 0 && size_dst == 0);
|
|
}
|
|
|
|
managed_bytes(managed_bytes&& o) noexcept
|
|
: _u(o._u)
|
|
{
|
|
if (external()) {
|
|
if (_u.ptr) {
|
|
_u.ptr->backref = &_u.ptr;
|
|
}
|
|
}
|
|
o._u.small.size = 0;
|
|
}
|
|
|
|
managed_bytes& operator=(managed_bytes&& o) noexcept {
|
|
if (this != &o) {
|
|
this->~managed_bytes();
|
|
new (this) managed_bytes(std::move(o));
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
managed_bytes& operator=(const managed_bytes& o) {
|
|
if (this != &o) {
|
|
managed_bytes tmp(o);
|
|
this->~managed_bytes();
|
|
new (this) managed_bytes(std::move(tmp));
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
bool operator==(const managed_bytes& o) const {
|
|
if (size() != o.size()) {
|
|
return false;
|
|
}
|
|
if (!external()) {
|
|
return bytes_view(*this) == bytes_view(o);
|
|
} else {
|
|
auto a = _u.ptr;
|
|
auto a_data = a->data;
|
|
auto a_remain = a->frag_size;
|
|
a = a->next;
|
|
auto b = o._u.ptr;
|
|
auto b_data = b->data;
|
|
auto b_remain = b->frag_size;
|
|
b = b->next;
|
|
while (a_remain || b_remain) {
|
|
auto now = std::min(a_remain, b_remain);
|
|
if (bytes_view(a_data, now) != bytes_view(b_data, now)) {
|
|
return false;
|
|
}
|
|
a_data += now;
|
|
a_remain -= now;
|
|
if (!a_remain && a) {
|
|
a_data = a->data;
|
|
a_remain = a->frag_size;
|
|
a = a->next;
|
|
}
|
|
b_data += now;
|
|
b_remain -= now;
|
|
if (!b_remain && b) {
|
|
b_data = b->data;
|
|
b_remain = b->frag_size;
|
|
b = b->next;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
bool operator!=(const managed_bytes& o) const {
|
|
return !(*this == o);
|
|
}
|
|
|
|
operator bytes_view() const {
|
|
return { data(), size() };
|
|
}
|
|
|
|
bool is_fragmented() const {
|
|
return external() && _u.ptr->next;
|
|
}
|
|
|
|
operator bytes_mutable_view() {
|
|
assert(!is_fragmented());
|
|
return { data(), size() };
|
|
};
|
|
|
|
bytes_view::value_type& operator[](size_type index) {
|
|
return value_at_index(index);
|
|
}
|
|
|
|
const bytes_view::value_type& operator[](size_type index) const {
|
|
return const_cast<const bytes_view::value_type&>(
|
|
const_cast<managed_bytes*>(this)->value_at_index(index));
|
|
}
|
|
|
|
size_type size() const {
|
|
if (external()) {
|
|
return _u.ptr->size;
|
|
} else {
|
|
return _u.small.size;
|
|
}
|
|
}
|
|
|
|
const blob_storage::char_type* begin() const {
|
|
return data();
|
|
}
|
|
|
|
const blob_storage::char_type* end() const {
|
|
return data() + size();
|
|
}
|
|
|
|
blob_storage::char_type* begin() {
|
|
return data();
|
|
}
|
|
|
|
blob_storage::char_type* end() {
|
|
return data() + size();
|
|
}
|
|
|
|
bool empty() const {
|
|
return _u.small.size == 0;
|
|
}
|
|
|
|
blob_storage::char_type* data() {
|
|
if (external()) {
|
|
assert(!_u.ptr->next); // must be linearized
|
|
return _u.ptr->data;
|
|
} else {
|
|
return _u.small.data;
|
|
}
|
|
}
|
|
|
|
const blob_storage::char_type* data() const {
|
|
return read_linearize();
|
|
}
|
|
|
|
// Returns the amount of external memory used.
|
|
size_t external_memory_usage() const {
|
|
if (external()) {
|
|
size_t mem = 0;
|
|
blob_storage* blob = _u.ptr;
|
|
while (blob) {
|
|
mem += blob->frag_size + sizeof(blob_storage);
|
|
blob = blob->next;
|
|
}
|
|
return mem;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
template <typename Func>
|
|
friend std::result_of_t<Func()> with_linearized_managed_bytes(Func&& func);
|
|
};
|
|
|
|
// Run func() while ensuring that reads of managed_bytes objects are
|
|
// temporarlily linearized
|
|
template <typename Func>
|
|
inline
|
|
std::result_of_t<Func()>
|
|
with_linearized_managed_bytes(Func&& func) {
|
|
managed_bytes::linearization_context_guard g;
|
|
return func();
|
|
}
|
|
|
|
namespace std {
|
|
|
|
template <>
|
|
struct hash<managed_bytes> {
|
|
size_t operator()(const managed_bytes& v) const {
|
|
return hash<bytes_view>()(v);
|
|
}
|
|
};
|
|
|
|
}
|
|
|
|
// blob_storage is a variable-size type
|
|
inline
|
|
size_t
|
|
size_for_allocation_strategy(const blob_storage& bs) {
|
|
return sizeof(bs) + bs.frag_size;
|
|
}
|