/* * Copyright (C) 2015 ScyllaDB */ /* * This file is part of Scylla. * * Scylla is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Scylla is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Scylla. If not, see . */ #pragma once #include #include #include "bytes.hh" #include "utils/allocation_strategy.hh" #include #include #include #include struct blob_storage { struct [[gnu::packed]] ref_type { blob_storage* ptr; ref_type() {} ref_type(blob_storage* ptr) : ptr(ptr) {} operator blob_storage*() const { return ptr; } blob_storage* operator->() const { return ptr; } blob_storage& operator*() const { return *ptr; } }; using size_type = uint32_t; using char_type = bytes_view::value_type; ref_type* backref; size_type size; size_type frag_size; ref_type next; char_type data[]; blob_storage(ref_type* backref, size_type size, size_type frag_size) noexcept : backref(backref) , size(size) , frag_size(frag_size) , next(nullptr) { *backref = this; } blob_storage(blob_storage&& o) noexcept : backref(o.backref) , size(o.size) , frag_size(o.frag_size) , next(o.next) { *backref = this; o.next = nullptr; if (next) { next->backref = &next; } memcpy(data, o.data, frag_size); } } __attribute__((packed)); // A managed version of "bytes" (can be used with LSA). class managed_bytes { struct linearization_context { unsigned _nesting = 0; // Map from first blob_storage address to linearized version // We use the blob_storage address to be insentive to moving // a managed_bytes object. std::unordered_map> _state; void enter() { ++_nesting; } void leave() { if (!--_nesting) { _state.clear(); } } void forget(const blob_storage* p) noexcept; }; static thread_local linearization_context _linearization_context; public: struct linearization_context_guard { linearization_context_guard() { _linearization_context.enter(); } ~linearization_context_guard() { _linearization_context.leave(); } }; private: static constexpr size_t max_inline_size = 15; struct small_blob { bytes_view::value_type data[max_inline_size]; int8_t size; // -1 -> use blob_storage }; union u { u() {} ~u() {} blob_storage::ref_type ptr; small_blob small; } _u; static_assert(sizeof(small_blob) > sizeof(blob_storage*), "inline size too small"); private: bool external() const { return _u.small.size < 0; } size_t max_seg(allocation_strategy& alctr) { return alctr.preferred_max_contiguous_allocation() - sizeof(blob_storage); } void free_chain(blob_storage* p) noexcept { if (p->next && _linearization_context._nesting) { _linearization_context.forget(p); } auto& alctr = current_allocator(); while (p) { auto n = p->next; alctr.destroy(p); p = n; } } const bytes_view::value_type* read_linearize() const { if (!external()) { return _u.small.data; } else if (!_u.ptr->next) { return _u.ptr->data; } else { return do_linearize(); } } bytes_view::value_type& value_at_index(blob_storage::size_type index) { if (!external()) { return _u.small.data[index]; } blob_storage* a = _u.ptr; while (index >= a->frag_size) { index -= a->frag_size; a = a->next; } return a->data[index]; } const bytes_view::value_type* do_linearize() const; public: using size_type = blob_storage::size_type; struct initialized_later {}; managed_bytes() { _u.small.size = 0; } managed_bytes(const blob_storage::char_type* ptr, size_type size) : managed_bytes(bytes_view(ptr, size)) {} managed_bytes(const bytes& b) : managed_bytes(static_cast(b)) {} managed_bytes(initialized_later, size_type size) { memory::on_alloc_point(); if (size <= max_inline_size) { _u.small.size = size; } else { _u.small.size = -1; auto& alctr = current_allocator(); auto maxseg = max_seg(alctr); auto now = std::min(size_t(size), maxseg); void* p = alctr.alloc(&standard_migrator::object, sizeof(blob_storage) + now, alignof(blob_storage)); auto first = new (p) blob_storage(&_u.ptr, size, now); auto last = first; size -= now; try { while (size) { auto now = std::min(size_t(size), maxseg); void* p = alctr.alloc(&standard_migrator::object, sizeof(blob_storage) + now, alignof(blob_storage)); last = new (p) blob_storage(&last->next, 0, now); size -= now; } } catch (...) { free_chain(first); throw; } } } managed_bytes(bytes_view v) : managed_bytes(initialized_later(), v.size()) { if (!external()) { std::copy(v.begin(), v.end(), _u.small.data); return; } auto p = v.data(); auto s = v.size(); auto b = _u.ptr; while (s) { memcpy(b->data, p, b->frag_size); p += b->frag_size; s -= b->frag_size; b = b->next; } assert(!b); } managed_bytes(std::initializer_list b) : managed_bytes(b.begin(), b.size()) {} ~managed_bytes() noexcept { if (external()) { free_chain(_u.ptr); } } managed_bytes(const managed_bytes& o) : managed_bytes(initialized_later(), o.size()) { if (!external()) { memcpy(data(), o.data(), size()); return; } auto s = size(); const blob_storage::ref_type* next_src = &o._u.ptr; blob_storage* blob_src = nullptr; size_type size_src = 0; size_type offs_src = 0; blob_storage::ref_type* next_dst = &_u.ptr; blob_storage* blob_dst = nullptr; size_type size_dst = 0; size_type offs_dst = 0; while (s) { if (!size_src) { blob_src = *next_src; next_src = &blob_src->next; size_src = blob_src->frag_size; offs_src = 0; } if (!size_dst) { blob_dst = *next_dst; next_dst = &blob_dst->next; size_dst = blob_dst->frag_size; offs_dst = 0; } auto now = std::min(size_src, size_dst); memcpy(blob_dst->data + offs_dst, blob_src->data + offs_src, now); s -= now; offs_src += now; size_src -= now; offs_dst += now; size_dst -= now; } assert(size_src == 0 && size_dst == 0); } managed_bytes(managed_bytes&& o) noexcept : _u(o._u) { if (external()) { if (_u.ptr) { _u.ptr->backref = &_u.ptr; } } o._u.small.size = 0; } managed_bytes& operator=(managed_bytes&& o) noexcept { if (this != &o) { this->~managed_bytes(); new (this) managed_bytes(std::move(o)); } return *this; } managed_bytes& operator=(const managed_bytes& o) { if (this != &o) { managed_bytes tmp(o); this->~managed_bytes(); new (this) managed_bytes(std::move(tmp)); } return *this; } bool operator==(const managed_bytes& o) const { if (size() != o.size()) { return false; } if (!external()) { return bytes_view(*this) == bytes_view(o); } else { auto a = _u.ptr; auto a_data = a->data; auto a_remain = a->frag_size; a = a->next; auto b = o._u.ptr; auto b_data = b->data; auto b_remain = b->frag_size; b = b->next; while (a_remain || b_remain) { auto now = std::min(a_remain, b_remain); if (bytes_view(a_data, now) != bytes_view(b_data, now)) { return false; } a_data += now; a_remain -= now; if (!a_remain && a) { a_data = a->data; a_remain = a->frag_size; a = a->next; } b_data += now; b_remain -= now; if (!b_remain && b) { b_data = b->data; b_remain = b->frag_size; b = b->next; } } return true; } } bool operator!=(const managed_bytes& o) const { return !(*this == o); } operator bytes_view() const { return { data(), size() }; } bool is_fragmented() const { return external() && _u.ptr->next; } operator bytes_mutable_view() { assert(!is_fragmented()); return { data(), size() }; }; bytes_view::value_type& operator[](size_type index) { return value_at_index(index); } const bytes_view::value_type& operator[](size_type index) const { return const_cast( const_cast(this)->value_at_index(index)); } size_type size() const { if (external()) { return _u.ptr->size; } else { return _u.small.size; } } const blob_storage::char_type* begin() const { return data(); } const blob_storage::char_type* end() const { return data() + size(); } blob_storage::char_type* begin() { return data(); } blob_storage::char_type* end() { return data() + size(); } bool empty() const { return _u.small.size == 0; } blob_storage::char_type* data() { if (external()) { assert(!_u.ptr->next); // must be linearized return _u.ptr->data; } else { return _u.small.data; } } const blob_storage::char_type* data() const { return read_linearize(); } // Returns the amount of external memory used. size_t external_memory_usage() const { if (external()) { size_t mem = 0; blob_storage* blob = _u.ptr; while (blob) { mem += blob->frag_size + sizeof(blob_storage); blob = blob->next; } return mem; } return 0; } template friend std::result_of_t with_linearized_managed_bytes(Func&& func); }; // Run func() while ensuring that reads of managed_bytes objects are // temporarlily linearized template inline std::result_of_t with_linearized_managed_bytes(Func&& func) { managed_bytes::linearization_context_guard g; return func(); } namespace std { template <> struct hash { size_t operator()(const managed_bytes& v) const { return hash()(v); } }; } // blob_storage is a variable-size type inline size_t size_for_allocation_strategy(const blob_storage& bs) { return sizeof(bs) + bs.frag_size; }