mirror of
https://github.com/scylladb/scylladb.git
synced 2026-04-23 10:00:35 +00:00
43f77c71c7
utils/chunked_vector::reserve_partial: fix usage in callers
The method reserve_partial(), when used as documented, quits before the
intended capacity can be reserved fully. This can lead to overallocation
of memory in the last chunk when data is inserted to the chunked vector.
The method itself doesn't have any bug but the way it is being used by
the callers needs to be updated to get the desired behaviour.
Instead of calling it repeatedly with the value returned from the
previous call until it returns zero, it should be repeatedly called with
the intended size until the vector's capacity reaches that size.
This PR updates the method comment and all the callers to use the
right way.
Fixes #19254
Closes scylladb/scylladb#19279
* github.com:scylladb/scylladb:
utils/large_bitset: remove unused includes identified by clangd
utils/large_bitset: use thread::maybe_yield()
test/boost/chunked_managed_vector_test: fix testcase tests_reserve_partial
utils/lsa/chunked_managed_vector: fix reserve_partial()
utils/chunked_vector: return void from reserve_partial and make_room
test/boost/chunked_vector_test: fix testcase tests_reserve_partial
utils/chunked_vector::reserve_partial: fix usage in callers
(cherry picked from commit b2ebc172d0)
Backported from #19308 to 5.4
Closes scylladb/scylladb#19355
366 lines
11 KiB
C++
366 lines
11 KiB
C++
/*
|
|
* Copyright (C) 2017-present ScyllaDB
|
|
*/
|
|
|
|
/*
|
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
*/
|
|
|
|
#define BOOST_TEST_MODULE core
|
|
|
|
#include <stdexcept>
|
|
#include <fmt/format.h>
|
|
|
|
#include <boost/test/included/unit_test.hpp>
|
|
#include <deque>
|
|
#include <random>
|
|
#include "utils/chunked_vector.hh"
|
|
#include "utils/amortized_reserve.hh"
|
|
|
|
#include <boost/range/algorithm/sort.hpp>
|
|
#include <boost/range/algorithm/equal.hpp>
|
|
#include <boost/range/algorithm/reverse.hpp>
|
|
#include <boost/range/irange.hpp>
|
|
|
|
using disk_array = utils::chunked_vector<uint64_t, 1024>;
|
|
|
|
|
|
using deque = std::deque<int>;
|
|
|
|
BOOST_AUTO_TEST_CASE(test_random_walk) {
|
|
auto rand = std::default_random_engine();
|
|
auto op_gen = std::uniform_int_distribution<unsigned>(0, 9);
|
|
auto nr_dist = std::geometric_distribution<size_t>(0.7);
|
|
deque d;
|
|
disk_array c;
|
|
for (auto i = 0; i != 1000000; ++i) {
|
|
auto op = op_gen(rand);
|
|
switch (op) {
|
|
case 0: {
|
|
auto n = rand();
|
|
c.push_back(n);
|
|
d.push_back(n);
|
|
break;
|
|
}
|
|
case 1: {
|
|
auto nr_pushes = nr_dist(rand);
|
|
for (auto i : boost::irange(size_t(0), nr_pushes)) {
|
|
(void)i;
|
|
auto n = rand();
|
|
c.push_back(n);
|
|
d.push_back(n);
|
|
}
|
|
break;
|
|
}
|
|
case 2: {
|
|
if (!d.empty()) {
|
|
auto n = d.back();
|
|
auto m = c.back();
|
|
BOOST_REQUIRE_EQUAL(n, m);
|
|
c.pop_back();
|
|
d.pop_back();
|
|
}
|
|
break;
|
|
}
|
|
case 3: {
|
|
c.reserve(nr_dist(rand));
|
|
break;
|
|
}
|
|
case 4: {
|
|
boost::sort(c);
|
|
boost::sort(d);
|
|
break;
|
|
}
|
|
case 5: {
|
|
if (!d.empty()) {
|
|
auto u = std::uniform_int_distribution<size_t>(0, d.size() - 1);
|
|
auto idx = u(rand);
|
|
auto m = c[idx];
|
|
auto n = c[idx];
|
|
BOOST_REQUIRE_EQUAL(m, n);
|
|
}
|
|
break;
|
|
}
|
|
case 6: {
|
|
c.clear();
|
|
d.clear();
|
|
break;
|
|
}
|
|
case 7: {
|
|
boost::reverse(c);
|
|
boost::reverse(d);
|
|
break;
|
|
}
|
|
case 8: {
|
|
c.clear();
|
|
d.clear();
|
|
break;
|
|
}
|
|
case 9: {
|
|
auto nr = nr_dist(rand);
|
|
c.resize(nr);
|
|
d.resize(nr);
|
|
break;
|
|
}
|
|
default:
|
|
abort();
|
|
}
|
|
BOOST_REQUIRE_EQUAL(c.size(), d.size());
|
|
BOOST_REQUIRE(boost::equal(c, d));
|
|
}
|
|
}
|
|
|
|
class exception_safety_checker {
|
|
int64_t _live_objects = 0;
|
|
int64_t _countdown = std::numeric_limits<int64_t>::max();
|
|
public:
|
|
bool ok() const {
|
|
return !_live_objects;
|
|
}
|
|
int64_t live_objects() const {
|
|
return _live_objects;
|
|
}
|
|
void set_countdown(unsigned x) {
|
|
_countdown = x;
|
|
}
|
|
void add_live_object() {
|
|
if (!_countdown--) { // auto-clears
|
|
throw "ouch";
|
|
}
|
|
++_live_objects;
|
|
}
|
|
void add_live_object_noexcept() noexcept {
|
|
++_live_objects;
|
|
}
|
|
void del_live_object() {
|
|
--_live_objects;
|
|
}
|
|
};
|
|
|
|
class exception_safe_class {
|
|
exception_safety_checker& _esc;
|
|
public:
|
|
explicit exception_safe_class(exception_safety_checker& esc) : _esc(esc) {
|
|
_esc.add_live_object();
|
|
}
|
|
exception_safe_class(const exception_safe_class& x) : _esc(x._esc) {
|
|
_esc.add_live_object();
|
|
}
|
|
exception_safe_class(exception_safe_class&& x) noexcept : _esc(x._esc) {
|
|
_esc.add_live_object_noexcept();
|
|
}
|
|
~exception_safe_class() {
|
|
_esc.del_live_object();
|
|
}
|
|
exception_safe_class& operator=(const exception_safe_class& x) {
|
|
if (this != &x) {
|
|
auto tmp = x;
|
|
this->~exception_safe_class();
|
|
*this = std::move(tmp);
|
|
}
|
|
return *this;
|
|
}
|
|
};
|
|
|
|
BOOST_AUTO_TEST_CASE(tests_constructor_exception_safety) {
|
|
auto checker = exception_safety_checker();
|
|
auto v = std::vector<exception_safe_class>(100, exception_safe_class(checker));
|
|
checker.set_countdown(5);
|
|
try {
|
|
auto u = utils::chunked_vector<exception_safe_class>(v.begin(), v.end());
|
|
BOOST_REQUIRE(false);
|
|
} catch (...) {
|
|
v.clear();
|
|
BOOST_REQUIRE(checker.ok());
|
|
}
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(tests_reserve_partial) {
|
|
auto rand = std::default_random_engine();
|
|
auto size_dist = std::uniform_int_distribution<unsigned>(1, 1 << 12);
|
|
|
|
for (int i = 0; i < 100; ++i) {
|
|
utils::chunked_vector<uint8_t, 512> v;
|
|
const auto size = size_dist(rand);
|
|
while (v.capacity() != size) {
|
|
v.reserve_partial(size);
|
|
}
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), size);
|
|
}
|
|
}
|
|
|
|
// Tests the case of make_room() invoked with last_chunk_capacity_deficit but _size not in
|
|
// the last reserved chunk.
|
|
BOOST_AUTO_TEST_CASE(test_shrinking_and_expansion_involving_chunk_boundary) {
|
|
using vector_type = utils::chunked_vector<std::unique_ptr<uint64_t>>;
|
|
vector_type v;
|
|
|
|
// Fill two chunks
|
|
v.reserve(vector_type::max_chunk_capacity() * 3 / 2);
|
|
for (uint64_t i = 0; i < vector_type::max_chunk_capacity() * 3 / 2; ++i) {
|
|
v.emplace_back(std::make_unique<uint64_t>(i));
|
|
}
|
|
|
|
// Make the last chunk smaller than max size to trigger the last_chunk_capacity_deficit path in make_room()
|
|
v.shrink_to_fit();
|
|
|
|
// Leave the last chunk reserved but empty
|
|
for (uint64_t i = 0; i < vector_type::max_chunk_capacity(); ++i) {
|
|
v.pop_back();
|
|
}
|
|
|
|
// Try to reserve more than the currently reserved capacity and trigger last_chunk_capacity_deficit path
|
|
// with _size not in the last chunk. Should not sigsegv.
|
|
v.reserve(vector_type::max_chunk_capacity() * 4);
|
|
|
|
for (uint64_t i = 0; i < vector_type::max_chunk_capacity() * 2; ++i) {
|
|
v.emplace_back(std::make_unique<uint64_t>(i));
|
|
}
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(test_amoritzed_reserve) {
|
|
utils::chunked_vector<int> v;
|
|
|
|
v.reserve(10);
|
|
amortized_reserve(v, 1);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 10);
|
|
BOOST_REQUIRE_EQUAL(v.size(), 0);
|
|
|
|
v = {};
|
|
amortized_reserve(v, 1);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 1);
|
|
BOOST_REQUIRE_EQUAL(v.size(), 0);
|
|
|
|
v = {};
|
|
amortized_reserve(v, 1);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 1);
|
|
amortized_reserve(v, 2);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 2);
|
|
amortized_reserve(v, 3);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 4);
|
|
amortized_reserve(v, 4);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 4);
|
|
amortized_reserve(v, 5);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 8);
|
|
amortized_reserve(v, 6);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 8);
|
|
amortized_reserve(v, 7);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 8);
|
|
amortized_reserve(v, 7);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 8);
|
|
amortized_reserve(v, 1);
|
|
BOOST_REQUIRE_EQUAL(v.capacity(), 8);
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(tests_insertion_exception_safety) {
|
|
constexpr size_t chunk_size = 512;
|
|
using chunked_vector = utils::chunked_vector<exception_safe_class, chunk_size>;
|
|
size_t max_chunk_capacity = chunked_vector::max_chunk_capacity();
|
|
|
|
// FIXME: convert to seastar test infstrature and use test::random
|
|
// for reproducibility
|
|
std::random_device r;
|
|
auto seed = r();
|
|
BOOST_TEST_MESSAGE(fmt::format("random-seed={}", seed));
|
|
auto rand = std::default_random_engine(seed);
|
|
auto size_dist = std::uniform_int_distribution<size_t>(1, 4 * max_chunk_capacity);
|
|
|
|
auto checker = exception_safety_checker();
|
|
auto count = size_dist(rand);
|
|
BOOST_TEST_MESSAGE(fmt::format("count={}", count));
|
|
checker.set_countdown(count - 1);
|
|
try {
|
|
chunked_vector v;
|
|
for (size_t i = 0; i < count; i++) {
|
|
v.emplace_back(checker);
|
|
}
|
|
BOOST_REQUIRE(false);
|
|
} catch (...) {
|
|
BOOST_REQUIRE_EQUAL(checker.live_objects(), 0);
|
|
}
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(tests_insertion_exception_safety_with_reserve) {
|
|
constexpr size_t chunk_size = 512;
|
|
using chunked_vector = utils::chunked_vector<exception_safe_class, chunk_size>;
|
|
size_t max_chunk_capacity = chunked_vector::max_chunk_capacity();
|
|
|
|
// FIXME: convert to seastar test infstrature and use test::random
|
|
// for reproducibility
|
|
std::random_device r;
|
|
auto seed = r();
|
|
BOOST_TEST_MESSAGE(fmt::format("random-seed={}", seed));
|
|
auto rand = std::default_random_engine(seed);
|
|
auto size_dist = std::uniform_int_distribution<size_t>(1, 4 * max_chunk_capacity);
|
|
auto count = size_dist(rand);
|
|
BOOST_TEST_MESSAGE(fmt::format("count={}", count));
|
|
auto checker = exception_safety_checker();
|
|
checker.set_countdown(count - 1);
|
|
try {
|
|
chunked_vector v;
|
|
auto reserve_count = size_dist(rand);
|
|
BOOST_TEST_MESSAGE(fmt::format("reserve_count={}", reserve_count));
|
|
v.reserve(reserve_count);
|
|
for (size_t i = 0; i < count; i++) {
|
|
v.emplace_back(checker);
|
|
}
|
|
BOOST_REQUIRE(false);
|
|
} catch (...) {
|
|
BOOST_REQUIRE_EQUAL(checker.live_objects(), 0);
|
|
}
|
|
}
|
|
|
|
// Reproduces https://github.com/scylladb/scylladb/issues/18635
|
|
BOOST_AUTO_TEST_CASE(tests_fill_constructor_exception_safety) {
|
|
constexpr size_t chunk_size = 512;
|
|
using chunked_vector = utils::chunked_vector<exception_safe_class, chunk_size>;
|
|
size_t max_chunk_capacity = chunked_vector::max_chunk_capacity();
|
|
|
|
// FIXME: convert to seastar test infstrature and use test::random
|
|
// for reproducibility
|
|
std::random_device r;
|
|
auto seed = r();
|
|
BOOST_TEST_MESSAGE(fmt::format("random-seed={}", seed));
|
|
auto rand = std::default_random_engine(seed);
|
|
auto size_dist = std::uniform_int_distribution<size_t>(1, 4 * max_chunk_capacity);
|
|
auto count = size_dist(rand);
|
|
BOOST_TEST_MESSAGE(fmt::format("count={}", count));
|
|
auto checker = exception_safety_checker();
|
|
auto filler = std::optional<exception_safe_class>(checker);
|
|
checker.set_countdown(count - 1);
|
|
try {
|
|
chunked_vector v(count, *filler);
|
|
BOOST_REQUIRE(false);
|
|
} catch (...) {
|
|
filler.reset();
|
|
BOOST_REQUIRE_EQUAL(checker.live_objects(), 0);
|
|
}
|
|
}
|
|
|
|
BOOST_AUTO_TEST_CASE(tests_copy_constructor_exception_safety) {
|
|
constexpr size_t chunk_size = 512;
|
|
using chunked_vector = utils::chunked_vector<exception_safe_class, chunk_size>;
|
|
size_t max_chunk_capacity = chunked_vector::max_chunk_capacity();
|
|
|
|
// FIXME: convert to seastar test infstrature and use test::random
|
|
// for reproducibility
|
|
std::random_device r;
|
|
auto seed = r();
|
|
BOOST_TEST_MESSAGE(fmt::format("random-seed={}", seed));
|
|
auto rand = std::default_random_engine(seed);
|
|
auto size_dist = std::uniform_int_distribution<size_t>(1, 4 * max_chunk_capacity);
|
|
auto count = size_dist(rand);
|
|
BOOST_TEST_MESSAGE(fmt::format("count={}", count));
|
|
auto checker = exception_safety_checker();
|
|
chunked_vector src(count, exception_safe_class(checker));
|
|
|
|
checker.set_countdown(count - 1);
|
|
try {
|
|
chunked_vector v(src);
|
|
BOOST_REQUIRE(false);
|
|
} catch (...) {
|
|
src.clear();
|
|
BOOST_REQUIRE_EQUAL(checker.live_objects(), 0);
|
|
}
|
|
}
|