Files
scylladb/test/boost/counter_test.cc
Tomasz Grabiec 51e3b9321b Merge ' mvcc: make schema upgrades gentle' from Michał Chojnowski
After a schema change, memtable and cache have to be upgraded to the new schema. Currently, they are upgraded (on the first access after a schema change) atomically, i.e. all rows of the entry are upgraded with one non-preemptible call. This is a one of the last vestiges of the times when partition were treated atomically, and it is a well known source of numerous large stalls.

This series makes schema upgrades gentle (preemptible). This is done by co-opting the existing MVCC machinery.
Before the series, all partition_versions in the partition_entry chain have the same schema, and an entry upgrade replaces the entire chain with a single squashed and upgraded version.
After the series, each partition_version has its own schema. A partition entry upgrade happens simply by adding an empty version with the new schema to the head of the chain. Row entries are upgraded to the current schema on-the-fly by the cursor during reads, and by the MVCC version merge ongoing in the background after the upgrade.

The series:
1. Does some code cleanup in the mutation_partition area.
2. Adds a schema field to partition_version and removes it from its containers (partition_snapshot, cache_entry, memtable_entry).
3. Adds upgrading variants of constructors and apply() for `row` and its wrappers.
4. Prepares partition_snapshot_row_cursor, mutation_partition_v2::apply_monotonically and partition_snapshot::merge_partition_versions for dealing with heterogeneous version chains.
5. Modifies partition_entry::upgrade to perform upgrades by extending the version chain with a new schema instead of squashing it to a single upgraded version.

Fixes #2577

Closes #13761

* github.com:scylladb/scylladb:
  test: mvcc_test: add a test for gentle schema upgrades
  partition_version: make partition_entry::upgrade() gentle
  partition_version: handle multi-schema snapshots in merge_partition_versions
  mutation_partition_v2: handle schema upgrades in apply_monotonically()
  partition_version: remove the unused "from" argument in partition_entry::upgrade()
  row_cache_test: prepare test_eviction_after_schema_change for gentle schema upgrades
  partition_version: handle multi-schema entries in partition_entry::squashed
  partition_snapshot_row_cursor: handle multi-schema snapshots
  partiton_version: prepare partition_snapshot::squashed() for multi-schema snapshots
  partition_version: prepare partition_snapshot::static_row() for multi-schema snapshots
  partition_version: add a logalloc::region argument to partition_entry::upgrade()
  memtable: propagate the region to memtable_entry::upgrade_schema()
  mutation_partition: add an upgrading variant of lazy_row::apply()
  mutation_partition: add an upgrading variant of rows_entry::rows_entry
  mutation_partition: switch an apply() call to apply_monotonically()
  mutation_partition: add an upgrading variant of rows_entry::apply_monotonically()
  mutation_fragment: add an upgrading variant of clustering_row::apply()
  mutation_partition: add an upgrading variant of row::row
  partition_version: remove _schema from partition_entry::operator<<
  partition_version: remove the schema argument from partition_entry::read()
  memtable: remove _schema from memtable_entry
  row_cache: remove _schema from cache_entry
  partition_version: remove the _schema field from partition_snapshot
  partition_version: add a _schema field to partition_version
  mutation_partition: change schema_ptr to schema& in mutation_partition::difference
  mutation_partition: change schema_ptr to schema& in mutation_partition constructor
  mutation_partition_v2: change schema_ptr to schema& in mutation_partition_v2 constructor
  mutation_partition: add upgrading variants of row::apply()
  partition_version: update the comment to apply_to_incomplete()
  mutation_partition_v2: clean up variants of apply()
  mutation_partition: remove apply_weak()
  mutation_partition_v2: remove a misleading comment in apply_monotonically()
  row_cache_test: add schema changes to test_concurrent_reads_and_eviction
  mutation_partition: fix mixed-schema apply()
2023-05-24 22:58:43 +02:00

583 lines
20 KiB
C++

/*
* Copyright (C) 2017-present ScyllaDB
*/
/*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
#include "counters.hh"
#include <random>
#include <seastar/core/thread.hh>
#include <boost/range/algorithm/sort.hpp>
#include <boost/range/algorithm/random_shuffle.hpp>
#include "test/lib/scylla_test_case.hh"
#include "test/lib/random_utils.hh"
#include "schema/schema_builder.hh"
#include "keys.hh"
#include "mutation/mutation.hh"
#include "mutation/frozen_mutation.hh"
#include "mutation/mutation_partition_view.hh"
std::ostream& boost_test_print_type(std::ostream& os, const counter_shard_view& csv) {
fmt::print(os, "{}", csv);
return os;
}
std::ostream& boost_test_print_type(std::ostream& os, const counter_cell_view& ccv) {
fmt::print(os, "{}", ccv);
return os;
}
void verify_shard_order(counter_cell_view ccv) {
if (ccv.shards().begin() == ccv.shards().end()) {
return;
}
auto it = ccv.shards().begin();
auto prev = it;
++it;
while (it != ccv.shards().end()) {
BOOST_REQUIRE_GT(it->id(), prev->id());
prev = it;
++it;
}
}
std::vector<counter_id> generate_ids(unsigned count) {
std::vector<counter_id> id;
std::generate_n(std::back_inserter(id), count, counter_id::create_random_id);
boost::range::sort(id);
return id;
}
SEASTAR_TEST_CASE(test_counter_cell) {
return seastar::async([] {
auto cdef = column_definition("name", counter_type, column_kind::regular_column);
auto id = generate_ids(3);
counter_cell_builder b1;
b1.add_shard(counter_shard(id[0], 5, 1));
b1.add_shard(counter_shard(id[1], -4, 1));
auto c1 = atomic_cell_or_collection(b1.build(0));
atomic_cell_or_collection c2;
{
counter_cell_view cv(c1.as_atomic_cell(cdef));
BOOST_REQUIRE_EQUAL(cv.total_value(), 1);
verify_shard_order(cv);
counter_cell_builder b2;
b2.add_shard(counter_shard(*cv.get_shard(id[0])).update(2, 1));
b2.add_shard(counter_shard(id[2], 1, 1));
c2 = atomic_cell_or_collection(b2.build(0));
}
{
counter_cell_view cv(c2.as_atomic_cell(cdef));
BOOST_REQUIRE_EQUAL(cv.total_value(), 8);
verify_shard_order(cv);
}
counter_cell_view::apply(cdef, c1, c2);
{
counter_cell_view cv(c1.as_atomic_cell(cdef));
BOOST_REQUIRE_EQUAL(cv.total_value(), 4);
verify_shard_order(cv);
}
});
}
SEASTAR_TEST_CASE(test_apply) {
return seastar::async([] {
auto cdef = column_definition("name", counter_type, column_kind::regular_column);
auto verify_apply = [&] (const atomic_cell_or_collection& a, const atomic_cell_or_collection& b, int64_t value) {
auto dst = a.copy(*cdef.type);
auto src = b.copy(*cdef.type);
counter_cell_view::apply(cdef, dst, src);
{
counter_cell_view cv(dst.as_atomic_cell(cdef));
BOOST_REQUIRE_EQUAL(cv.total_value(), value);
BOOST_REQUIRE_EQUAL(cv.timestamp(), std::max(dst.as_atomic_cell(cdef).timestamp(), src.as_atomic_cell(cdef).timestamp()));
}
};
auto id = generate_ids(5);
counter_cell_builder b1;
b1.add_shard(counter_shard(id[0], 3, 1));
b1.add_shard(counter_shard(id[2], 2, 2));
b1.add_shard(counter_shard(id[4], 1, 3));
auto c1 = atomic_cell_or_collection(b1.build(1));
auto c2 = atomic_cell_or_collection(
counter_cell_builder::from_single_shard(2, counter_shard(id[2], 8, 3))
);
verify_apply(c1, c2, 12);
verify_apply(c2, c1, 12);
counter_cell_builder b2;
b2.add_shard(counter_shard(id[1], 4, 5));
b2.add_shard(counter_shard(id[3], 5, 4));
auto c3 = atomic_cell_or_collection(b2.build(2));
verify_apply(c1, c3, 15);
verify_apply(c3, c1, 15);
auto c4 = atomic_cell_or_collection(
counter_cell_builder::from_single_shard(0, counter_shard(id[2], 8, 1))
);
verify_apply(c1, c4, 6);
verify_apply(c4, c1, 6);
counter_cell_builder b3;
b3.add_shard(counter_shard(id[0], 9, 0));
b3.add_shard(counter_shard(id[2], 12, 3));
b3.add_shard(counter_shard(id[3], 5, 4));
auto c5 = atomic_cell_or_collection(b3.build(2));
verify_apply(c1, c5, 21);
verify_apply(c5, c1, 21);
auto c6 = atomic_cell_or_collection(
counter_cell_builder::from_single_shard(3, counter_shard(id[2], 8, 1))
);
verify_apply(c1, c6, 6);
verify_apply(c6, c1, 6);
});
}
schema_ptr get_schema() {
return schema_builder("ks", "cf")
.with_column("pk", int32_type, column_kind::partition_key)
.with_column("ck", int32_type, column_kind::clustering_key)
.with_column("s1", counter_type, column_kind::static_column)
.with_column("c1", counter_type)
.build();
}
atomic_cell_view get_counter_cell(mutation& m) {
auto& mp = m.partition();
BOOST_REQUIRE_EQUAL(mp.clustered_rows().calculate_size(), 1);
const auto& cells = mp.clustered_rows().begin()->row().cells();
BOOST_REQUIRE_EQUAL(cells.size(), 1);
std::optional<atomic_cell_view> acv;
cells.for_each_cell([&] (column_id id, const atomic_cell_or_collection& ac_o_c) {
acv = ac_o_c.as_atomic_cell(m.schema()->regular_column_at(id));
});
BOOST_REQUIRE(bool(acv));
return *acv;
};
atomic_cell_view get_static_counter_cell(mutation& m) {
auto& mp = m.partition();
const auto& cells = mp.static_row();
BOOST_REQUIRE_EQUAL(cells.size(), 1);
std::optional<atomic_cell_view> acv;
cells.for_each_cell([&] (column_id id, const atomic_cell_or_collection& ac_o_c) {
acv = ac_o_c.as_atomic_cell(m.schema()->static_column_at(id));
});
BOOST_REQUIRE(bool(acv));
return *acv;
};
SEASTAR_TEST_CASE(test_counter_mutations) {
return seastar::async([] {
auto s = get_schema();
auto id = generate_ids(4);
auto pk = partition_key::from_single_value(*s, int32_type->decompose(0));
auto ck = clustering_key::from_single_value(*s, int32_type->decompose(0));
auto& col = *s->get_column_definition(utf8_type->decompose(sstring("c1")));
auto& scol = *s->get_column_definition(utf8_type->decompose(sstring("s1")));
mutation m1(s, pk);
counter_cell_builder b1;
b1.add_shard(counter_shard(id[0], 1, 1));
b1.add_shard(counter_shard(id[1], 2, 1));
b1.add_shard(counter_shard(id[2], 3, 1));
m1.set_clustered_cell(ck, col, b1.build(api::new_timestamp()));
counter_cell_builder b1s;
b1s.add_shard(counter_shard(id[1], 4, 3));
b1s.add_shard(counter_shard(id[2], 5, 1));
b1s.add_shard(counter_shard(id[3], 6, 2));
m1.set_static_cell(scol, b1s.build(api::new_timestamp()));
mutation m2(s, pk);
counter_cell_builder b2;
b2.add_shard(counter_shard(id[0], 1, 1));
b2.add_shard(counter_shard(id[2], -5, 4));
b2.add_shard(counter_shard(id[3], -100, 1));
m2.set_clustered_cell(ck, col, b2.build(api::new_timestamp()));
counter_cell_builder b2s;
b2s.add_shard(counter_shard(id[0], 8, 8));
b2s.add_shard(counter_shard(id[1], 1, 4));
b2s.add_shard(counter_shard(id[3], 9, 1));
m2.set_static_cell(scol, b2s.build(api::new_timestamp()));
mutation m3(s, pk);
m3.set_clustered_cell(ck, col, atomic_cell::make_dead(1, gc_clock::now()));
m3.set_static_cell(scol, atomic_cell::make_dead(1, gc_clock::now()));
mutation m4(s, pk);
m4.partition().apply(tombstone(0, gc_clock::now()));
// Apply
auto m = m1;
m.apply(m2);
auto ac = get_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), -102);
verify_shard_order(ccv);
}
ac = get_static_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 20);
verify_shard_order(ccv);
}
m.apply(m3);
ac = get_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
ac = get_static_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
m = m1;
m.apply(m4);
m.partition().compact_for_query(*s, m.decorated_key(), gc_clock::now(), { query::clustering_range::make_singular(ck) },
false, false, query::max_rows);
BOOST_REQUIRE_EQUAL(m.partition().clustered_rows().calculate_size(), 0);
BOOST_REQUIRE(m.partition().static_row().empty());
// Difference
m = mutation(s, m1.decorated_key(), m1.partition().difference(*s, m2.partition()));
ac = get_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 2);
verify_shard_order(ccv);
}
ac = get_static_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 11);
verify_shard_order(ccv);
}
m = mutation(s, m1.decorated_key(), m2.partition().difference(*s, m1.partition()));
ac = get_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), -105);
verify_shard_order(ccv);
}
ac = get_static_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 9);
verify_shard_order(ccv);
}
m = mutation(s, m1.decorated_key(), m1.partition().difference(*s, m3.partition()));
BOOST_REQUIRE_EQUAL(m.partition().clustered_rows().calculate_size(), 0);
BOOST_REQUIRE(m.partition().static_row().empty());
m = mutation(s, m1.decorated_key(), m3.partition().difference(*s, m1.partition()));
ac = get_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
ac = get_static_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
// Freeze
auto fm1 = freeze(m1);
auto fm2 = freeze(m2);
auto fm3 = freeze(m3);
BOOST_REQUIRE_EQUAL(fm1.unfreeze(s), m1);
BOOST_REQUIRE_EQUAL(fm2.unfreeze(s), m2);
BOOST_REQUIRE_EQUAL(fm3.unfreeze(s), m3);
mutation_application_stats app_stats;
auto m0 = m1;
m0.partition().apply(*s, fm2.partition(), *s, app_stats);
m = m1;
m.apply(m2);
BOOST_REQUIRE_EQUAL(m, m0);
m0 = m2;
m0.partition().apply(*s, fm1.partition(), *s, app_stats);
m = m2;
m.apply(m1);
BOOST_REQUIRE_EQUAL(m, m0);
m0 = m1;
m0.partition().apply(*s, fm3.partition(), *s, app_stats);
m = m1;
m.apply(m3);
BOOST_REQUIRE_EQUAL(m, m0);
m0 = m3;
m0.partition().apply(*s, fm1.partition(), *s, app_stats);
m = m3;
m.apply(m1);
BOOST_REQUIRE_EQUAL(m, m0);
});
}
SEASTAR_TEST_CASE(test_counter_update_mutations) {
return seastar::async([] {
auto s = get_schema();
auto pk = partition_key::from_single_value(*s, int32_type->decompose(0));
auto ck = clustering_key::from_single_value(*s, int32_type->decompose(0));
auto& col = *s->get_column_definition(utf8_type->decompose(sstring("c1")));
auto& scol = *s->get_column_definition(utf8_type->decompose(sstring("s1")));
auto c1 = atomic_cell::make_live_counter_update(api::new_timestamp(), 5);
auto s1 = atomic_cell::make_live_counter_update(api::new_timestamp(), 4);
mutation m1(s, pk);
m1.set_clustered_cell(ck, col, std::move(c1));
m1.set_static_cell(scol, std::move(s1));
auto c2 = atomic_cell::make_live_counter_update(api::new_timestamp(), 9);
auto s2 = atomic_cell::make_live_counter_update(api::new_timestamp(), 8);
mutation m2(s, pk);
m2.set_clustered_cell(ck, col, std::move(c2));
m2.set_static_cell(scol, std::move(s2));
auto c3 = atomic_cell::make_dead(api::new_timestamp() / 2, gc_clock::now());
mutation m3(s, pk);
m3.set_clustered_cell(ck, col, atomic_cell(*counter_type, c3));
m3.set_static_cell(scol, std::move(c3));
auto m12 = m1;
m12.apply(m2);
auto ac = get_counter_cell(m12);
BOOST_REQUIRE(ac.is_live());
BOOST_REQUIRE(ac.is_counter_update());
BOOST_REQUIRE_EQUAL(ac.counter_update_value(), 14);
ac = get_static_counter_cell(m12);
BOOST_REQUIRE(ac.is_live());
BOOST_REQUIRE(ac.is_counter_update());
BOOST_REQUIRE_EQUAL(ac.counter_update_value(), 12);
auto m123 = m12;
m123.apply(m3);
ac = get_counter_cell(m123);
BOOST_REQUIRE(!ac.is_live());
ac = get_static_counter_cell(m123);
BOOST_REQUIRE(!ac.is_live());
});
}
SEASTAR_TEST_CASE(test_transfer_updates_to_shards) {
return seastar::async([] {
auto s = get_schema();
auto pk = partition_key::from_single_value(*s, int32_type->decompose(0));
auto ck = clustering_key::from_single_value(*s, int32_type->decompose(0));
auto& col = *s->get_column_definition(utf8_type->decompose(sstring("c1")));
auto& scol = *s->get_column_definition(utf8_type->decompose(sstring("s1")));
auto c1 = atomic_cell::make_live_counter_update(api::new_timestamp(), 5);
auto s1 = atomic_cell::make_live_counter_update(api::new_timestamp(), 4);
mutation m1(s, pk);
m1.set_clustered_cell(ck, col, std::move(c1));
m1.set_static_cell(scol, std::move(s1));
auto c2 = atomic_cell::make_live_counter_update(api::new_timestamp(), 9);
auto s2 = atomic_cell::make_live_counter_update(api::new_timestamp(), 8);
mutation m2(s, pk);
m2.set_clustered_cell(ck, col, std::move(c2));
m2.set_static_cell(scol, std::move(s2));
auto c3 = atomic_cell::make_dead(api::new_timestamp() / 2, gc_clock::now());
mutation m3(s, pk);
m3.set_clustered_cell(ck, col, atomic_cell(*counter_type, c3));
m3.set_static_cell(scol, std::move(c3));
auto m0 = m1;
transform_counter_updates_to_shards(m0, nullptr, 0, locator::host_id::create_null_id());
auto empty = mutation(s, pk);
auto m = m1;
transform_counter_updates_to_shards(m, &empty, 0, locator::host_id::create_null_id());
BOOST_REQUIRE_EQUAL(m, m0);
auto ac = get_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 5);
verify_shard_order(ccv);
}
ac = get_static_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 4);
verify_shard_order(ccv);
}
m = m2;
transform_counter_updates_to_shards(m, &m0, 0, locator::host_id::create_null_id());
ac = get_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 14);
verify_shard_order(ccv);
}
ac = get_static_counter_cell(m);
BOOST_REQUIRE(ac.is_live());
{
counter_cell_view ccv(ac);
BOOST_REQUIRE_EQUAL(ccv.total_value(), 12);
verify_shard_order(ccv);
}
m = m3;
transform_counter_updates_to_shards(m, &m0, 0, locator::host_id::create_null_id());
ac = get_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
ac = get_static_counter_cell(m);
BOOST_REQUIRE(!ac.is_live());
});
}
SEASTAR_TEST_CASE(test_sanitize_corrupted_cells) {
return seastar::async([] {
auto& gen = seastar::testing::local_random_engine;
std::uniform_int_distribution<unsigned> shard_count_dist(2, 64);
std::uniform_int_distribution<int64_t> logical_clock_dist(1, 1024 * 1024);
std::uniform_int_distribution<int64_t> value_dist(-1024 * 1024, 1024 * 1024);
for (auto i = 0; i < 100; i++) {
auto cdef = column_definition("name", counter_type, column_kind::regular_column);
auto shard_count = shard_count_dist(gen);
auto ids = generate_ids(shard_count);
// Create a valid counter cell
std::vector<counter_shard> shards;
for (auto id : ids) {
shards.emplace_back(id, value_dist(gen), logical_clock_dist(gen));
}
counter_cell_builder b1;
for (auto&& cs : shards) {
b1.add_shard(cs);
}
auto c1 = atomic_cell_or_collection(b1.build(0));
// Corrupt it by changing shard order and adding duplicates
boost::range::random_shuffle(shards);
std::uniform_int_distribution<unsigned> duplicate_count_dist(1, shard_count / 2);
auto duplicate_count = duplicate_count_dist(gen);
for (auto i = 0u; i < duplicate_count; i++) {
auto cs = shards[i];
shards.emplace_back(cs);
}
boost::range::random_shuffle(shards);
// Sanitize
counter_cell_builder b2;
for (auto&& cs : shards) {
b2.add_maybe_unsorted_shard(cs);
}
b2.sort_and_remove_duplicates();
auto c2 = atomic_cell_or_collection(b2.build(0));
// Compare
{
counter_cell_view cv1(c1.as_atomic_cell(cdef));
counter_cell_view cv2(c2.as_atomic_cell(cdef));
BOOST_REQUIRE_EQUAL(cv1, cv2);
BOOST_REQUIRE_EQUAL(cv1.total_value(), cv2.total_value());
verify_shard_order(cv1);
verify_shard_order(cv2);
}
}
});
}
SEASTAR_TEST_CASE(test_counter_id_ordering) {
return seastar::async([] {
const char* ids[] = {
"00000000-0000-0000-0000-000000000000",
"00000000-0000-0000-0000-000000000001",
"0290003c-977e-397c-ac3e-fdfdc01d626b",
"0290003c-987e-397c-ac3e-fdfdc01d626b",
"0eeeddcc-aa99-8877-6655-443322110000",
"0feeddcc-aa99-8877-6655-443322110000",
"0feeddcc-aa99-8877-8655-443322110000",
"3bf296f0-6e46-4481-87dc-ca53e61a8f08",
"e41baa44-b178-48fc-ab75-11e9664409be",
"f2ad405d-1658-484f-9418-6314ae2cedcf",
"ffeeddcc-aa99-8877-6655-443322110000",
"ffeeddcc-aa99-8877-6655-443322110001",
"ffeeddcc-aa99-8878-6655-443322110000",
};
auto counter_ids = boost::copy_range<std::vector<counter_id>>(
ids | boost::adaptors::transformed([] (auto id) {
return counter_id(utils::UUID(id));
})
);
for (auto it = counter_ids.begin(); it != counter_ids.end(); ++it) {
BOOST_REQUIRE_EQUAL(*it, *it);
BOOST_REQUIRE(!(*it < *it));
BOOST_REQUIRE(!(*it > *it));
for (auto it2 = counter_ids.begin(); it2 != it; ++it2) {
BOOST_REQUIRE(*it2 < *it);
BOOST_REQUIRE(*it2 != *it);
BOOST_REQUIRE(!(*it2 > *it));
BOOST_REQUIRE(!(*it2 == *it));
}
for (auto it2 = std::next(it); it2 != counter_ids.end(); ++it2) {
BOOST_REQUIRE(*it2 > *it);
BOOST_REQUIRE(*it2 != *it);
BOOST_REQUIRE(!(*it2 < *it));
BOOST_REQUIRE(!(*it2 == *it));
}
}
});
}