db/system_keyspace: use user memory limits for local.paxos table

Treat writes to local.paxos as user memory, as the number of writes is dependent on the amount of user data written with LWT. Fixes #5682 Message-Id: <20200130150048.GW26048@scylladb.com> (cherry picked from commit b08679e1d3)
types: Fix encoding of negative varint
2020-02-02 17:36:52 +02:00 · 2020-02-02 16:00:58 +02:00 · 2020-02-01 13:20:22 +02:00 · 2020-01-30 14:00:51 +02:00 · 2020-01-28 18:34:48 +02:00 · 2020-01-28 18:14:01 +02:00
8 changed files with 20 additions and 10 deletions
--- a/2
+++ b/2
@@ -1,7 +1,7 @@
 #!/bin/sh

 PRODUCT=scylla
-VERSION=666.development
+VERSION=3.3.rc0

 if test -f version
 then
--- a/db/system_keyspace.cc
+++ b/db/system_keyspace.cc
@@ -1792,7 +1792,7 @@ static void maybe_add_virtual_reader(schema_ptr s, database& db) {
 }

 static bool maybe_write_in_user_memory(schema_ptr s, database& db) {
-    return (s.get() == batchlog().get())
+    return (s.get() == batchlog().get()) || (s.get() == paxos().get())
            || s == v3::scylla_views_builds_in_progress();
 }

--- a/db/view/view.cc
+++ b/db/view/view.cc
@@ -311,7 +311,7 @@ deletable_row& view_updates::get_view_row(const partition_key& base_key, const c
            if (!cdef.is_computed()) {
                //FIXME(sarna): this legacy code is here for backward compatibility and should be removed
                // once "computed_columns feature" is supported by every node
-                if (!service::get_local_storage_service().db().local().find_column_family(_base->id()).get_index_manager().is_index(*_base)) {
+                if (!service::get_local_storage_service().db().local().find_column_family(_base->id()).get_index_manager().is_index(*_view)) {
                    throw std::logic_error(format("Column {} doesn't exist in base and this view is not backing a secondary index", cdef.name_as_text()));
                }
                computed_value = token_column_computation().compute_value(*_base, base_key, update);
--- a/dist/debian/build_deb.sh
+++ b/dist/debian/build_deb.sh
@@ -125,7 +125,7 @@ if [ -z "$TARGET" ]; then
 fi
 RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG)
 RELOC_PKG_BASENAME=$(basename $RELOC_PKG)
-SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE)
+SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/')
 SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE)

 ln -fv $RELOC_PKG_FULLPATH ../$PRODUCT-server_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz
--- a/test/boost/types_test.cc
+++ b/test/boost/types_test.cc
@@ -415,6 +415,8 @@ BOOST_AUTO_TEST_CASE(test_varint) {
    BOOST_CHECK_EQUAL(value_cast<boost::multiprecision::cpp_int>(varint_type->deserialize(from_hex("00deadbeef"))), boost::multiprecision::cpp_int("0xdeadbeef"));
    BOOST_CHECK_EQUAL(value_cast<boost::multiprecision::cpp_int>(varint_type->deserialize(from_hex("00ffffffffffffffffffffffffffffffff"))), boost::multiprecision::cpp_int("340282366920938463463374607431768211455"));

+    BOOST_REQUIRE_EQUAL(from_hex("80000000"), varint_type->decompose(boost::multiprecision::cpp_int(-2147483648)));
+
    test_parsing_fails(varint_type, "1A");
 }

--- a/test/lib/eventually.hh
+++ b/test/lib/eventually.hh
@@ -25,7 +25,7 @@
 #include <seastar/util/noncopyable_function.hh>

 inline
-void eventually(noncopyable_function<void ()> f, size_t max_attempts = 12) {
+void eventually(noncopyable_function<void ()> f, size_t max_attempts = 17) {
    size_t attempts = 0;
    while (true) {
        try {
@@ -43,7 +43,7 @@ void eventually(noncopyable_function<void ()> f, size_t max_attempts = 12) {

 inline
 bool eventually_true(noncopyable_function<bool ()> f) {
-    const unsigned max_attempts = 10;
+    const unsigned max_attempts = 15;
    unsigned attempts = 0;
    while (true) {
        if (f()) {
--- a/tools/toolchain/dbuild
+++ b/tools/toolchain/dbuild
@@ -133,6 +133,7 @@ fi
 tmpdir=$(mktemp -d)

 docker_common_args+=(
+       --pids-limit -1 \
       --network host \
       --cap-add SYS_PTRACE \
       -v "$PWD:$PWD:z" \
--- a/types.cc
+++ b/types.cc
@@ -2082,12 +2082,19 @@ static size_t concrete_serialized_size(const string_type_impl::native_type& v) {
 static size_t concrete_serialized_size(const bytes_type_impl::native_type& v) { return v.size(); }
 static size_t concrete_serialized_size(const inet_addr_type_impl::native_type& v) { return v.get().size(); }

-static size_t concrete_serialized_size(const boost::multiprecision::cpp_int& num) {
-    if (!num) {
+static size_t concrete_serialized_size_aux(const boost::multiprecision::cpp_int& num) {
+    if (num) {
+        return align_up(boost::multiprecision::msb(num) + 2, 8u) / 8;
+    } else {
        return 1;
    }
-    auto pnum = abs(num);
-    return align_up(boost::multiprecision::msb(pnum) + 2, 8u) / 8;
+}
+
+static size_t concrete_serialized_size(const boost::multiprecision::cpp_int& num) {
+    if (num < 0) {
+        return concrete_serialized_size_aux(-num - 1);
+    }
+    return concrete_serialized_size_aux(num);
 }

 static size_t concrete_serialized_size(const varint_type_impl::native_type& v) {
Author	SHA1	Message	Date
Gleb Natapov	00b3f28199	db/system_keyspace: use user memory limits for local.paxos table Treat writes to local.paxos as user memory, as the number of writes is dependent on the amount of user data written with LWT. Fixes #5682 Message-Id: <20200130150048.GW26048@scylladb.com> (cherry picked from commit `b08679e1d3`)	2020-02-02 17:36:52 +02:00
Rafael Ávila de Espíndola	1bbe619689	types: Fix encoding of negative varint We would sometimes produce an unnecessary extra 0xff prefix byte. The new encoding matches what cassandra does. This was both a efficiency and correctness issue, as using varint in a key could produce different tokens. Fixes #5656 Signed-off-by: Rafael Ávila de Espíndola <espindola@scylladb.com> (cherry picked from commit `c89c90d07f`)	2020-02-02 16:00:58 +02:00
Avi Kivity	c36f71c783	test: make eventually() more patient We use eventually() in tests to wait for eventually consistent data to become consistent. However, we see spurious failures indicating that we wait too little. Increasing the timeout has a negative side effect in that tests that fail will now take longer to do so. However, this negative side effect is negligible to false-positive failures, since they throw away large test efforts and sometimes require a person to investigate the problem, only to conclude it is a false positive. This patch therefore makes eventually() more patient, by a factor of 32. Fixes #4707. Message-Id: <20200130162745.45569-1-avi@scylladb.com> (cherry picked from commit `ec5b721db7`)	2020-02-01 13:20:22 +02:00
Pekka Enberg	f5471d268b	release: prepare for 3.3.rc0	2020-01-30 14:00:51 +02:00
Takuya ASADA	fd5c65d9dc	dist/debian: Use tilde for release candidate builds We need to add '~' to handle rcX version correctly on Debian variants (merged at `ae33e9f`), but when we moved to relocated package we mistakenly dropped the code, so add the code again. Fixes #5641 (cherry picked from commit `dd81fd3454`)	2020-01-28 18:34:48 +02:00
Avi Kivity	3aa406bf00	tools: toolchain: dbuild: relax process limit in container Docker restricts the number of processes in a container to some limit it calculates. This limit turns out to be too low on large machines, since we run multiple links in parallel, and each link runs many threads. Remove the limit by specifying --pids-limit -1. Since dbuild is meant to provide a build environment, not a security barrier, this is okay (the container is still restricted by host limits). I checked that --pids-limit is supported by old versions of docker and by podman. Fixes #5651. Message-Id: <20200127090807.3528561-1-avi@scylladb.com> (cherry picked from commit `897320f6ab`)	2020-01-28 18:14:01 +02:00
Piotr Sarna	c0253d9221	db,view: fix checking for secondary index special columns A mistake in handling legacy checks for special 'idx_token' column resulted in not recognizing materialized views backing secondary indexes properly. The mistake is really a typo, but with bad consequences - instead of checking the view schema for being an index, we asked for the base schema, which is definitely not an index of itself. Branches 3.1,3.2 (asap) Fixes #5621 Fixes #4744 (cherry picked from commit `9b379e3d63`)	2020-01-21 23:32:11 +02:00