dht: persist values with system time

.github/workflows/ccpp.yml

@@ -7,7 +7,7 @@ env:
   UBUNTU_FULL_DEPS: "libncurses5-dev libreadline-dev nettle-dev libasio-dev libgnutls28-dev libuv1-dev python3-pip python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev libfmt-dev libmsgpack-dev libargon2-dev"
   UBUNTU_MESON_DEPS: "ninja-build libncurses5-dev libreadline-dev nettle-dev libasio-dev libgnutls28-dev libuv1-dev python3-dev python3-setuptools python3-pip libcppunit-dev libjsoncpp-dev libfmt-dev libmsgpack-dev libargon2-dev"
   UBUNTU_MINIMAL_DEPS: "libncurses5-dev libreadline-dev nettle-dev libfmt-dev libgnutls28-dev libcppunit-dev libmsgpack-dev libargon2-dev"
-  DEBIAN13_DEPS: "git libtool pkg-config wget cmake libncurses5-dev libreadline-dev nettle-dev libasio-dev libgnutls28-dev libuv1-dev python3-pip python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev libfmt-dev libmsgpack-dev libargon2-dev"
+  DEBIAN12_DEPS: "git libtool pkg-config wget cmake libncurses5-dev libreadline-dev nettle-dev libasio-dev libgnutls28-dev libuv1-dev python3-pip python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev libfmt-dev libmsgpack-dev libargon2-dev"
 
 on: [push, pull_request]
 
@@ -65,12 +65,12 @@ jobs:
       - name: Run Unit tests
         run: ctest --output-on-failure --test-dir build
 
-  build-debian13:
-    name: Debian 13/GCC build
+  build-debian12:
+    name: Debian 12/GCC build
     runs-on: ubuntu-latest
 
     container:
-      image: debian:13
+      image: debian:12
       volumes:
         - .:/workspace
       options: --workdir /workspace
@@ -83,7 +83,7 @@ jobs:
       - name: Install Dependencies
         run: |
           apt-get update
-          apt-get install -y $DEBIAN13_DEPS
+          apt-get install -y $DEBIAN12_DEPS
 
       - name: Build llhttp dependency
         run: |

CMakeLists.txt

@@ -10,7 +10,7 @@ if(POLICY CMP0167)
 endif()
 
 set (opendht_VERSION_MAJOR 4)
-set (opendht_VERSION_MINOR 3.0)
+set (opendht_VERSION_MINOR 2.0)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")

include/opendht/network_engine.h

@@ -489,12 +489,8 @@ private:
     static constexpr size_t MTU {1280};
     static constexpr size_t MAX_PACKET_VALUE_SIZE {600};
     static constexpr size_t MAX_MESSAGE_VALUE_SIZE {56 * 1024};
-    static constexpr size_t RX_MAX_EARLY_BYTES {MAX_MESSAGE_VALUE_SIZE + MAX_VALUE_SIZE + 32};
-    static constexpr size_t RX_MAX_EARLY_PARTS {RX_MAX_EARLY_BYTES / MTU + 2};
 
     void process(std::unique_ptr<ParsedMessage>&&, const SockAddr& from);
-    bool isExpectedReply(const ParsedMessage&, const SockAddr& from);
-    bool tryCompleteMessage(PartialMessage&, const SockAddr& from);
 
     bool rateLimit(const SockAddr& addr);
 
@@ -531,8 +527,7 @@ private:
     {
         return packValueHeader(buf, values.begin(), values.end());
     }
-    using PartialMessageKey = std::pair<SockAddr, Tid>;
-    void maintainRxBuffer(const PartialMessageKey& key);
+    void maintainRxBuffer(Tid tid);
 
     /*************
      *  Answers  *
@@ -578,7 +573,7 @@ private:
 
     // requests handling
     std::map<Tid, Sp<Request>> requests {};
-    std::map<PartialMessageKey, PartialMessage> partial_messages;
+    std::map<Tid, PartialMessage> partial_messages;
 
     MessageStats in_stats {}, out_stats {};
     std::set<SockAddr> blacklist {};

meson.build

@@ -5,7 +5,7 @@ project(
     'cython',
     license: 'MIT',
     license_files: 'LICENSE',
-    version: '4.3.0',
+    version: '4.2.0',
     meson_version: '>=1.1.0',
     default_options: ['cpp_std=c++20', 'warning_level=3'],
 )

src/dhtrunner.cpp

@@ -554,14 +554,8 @@ DhtRunner::registerInsecureType(const ValueType& type)
 void
 DhtRunner::importValues(const std::vector<ValuesExport>& values)
 {
-    {
-        std::lock_guard lck(dht_mtx);
-        dht_->importValues(values);
-    }
-    // Recompute the runner wakeup after import schedules value expirations.
-    std::lock_guard lck(storage_mtx);
-    pending_ops_prio.emplace([](SecureDht&) {});
-    cv.notify_all();
+    std::lock_guard lck(dht_mtx);
+    dht_->importValues(values);
 }
 
 unsigned

src/http.cpp

@@ -308,7 +308,7 @@ issuer_from_chain(STACK_OF(X509) * fullchain)
     if ((issuer_name = X509_get_issuer_name(cert)) == nullptr)
         return nullptr;
 
-    issuer = X509_find_by_subject(fullchain, const_cast<X509_NAME*>(issuer_name));
+    issuer = X509_find_by_subject(fullchain, issuer_name);
     return issuer;
 }
 

src/network_engine.cpp

@@ -40,44 +40,8 @@ struct NetworkEngine::PartialMessage
     time_point start;
     time_point last_part;
     std::unique_ptr<ParsedMessage> msg;
-    std::vector<std::unique_ptr<ParsedMessage>> early_parts;
-    size_t early_bytes {};
-    bool request_quota_consumed {};
 };
 
-bool
-NetworkEngine::isExpectedReply(const ParsedMessage& msg, const SockAddr& from)
-{
-    auto request = requests.find(msg.tid);
-    if (request != requests.end() && request->second->pending() && !request->second->node->id
-        && request->second->node->getAddr() == from)
-        return true;
-
-    auto node = cache.getNode(msg.id, from.getFamily());
-    if (!node || node->getAddr() != from)
-        return false;
-
-    auto pending = node->getRequest(msg.tid);
-    return (pending && pending->pending()) || node->getSocket(msg.tid);
-}
-
-bool
-NetworkEngine::tryCompleteMessage(PartialMessage& partial, const SockAddr& from)
-{
-    try {
-        if (!partial.msg->complete())
-            return false;
-        process(std::move(partial.msg), from);
-    } catch (const std::exception& e) {
-        if (logger_)
-            logger_->warn("Error while processing partial message: {}", e.what());
-    } catch (...) {
-        if (logger_)
-            logger_->warn("Unknown error while processing partial message");
-    }
-    return true;
-}
-
 std::vector<Blob>
 serializeValues(const std::vector<Sp<Value>>& st)
 {
@@ -502,58 +466,35 @@ NetworkEngine::processMessage(const uint8_t* buf, size_t buflen, SockAddr f)
 
     // partial value data
     if (msg->type == MessageType::ValueData) {
-        if (msg->fragment_parts.empty()
-            || std::any_of(msg->fragment_parts.begin(), msg->fragment_parts.end(), [](const auto& fragment) {
-                   return fragment.second.second.empty();
-               })) {
+        auto pmsg_it = partial_messages.find(msg->tid);
+        if (pmsg_it == partial_messages.end()) {
+            if (logIncoming_)
+                if (logger_)
+                    logger_->debug("Unable to find partial message");
             rateLimit(from);
             return;
         }
-        PartialMessageKey key {from, msg->tid};
-        auto pmsg_it = partial_messages.find(key);
-        if (pmsg_it == partial_messages.end()) {
-            if (!rateLimit(from))
-                return;
-            auto inserted = partial_messages.emplace(key, PartialMessage {});
-            pmsg_it = inserted.first;
-            pmsg_it->second.from = from;
-            pmsg_it->second.start = now;
-            pmsg_it->second.last_part = now;
-            pmsg_it->second.request_quota_consumed = true;
-            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, key));
-        }
-        auto& pmsg = pmsg_it->second;
-        if (!pmsg.msg) {
-            if (pmsg.early_parts.size() >= RX_MAX_EARLY_PARTS) {
-                partial_messages.erase(pmsg_it);
-                return;
-            }
-            size_t fragment_bytes {};
-            for (const auto& [index, fragment] : msg->fragment_parts) {
-                (void) index;
-                if (fragment.second.size() > RX_MAX_EARLY_BYTES - fragment_bytes) {
-                    partial_messages.erase(pmsg_it);
-                    return;
-                }
-                fragment_bytes += fragment.second.size();
-            }
-            if (fragment_bytes > RX_MAX_EARLY_BYTES - pmsg.early_bytes) {
-                partial_messages.erase(pmsg_it);
-                return;
-            }
-            pmsg.early_bytes += fragment_bytes;
-            pmsg.early_parts.emplace_back(std::move(msg));
-            pmsg.last_part = now;
-            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, key));
+        if (!pmsg_it->second.from.equals(from)) {
+            if (logger_)
+                logger_->debug("Received partial message data from unexpected IP address");
+            rateLimit(from);
             return;
         }
         // append data block
-        if (pmsg.msg->append(*msg)) {
-            pmsg.last_part = now;
-            if (tryCompleteMessage(pmsg, from)) {
+        if (pmsg_it->second.msg->append(*msg)) {
+            pmsg_it->second.last_part = now;
+            // check data completion
+            if (pmsg_it->second.msg->complete()) {
+                try {
+                    // process the full message
+                    process(std::move(pmsg_it->second.msg), from);
+                } catch (const std::exception& e) {
+                    if (logger_)
+                        logger_->warn("Error while processing partial message: {}", e.what());
+                }
                 partial_messages.erase(pmsg_it);
             } else
-                scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, key));
+                scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, msg->tid));
         }
         return;
     }
@@ -564,32 +505,7 @@ NetworkEngine::processMessage(const uint8_t* buf, size_t buflen, SockAddr f)
         return;
     }
 
-    if (msg->value_parts.empty()) {
-        if (msg->type > MessageType::Reply) {
-            /* Rate limit requests. */
-            if (!rateLimit(from)) {
-                if (logger_)
-                    logger_->warn("Dropping request due to rate limiting");
-                return;
-            }
-        }
-        try {
-            process(std::move(msg), from);
-        } catch (const std::exception& e) {
-            if (logger_)
-                logger_->warn("Error while processing message: {}", e.what());
-        }
-        return;
-    }
-
-    PartialMessageKey partial_key {from, msg->tid};
-    if (msg->type == MessageType::Reply && !isExpectedReply(*msg, from))
-        return;
-
-    const auto partial = partial_messages.find(partial_key);
-    const bool request_quota_consumed = partial != partial_messages.end() && !partial->second.msg
-                                        && partial->second.request_quota_consumed;
-    if (msg->type > MessageType::Reply && !request_quota_consumed) {
+    if (msg->type > MessageType::Reply) {
         /* Rate limit requests. */
         if (!rateLimit(from)) {
             if (logger_)
@@ -598,28 +514,28 @@ NetworkEngine::processMessage(const uint8_t* buf, size_t buflen, SockAddr f)
         }
     }
 
-    // starting partial message session
-    auto inserted = partial_messages.emplace(partial_key, PartialMessage {});
-    auto& pmsg = inserted.first->second;
-    if (not pmsg.msg) {
-        if (inserted.second) {
+    if (msg->value_parts.empty()) {
+        try {
+            process(std::move(msg), from);
+        } catch (const std::exception& e) {
+            if (logger_)
+                logger_->warn("Error while processing message: {}", e.what());
+            return;
+        }
+    } else {
+        // starting partial message session
+        auto k = msg->tid;
+        auto& pmsg = partial_messages[k];
+        if (not pmsg.msg) {
             pmsg.from = from;
+            pmsg.msg = std::move(msg);
             pmsg.start = now;
-            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, partial_key));
-        }
-        pmsg.msg = std::move(msg);
-        pmsg.last_part = now;
-        for (const auto& part : pmsg.early_parts)
-            pmsg.msg->append(*part);
-        pmsg.early_parts.clear();
-        pmsg.early_bytes = 0;
-        if (tryCompleteMessage(pmsg, from)) {
-            partial_messages.erase(inserted.first);
-        } else {
-            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, partial_key));
-        }
-    } else if (logger_)
-        logger_->error("Partial message from {} with given TID {} already exists", from.toString(), msg->tid);
+            pmsg.last_part = now;
+            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
+            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
+        } else if (logger_)
+            logger_->error("Partial message with given TID {} already exists", k);
+    }
 }
 
 void
@@ -1698,9 +1614,9 @@ NetworkEngine::sendError(const SockAddr& addr, Tid tid, uint16_t code, const std
 }
 
 void
-NetworkEngine::maintainRxBuffer(const PartialMessageKey& key)
+NetworkEngine::maintainRxBuffer(Tid tid)
 {
-    auto msg = partial_messages.find(key);
+    auto msg = partial_messages.find(tid);
     if (msg != partial_messages.end()) {
         const auto& now = scheduler.time();
         if (msg->second.start + RX_MAX_PACKET_TIME < now || msg->second.last_part + RX_TIMEOUT < now) {

src/parsed_message.h

@@ -274,21 +274,14 @@ ParsedMessage::msgpack_unpack(const msgpack::object& msg)
 
     if (!parsed.a && !parsed.r && !parsed.e && !parsed.u)
         throw msgpack::type_error();
+    auto& req = parsed.a ? *parsed.a : (parsed.r ? *parsed.r : (parsed.u ? *parsed.u : *parsed.e));
 
     if (parsed.e) {
-        if (parsed.e->type != msgpack::type::ARRAY or parsed.e->via.array.size < 1)
+        if (parsed.e->type != msgpack::type::ARRAY)
             throw msgpack::type_error();
         error_code = parsed.e->via.array.ptr[0].as<uint16_t>();
     }
 
-    /* An error-only message carries no request map. The error array must never
-       be reached through the map view of the msgpack object union, and a
-       request field that is not a map must be rejected rather than
-       reinterpreted. */
-    const msgpack::object* req = parsed.a ? parsed.a : (parsed.r ? parsed.r : parsed.u);
-    if (req and req->type != msgpack::type::MAP)
-        throw msgpack::type_error();
-
     struct ParsedReq
     {
         msgpack::object* values;
@@ -297,9 +290,8 @@ ParsedMessage::msgpack_unpack(const msgpack::object& msg)
         msgpack::object* want;
     } parsedReq {};
 
-    const uint32_t req_size = req ? req->via.map.size : 0;
-    for (unsigned i = 0; i < req_size; i++) {
-        auto& o = req->via.map.ptr[i];
+    for (unsigned i = 0; i < req.via.map.size; i++) {
+        auto& o = req.via.map.ptr[i];
         if (o.key.type != msgpack::type::STR)
             continue;
         auto key = o.key.as<std::string_view>();
@@ -378,8 +370,6 @@ ParsedMessage::msgpack_unpack(const msgpack::object& msg)
     } else if (parsedReq.fields) {
         if (auto rfields = findMapValue(*parsedReq.fields, "f"sv)) {
             auto vfields = rfields->as<std::set<Value::Field>>();
-            if (vfields.empty())
-                throw msgpack::type_error();
             if (auto rvalues = findMapValue(*parsedReq.fields, "v"sv)) {
                 if (rvalues->type != msgpack::type::ARRAY)
                     throw msgpack::type_error();

tests/test_dhtrunner.cpp

@@ -303,25 +303,14 @@ DhtRunnerTester::testImportValuesPreservesStoredExpiration()
 void
 DhtRunnerTester::testImportPermanentValueExpires()
 {
-    static const dht::ValueType SHORT_TYPE {201, "Short perm", 500ms};
-
-    dht::DhtRunner::Config config;
-    config.dht_config.node_config.max_peer_req_per_sec = -1;
-    config.dht_config.node_config.max_req_per_sec = -1;
-    config.dht_config.node_config.max_store_size = -1;
-    config.dht_config.node_config.max_store_keys = -1;
-    dht::DhtRunner restored;
-    restored.run(0, config);
-    restored.registerInsecureType(SHORT_TYPE);
-
-    // Ensure the runner is waiting on a previously computed scheduler deadline.
-    std::promise<void> loopCompleted;
-    auto loopCompletedFuture = loopCompleted.get_future();
-    restored.getNodeInfo([&loopCompleted](auto) { loopCompleted.set_value(); });
-    getFutureValue(std::move(loopCompletedFuture));
+    // Simulate importing a value that was originally stored as permanent
+    // (expiration = time_point::max()). After import the permanent flag is lost,
+    // so the value must expire within the type's normal expiration window.
+    static const dht::ValueType SHORT_TYPE {201, "Short perm", 2s};
 
     auto key = dht::InfoHash::get("import-permanent-expiration");
     auto now = dht::clock::now();
+
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack_array(1);
@@ -329,33 +318,63 @@ DhtRunnerTester::testImportPermanentValueExpires()
     pk.pack(now.time_since_epoch().count());
     dht::Value {SHORT_TYPE.id, std::string("was-permanent")}.msgpack_pack(pk);
     pk.pack_bin(0);
+    // Permanent values are exported with time_point::max() expiration
     pk.pack(dht::clock::time_point::max().time_since_epoch().count());
 
+    dht::DhtRunner restored;
+    dht::DhtRunner::Config config;
+    config.dht_config.node_config.max_peer_req_per_sec = -1;
+    config.dht_config.node_config.max_req_per_sec = -1;
+    config.dht_config.node_config.max_store_size = -1;
+    config.dht_config.node_config.max_store_keys = -1;
+    restored.run(0, config);
+
     restored.importValues({
         dht::ValuesExport {key, {buffer.data(), buffer.data() + buffer.size()}}
     });
 
-    auto containsValue = [&] {
-        for (const auto& [exportedKey, data] : restored.exportValues()) {
-            if (exportedKey != key)
-                continue;
+    // Value should be present right after import
+    auto exportedBefore = restored.exportValues();
+    CPPUNIT_ASSERT(!exportedBefore.empty());
+
+    // Wait longer than the default USER_DATA expiration (10 min) is not
+    // practical, but the type is unregistered so the default 10 min expiration
+    // applies.  Instead, register the short type so the cap uses 2 s.
+    // Re-import into a fresh node that knows the short type.
+    dht::DhtRunner restored2;
+    restored2.run(0, config);
+    restored2.registerInsecureType(SHORT_TYPE);
+    restored2.importValues({
+        dht::ValuesExport {key, {buffer.data(), buffer.data() + buffer.size()}}
+    });
+
+    CPPUNIT_ASSERT(!restored2.exportValues().empty());
+
+    // Wait for the short expiration to pass
+    std::this_thread::sleep_for(3s);
+
+    auto restoredValues = restored2.exportValues();
+    bool found = false;
+    for (const auto& ve : restoredValues) {
+        if (ve.first == key) {
             msgpack::unpacked msg;
-            msgpack::unpack(msg, reinterpret_cast<const char*>(data.data()), data.size());
+            msgpack::unpack(msg, (const char*) ve.second.data(), ve.second.size());
             auto valarr = msg.get();
             CPPUNIT_ASSERT(valarr.type == msgpack::type::ARRAY);
-            return valarr.via.array.size != 0;
+            // The value should have expired
+            CPPUNIT_ASSERT_EQUAL(uint32_t(0), valarr.via.array.size);
+            found = true;
         }
-        return false;
-    };
-
-    CPPUNIT_ASSERT(containsValue());
-    auto timeout = dht::clock::now() + 2s;
-    while (containsValue() and dht::clock::now() < timeout)
-        std::this_thread::sleep_for(10ms);
-    CPPUNIT_ASSERT(!containsValue());
+    }
+    // Either the key was cleaned up entirely or it was found with 0 values
+    CPPUNIT_ASSERT(found || restoredValues.empty()
+                   || std::none_of(restoredValues.begin(), restoredValues.end(),
+                                   [&](const auto& ve) { return ve.first == key; }));
 
     restored.shutdown();
     restored.join();
+    restored2.shutdown();
+    restored2.join();
 }
 
 void

tests/test_networkengine.cpp

@@ -214,12 +214,9 @@ makeEngine(std::unique_ptr<TestDatagramSocket>&& socket,
            std::mt19937_64& rd,
            int& onNewNodeCalls)
 {
-    net::NetworkConfig config {};
-    config.max_req_per_sec = -1;
-    config.max_peer_req_per_sec = -1;
     return net::NetworkEngine(
         myid,
-        config,
+        {},
         std::move(socket),
         {},
         rd,
@@ -237,21 +234,6 @@ makeEngine(std::unique_ptr<TestDatagramSocket>&& socket,
         [](Sp<Node>, const InfoHash&, const Blob&, const Value::Id&) { return net::RequestAnswer {}; });
 }
 
-static Sp<Node>
-addPendingGetRequest(net::NetworkEngine& engine, const InfoHash& id, const SockAddr& addr, Tid tid)
-{
-    auto node = engine.insertNode(id, addr);
-    auto request = std::make_shared<net::Request>(
-        MessageType::GetValues,
-        tid,
-        node,
-        Blob {},
-        [](const net::Request&, ParsedMessage&&) {},
-        [](const net::Request&, bool) {});
-    node->requested(request);
-    return node;
-}
-
 void
 NetworkEngineTester::setUp()
 {}
@@ -261,130 +243,24 @@ NetworkEngineTester::tearDown()
 {}
 
 void
-NetworkEngineTester::testCompletesPartialSessionWithDataBeforeHeader()
+NetworkEngineTester::testIgnoresUnknownPartialData()
 {
     Scheduler scheduler;
     std::mt19937_64 rd(0);
     InfoHash myid = InfoHash::getRandom(rd);
-    InfoHash remoteId = InfoHash::getRandom(rd);
     int onNewNodeCalls = 0;
     auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
 
-    auto serialized = serializeValue("fragment before header");
+    Blob fragment = {1, 2, 3};
+    auto packet = makeValueDataPacketBlob(99, 0, 0, fragment);
     auto from = makeIPv4("127.0.0.2", 5000);
-    addPendingGetRequest(engine, remoteId, from, 99);
-    auto initialOnNewNodeCalls = onNewNodeCalls;
-    auto fragment = makeValueDataPacketBlob(99, 0, 0, serialized);
-    auto header = makeReplyHeaderPacketBlob(remoteId, 99, {serialized.size()});
 
-    engine.processMessage(fragment.data(), fragment.size(), from);
-    CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
-    CPPUNIT_ASSERT_EQUAL(initialOnNewNodeCalls, onNewNodeCalls);
-
-    engine.processMessage(header.data(), header.size(), from);
-
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-    CPPUNIT_ASSERT(onNewNodeCalls > initialOnNewNodeCalls);
-}
-
-void
-NetworkEngineTester::testBuffersLargeValueBeforeHeader()
-{
-    Scheduler scheduler;
-    std::mt19937_64 rd(8);
-    InfoHash myid = InfoHash::getRandom(rd);
-    InfoHash remoteId = InfoHash::getRandom(rd);
-    int onNewNodeCalls = 0;
-    auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
-
-    auto serialized = serializeValue(std::string(57 * 1024, 'l'));
-    auto from = makeIPv4("127.0.0.2", 5008);
-    addPendingGetRequest(engine, remoteId, from, 101);
-    auto initialOnNewNodeCalls = onNewNodeCalls;
-    for (size_t offset = 0; offset < serialized.size(); offset += TEST_MTU) {
-        auto end = std::min(offset + TEST_MTU, serialized.size());
-        Blob data(serialized.begin() + offset, serialized.begin() + end);
-        auto fragment = makeValueDataPacketBlob(101, 0, offset, data);
-        engine.processMessage(fragment.data(), fragment.size(), from);
-    }
-    CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
-
-    auto header = makeReplyHeaderPacketBlob(remoteId, 101, {serialized.size()});
-    engine.processMessage(header.data(), header.size(), from);
-
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-    CPPUNIT_ASSERT(onNewNodeCalls > initialOnNewNodeCalls);
-}
-
-void
-NetworkEngineTester::testIgnoresEmptyPartialData()
-{
-    Scheduler scheduler;
-    std::mt19937_64 rd(7);
-    InfoHash myid = InfoHash::getRandom(rd);
-    int onNewNodeCalls = 0;
-    auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
-
-    auto packet = makeValueDataPacketBlob(100, 0, 0, {});
-    auto from = makeIPv4("127.0.0.2", 5007);
     engine.processMessage(packet.data(), packet.size(), from);
 
     CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
     CPPUNIT_ASSERT_EQUAL(0, onNewNodeCalls);
 }
 
-void
-NetworkEngineTester::testRejectsUnsolicitedPartialReply()
-{
-    Scheduler scheduler;
-    std::mt19937_64 rd(9);
-    InfoHash myid = InfoHash::getRandom(rd);
-    InfoHash remoteId = InfoHash::getRandom(rd);
-    int onNewNodeCalls = 0;
-    auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
-
-    auto from = makeIPv4("127.0.0.2", 5009);
-    auto header = makeReplyHeaderPacketBlob(remoteId, 102, {TEST_MTU});
-    engine.processMessage(header.data(), header.size(), from);
-
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getNodeCacheSize());
-}
-
-void
-NetworkEngineTester::testDropsMalformedCompletedPartialMessage()
-{
-    Scheduler scheduler;
-    std::mt19937_64 rd(10);
-    InfoHash myid = InfoHash::getRandom(rd);
-    InfoHash remoteId = InfoHash::getRandom(rd);
-    int onNewNodeCalls = 0;
-    auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
-
-    auto from = makeIPv4("127.0.0.2", 5010);
-    addPendingGetRequest(engine, remoteId, from, 103);
-    Blob malformed {0xc1};
-    auto header = makeReplyHeaderPacketBlob(remoteId, 103, {malformed.size()});
-    auto fragment = makeValueDataPacketBlob(103, 0, 0, malformed);
-
-    engine.processMessage(header.data(), header.size(), from);
-    CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
-    engine.processMessage(fragment.data(), fragment.size(), from);
-
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-
-    auto earlyFrom = makeIPv4("127.0.0.2", 5011);
-    addPendingGetRequest(engine, remoteId, earlyFrom, 104);
-    auto earlyHeader = makeReplyHeaderPacketBlob(remoteId, 104, {malformed.size()});
-    auto earlyFragment = makeValueDataPacketBlob(104, 0, 0, malformed);
-
-    engine.processMessage(earlyFragment.data(), earlyFragment.size(), earlyFrom);
-    CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
-    engine.processMessage(earlyHeader.data(), earlyHeader.size(), earlyFrom);
-
-    CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-}
-
 void
 NetworkEngineTester::testCompletesPartialSessionFromSameSource()
 {
@@ -398,8 +274,6 @@ NetworkEngineTester::testCompletesPartialSessionFromSameSource()
     std::string data(TEST_MTU * 2 + 64, 'z');
     auto serialized = serializeValue(data);
     auto from = makeIPv4("127.0.0.2", 5001);
-    addPendingGetRequest(engine, remoteId, from, 7);
-    auto initialOnNewNodeCalls = onNewNodeCalls;
     auto header = makeReplyHeaderPacketBlob(remoteId, 7, {serialized.size()});
 
     engine.processMessage(header.data(), header.size(), from);
@@ -413,40 +287,45 @@ NetworkEngineTester::testCompletesPartialSessionFromSameSource()
     }
 
     CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-    CPPUNIT_ASSERT(onNewNodeCalls > initialOnNewNodeCalls);
+    CPPUNIT_ASSERT(onNewNodeCalls > 0);
 }
 
 void
-NetworkEngineTester::testSeparatesPartialSessionsBySource()
+NetworkEngineTester::testKeepsSessionForWrongSourceFragment()
 {
     Scheduler scheduler;
     std::mt19937_64 rd(2);
     InfoHash myid = InfoHash::getRandom(rd);
-    InfoHash firstRemoteId = InfoHash::getRandom(rd);
-    InfoHash secondRemoteId = InfoHash::getRandom(rd);
+    InfoHash remoteId = InfoHash::getRandom(rd);
     int onNewNodeCalls = 0;
     auto engine = makeEngine(std::make_unique<TestDatagramSocket>(), scheduler, myid, rd, onNewNodeCalls);
 
-    auto serialized = serializeValue("same transaction, different peers");
-    auto firstFrom = makeIPv4("127.0.0.2", 5002);
-    auto secondFrom = makeIPv4("127.0.0.3", 5002);
-    addPendingGetRequest(engine, firstRemoteId, firstFrom, 11);
-    addPendingGetRequest(engine, secondRemoteId, secondFrom, 11);
-    auto initialOnNewNodeCalls = onNewNodeCalls;
-    auto firstHeader = makeReplyHeaderPacketBlob(firstRemoteId, 11, {serialized.size()});
-    auto secondHeader = makeReplyHeaderPacketBlob(secondRemoteId, 11, {serialized.size()});
+    std::string data(TEST_MTU + 32, 'w');
+    auto serialized = serializeValue(data);
+    auto goodFrom = makeIPv4("127.0.0.2", 5002);
+    auto badFrom = makeIPv4("127.0.0.3", 5002);
+    auto header = makeReplyHeaderPacketBlob(remoteId, 11, {serialized.size()});
 
-    engine.processMessage(firstHeader.data(), firstHeader.size(), firstFrom);
-    engine.processMessage(secondHeader.data(), secondHeader.size(), secondFrom);
-    CPPUNIT_ASSERT_EQUAL((size_t) 2, engine.getPartialCount());
-
-    auto fragment = makeValueDataPacketBlob(11, 0, 0, serialized);
-    engine.processMessage(fragment.data(), fragment.size(), firstFrom);
+    engine.processMessage(header.data(), header.size(), goodFrom);
     CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
-    engine.processMessage(fragment.data(), fragment.size(), secondFrom);
+
+    auto firstEnd = std::min(TEST_MTU, serialized.size());
+    Blob firstFragment(serialized.begin(), serialized.begin() + firstEnd);
+    auto firstPacket = makeValueDataPacketBlob(11, 0, 0, firstFragment);
+    engine.processMessage(firstPacket.data(), firstPacket.size(), badFrom);
+
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, engine.getPartialCount());
+    CPPUNIT_ASSERT_EQUAL(0, onNewNodeCalls);
+
+    engine.processMessage(firstPacket.data(), firstPacket.size(), goodFrom);
+    if (firstEnd < serialized.size()) {
+        Blob tail(serialized.begin() + firstEnd, serialized.end());
+        auto tailPacket = makeValueDataPacketBlob(11, 0, firstEnd, tail);
+        engine.processMessage(tailPacket.data(), tailPacket.size(), goodFrom);
+    }
 
     CPPUNIT_ASSERT_EQUAL((size_t) 0, engine.getPartialCount());
-    CPPUNIT_ASSERT(onNewNodeCalls > initialOnNewNodeCalls);
+    CPPUNIT_ASSERT(onNewNodeCalls > 0);
 }
 
 void

tests/test_networkengine.h

@@ -13,13 +13,9 @@ class NetworkEngineTester : public CppUnit::TestFixture
 #ifdef _MSC_VER
     CPPUNIT_TEST(testDisabledOnMsvc);
 #else
-    CPPUNIT_TEST(testCompletesPartialSessionWithDataBeforeHeader);
-    CPPUNIT_TEST(testBuffersLargeValueBeforeHeader);
-    CPPUNIT_TEST(testIgnoresEmptyPartialData);
-    CPPUNIT_TEST(testRejectsUnsolicitedPartialReply);
-    CPPUNIT_TEST(testDropsMalformedCompletedPartialMessage);
+    CPPUNIT_TEST(testIgnoresUnknownPartialData);
     CPPUNIT_TEST(testCompletesPartialSessionFromSameSource);
-    CPPUNIT_TEST(testSeparatesPartialSessionsBySource);
+    CPPUNIT_TEST(testKeepsSessionForWrongSourceFragment);
     CPPUNIT_TEST(testListenConfirmationCarriesToken);
     CPPUNIT_TEST(testListenConfirmationUpdatesSearchNodeToken);
     CPPUNIT_TEST(testListenReopensSocketAfterNodeExpiration);
@@ -34,13 +30,9 @@ public:
 #ifdef _MSC_VER
     void testDisabledOnMsvc();
 #else
-    void testCompletesPartialSessionWithDataBeforeHeader();
-    void testBuffersLargeValueBeforeHeader();
-    void testIgnoresEmptyPartialData();
-    void testRejectsUnsolicitedPartialReply();
-    void testDropsMalformedCompletedPartialMessage();
+    void testIgnoresUnknownPartialData();
     void testCompletesPartialSessionFromSameSource();
-    void testSeparatesPartialSessionsBySource();
+    void testKeepsSessionForWrongSourceFragment();
     void testListenConfirmationCarriesToken();
     void testListenConfirmationUpdatesSearchNodeToken();
     void testListenReopensSocketAfterNodeExpiration();

tests/test_parsedmessage.cpp

@@ -328,104 +328,6 @@ ParsedMessageTester::testParseRejectsInvalidValuesField()
     CPPUNIT_ASSERT_THROW(parsed.msgpack_unpack(msg.get()), msgpack::type_error);
 }
 
-void
-ParsedMessageTester::testParseRejectsEmptyFieldSelection()
-{
-    msgpack::sbuffer buffer;
-    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-
-    pk.pack_map(3);
-    pk.pack(KEY_R);
-    pk.pack_map(1);
-    pk.pack(KEY_REQ_FIELDS);
-    pk.pack_map(2);
-    pk.pack("f");
-    pk.pack_array(0);
-    pk.pack("v");
-    pk.pack_array(0);
-    pk.pack(KEY_TID);
-    pk.pack(1u);
-    pk.pack(KEY_Y);
-    pk.pack(KEY_R);
-
-    auto msg = msgpack::unpack(buffer.data(), buffer.size());
-    ParsedMessage parsed;
-    CPPUNIT_ASSERT_THROW(parsed.msgpack_unpack(msg.get()), msgpack::type_error);
-}
-
-void
-ParsedMessageTester::testParseRejectsEmptyErrorArray()
-{
-    msgpack::sbuffer buffer;
-    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-
-    pk.pack_map(3);
-    pk.pack(KEY_E);
-    pk.pack_array(0);
-    pk.pack(KEY_TID);
-    pk.pack(1u);
-    pk.pack(KEY_Y);
-    pk.pack(KEY_E);
-
-    auto msg = msgpack::unpack(buffer.data(), buffer.size());
-    ParsedMessage parsed;
-    CPPUNIT_ASSERT_THROW(parsed.msgpack_unpack(msg.get()), msgpack::type_error);
-}
-
-void
-ParsedMessageTester::testParseRejectsNonMapRequest()
-{
-    msgpack::sbuffer buffer;
-    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-
-    pk.pack_map(4);
-    pk.pack(KEY_A);
-    pk.pack_array(2);
-    pk.pack(1u);
-    pk.pack(2u);
-    pk.pack(KEY_Q);
-    pk.pack(QUERY_PING);
-    pk.pack(KEY_TID);
-    pk.pack(1u);
-    pk.pack(KEY_Y);
-    pk.pack("q");
-
-    auto msg = msgpack::unpack(buffer.data(), buffer.size());
-    ParsedMessage parsed;
-    CPPUNIT_ASSERT_THROW(parsed.msgpack_unpack(msg.get()), msgpack::type_error);
-}
-
-void
-ParsedMessageTester::testParseErrorOnlyMessageKeepsRequestFieldsEmpty()
-{
-    msgpack::sbuffer buffer;
-    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-
-    // An error message with no request map: the error array must not be walked
-    // as if it were a sequence of key/value pairs.
-    pk.pack_map(3);
-    pk.pack(KEY_E);
-    pk.pack_array(2);
-    pk.pack(401u);
-    pk.pack("unauthorized");
-    pk.pack(KEY_TID);
-    pk.pack(9u);
-    pk.pack(KEY_Y);
-    pk.pack(KEY_E);
-
-    auto msg = msgpack::unpack(buffer.data(), buffer.size());
-    ParsedMessage parsed;
-    parsed.msgpack_unpack(msg.get());
-
-    CPPUNIT_ASSERT(parsed.type == MessageType::Error);
-    CPPUNIT_ASSERT_EQUAL((uint16_t) 401, parsed.error_code);
-    CPPUNIT_ASSERT(not parsed.id);
-    CPPUNIT_ASSERT(parsed.token.empty());
-    CPPUNIT_ASSERT(parsed.values.empty());
-    CPPUNIT_ASSERT(parsed.nodes4_raw.empty());
-    CPPUNIT_ASSERT(parsed.nodes6_raw.empty());
-}
-
 void
 ParsedMessageTester::testParseIgnoresIncompleteValueDataEntry()
 {

tests/test_parsedmessage.h

@@ -24,10 +24,6 @@ class ParsedMessageTester : public CppUnit::TestFixture
     CPPUNIT_TEST(testParseRejectsNonMapPacket);
     CPPUNIT_TEST(testParseRejectsInvalidValueDataPayload);
     CPPUNIT_TEST(testParseRejectsInvalidValuesField);
-    CPPUNIT_TEST(testParseRejectsEmptyFieldSelection);
-    CPPUNIT_TEST(testParseRejectsEmptyErrorArray);
-    CPPUNIT_TEST(testParseRejectsNonMapRequest);
-    CPPUNIT_TEST(testParseErrorOnlyMessageKeepsRequestFieldsEmpty);
     CPPUNIT_TEST(testParseIgnoresIncompleteValueDataEntry);
     CPPUNIT_TEST(testCompleteEmpty);
     CPPUNIT_TEST(testCompleteIncomplete);
@@ -57,10 +53,6 @@ public:
     void testParseRejectsNonMapPacket();
     void testParseRejectsInvalidValueDataPayload();
     void testParseRejectsInvalidValuesField();
-    void testParseRejectsEmptyFieldSelection();
-    void testParseRejectsEmptyErrorArray();
-    void testParseRejectsNonMapRequest();
-    void testParseErrorOnlyMessageKeepsRequestFieldsEmpty();
     void testParseIgnoresIncompleteValueDataEntry();
     void testCompleteEmpty();
     void testCompleteIncomplete();

vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "opendht",
-  "version-string": "4.3.0",
+  "version-string": "4.2.0",
   "port-version": 1,
   "description": "OpenDHT: A lightweight C++17 Distributed Hash Table library.",
   "homepage": "https://opendht.net",