ci/github: cleanup

This reverts commit 965fbb43c7.

.github/workflows/ccpp.yml

@@ -4,55 +4,63 @@ on: [push, pull_request]
 
 jobs:
   build-ubuntu:
-    name: Ubuntu/GCC build
+    name: Ubuntu/GCC Autotools build
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: deps
+    - name: Dependencies
       run: |
-        sudo apt install libncurses5-dev libreadline-dev nettle-dev \
+        sudo apt install libncurses5-dev libreadline-dev nettle-dev libasio-dev \
         libgnutls28-dev libuv1-dev cython3 python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev \
         autotools-dev autoconf libfmt-dev libhttp-parser-dev libmsgpack-dev libargon2-0-dev
-    - name: autogen
-      run: ./autogen.sh
-    - name: asio
+    - name: Configure
       run: |
-        wget https://github.com/aberaud/asio/archive/b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz \
-        && tar -xvf b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz && cd asio-b2b7a1c166390459e1c169c8ae9ef3234b361e3f/asio \
-        && ./autogen.sh && ./configure --prefix=/usr --without-boost --disable-examples --disable-tests  \
-        && sudo make install \
-        && cd ../../ && rm -rf asio*
-    - name: configure
-      run: ./configure
-    - name: make
+        ./autogen.sh
+        ./configure
+    - name: Build
       run: make
-    - name: make check
-      run: make check
+
+  build-ubuntu-meson:
+    name: Ubuntu/GCC Meson build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Dependencies
+      run: |
+        sudo apt install meson ninja-build libncurses5-dev libreadline-dev nettle-dev libasio-dev \
+        libgnutls28-dev libuv1-dev cython3 python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev \
+        libfmt-dev libhttp-parser-dev libmsgpack-dev libargon2-0-dev
+    - name: Configure
+      run: meson setup build .
+    - name: Build
+      run: cd build && ninja
 
   build-ubuntu-minimal:
     name: Ubuntu/GCC minimal build
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: deps
+    - name: Dependencies
       run: |
         sudo apt install libncurses5-dev libreadline-dev nettle-dev \
         libgnutls28-dev libcppunit-dev libmsgpack-dev libargon2-0-dev
-    - name: cmake
+    - name: Configure
       run: |
         mkdir build && cd build
         cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Debug \
                  -DOPENDHT_C=Off -DOPENDHT_TESTS=On -DOPENDHT_PEER_DISCOVERY=Off -DOPENDHT_PYTHON=Off \
                  -DOPENDHT_TOOLS=On -DOPENDHT_PROXY_SERVER=Off -DOPENDHT_PROXY_CLIENT=Off
-    - name: make
+    - name: Build
       run: cd build && make
+    - name: Unit tests
+      run: cd build && ./opendht_unit_tests
 
   build-macos:
     name: macOS/Clang build
     runs-on: macos-11
     steps:
     - uses: actions/checkout@v3
-    - name: deps
+    - name: Dependencies
       run: |
         brew install msgpack-cxx asio gnutls nettle readline fmt jsoncpp argon2 openssl http-parser cppunit
 
@@ -68,7 +76,7 @@ jobs:
         make -j8 && sudo make install
         cd ../../.. && rm -rf restinio
 
-    - name: cmake
+    - name: Configure
       run: |
         mkdir build && cd build
         export PATH="/opt/homebrew/opt/openssl@3/bin:$PATH"
@@ -79,6 +87,6 @@ jobs:
                  -DOPENDHT_C=On -DOPENDHT_TESTS=On -DOPENDHT_PEER_DISCOVERY=On -DOPENDHT_PYTHON=Off \
                  -DOPENDHT_TOOLS=On -DOPENDHT_PROXY_SERVER=On -DOPENDHT_PROXY_CLIENT=On -DOPENDHT_PUSH_NOTIFICATIONS=On
 
-    - name: make
+    - name: Build
       run: cd build && make
 

.github/workflows/release-package.yml

@@ -13,6 +13,7 @@ env:
   IMAGE_NAME_LLVM: ${{ github.repository }}/opendht-llvm
   IMAGE_NAME_DHTNODE: ${{ github.repository }}/dhtnode
   IMAGE_NAME_ALPINE: ${{ github.repository }}/opendht-alpine
+  IMAGE_NAME_ALPINE_DEPS: ${{ github.repository }}/opendht-deps-alpine
 
 jobs:
   build-and-push-deps-image:
@@ -183,6 +184,40 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
+  build-and-push-image-alpine-deps:
+    name: Alpine Deps Docker image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_ALPINE_DEPS }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: docker/DockerfileAlpineDeps
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
   build-and-push-image-alpine:
     name: Alpine Docker image
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -15,7 +15,7 @@ include(cmake/CheckAtomic.cmake)
 include(CTest)
 
 set (opendht_VERSION_MAJOR 2)
-set (opendht_VERSION_MINOR 4.12)
+set (opendht_VERSION_MINOR 5.0)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -178,7 +178,6 @@ set (top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
 # Sources
 list (APPEND opendht_SOURCES
     src/utils.cpp
-    src/infohash.cpp
     src/crypto.cpp
     src/default_types.cpp
     src/node.cpp

c/opendht.cpp

@@ -261,7 +261,7 @@ dht_pkid dht_certificate_get_long_id(const dht_certificate* c) {
 
 dht_publickey* dht_certificate_get_publickey(const dht_certificate* c) {
     const auto& cert = *reinterpret_cast<const CertSp*>(c);
-    return reinterpret_cast<dht_publickey*>(new PubkeySp(std::make_shared<dht::crypto::PublicKey>(cert->getPublicKey())));
+    return reinterpret_cast<dht_publickey*>(new PubkeySp(cert->getSharedPublicKey()));
 }
 
 // dht::crypto::Identity

configure.ac

@@ -1,7 +1,7 @@
 dnl define macros
 m4_define([opendht_major_version], 2)
-m4_define([opendht_minor_version], 4)
-m4_define([opendht_patch_version], 12)
+m4_define([opendht_minor_version], 5)
+m4_define([opendht_patch_version], 0)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
 

docker/DockerfileAlpineDeps

@@ -0,0 +1,23 @@
+FROM alpine:3.17
+LABEL maintainer="Adrien Béraud <adrien.beraud@savoirfairelinux.com>"
+LABEL org.opencontainers.image.source https://github.com/savoirfairelinux/opendht
+
+RUN apk add --no-cache \
+        build-base cmake ninja git wget \
+        cython python3-dev py3-setuptools \
+        ncurses-dev readline-dev nettle-dev \
+        cppunit-dev gnutls-dev jsoncpp-dev \
+        argon2-dev openssl-dev fmt-dev \
+        http-parser-dev asio-dev msgpack-cxx-dev \
+    && rm -rf /var/cache/apk/*
+
+RUN echo "*** Downloading RESTinio ***" \
+    && mkdir restinio && cd restinio \
+    && wget https://github.com/aberaud/restinio/archive/bbaa034dbcc7555ce67df0f8a1475591a7441733.tar.gz \
+    && tar -xzf bbaa034dbcc7555ce67df0f8a1475591a7441733.tar.gz \
+    && cd restinio-bbaa034dbcc7555ce67df0f8a1475591a7441733/dev \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr -DRESTINIO_TEST=OFF -DRESTINIO_SAMPLE=OFF \
+             -DRESTINIO_INSTALL_SAMPLES=OFF -DRESTINIO_BENCH=OFF -DRESTINIO_INSTALL_BENCHES=OFF \
+             -DRESTINIO_FIND_DEPS=ON -DRESTINIO_ALLOW_SOBJECTIZER=Off -DRESTINIO_USE_BOOST_ASIO=none . \
+    && make -j8 && make install \
+    && cd ../../.. && rm -rf restinio

include/opendht/crypto.h

@@ -390,7 +390,11 @@ struct OPENDHT_PUBLIC Certificate {
      */
     Certificate(gnutls_x509_crt_t crt) noexcept : cert(crt) {}
 
-    Certificate(Certificate&& o) noexcept : cert(o.cert), issuer(std::move(o.issuer)) { o.cert = nullptr; };
+    Certificate(Certificate&& o) noexcept
+        : cert(o.cert)
+        , issuer(std::move(o.issuer))
+        , publicKey_(std::move(o.publicKey_))
+        { o.cert = nullptr; };
 
     /**
      * Import certificate (PEM or DER) or certificate chain (PEM),
@@ -498,7 +502,8 @@ struct OPENDHT_PUBLIC Certificate {
     void msgpack_unpack(const msgpack::object& o);
 
     explicit operator bool() const { return cert; }
-    PublicKey getPublicKey() const;
+    const PublicKey& getPublicKey() const;
+    const std::shared_ptr<PublicKey>& getSharedPublicKey() const;
 
     /** Same as getPublicKey().getId() */
     const InfoHash& getId() const;
@@ -631,6 +636,9 @@ private:
     };
 
     std::set<std::shared_ptr<RevocationList>, crlNumberCmp> revocation_lists;
+
+    mutable std::mutex publicKeyMutex_ {};
+    mutable std::shared_ptr<PublicKey> publicKey_ {};
 };
 
 struct OPENDHT_PUBLIC TrustList

include/opendht/dht.h

@@ -128,7 +128,7 @@ public:
      */
     void insertNode(const InfoHash& id, const SockAddr&) override;
     void insertNode(const NodeExport& n) override {
-        insertNode(n.id, SockAddr(n.ss, n.sslen));
+        insertNode(n.id, n.addr);
     }
 
     void pingNode(SockAddr, DoneCallbackSimple&& cb={}) override;

include/opendht/dht_interface.h

@@ -20,6 +20,7 @@
 
 #include "infohash.h"
 #include "log_enable.h"
+#include "node_export.h"
 
 #include <queue>
 

include/opendht/dhtrunner.h

@@ -27,6 +27,7 @@
 #include "sockaddr.h"
 #include "log_enable.h"
 #include "network_utils.h"
+#include "node_export.h"
 
 #include <thread>
 #include <mutex>

include/opendht/infohash.h

@@ -44,6 +44,7 @@ typedef uint16_t in_port_t;
 #include <algorithm>
 #include <stdexcept>
 #include <sstream>
+
 #include <cstring>
 #include <cstddef>
 
@@ -387,25 +388,4 @@ Hash<N>::toString() const
     return std::string(to_c_str(), N*2);
 }
 
-struct OPENDHT_PUBLIC NodeExport {
-    InfoHash id;
-    sockaddr_storage ss;
-    socklen_t sslen;
-
-    template <typename Packer>
-    void msgpack_pack(Packer& pk) const
-    {
-        pk.pack_map(2);
-        pk.pack(std::string("id"));
-        pk.pack(id);
-        pk.pack(std::string("addr"));
-        pk.pack_bin(sslen);
-        pk.pack_bin_body((char*)&ss, sslen);
-    }
-
-    void msgpack_unpack(msgpack::object o);
-
-    OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const NodeExport& h);
-};
-
 }

include/opendht/network_engine.h

@@ -416,12 +416,19 @@ public:
      *
      * @return the request with information concerning its success.
      */
-    Sp<Request> sendUpdateValues(const Sp<Node>& n,
+    void sendUpdateValues(const Sp<Node>& n,
                                  const InfoHash& infohash,
-                                 const std::vector<Sp<Value>>& values,
+                                 std::vector<Sp<Value>>&& values,
                                  time_point created,
                                  const Blob& token,
-                                 const size_t& sid);
+                                 size_t sid);
+    Sp<Request> sendUpdateValues(const Sp<Node>& n,
+                                 const InfoHash& infohash,
+                                 std::vector<Sp<Value>>::iterator begin,
+                                 std::vector<Sp<Value>>::iterator end,
+                                 time_point created,
+                                 const Blob& token,
+                                 size_t sid);
 
     /**
      * Parses a message and calls appropriate callbacks.
@@ -492,6 +499,7 @@ private:
 
     static constexpr size_t MTU {1280};
     static constexpr size_t MAX_PACKET_VALUE_SIZE {600};
+    static constexpr size_t MAX_MESSAGE_VALUE_SIZE {56 * 1024};
 
     static const std::string my_v;
 
@@ -525,7 +533,10 @@ private:
     int send(const SockAddr& addr, const char *buf, size_t len, bool confirmed = false);
 
     void sendValueParts(Tid tid, const std::vector<Blob>& svals, const SockAddr& addr);
-    std::vector<Blob> packValueHeader(msgpack::sbuffer&, const std::vector<Sp<Value>>&);
+    std::vector<Blob> packValueHeader(msgpack::sbuffer&, std::vector<Sp<Value>>::const_iterator, std::vector<Sp<Value>>::const_iterator) const;
+    std::vector<Blob> packValueHeader(msgpack::sbuffer& buf, const std::vector<Sp<Value>>& values) const {
+        return packValueHeader(buf, values.begin(), values.end());
+    }
     void maintainRxBuffer(Tid tid);
 
     /*************

include/opendht/node.h

@@ -22,6 +22,7 @@
 #include "infohash.h" // includes socket structures
 #include "utils.h"
 #include "sockaddr.h"
+#include "node_export.h"
 
 #include <list>
 #include <map>
@@ -52,7 +53,7 @@ struct Node {
     Node(const InfoHash& id, const sockaddr* sa, socklen_t salen, std::mt19937_64& rd)
         : Node(id, SockAddr(sa, salen), rd) {}
 
-    InfoHash getId() const {
+    const InfoHash& getId() const {
         return id;
     }
     const SockAddr& getAddr() const { return addr; }
@@ -92,8 +93,7 @@ struct Node {
     NodeExport exportNode() const {
         NodeExport ne;
         ne.id = id;
-        ne.sslen = addr.getLength();
-        std::memcpy(&ne.ss, addr.get(), ne.sslen);
+        ne.addr = addr;
         return ne;
     }
     sa_family_t getFamily() const { return addr.getFamily(); }

include/opendht/node_export.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (C) 2014-2023 Savoir-faire Linux Inc.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "def.h"
+#include "infohash.h"
+#include "sockaddr.h"
+
+#include <string_view>
+
+namespace dht {
+using namespace std::literals;
+
+struct OPENDHT_PUBLIC NodeExport {
+    InfoHash id;
+    SockAddr addr;
+
+    template <typename Packer>
+    void msgpack_pack(Packer& pk) const
+    {
+        pk.pack_map(2);
+        pk.pack("id"sv);
+        pk.pack(id);
+        pk.pack("addr"sv);
+        pk.pack_bin(addr.getLength());
+        pk.pack_bin_body((const char*)addr.get(), (size_t)addr.getLength());
+    }
+
+    void msgpack_unpack(msgpack::object o);
+
+    OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const NodeExport& h);
+};
+
+}

include/opendht/value.h

@@ -475,7 +475,7 @@ struct OPENDHT_PUBLIC Value
     OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Value& v);
 
     inline std::string toString() const {
-        std::stringstream ss;
+        std::ostringstream ss;
         ss << *this;
         return ss.str();
     }
@@ -774,7 +774,7 @@ struct OPENDHT_PUBLIC Select
     }
 
     std::string toString() const {
-        std::stringstream ss;
+        std::ostringstream ss;
         ss << *this;
         return ss.str();
     }
@@ -897,7 +897,7 @@ struct OPENDHT_PUBLIC Where
     }
 
     std::string toString() const {
-        std::stringstream ss;
+        std::ostringstream ss;
         ss << *this;
         return ss.str();
     }
@@ -963,7 +963,7 @@ struct OPENDHT_PUBLIC Query
     void msgpack_unpack(const msgpack::object& o);
 
     std::string toString() const {
-        std::stringstream ss;
+        std::ostringstream ss;
         ss << *this;
         return ss.str();
     }

meson.build

@@ -7,7 +7,7 @@ project('opendht', 'c', 'cpp',
 
 gnutls = dependency('gnutls')
 nettle = dependency('nettle')
-msgpack = dependency('msgpack-cxx')
+msgpack = dependency('msgpack-cxx', required : false)
 argon2 = dependency('libargon2')
 openssl = dependency('openssl', required: get_option('proxy_client'))
 jsoncpp = dependency('jsoncpp', required: get_option('proxy_client'))
@@ -25,9 +25,9 @@ add_project_arguments('-DMSGPACK_NO_BOOST', language : 'cpp')
 add_project_arguments(['-Wno-return-type','-Wno-deprecated','-Wnon-virtual-dtor','-pedantic-errors','-fvisibility=hidden'], language : 'cpp')
 
 opendht_inc = include_directories('include/opendht')
+opendht_interface_inc = include_directories('include', is_system: true)
 opendht_src = [
     'src/utils.cpp',
-    'src/infohash.cpp',
     'src/crypto.cpp',
     'src/default_types.cpp',
     'src/node.cpp',
@@ -66,10 +66,23 @@ if get_option('peer_discovery').enabled()
     add_project_arguments('-DOPENDHT_PEER_DISCOVERY', language : 'cpp')
 endif
 opendht = shared_library('opendht',
-    opendht_src, 
+    opendht_src,
     include_directories : opendht_inc,
     dependencies : deps,
-    cpp_args : ['-DOPENHT_BUILD', '-Dopendht_EXPORTS'])
+    cpp_args : ['-DOPENHT_BUILD', '-Dopendht_EXPORTS'],
+    install : true)
 
-readline = meson.get_compiler('c').find_library('readline')
-dhtnode = executable('dhtnode', 'tools/dhtnode.cpp', link_with : opendht, dependencies : [readline, msgpack])
+readline = meson.get_compiler('c').find_library('readline', required: get_option('tools'))
+if get_option('tools').enabled()
+    dhtnode = executable('dhtnode', 'tools/dhtnode.cpp',
+        include_directories : opendht_interface_inc,
+        link_with : opendht,
+        dependencies : [readline, msgpack],
+        install : true)
+    if http_parser.found()
+        durl = executable('durl', 'tools/durl.cpp',
+        include_directories : opendht_interface_inc,
+        link_with : opendht,
+        dependencies : [msgpack, openssl])
+    endif
+endif

meson_options.txt

@@ -1,3 +1,4 @@
 option('proxy_client', type : 'feature', value : 'disabled')
 option('proxy_server', type : 'feature', value : 'disabled')
 option('peer_discovery', type : 'feature', value : 'enabled')
+option('tools', type : 'feature', value : 'enabled')

src/Makefile.am

@@ -19,7 +19,6 @@ libopendht_la_SOURCES  = \
         routing_table.cpp \
         network_engine.cpp \
         utils.cpp \
-        infohash.cpp \
         node.cpp \
         value.cpp \
         crypto.cpp \

src/callbacks.cpp

@@ -92,7 +92,7 @@ bindDoneCbSimple(DoneCallbackSimpleRaw raw_cb, void* user_data) {
 std::string
 NodeStats::toString() const
 {
-    std::stringstream ss;
+    std::ostringstream ss;
     ss << "Known nodes: " << good_nodes << " good, " << dubious_nodes << " dubious, " << incoming_nodes << " incoming." << std::endl;
     ss << searches << " searches, " << node_cache_size << " total cached nodes" << std::endl;
     if (table_depth > 1) {

src/crypto.cpp

@@ -366,7 +366,7 @@ PrivateKey::getSharedPublicKey() const
         auto pk = std::make_shared<PublicKey>();
         if (auto err = gnutls_pubkey_import_privkey(pk->pk, key, GNUTLS_KEY_KEY_CERT_SIGN | GNUTLS_KEY_CRL_SIGN, 0))
             throw CryptoException(std::string("Can't retreive public key: ") + gnutls_strerror(err));
-        publicKey_ = pk;
+        publicKey_ = std::move(pk);
     }
     return publicKey_;
 }
@@ -824,13 +824,23 @@ Certificate::~Certificate()
     }
 }
 
-PublicKey
+const PublicKey&
 Certificate::getPublicKey() const
 {
-    PublicKey pk_ret;
-    if (auto err = gnutls_pubkey_import_x509(pk_ret.pk, cert, 0))
-        throw CryptoException(std::string("Can't get certificate public key: ") + gnutls_strerror(err));
-    return pk_ret;
+    return *getSharedPublicKey();
+}
+
+const std::shared_ptr<PublicKey>&
+Certificate::getSharedPublicKey() const
+{
+    std::lock_guard<std::mutex> lock(publicKeyMutex_);
+    if (not publicKey_) {
+        auto pk = std::make_shared<PublicKey>();
+        if (auto err = gnutls_pubkey_import_x509(pk->pk, cert, 0))
+            throw CryptoException(std::string("Can't get certificate public key: ") + gnutls_strerror(err));
+        publicKey_ = std::move(pk);
+    }
+    return publicKey_;
 }
 
 const InfoHash&

src/dht.cpp

@@ -1642,7 +1642,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const
 void
 Dht::dumpTables() const
 {
-    std::stringstream out;
+    std::ostringstream out;
     out << "My id " << myid << std::endl;
 
     out << "Buckets IPv4 :" << std::endl;
@@ -1694,7 +1694,7 @@ Dht::getStorageLog(const InfoHash& h) const
 {
     auto s = store.find(h);
     if (s == store.end()) {
-        std::stringstream out;
+        std::ostringstream out;
         out << "Storage " << h << " empty" << std::endl;
         return out.str();
     }
@@ -1704,7 +1704,7 @@ Dht::getStorageLog(const InfoHash& h) const
 std::string
 Dht::printStorageLog(const decltype(store)::value_type& s) const
 {
-    std::stringstream out;
+    std::ostringstream out;
     using namespace std::chrono;
     const auto& st = s.second;
     out << "Storage " << s.first << " "
@@ -1723,7 +1723,7 @@ Dht::printStorageLog(const decltype(store)::value_type& s) const
 std::string
 Dht::getRoutingTablesLog(sa_family_t af) const
 {
-    std::stringstream out;
+    std::ostringstream out;
     for (const auto& b : buckets(af))
         dumpBucket(b, out);
     return out.str();
@@ -1732,7 +1732,7 @@ Dht::getRoutingTablesLog(sa_family_t af) const
 std::string
 Dht::getSearchesLog(sa_family_t af) const
 {
-    std::stringstream out;
+    std::ostringstream out;
     auto num_searches = dht4.searches.size() + dht6.searches.size();
     if (num_searches > 8) {
         if (not af or af == AF_INET)
@@ -1757,7 +1757,7 @@ Dht::getSearchesLog(sa_family_t af) const
 std::string
 Dht::getSearchLog(const InfoHash& id, sa_family_t af) const
 {
-    std::stringstream out;
+    std::ostringstream out;
     if (af == AF_UNSPEC) {
         out << getSearchLog(id, AF_INET) << getSearchLog(id, AF_INET6);
     } else {
@@ -2638,7 +2638,7 @@ Dht::loadState(const std::string& path)
             std::vector<Sp<Node>> tmpNodes;
             tmpNodes.reserve(state.nodes.size());
             for (const auto& node : state.nodes)
-                tmpNodes.emplace_back(network_engine.insertNode(node.id, SockAddr(node.ss, node.sslen)));
+                tmpNodes.emplace_back(network_engine.insertNode(node.id, node.addr));
             importValues(state.values);
         }
     } catch (const std::exception& e) {

src/dht_proxy_server.cpp

@@ -704,7 +704,7 @@ DhtProxyServer::get(restinio::request_handle_t request,
             initHttpResponse(request->create_response<ResponseByParts>()));
         response->flush();
         dht_->get(infoHash, [this, response](const std::vector<Sp<Value>>& values) {
-            std::stringstream output;
+            std::ostringstream output;
             for (const auto& value : values) {
                 output << Json::writeString(jsonBuilder_, value->toJson()) << "\n";
             }

src/dhtrunner.cpp

@@ -1088,6 +1088,7 @@ DhtRunner::setProxyServer(const std::string& proxy, const std::string& pushNodeI
 #else
     if (not proxy.empty())
         throw std::runtime_error("DHT proxy requested but OpenDHT built without proxy support.");
+    (void) pushNodeId;
 #endif
 }
 

src/http.cpp

@@ -87,7 +87,7 @@ Url::Url(const std::string& url): url(url)
 std::string
 Url::toString() const
 {
-    std::stringstream ss;
+    std::ostringstream ss;
     if (not protocol.empty()) {
         ss << protocol << "://";
     }
@@ -1044,7 +1044,7 @@ Request::set_auth(const std::string& username, const std::string& password)
 void
 Request::build()
 {
-    std::stringstream request;
+    std::ostringstream request;
     bool append_body = !body_.empty();
 
     // first header

src/indexation/pht.cpp

@@ -32,7 +32,7 @@ namespace indexation {
  * @return string that represent the blob into a readable way
  */
 static std::string blobToString(const Blob &bl) {
-    std::stringstream ss;
+    std::ostringstream ss;
     auto bn = bl.size() % 8;
     auto n = bl.size() / 8;
 
@@ -46,7 +46,7 @@ static std::string blobToString(const Blob &bl) {
 }
 
 std::string Prefix::toString() const {
-    std::stringstream ss;
+    std::ostringstream ss;
 
     ss << "Prefix : " << std::endl << "\tContent_ : \"";
     ss << blobToString(content_);

src/infohash.cpp

@@ -1,58 +0,0 @@
-/*
- *  Copyright (C) 2014-2022 Savoir-faire Linux Inc.
- *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program. If not, see <https://www.gnu.org/licenses/>.
- */
-
-#include "infohash.h"
-
-#include <functional>
-#include <sstream>
-#include <cstdio>
-
-using namespace std::literals;
-
-namespace dht {
-
-const HexMap hex_map = {};
-
-void
-NodeExport::msgpack_unpack(msgpack::object o)
-{
-    if (o.type != msgpack::type::MAP)
-        throw msgpack::type_error();
-    if (o.via.map.size < 2)
-        throw msgpack::type_error();
-    if (o.via.map.ptr[0].key.as<std::string_view>() != "id"sv)
-        throw msgpack::type_error();
-    if (o.via.map.ptr[1].key.as<std::string_view>() != "addr"sv)
-        throw msgpack::type_error();
-    const auto& addr = o.via.map.ptr[1].val;
-    if (addr.type != msgpack::type::BIN)
-        throw msgpack::type_error();
-    if (addr.via.bin.size > sizeof(sockaddr_storage))
-        throw msgpack::type_error();
-    id.msgpack_unpack(o.via.map.ptr[0].val);
-    sslen = addr.via.bin.size;
-    std::copy_n(addr.via.bin.ptr, addr.via.bin.size, (char*)&ss);
-}
-
-std::ostream& operator<< (std::ostream& s, const NodeExport& h)
-{
-    msgpack::pack(s, h);
-    return s;
-}
-
-}

src/network_engine.cpp

@@ -124,9 +124,9 @@ NetworkEngine::tellListener(const Sp<Node>& node, Tid socket_id, const InfoHash&
     auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
     try {
         if (version >= 1) {
-            sendUpdateValues(node, hash, values, scheduler.time(), ntoken, socket_id);
+            sendUpdateValues(node, hash, std::move(values), scheduler.time(), ntoken, socket_id);
         } else {
-            sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, values, query, ntoken);
+            sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, std::move(values), query, ntoken);
         }
     } catch (const std::overflow_error& e) {
         if (logger_)
@@ -501,9 +501,8 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
             pmsg.last_part = now;
             scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
             scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
-        } else
-            if (logger_)
-                logger_->e("Partial message with given TID already exists");
+        } else if (logger_)
+            logger_->e("Partial message with given TID %u already exists", k);
     }
 }
 
@@ -912,12 +911,16 @@ NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) {
 }
 
 std::vector<Blob>
-NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Value>>& st)
+NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, std::vector<Sp<Value>>::const_iterator b, std::vector<Sp<Value>>::const_iterator e) const
 {
-    auto svals = serializeValues(st);
+    std::vector<Blob> svals;
     size_t total_size = 0;
-    for (const auto& v : svals)
-        total_size += v.size();
+
+    svals.reserve(std::distance(b, e));
+    for (; b != e; ++b) {
+        svals.emplace_back(packMsg(*b));
+        total_size += svals.back().size();
+    }
 
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
     pk.pack(KEY_REQ_VALUES);
@@ -1200,13 +1203,39 @@ NetworkEngine::sendAnnounceValue(const Sp<Node>& n,
     return req;
 }
 
+void
+NetworkEngine::sendUpdateValues(const Sp<Node>& n,
+                                const InfoHash& infohash,
+                                std::vector<Sp<Value>>&& values,
+                                time_point created,
+                                const Blob& token,
+                                size_t socket_id)
+{
+    size_t total_size = 0;
+
+    auto b = values.begin(), e = values.begin();
+    while (e != values.end()) {
+        if (total_size >= MAX_MESSAGE_VALUE_SIZE) {
+            sendUpdateValues(n, infohash, b, e, created, token, socket_id);
+            b = e;
+            total_size = 0;
+        }
+        total_size += (*e)->size();
+        ++e;
+    }
+    if (b != e) {
+        sendUpdateValues(n, infohash, b, e, created, token, socket_id);
+    }
+}
+
 Sp<Request>
 NetworkEngine::sendUpdateValues(const Sp<Node>& n,
                                 const InfoHash& infohash,
-                                const std::vector<Sp<Value>>& values,
+                                std::vector<Sp<Value>>::iterator begin,
+                                std::vector<Sp<Value>>::iterator end,
                                 time_point created,
                                 const Blob& token,
-                                const size_t& socket_id)
+                                size_t socket_id)
 {
     Tid tid (n->getNewTid());
     Tid sid (socket_id);
@@ -1220,7 +1249,7 @@ NetworkEngine::sendUpdateValues(const Sp<Node>& n,
       pk.pack(KEY_VERSION);    pk.pack(1);
       pk.pack(KEY_REQ_H);      pk.pack(infohash);
       pk.pack(KEY_REQ_SID);   pk.pack(sid);
-      auto v = packValueHeader(buffer, values);
+      auto v = packValueHeader(buffer, begin, end);
       if (created < scheduler.time()) {
           pk.pack(KEY_REQ_CREATION);
           pk.pack(to_time_t(created));

src/node.cpp

@@ -160,7 +160,7 @@ Node::closeSocket(Tid id)
 std::string
 Node::toString() const
 {
-    std::stringstream ss;
+    std::ostringstream ss;
     ss << (*this);
     return ss.str();
 }
@@ -171,4 +171,30 @@ std::ostream& operator<< (std::ostream& s, const Node& h)
     return s;
 }
 
+void
+NodeExport::msgpack_unpack(msgpack::object o)
+{
+    if (o.type != msgpack::type::MAP)
+        throw msgpack::type_error();
+    if (o.via.map.size < 2)
+        throw msgpack::type_error();
+    if (o.via.map.ptr[0].key.as<std::string_view>() != "id"sv)
+        throw msgpack::type_error();
+    if (o.via.map.ptr[1].key.as<std::string_view>() != "addr"sv)
+        throw msgpack::type_error();
+    const auto& maddr = o.via.map.ptr[1].val;
+    if (maddr.type != msgpack::type::BIN)
+        throw msgpack::type_error();
+    if (maddr.via.bin.size > sizeof(sockaddr_storage))
+        throw msgpack::type_error();
+    id.msgpack_unpack(o.via.map.ptr[0].val);
+    addr = {(const sockaddr*)maddr.via.bin.ptr, (socklen_t)maddr.via.bin.size};
+}
+
+std::ostream& operator<< (std::ostream& s, const NodeExport& h)
+{
+    msgpack::pack(s, h);
+    return s;
+}
+
 }

src/parsed_message.h

@@ -152,9 +152,10 @@ bool
 ParsedMessage::complete()
 {
     for (auto& e : value_parts) {
-        //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl;
-        if (e.second.first > e.second.second.size())
+        if (e.second.first > e.second.second.size()) {
+            //std::cout << "uncomplete part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl;
             return false;
+        }
     }
     for (auto& e : value_parts) {
         msgpack::unpacked msg;

src/securedht.cpp

@@ -121,7 +121,7 @@ Sp<crypto::PublicKey>
 SecureDht::getPublicKey(const InfoHash& node) const
 {
     if (node == getId())
-        return std::make_shared<crypto::PublicKey>(certificate_->getPublicKey());
+        return certificate_->getSharedPublicKey();
     auto it = nodesPubKeys_.find(node);
     if (it == nodesPubKeys_.end())
         return nullptr;
@@ -217,7 +217,7 @@ SecureDht::findPublicKey(const InfoHash& node, const std::function<void(const Sp
     }
     findCertificate(node, [=](const Sp<crypto::Certificate>& crt) {
         if (crt && *crt) {
-            auto pk = std::make_shared<crypto::PublicKey>(crt->getPublicKey());
+            auto pk = crt->getSharedPublicKey();
             if (*pk) {
                 nodesPubKeys_[pk->getId()] = pk;
                 if (cb) cb(pk);

src/thread_pool.cpp

@@ -26,7 +26,7 @@
 
 namespace dht {
 
-constexpr const size_t IO_THREADS_MAX {64};
+constexpr const size_t IO_THREADS_MAX {512};
 
 ThreadPool&
 ThreadPool::computation()
@@ -65,33 +65,39 @@ ThreadPool::run(std::function<void()>&& cb)
 
     // launch new thread if necessary
     if (not readyThreads_ && threads_.size() < maxThreads_) {
-        threads_.emplace_back(std::make_unique<std::thread>([this]() {
-            while (true) {
-                std::function<void()> task;
+        try {
+            threads_.emplace_back(std::make_unique<std::thread>([this]() {
+                while (true) {
+                    std::function<void()> task;
 
-                // pick task from queue
-                {
-                    std::unique_lock<std::mutex> l(lock_);
-                    readyThreads_++;
-                    cv_.wait(l, [&](){
-                        return not running_ or not tasks_.empty();
-                    });
-                    readyThreads_--;
-                    if (not running_)
-                        break;
-                    task = std::move(tasks_.front());
-                    tasks_.pop();
-                }
+                    // pick task from queue
+                    {
+                        std::unique_lock<std::mutex> l(lock_);
+                        readyThreads_++;
+                        cv_.wait(l, [&](){
+                            return not running_ or not tasks_.empty();
+                        });
+                        readyThreads_--;
+                        if (not running_)
+                            break;
+                        task = std::move(tasks_.front());
+                        tasks_.pop();
+                    }
 
-                // run task
-                try {
-                    task();
-                } catch (const std::exception& e) {
-                    // LOG_ERR("Exception running task: %s", e.what());
-                    std::cerr << "Exception running task: " << e.what() << std::endl;
+                    // run task
+                    try {
+                        task();
+                    } catch (const std::exception& e) {
+                        // LOG_ERR("Exception running task: %s", e.what());
+                        std::cerr << "Exception running task: " << e.what() << std::endl;
+                    }
                 }
-            }
-        }));
+            }));
+        } catch(const std::exception& e) {
+            std::cerr << "Exception starting thread: " << e.what() << std::endl;
+            if (threads_.empty())
+                throw;
+        }
     }
 
     // push task to queue

src/utils.cpp

@@ -35,12 +35,14 @@
 
 namespace dht {
 
-static constexpr std::array<uint8_t, 12> MAPPED_IPV4_PREFIX {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}};
-
 const char* version() {
     return PACKAGE_VERSION;
 }
 
+const HexMap hex_map = {};
+
+static constexpr std::array<uint8_t, 12> MAPPED_IPV4_PREFIX {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}};
+
 std::pair<std::string, std::string>
 splitPort(const std::string& s) {
     if (s.empty())

src/value.cpp

@@ -98,7 +98,7 @@ ValueType::DEFAULT_STORE_POLICY(InfoHash, const std::shared_ptr<Value>& v, const
 size_t
 Value::size() const
 {
-    return cypher.size() + data.size() + signature.size()  + user_type.size();
+    return cypher.size() + data.size() + signature.size() + user_type.size();
 }
 
 void