dht: move socket to SearchNode

dht: add error handler for sendRefreshValue
network engine: complete request in case of error
2026-11-08 20:37:23 +00:00 · 2020-06-12 02:51:08 -04:00 · 2020-06-12 02:51:08 -04:00 · 2020-06-12 02:51:08 -04:00 · 2020-06-12 02:51:08 -04:00 · 2020-06-11 15:10:48 -04:00
115 changed files with 3585 additions and 2209 deletions
@@ -16,13 +16,12 @@ env:

 before_install:
  - |
-    # non llvm builds
-    if [[ "$OPENDHT_TEST_JOB" != *"opendht.llvm"* ]]; then
+    if [[ "$OPENDHT_TEST_JOB" == *"opendht.classic"* ]]; then
+      docker pull aberaud/opendht-deps-bionic;
+    fi
+    if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyclient"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyserver"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.push"* ]]; then
      docker pull aberaud/opendht-deps;
    fi
-
-  - |
-    # classic build
    if [[ "$OPENDHT_TEST_JOB" == *"opendht.llvm"* ]]; then
      docker pull aberaud/opendht-deps-llvm
    fi
@@ -1,112 +1,98 @@
 cmake_minimum_required (VERSION 3.1)
+include(CMakePackageConfigHelpers)
+include(CMakeDependentOption)
+
 project (opendht)
 set (opendht_VERSION_MAJOR 2)
-set (opendht_VERSION_MINOR 0.0)
+set (opendht_VERSION_MINOR 1.3)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")

 # Options
-option (OPENDHT_LIGHT "Build minimal version of the library" OFF)
-
 option (OPENDHT_STATIC "Build static library" ON)
-
 option (OPENDHT_SHARED "Build shared library" ON)
+option (OPENDHT_LOG "Build with logs" ON)
 option (OPENDHT_PYTHON "Build Python bindings" OFF)
 option (OPENDHT_TOOLS "Build DHT tools" ON)
 option (OPENDHT_SYSTEMD "Install systemd module" OFF)
 option (OPENDHT_ARGON2 "Use included argon2 sources" OFF)
+option (OPENDHT_LTO "Build with LTO" OFF)
+option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF)
 option (OPENDHT_PROXY_SERVER "Enable DHT proxy server, use Restinio and jsoncpp" OFF)
 option (OPENDHT_PUSH_NOTIFICATIONS "Enable push notifications support" OFF)
 option (OPENDHT_PROXY_SERVER_IDENTITY "Allow clients to use the node identity" OFF)
 option (OPENDHT_PROXY_CLIENT "Enable DHT proxy client, use Restinio and jsoncpp" OFF)
 option (OPENDHT_PROXY_OPENSSL "Build DHT proxy with OpenSSL" ON)
 option (OPENDHT_PROXY_HTTP_PARSER_FORK "Build DHT proxy with custom http_parser to support old API" OFF)
+CMAKE_DEPENDENT_OPTION(OPENDHT_HTTP "Build embedded http(s) client" OFF "NOT OPENDHT_PROXY_SERVER;NOT OPENDHT_PROXY_CLIENT" ON)
 option (OPENDHT_PEER_DISCOVERY "Enable multicast peer discovery" ON)
 option (OPENDHT_INDEX "Build DHT indexation feature" OFF)
-option (OPENDHT_C "Build C bindings" OFF)
 option (OPENDHT_TESTS "Add unit tests executable" OFF)
-
-if (OPENDHT_LIGHT)
-    set(OPENDHT_SHARED OFF)
-    set(OPENDHT_TOOLS OFF)
-    set(OPENDHT_PROXY_SERVER OFF)
-    set(OPENDHT_PROXY_CLIENT OFF)
-    set(OPENDHT_PEER_DISCOVERY OFF)
-    set(OPENDHT_INDEX OFF)
-    set(OPENDHT_PYTHON OFF)
-    add_definitions(-DOPENDHT_LIGHT)
-endif()
-
-option (OPENDHT_LOG "Build with logs" ON)
-option (OPENDHT_LTO "Build with LTO" OFF)
-option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF)
+option (OPENDHT_C "Build C bindings" OFF)

 find_package(Doxygen)
 option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND})

 # Dependencies
 list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
-find_package (PkgConfig)
+find_package (Threads)
+find_package (PkgConfig REQUIRED)
+find_package (GnuTLS 3.3 REQUIRED)
+pkg_search_module (Nettle nettle)
 find_package (Msgpack 1.2 REQUIRED)
-if (NOT OPENDHT_LIGHT)
-    find_package (Threads)
-    find_package (GnuTLS 3.3 REQUIRED)
-    pkg_search_module (Nettle nettle)
-    if (OPENDHT_TOOLS)
-        find_package (Readline 6 REQUIRED)
-    endif ()
-    if (NOT OPENDHT_ARGON2)
-        find_package(PkgConfig)
-        pkg_search_module(argon2 libargon2)
-        if (argon2_FOUND)
-            message("-- Found Argon2: " ${argon2_LIBRARY_DIRS} " (found version \"" ${argon2_VERSION} "\")")
-            link_directories (${argon2_LIBRARY_DIRS})
+if (OPENDHT_TOOLS)
+    find_package (Readline 6 REQUIRED)
+endif ()
+if (NOT OPENDHT_ARGON2)
+    pkg_search_module(argon2 libargon2)
+    if (argon2_FOUND)
+        message("-- Found Argon2: " ${argon2_LIBRARY_DIRS} " (found version \"" ${argon2_VERSION} "\")")
+        link_directories (${argon2_LIBRARY_DIRS})
+    else ()
+        message("Argon2 not found, using included version.")
+        set(OPENDHT_ARGON2 ON)
+    endif()
+endif ()
+
+pkg_search_module(Jsoncpp jsoncpp)
+if (Jsoncpp_FOUND)
+    add_definitions(-DOPENDHT_JSONCPP)
+    list (APPEND opendht_SOURCES
+      src/base64.h
+      src/base64.cpp
+    )
+endif()
+
+if (OPENDHT_HTTP)
+    find_package(Restinio REQUIRED)
+    if (Restinio_FOUND)
+        find_library(FMT_LIBRARY fmt)
+        add_library(fmt SHARED IMPORTED)
+        find_library(HTTP_PARSER_LIBRARY http_parser)
+        add_library(http_parser SHARED IMPORTED)
+    endif()
+    if (NOT Jsoncpp_FOUND)
+        message(SEND_ERROR "Jsoncpp is required for DHT proxy support")
+    endif()
+    if (OPENDHT_PROXY_OPENSSL)
+        # https://cmake.org/cmake/help/latest/module/FindOpenSSL.html
+        pkg_search_module(OPENSSL REQUIRED openssl)
+        if (OPENSSL_FOUND)
+            message(STATUS "Found OpenSSL ${OPENSSL_VERSION}")
+            include_directories(${OPENSSL_INCLUDE_DIRS})
        else ()
-            message("Argon2 not found, using included version.")
-            set(OPENDHT_ARGON2 ON)
+            message(SEND_ERROR "OpenSSL is required for DHT proxy as specified")
        endif()
-    endif ()
-
-    pkg_search_module(Jsoncpp jsoncpp)
-    if (Jsoncpp_FOUND)
-        add_definitions(-DOPENDHT_JSONCPP)
-        list (APPEND opendht_SOURCES
-        src/base64.h
-        src/base64.cpp
-        )
    endif()
-
-    if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT)
-        find_package(Restinio REQUIRED)
-        if (Restinio_FOUND)
-            find_library(FMT_LIBRARY fmt)
-            add_library(fmt SHARED IMPORTED)
-            find_library(HTTP_PARSER_LIBRARY http_parser)
-            add_library(http_parser SHARED IMPORTED)
-        endif()
-        if (NOT Jsoncpp_FOUND)
-            message(SEND_ERROR "Jsoncpp is required for DHT proxy support")
-        endif()
-        if (OPENDHT_PROXY_OPENSSL)
-            # https://cmake.org/cmake/help/latest/module/FindOpenSSL.html
-            pkg_search_module(OPENSSL REQUIRED openssl)
-            if (OPENSSL_FOUND)
-                message(STATUS "Found OpenSSL ${OPENSSL_VERSION}")
-                include_directories(${OPENSSL_INCLUDE_DIRS})
-            else ()
-                message(SEND_ERROR "OpenSSL is required for DHT proxy as specified")
-            endif()
-        endif()
-        if (OPENDHT_PROXY_HTTP_PARSER_FORK)
-            add_definitions(-DOPENDHT_PROXY_HTTP_PARSER_FORK)
-        endif()
-    else()
-        set(OPENDHT_PROXY_OPENSSL OFF)
-    endif()
-    if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT OR OPENDHT_PEER_DISCOVERY)
-        add_definitions(-DASIO_STANDALONE)
+    if (OPENDHT_PROXY_HTTP_PARSER_FORK)
+        add_definitions(-DOPENDHT_PROXY_HTTP_PARSER_FORK)
    endif()
+else()
+    set(OPENDHT_PROXY_OPENSSL OFF)
+endif()
+if (OPENDHT_HTTP OR OPENDHT_PEER_DISCOVERY)
+    add_definitions(-DASIO_STANDALONE)
 endif()

 # Build flags
@@ -178,6 +164,7 @@ set (top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
 list (APPEND opendht_SOURCES
    src/utils.cpp
    src/infohash.cpp
+    src/crypto.cpp
    src/default_types.cpp
    src/node.cpp
    src/value.cpp
@@ -195,7 +182,11 @@ list (APPEND opendht_SOURCES
    src/routing_table.cpp
    src/node_cache.cpp
    src/network_engine.cpp
+    src/securedht.cpp
+    src/dhtrunner.cpp
    src/log.cpp
+    src/network_utils.cpp
+    src/thread_pool.cpp
 )

 list (APPEND opendht_HEADERS
@@ -203,6 +194,7 @@ list (APPEND opendht_HEADERS
    include/opendht/utils.h
    include/opendht/sockaddr.h
    include/opendht/rng.h
+    include/opendht/crypto.h
    include/opendht/infohash.h
    include/opendht/default_types.h
    include/opendht/node.h
@@ -218,27 +210,11 @@ list (APPEND opendht_HEADERS
    include/opendht/securedht.h
    include/opendht/log.h
    include/opendht/log_enable.h
+    include/opendht/thread_pool.h
+    include/opendht/network_utils.h
    include/opendht.h
 )

-if (NOT OPENDHT_LIGHT)
-    list (APPEND opendht_SOURCES
-        src/crypto.cpp
-        src/securedht.cpp
-        src/dhtrunner.cpp
-        src/network_utils.cpp
-        src/thread_pool.cpp
-    )
-
-    list (APPEND opendht_HEADERS
-        include/opendht/crypto.h
-        include/opendht/securedht.h
-        include/opendht/dhtrunner.h
-        include/opendht/thread_pool.h
-        include/opendht/network_utils.h
-    )
-endif()
-
 if (OPENDHT_PEER_DISCOVERY)
    list (APPEND opendht_SOURCES src/peer_discovery.cpp)
    list (APPEND opendht_HEADERS include/opendht/peer_discovery.h)
@@ -278,7 +254,7 @@ if (OPENDHT_PROXY_CLIENT)
  )
 endif ()

-if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT)
+if (OPENDHT_HTTP)
  if (OPENDHT_PUSH_NOTIFICATIONS)
    message("Using push notification")
    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS)
@@ -323,9 +299,9 @@ if (OPENDHT_STATIC)
    endif ()
    target_link_libraries(opendht-static
        PRIVATE  ${argon2_LIBRARIES}
-        PUBLIC ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}
-               ${Jsoncpp_LIBRARIES} ${FMT_LIBRARY} ${HTTP_PARSER_LIBRARY}
-               ${OPENSSL_LIBRARIES})
+        PUBLIC ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_STATIC_LIBRARIES}
+               ${Jsoncpp_STATIC_LIBRARIES} ${FMT_LIBRARY} ${HTTP_PARSER_LIBRARY}
+               ${OPENSSL_STATIC_LIBRARIES})
    install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()

@@ -384,7 +360,6 @@ if (OPENDHT_PYTHON)
 endif ()

 # CMake module
-include(CMakePackageConfigHelpers)
 write_basic_package_version_file("${CMAKE_CURRENT_BINARY_DIR}/opendhtConfigVersion.cmake"
  VERSION ${opendht_VERSION}
  COMPATIBILITY AnyNewerVersion
@@ -404,24 +379,20 @@ install (FILES ${CMAKE_CURRENT_BINARY_DIR}/opendhtConfigVersion.cmake DESTINATIO

 # Unit tests
 if (OPENDHT_TESTS)
-    FIND_PACKAGE(Cppunit REQUIRED)
+    pkg_search_module(Cppunit REQUIRED cppunit)
    # unit testing
    list (APPEND test_FILES
        tests/infohashtester.h
        tests/infohashtester.cpp
        tests/valuetester.h
        tests/valuetester.cpp
+        tests/cryptotester.h
+        tests/cryptotester.cpp
+        tests/dhtrunnertester.h
+        tests/dhtrunnertester.cpp
+        tests/threadpooltester.h
+        tests/threadpooltester.cpp
    )
-    if (NOT OPENDHT_LIGHT)
-        list (APPEND test_FILES
-            tests/cryptotester.h
-            tests/cryptotester.cpp
-            tests/dhtrunnertester.h
-            tests/dhtrunnertester.cpp
-            tests/threadpooltester.h
-            tests/threadpooltester.cpp
-        )
-    endif()
    if (OPENDHT_PROXY_SERVER AND OPENDHT_PROXY_CLIENT)
        list (APPEND test_FILES
            tests/httptester.h
@@ -440,7 +411,8 @@ if (OPENDHT_TESTS)
        tests/tests_runner.cpp
        ${test_FILES}
    )
-    target_include_directories(opendht_unit_tests SYSTEM PRIVATE ${CPPUNIT_INCLUDE_DIRS})
+    target_include_directories(opendht_unit_tests SYSTEM PRIVATE ${Cppunit_INCLUDE_DIR})
+    target_link_directories(opendht_unit_tests PRIVATE ${Cppunit_LIBRARY_DIRS})
    if (OPENDHT_SHARED)
        target_link_libraries(opendht_unit_tests opendht)
    else ()
@@ -448,7 +420,7 @@ if (OPENDHT_TESTS)
    endif ()
    target_link_libraries(opendht_unit_tests
       ${CMAKE_THREAD_LIBS_INIT}
-       ${CPPUNIT_LIBRARIES}
+       ${Cppunit_LIBRARIES}
       ${GNUTLS_LIBRARIES}
       ${Jsoncpp_LIBRARIES}
    )
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Andreas Traczyk <andreas.traczyk@savoirfairelinux.com>
 *
@@ -14,9 +14,9 @@ Values are distributed over the network, with redundancy.
 * High resilience to network disruption
 * Public key cryptography layer providing optional data signature and encryption (using GnuTLS)
 * IPv4 and IPv6 support
- * Clean and powerful C++14 map API
- * Python 3 bindings
- * REST API
+ * Clean and powerful **C++14** map API
+ * Bindings for **C, Rust & Python 3**
+ * REST API with optional HTTP client+server with push notification support

 ## Documentation
 See the wiki: <https://github.com/savoirfairelinux/opendht/wiki>
@@ -27,7 +27,7 @@ Build instructions: <https://github.com/savoirfairelinux/opendht/wiki/Build-the-

 #### How-to build a simple client app
 ```bash
-g++ main.cpp -std=c++14 -lopendht -lgnutls
+g++ main.cpp -std=c++14 -lopendht
 ```

 ## Examples
@@ -51,7 +51,7 @@ int main()

    // Join the network through any running node,
    // here using a known bootstrap node.
-    node.bootstrap("bootstrap.ring.cx", "4222");
+    node.bootstrap("bootstrap.jami.net", "4222");

    // put some data on the dht
    std::vector<uint8_t> some_data(5, 10);
@@ -82,7 +82,7 @@ node.run()

 # Join the network through any running node,
 # here using a known bootstrap node.
-node.bootstrap("bootstrap.ring.cx", "4222")
+node.bootstrap("bootstrap.jami.net", "4222")

 # blocking call (provide callback arguments to make the call non-blocking)
 node.put(dht.InfoHash.get("unique_key"), dht.Value(b'some binary data'))
@@ -106,7 +106,7 @@ for r in results:
 IRC: join us on Freenode at [`#opendht`](https://webchat.freenode.net/?channels=%23opendht).

 ## License
-Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+Copyright (C) 2014-2020 Savoir-faire Linux Inc.

 OpenDHT is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.

@@ -124,7 +124,7 @@ void dht_publickey_delete(dht_publickey* pk) {

 int dht_publickey_export(const dht_publickey* pk, char* out, size_t* outlen) {
    const auto& pkey = *reinterpret_cast<const PubkeySp*>(pk);
-    return gnutls_pubkey_export(pkey->pk, GNUTLS_X509_FMT_DER, out, outlen);
+    return pkey->pack((uint8_t*)out, outlen);
 }

 dht_infohash dht_publickey_get_id(const dht_publickey* pk) {
@@ -1,28 +0,0 @@
-INCLUDE(FindPkgConfig)
-PKG_CHECK_MODULES(PC_CPPUNIT "cppunit")
-
-FIND_PATH(CPPUNIT_INCLUDE_DIRS
-    NAMES cppunit/TestCase.h
-    HINTS ${PC_CPPUNIT_INCLUDE_DIR}
-    ${CMAKE_INSTALL_PREFIX}/include
-    PATHS
-    /usr/local/include
-    /usr/include
-)
-
-FIND_LIBRARY(CPPUNIT_LIBRARIES
-    NAMES cppunit
-    HINTS ${PC_CPPUNIT_LIBDIR}
-    ${CMAKE_INSTALL_PREFIX}/lib
-    ${CMAKE_INSTALL_PREFIX}/lib64
-    PATHS
-    ${CPPUNIT_INCLUDE_DIRS}/../lib
-    /usr/local/lib
-    /usr/lib
-)
-
-LIST(APPEND CPPUNIT_LIBRARIES ${CMAKE_DL_LIBS})
-
-INCLUDE(FindPackageHandleStandardArgs)
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(CPPUNIT DEFAULT_MSG CPPUNIT_LIBRARIES CPPUNIT_INCLUDE_DIRS)
-MARK_AS_ADVANCED(CPPUNIT_LIBRARIES CPPUNIT_INCLUDE_DIRS)
@@ -1,7 +1,7 @@
 dnl define macros
 m4_define([opendht_major_version], 2)
-m4_define([opendht_minor_version], 0)
-m4_define([opendht_patch_version], 0)
+m4_define([opendht_minor_version], 1)
+m4_define([opendht_patch_version], 3)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])

@@ -140,6 +140,7 @@ AS_IF([test "x$with_jsoncpp" != "xno"],
 AS_IF([test "x$have_jsoncpp" = "xyes"], [
    AC_MSG_NOTICE([Using JsonCpp])
    CPPFLAGS+=" -DOPENDHT_JSONCPP"
+    AC_SUBST(jsoncpp_lib, [", jsoncpp"])
 ], [
    AC_MSG_NOTICE([Not using JsonCpp])
    AM_COND_IF(PROXY_CLIENT_OR_SERVER, AC_MSG_ERROR(["JsonCpp is required for proxy/push notification support"]))
@@ -151,6 +152,7 @@ AS_IF([test "x$with_openssl" != "xno"],
      [have_openssl=no])
 AS_IF([test "x$have_openssl" = "xyes"], [
    AC_MSG_NOTICE([Using OpenSSL])
+    AC_SUBST(openssl_lib, [", openssl"])
 ], [
    AC_MSG_NOTICE([Not using OpenSSL])
 ])
@@ -175,6 +177,7 @@ AM_COND_IF([PROXY_CLIENT_OR_SERVER], [
    # http_parser has no pkgconfig, instead we check with:
    AC_CHECK_LIB(http_parser, exit,, AC_MSG_ERROR([Missing HttpParser library files]))
    AC_CHECK_HEADERS([http_parser.h], [http_parser_headers=yes; break;])
+    AC_SUBST(http_parser_lib, ["-lhttp_parser"])
    AS_IF([test "x$http_parser_headers" != "xyes"], AC_MSG_ERROR([Missing HttpParser headers files]))
 ])

@@ -1,33 +1,20 @@
-FROM ubuntu:16.04
+FROM ubuntu:20.04
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
 RUN apt-get update && apt-get install -y \
-        build-essential cmake git wget libncurses5-dev libreadline-dev nettle-dev \
-        libgnutls28-dev libuv1-dev cython3 python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev \
-        autotools-dev autoconf libssl-dev \
+        dialog apt-utils \
+    && apt-get clean \
+    && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN apt-get update && apt-get install -y \
+        build-essential pkg-config cmake git wget \
+        autotools-dev autoconf \
+        cython3 python3-dev python3-setuptools \
+        libncurses5-dev libreadline-dev nettle-dev libcppunit-dev \
+        libgnutls28-dev libuv1-dev libjsoncpp-dev libargon2-dev \
+        libssl-dev libfmt-dev libhttp-parser-dev libasio-dev libmsgpack-dev \
    && apt-get clean

-RUN echo "*** Installing libfmt ***" \
-    && wget https://github.com/fmtlib/fmt/archive/5.3.0.tar.gz \
-    && tar -xzf 5.3.0.tar.gz && cd fmt-5.3.0/ \
-    && cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=On -DFMT_TEST=Off -DFMT_DOC=Off . \
-    && make -j8 && make install \
-    && cd ../ && rm -rf fmt*
-
-# libhttp-parser-dev is 2.1-2 which is too old
-RUN echo "*** Building nodejs/http_parser dependency ***" \
-    && wget https://github.com/nodejs/http-parser/archive/v2.9.2.tar.gz \
-    && tar xvf v2.9.2.tar.gz && cd http-parser-2.9.2/ \
-    && PREFIX=/usr make -j8 install \
-    && cd ../ && rm -rf http-parser*
-
-# libasio-dev (1.10.6-3) is too old
-RUN echo "** Building a recent version of asio ***" \
-    && wget https://github.com/aberaud/asio/archive/b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz \
-    && tar -xvf b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz && cd asio-b2b7a1c166390459e1c169c8ae9ef3234b361e3f/asio \
-    && ./autogen.sh && ./configure --prefix=/usr --without-boost --disable-examples --disable-tests  \
-    && make install \
-    && cd ../../ && rm -rf asio*
-
 RUN echo "*** Downloading RESTinio ***" \
    && mkdir restinio && cd restinio \
    && wget https://github.com/aberaud/restinio/archive/a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
@@ -38,19 +25,3 @@ RUN echo "*** Downloading RESTinio ***" \
             -DRESTINIO_FIND_DEPS=ON -DRESTINIO_ALLOW_SOBJECTIZER=Off -DRESTINIO_USE_BOOST_ASIO=none . \
    && make -j8 && make install \
    && cd ../../ && rm -rf restinio*
-
-
-# build restbed from sources
-RUN git clone --recursive https://github.com/corvusoft/restbed.git \
-	&& cd restbed && mkdir build && cd build \
-	&& cmake -DBUILD_TESTS=NO -DBUILD_EXAMPLES=NO -DBUILD_SSL=NO -DBUILD_SHARED=YES -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib .. \
-	&& make -j8 install \
-	&& cd .. && rm -rf restbed
-
-#build msgpack from source
-RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.1.5/msgpack-2.1.5.tar.gz \
-    && tar -xzf msgpack-2.1.5.tar.gz \
-    && cd msgpack-2.1.5 && mkdir build && cd build \
-    && cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
-    && make -j8 && make install \
-    && cd ../.. && rm -rf msgpack-2.1.5 msgpack-2.1.5.tar.gz
@@ -3,15 +3,17 @@ MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>

 RUN echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries
 RUN apt-get update && apt-get install -y \
-        apt-transport-https build-essential cmake git wget libncurses5-dev libreadline-dev nettle-dev \
+        apt-transport-https build-essential pkg-config git wget libncurses5-dev libreadline-dev nettle-dev \
        libgnutls28-dev libuv1-dev cython3 python3-dev python3-setuptools libcppunit-dev libjsoncpp-dev \
-        autotools-dev autoconf libfmt-dev libhttp-parser-dev libmsgpack-dev \
+        autotools-dev autoconf libfmt-dev libhttp-parser-dev libmsgpack-dev libssl-dev \
    && apt-get clean

+RUN apt-get update && apt-get install -y python3-pip && pip3 install --upgrade cmake
+
 # libasio-dev (1.10) is too old
 RUN echo "** Building a recent version of asio ***" \
-    && wget https://github.com/aberaud/asio/archive/b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz \
-    && tar -xvf b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz && cd asio-b2b7a1c166390459e1c169c8ae9ef3234b361e3f/asio \
+    && wget https://github.com/aberaud/asio/archive/a7d66ef4017d8f1b7f2cef1bb4ba8e23b0961571.tar.gz \
+    && tar -xvf a7d66ef4017d8f1b7f2cef1bb4ba8e23b0961571.tar.gz && cd asio-a7d66ef4017d8f1b7f2cef1bb4ba8e23b0961571/asio \
    && ./autogen.sh && ./configure --prefix=/usr --without-boost --disable-examples --disable-tests  \
    && make install \
    && cd ../../ && rm -rf asio*
@@ -1,40 +1,21 @@
-FROM ubuntu:16.04
+FROM ubuntu:20.04
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN apt-get update && apt-get install -y \
+        dialog apt-utils \
+    && apt-get clean \
+    && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
 RUN apt-get update \
-    && apt-get install -y llvm llvm-dev clang make cmake git wget libncurses5-dev libreadline-dev \
+    && apt-get install -y llvm llvm-dev clang make cmake pkg-config git wget libncurses5-dev libreadline-dev \
       nettle-dev libgnutls28-dev libuv1-dev libmsgpack-dev libjsoncpp-dev cython3 python3-dev \
       python3-setuptools libcppunit-dev python3-pip \
-       autotools-dev autoconf libssl-dev \
+       autotools-dev autoconf libssl-dev libargon2-dev \
+       libfmt-dev libhttp-parser-dev libasio-dev \
    && apt-get remove -y gcc g++ && apt-get autoremove -y && apt-get clean

 ENV CC cc
 ENV CXX c++

-RUN echo "*** Installing libfmt ***" \
-    && wget https://github.com/fmtlib/fmt/archive/5.3.0.tar.gz \
-    && tar -xzf 5.3.0.tar.gz && cd fmt-5.3.0/ \
-    && cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=On -DFMT_TEST=Off -DFMT_DOC=Off . \
-    && make -j8 && make install \
-    && cd ../ && rm -rf fmt*
-
-# libhttp-parser-dev is 2.1-2 which is too old
-RUN echo "*** Building nodejs/http_parser dependency ***" \
-    && wget https://github.com/nodejs/http-parser/archive/v2.9.2.tar.gz \
-    && tar xvf v2.9.2.tar.gz && cd http-parser-2.9.2/ \
-    && PREFIX=/usr make -j8 install \
-    && cd ../ && rm -rf http-parser*
-
-# libasio-dev (1.10.6-3) is too old
-RUN echo "** Building a recent version of asio ***" \
-    && wget https://github.com/aberaud/asio/archive/b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz \
-    && tar -xvf b2b7a1c166390459e1c169c8ae9ef3234b361e3f.tar.gz && cd asio-b2b7a1c166390459e1c169c8ae9ef3234b361e3f/asio \
-    && ./autogen.sh && ./configure --prefix=/usr --without-boost --disable-examples --disable-tests  \
-    && make install \
-    && cd ../../ && rm -rf asio*
-
-#patch for https://github.com/Stiffstream/restinio-conan-example/issues/2
-RUN apt-get update && apt-get install -y python3-pip && pip3 install --upgrade cmake
-
 RUN echo "*** Downloading RESTinio ***" \
    && mkdir restinio && cd restinio \
    && wget https://github.com/aberaud/restinio/archive/a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
@@ -45,18 +26,3 @@ RUN echo "*** Downloading RESTinio ***" \
             -DRESTINIO_FIND_DEPS=ON -DRESTINIO_ALLOW_SOBJECTIZER=Off -DRESTINIO_USE_BOOST_ASIO=none . \
    && make -j8 && make install \
    && cd ../../ && rm -rf restinio*
-
-# build restbed from sources
-RUN git clone --recursive https://github.com/corvusoft/restbed.git \
-	&& cd restbed && mkdir build && cd build \
-	&& cmake -DBUILD_TESTS=NO -DBUILD_EXAMPLES=NO -DBUILD_SSL=NO -DBUILD_SHARED=YES -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib .. \
-	&& make -j8 install \
-	&& cd .. && rm -rf restbed
-
-#build msgpack from source
-RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.1.5/msgpack-2.1.5.tar.gz \
-    && tar -xzf msgpack-2.1.5.tar.gz \
-    && cd msgpack-2.1.5 && mkdir build && cd build \
-    && cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
-    && make -j8 && make install \
-    && cd ../.. && rm -rf msgpack-2.1.5 msgpack-2.1.5.tar.gz
@@ -0,0 +1,6 @@
+FROM aberaud/opendht-deps-llvm
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN git clone https://github.com/savoirfairelinux/opendht.git \
+	&& cd opendht && mkdir build && cd build \
+	&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On && make -j8 && make install \
+	&& cd ../.. && rm -rf opendht
@@ -1,7 +1,9 @@
-FROM aberaud/opendht-deps
+FROM aberaud/opendht-deps-bionic
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>

+RUN apt-get update && apt-get install -y python3-pip && pip3 install --upgrade cmake
+
 COPY . /root/opendht
 RUN cd /root/opendht && mkdir build && cd build \
-	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On -DOPENDHT_TESTS=ON ..  \
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_C=On -DOPENDHT_LTO=On -DOPENDHT_TESTS=ON ..  \
 	&& make -j8 && ./opendht_unit_tests && make install
@@ -3,5 +3,5 @@ MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>

 COPY . /root/opendht
 RUN cd /root/opendht && mkdir build && cd build \
-	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_TESTS=ON .. \
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_C=On -DOPENDHT_TESTS=ON .. \
 	&& make -j8 && ./opendht_unit_tests && make install
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -26,6 +26,7 @@
 #include <vector>
 #include <memory>
 #include <functional>
+#include <string>

 #ifdef OPENDHT_JSONCPP
 #include <json/json.h>
@@ -57,6 +58,8 @@ struct OPENDHT_PUBLIC NodeStats {
             cached_nodes {0},
             incoming_nodes {0};
    unsigned table_depth {0};
+    unsigned searches {0};
+    unsigned node_cache_size {0};
    unsigned getKnownNodes() const { return good_nodes + dubious_nodes; }
    unsigned long getNetworkSizeEstimation() const { return 8 * std::exp2(table_depth); }
    std::string toString() const;
@@ -70,7 +73,7 @@ struct OPENDHT_PUBLIC NodeStats {
    explicit NodeStats(const Json::Value& v);
 #endif

-    MSGPACK_DEFINE_MAP(good_nodes, dubious_nodes, cached_nodes, incoming_nodes, table_depth)
+    MSGPACK_DEFINE_MAP(good_nodes, dubious_nodes, cached_nodes, incoming_nodes, table_depth, searches, node_cache_size)
 };

 struct OPENDHT_PUBLIC NodeInfo {
@@ -79,6 +82,8 @@ struct OPENDHT_PUBLIC NodeInfo {
    NodeStats ipv4 {};
    NodeStats ipv6 {};
    size_t ongoing_ops {0};
+    in_port_t bound4 {0};
+    in_port_t bound6 {0};

 #ifdef OPENDHT_JSONCPP
    /**
@@ -115,14 +120,26 @@ struct OPENDHT_PUBLIC Config {
    /** If set, the dht will load its state from this file on start and save its state in this file on shutdown */
    std::string persist_path {};

-    /** If non-0, overrides the default global rate-limit.-1 means no limit. */
+    /** If non-0, overrides the default global rate-limit. -1 means no limit. */
    ssize_t max_req_per_sec {0};

    /** If non-0, overrides the default per-IP address rate-limit. -1 means no limit. */
    ssize_t max_peer_req_per_sec {0};
+
+    /* If non-0, overrides the default maximum number of searches. -1 means no limit.  */
+    ssize_t max_searches {0};
+
+    /* If non-0, overrides the default maximum store size. -1 means no limit.  */
+    ssize_t max_store_size {0};
+
+    /** 
+     * Use appropriate bahavior for a public IP, stable node:
+     *   - No connectivity change triggered when a search fails
+     *   - Larger listen refresh time
+     */
+    bool public_stable {false};
 };

-#ifndef OPENDHT_LIGHT
 /**
 * SecureDht configuration.
 */
@@ -130,8 +147,13 @@ struct OPENDHT_PUBLIC SecureDhtConfig
 {
    Config node_config {};
    crypto::Identity id {};
+
+    /** 
+     * Cache all encountered public keys and certificates,
+     * for use by the certificate store, putEncrypted and putSigned
+     */
+    bool cert_cache_all {false};
 };
-#endif

 static constexpr size_t DEFAULT_STORAGE_LIMIT {1024 * 1024 * 64};

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -114,6 +114,7 @@ struct OPENDHT_PUBLIC PublicKey
    }

    void pack(Blob& b) const;
+    int pack(uint8_t* out, size_t* out_len) const;
    void unpack(const uint8_t* dat, size_t dat_size);

    std::string toString() const;
@@ -127,7 +128,7 @@ struct OPENDHT_PUBLIC PublicKey
        p.pack_bin_body((const char*)b.data(), b.size());
    }

-    void msgpack_unpack(msgpack::object o);
+    void msgpack_unpack(const msgpack::object& o);

    gnutls_digest_algorithm_t getPreferredDigest() const;

@@ -225,7 +226,7 @@ public:
        p.pack_bin_body((const char*)b.data(), b.size());
    }

-    void msgpack_unpack(msgpack::object o);
+    void msgpack_unpack(const msgpack::object& o);

    void revoke(const Certificate& crt, time_point t = time_point::min());

@@ -423,7 +424,7 @@ struct OPENDHT_PUBLIC Certificate {
        p.pack_bin_body((const char*)b.data(), b.size());
    }

-    void msgpack_unpack(msgpack::object o);
+    void msgpack_unpack(const msgpack::object& o);

    explicit operator bool() const { return cert; }
    PublicKey getPublicKey() const;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -58,7 +58,6 @@ public:
    MSGPACK_DEFINE(service, data)
 };

-#ifndef OPENDHT_LIGHT
 template <typename T>
 class OPENDHT_PUBLIC SignedValue : public Value::Serializable<T>
 {
@@ -103,6 +102,7 @@ public:



+
 class OPENDHT_PUBLIC ImMessage : public SignedValue<ImMessage>
 {
 private:
@@ -177,11 +177,19 @@ public:
    {
        pk.pack_array(2);
        pk.pack(id);
+#if 1
        pk.pack_bin(ice_data.size());
        pk.pack_bin_body((const char*)ice_data.data(), ice_data.size());
+#else
+        // hack for backward compatibility with old opendht compiled with msgpack 1.0
+        // remove when enough people have moved to new versions
+        pk.pack_array(ice_data.size());
+        for (uint8_t b : ice_data)
+            pk.pack(b);
+#endif
    }

-    virtual void msgpack_unpack(msgpack::object o)
+    virtual void msgpack_unpack(const msgpack::object& o)
    {
        if (o.type != msgpack::type::ARRAY) throw msgpack::type_error();
        if (o.via.array.size < 2) throw msgpack::type_error();
@@ -192,7 +200,6 @@ public:
    Value::Id id {0};
    Blob ice_data;
 };
-#endif

 /* "Peer" announcement
 */
@@ -222,7 +229,7 @@ public:
        pk.pack_bin_body((const char*)addr.get(), addr.getLength());
    }

-    virtual void msgpack_unpack(msgpack::object o)
+    virtual void msgpack_unpack(const msgpack::object& o)
    {
        if (o.type == msgpack::type::BIN)
            addr = {(sockaddr*)o.via.bin.ptr, (socklen_t)o.via.bin.size};
@@ -254,11 +261,8 @@ private:
    SockAddr addr;
 };

-#ifndef OPENDHT_LIGHT
+
 OPENDHT_PUBLIC extern const std::array<std::reference_wrapper<const ValueType>, 5> DEFAULT_TYPES;
-#else
-OPENDHT_PUBLIC extern const std::array<std::reference_wrapper<const ValueType>, 1> DEFAULT_TYPES;
-#endif

 OPENDHT_PUBLIC extern const std::array<std::reference_wrapper<const ValueType>, 1> DEFAULT_INSECURE_TYPES;

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -68,7 +68,11 @@ public:
     * Initialise the Dht with two open sockets (for IPv4 and IP6)
     * and an ID for the node.
     */
-    Dht(std::unique_ptr<net::DatagramSocket>&& sock, const Config& config, const Logger& l = {});
+    Dht(std::unique_ptr<net::DatagramSocket>&& sock, const Config& config, const Sp<Logger>& l = {});
+
+    Dht(std::unique_ptr<net::DatagramSocket>&& sock, const Config& config, const Logger& l = {})
+        : Dht(std::move(sock), config, std::make_shared<Logger>(l)) {}
+
    virtual ~Dht();

    /**
@@ -76,10 +80,14 @@ public:
     */
    inline const InfoHash& getNodeId() const override { return myid; }

+    NodeStatus updateStatus(sa_family_t af) override;
+
    /**
     * Get the current status of the node for the given family.
     */
-    NodeStatus getStatus(sa_family_t af) const override;
+    NodeStatus getStatus(sa_family_t af) const override {
+        return dht(af).status;
+    }

    NodeStatus getStatus() const override {
        return std::max(getStatus(AF_INET), getStatus(AF_INET6));
@@ -107,6 +115,15 @@ public:
        return types.getType(type_id);
    }

+    void addBootstrap(const std::string& host, const std::string& service) override {
+        bootstrap_nodes.emplace_back(host, service);
+        onDisconnected();
+    }
+
+    void clearBootstrap() override {
+        bootstrap_nodes.clear();
+    }
+
    /**
     * Insert a node in the main routing table.
     * The node is not pinged, so this should be
@@ -119,9 +136,9 @@ public:

    void pingNode(SockAddr, DoneCallbackSimple&& cb={}) override;

-    time_point periodic(const uint8_t *buf, size_t buflen, SockAddr) override;
-    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) override {
-        return periodic(buf, buflen, SockAddr(from, fromlen));
+    time_point periodic(const uint8_t *buf, size_t buflen, SockAddr, const time_point& now) override;
+    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen, const time_point& now) override {
+        return periodic(buf, buflen, SockAddr(from, fromlen), now);
    }

    /**
@@ -326,10 +343,10 @@ private:
    static constexpr unsigned LISTEN_NODES {4};

    /* The maximum number of hashes we're willing to track. */
-    static constexpr unsigned MAX_HASHES {64 * 1024};
+    static constexpr unsigned MAX_HASHES {1024 * 1024};

    /* The maximum number of searches we keep data about. */
-    static constexpr unsigned MAX_SEARCHES {64 * 1024};
+    static constexpr unsigned MAX_SEARCHES {1024 * 1024};

    static constexpr std::chrono::minutes MAX_STORAGE_MAINTENANCE_EXPIRE_TIME {10};

@@ -337,9 +354,10 @@ private:
    static constexpr std::chrono::minutes SEARCH_EXPIRE_TIME {62};

    /* Timeout for listen */
-    static constexpr std::chrono::seconds LISTEN_EXPIRE_TIME {30};
+    static constexpr duration LISTEN_EXPIRE_TIME {std::chrono::seconds(30)};
+    static constexpr duration LISTEN_EXPIRE_TIME_PUBLIC {std::chrono::minutes(5)};

-    static constexpr std::chrono::seconds REANNOUNCE_MARGIN {10};
+    static constexpr duration REANNOUNCE_MARGIN {std::chrono::seconds(10)};

    static constexpr size_t TOKEN_SIZE {32};

@@ -353,6 +371,8 @@ private:
    Dht(const Dht&) = delete;
    Dht& operator=(const Dht&) = delete;

+    std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};
+
    InfoHash myid {};

    uint64_t secret {};
@@ -361,40 +381,49 @@ private:
    // registred types
    TypeStore types;

-    // the stuff
-    RoutingTable buckets4 {};
-    RoutingTable buckets6 {};
+    using SearchMap = std::map<InfoHash, Sp<Search>>;
+    struct Kad {
+        RoutingTable buckets {};
+        SearchMap searches {};
+        unsigned pending_pings {0};
+        NodeStatus status;
+
+        NodeStatus getStatus(time_point now) const;
+        NodeStats getNodesStats(time_point now, const InfoHash& myid) const;
+    };
+
+    Kad dht4 {};
+    Kad dht6 {};
+
+    std::vector<std::pair<std::string,std::string>> bootstrap_nodes {};
+    std::chrono::steady_clock::duration bootstrap_period {std::chrono::seconds(10)};
+    Sp<Scheduler::Job> bootstrapJob {};

    std::map<InfoHash, Storage> store;
    std::map<SockAddr, StorageBucket, SockAddr::ipCmp> store_quota;
    size_t total_values {0};
    size_t total_store_size {0};
+    size_t max_store_keys {MAX_HASHES};
    size_t max_store_size {DEFAULT_STORAGE_LIMIT};

-    using SearchMap = std::map<InfoHash, Sp<Search>>;
-    SearchMap searches4 {};
-    SearchMap searches6 {};
-    uint16_t search_id {0};
+    size_t max_searches {MAX_SEARCHES};
+    size_t search_id {0};

    // map a global listen token to IPv4, IPv6 specific listen tokens.
    // 0 is the invalid token.
    std::map<size_t, std::tuple<size_t, size_t, size_t>> listeners {};
    size_t listener_token {1};

+
    // timing
    Scheduler scheduler;
    Sp<Scheduler::Job> nextNodesConfirmation {};
    Sp<Scheduler::Job> nextStorageMaintenance {};

    net::NetworkEngine network_engine;
-    unsigned pending_pings4 {0};
-    unsigned pending_pings6 {0};
-
    using ReportedAddr = std::pair<unsigned, SockAddr>;
    std::vector<ReportedAddr> reported_addr;

-    std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};
-
    std::string persistPath;

    // are we a bootstrap node ?
@@ -403,6 +432,11 @@ private:
    //       be put in bootstrap mode.
    const bool is_bootstrap {false};
    const bool maintain_storage {false};
+    const bool public_stable {false};
+
+    inline const duration& getListenExpiration() const {
+        return public_stable ? LISTEN_EXPIRE_TIME_PUBLIC : LISTEN_EXPIRE_TIME;
+    }

    void rotateSecrets();

@@ -412,7 +446,7 @@ private:
    void reportedAddr(const SockAddr&);

    // Storage
-    void storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t tid, Query&& = {});
+    void storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t tid, Query&& = {}, int version = 0);
    bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa = {}, bool permanent = false);
    bool storageErase(const InfoHash& id, Value::Id vid);
    bool storageRefresh(const InfoHash& id, Value::Id vid);
@@ -432,8 +466,10 @@ private:
    size_t maintainStorage(decltype(store)::value_type&, bool force=false, const DoneCallback& donecb={});

    // Buckets
-    RoutingTable& buckets(sa_family_t af) { return af == AF_INET ? buckets4 : buckets6; }
-    const RoutingTable& buckets(sa_family_t af) const { return af == AF_INET ? buckets4 : buckets6; }
+    Kad& dht(sa_family_t af) { return af == AF_INET ? dht4 : dht6; }
+    const Kad& dht(sa_family_t af) const { return af == AF_INET ? dht4 : dht6; }
+    RoutingTable& buckets(sa_family_t af) { return dht(af).buckets; }
+    const RoutingTable& buckets(sa_family_t af) const { return dht(af).buckets; }
    Bucket* findBucket(const InfoHash& id, sa_family_t af) {
        auto& b = buckets(af);
        auto it = b.findBucket(id);
@@ -455,8 +491,8 @@ private:

    // Searches

-    inline SearchMap& searches(sa_family_t af) { return af == AF_INET ? searches4 : searches6; }
-    inline const SearchMap& searches(sa_family_t af) const { return af == AF_INET ? searches4 : searches6; }
+    inline SearchMap& searches(sa_family_t af) { return dht(af).searches; }
+    inline const SearchMap& searches(sa_family_t af) const { return dht(af).searches; }

    /**
     * Low-level method that will perform a search on the DHT for the specified
@@ -479,6 +515,7 @@ private:

    void confirmNodes();
    void expire();
+    void onDisconnected();

    /**
     * Generic function to execute when a 'get' request has completed.
@@ -564,7 +601,8 @@ private:
            const InfoHash& hash,
            const Blob& token,
            size_t socket_id,
-            const Query& query);
+            const Query& query,
+            int version = 0);
    void onListenDone(const Sp<Node>& status,
            net::RequestAnswer& a,
            Sp<Search>& sr);
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -30,7 +30,8 @@ namespace net {
 class OPENDHT_PUBLIC DhtInterface {
 public:
    DhtInterface() = default;
-    DhtInterface(const Logger& l) : DHT_LOG(l) {};
+    DhtInterface(const Logger& l) : logger_(std::make_shared<Logger>(l)) {};
+    DhtInterface(const std::shared_ptr<Logger>& l) : logger_(l) {};
    virtual ~DhtInterface() = default;

    // [[deprecated]]
@@ -41,6 +42,7 @@ public:
    /**
     * Get the current status of the node for the given family.
     */
+    virtual NodeStatus updateStatus(sa_family_t af) { return getStatus(af); };
    virtual NodeStatus getStatus(sa_family_t af) const = 0;
    virtual NodeStatus getStatus() const = 0;

@@ -68,6 +70,9 @@ public:

    virtual const ValueType& getType(ValueType::Id type_id) const = 0;

+    virtual void addBootstrap(const std::string& /*host*/, const std::string& /*service*/) {};
+    virtual void clearBootstrap() {};
+
    /**
     * Insert a node in the main routing table.
     * The node is not pinged, so this should be
@@ -78,8 +83,8 @@ public:

    virtual void pingNode(SockAddr, DoneCallbackSimple&& cb={}) = 0;

-    virtual time_point periodic(const uint8_t *buf, size_t buflen, SockAddr) = 0;
-    virtual time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) = 0;
+    virtual time_point periodic(const uint8_t *buf, size_t buflen, SockAddr, const time_point& now) = 0;
+    virtual time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen, const time_point& now) = 0;

    /**
     * Get a value by searching on all available protocols (IPv4, IPv6),
@@ -221,15 +226,23 @@ public:
     * Enable or disable logging of DHT internal messages
     */
    virtual void setLoggers(LogMethod error = {}, LogMethod warn = {}, LogMethod debug = {}) {
-        DHT_LOG.DBG = debug;
-        DHT_LOG.WARN = warn;
-        DHT_LOG.ERR = error;
+        if (logger_) {
+            logger_->DBG = std::move(debug);
+            logger_->WARN = std::move(warn);
+            logger_->ERR = std::move(error);
+        } else
+            logger_= std::make_shared<Logger>(std::move(error), std::move(warn), std::move(debug));
    }

    virtual void setLogger(const Logger& l) {
-        DHT_LOG.DBG = l.DBG;
-        DHT_LOG.WARN = l.WARN;
-        DHT_LOG.ERR = l.ERR;
+        if (logger_)
+            *logger_ = l;
+        else
+            logger_= std::make_shared<Logger>(l);
+    }
+
+    virtual void setLogger(const std::shared_ptr<Logger>& l) {
+        logger_ = l;
    }

    /**
@@ -237,7 +250,8 @@ public:
     */
    virtual void setLogFilter(const InfoHash& f)
    {
-        DHT_LOG.setFilter(f);
+        if (logger_)
+            logger_->setFilter(f);
    }

    virtual void setPushNotificationToken(const std::string&) {};
@@ -249,7 +263,7 @@ public:
    virtual void pushNotificationReceived(const std::map<std::string, std::string>& data) = 0;

 protected:
-    Logger DHT_LOG;
+    std::shared_ptr<Logger> logger_ {};
 };

 } // namespace dht
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *          Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *          Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
@@ -197,9 +197,9 @@ public:
     */
    void pushNotificationReceived(const std::map<std::string, std::string>& notification) override;

-    time_point periodic(const uint8_t*, size_t, SockAddr) override;
-    time_point periodic(const uint8_t* buf, size_t buflen, const sockaddr* from, socklen_t fromlen) override {
-        return periodic(buf, buflen, SockAddr(from, fromlen));
+    time_point periodic(const uint8_t*, size_t, SockAddr, const time_point& now) override;
+    time_point periodic(const uint8_t* buf, size_t buflen, const sockaddr* from, socklen_t fromlen, const time_point& now) override {
+        return periodic(buf, buflen, SockAddr(from, fromlen), now);
    }

    /**
@@ -285,7 +285,7 @@ private:
     */
    struct InfoState;
    void getProxyInfos();
-    void queryProxyInfo(std::shared_ptr<InfoState> infoState, sa_family_t family, std::shared_ptr<http::Resolver> resolver);
+    void queryProxyInfo(const std::shared_ptr<InfoState>& infoState, const std::shared_ptr<http::Resolver>& resolver, sa_family_t family);
    void onProxyInfos(const Json::Value& val, const sa_family_t family);
    SockAddr parsePublicAddress(const Json::Value& val);

@@ -300,10 +300,12 @@ private:
        SUBSCRIBE,
        RESUBSCRIBE,
    };
+    using CacheValueCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values, bool expired, system_clock::time_point)>;
+
    /**
     * Send Listen with httpClient_
     */
-    void sendListen(const restinio::http_request_header_t header, const ValueCallback& cb,
+    void sendListen(const restinio::http_request_header_t& header, const CacheValueCallback& cb,
                    const Sp<OperationState>& opstate, Listener& listener, ListenMethod method = ListenMethod::LISTEN);
    void handleResubscribe(const asio::error_code& ec, const InfoHash& key,
                           const size_t token, std::shared_ptr<OperationState> opstate);
@@ -327,6 +329,7 @@ private:
    std::shared_ptr<dht::crypto::Certificate> serverCertificate_;
    //std::pair<std::string, std::string> serverHostService_;
    std::string pushClientId_;
+    std::string pushSessionId_;

    mutable std::mutex lockCurrentProxyInfos_;
    NodeStatus statusIpv4_ {NodeStatus::Disconnected};
@@ -405,7 +408,6 @@ private:

    Json::StreamWriterBuilder jsonBuilder_;
    std::unique_ptr<Json::CharReader> jsonReader_;
-    std::shared_ptr<dht::Logger> logger_;

    std::shared_ptr<http::Request> buildRequest(const std::string& target = {});
 };
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2017-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *          Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *          Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
@@ -36,6 +36,15 @@
 #include <memory>
 #include <mutex>

+namespace dht {
+enum class PushType {
+    None = 0,
+    Android,
+    iOS
+};
+}
+MSGPACK_ADD_ENUM(dht::PushType)
+
 namespace http {
 class Request;
 struct ListenerSession;
@@ -52,6 +61,13 @@ class DhtRunner;
 using RestRouter = restinio::router::express_router_t<>;
 using RequestStatus = restinio::request_handling_status_t;

+struct ProxyServerConfig {
+    in_port_t port {8000};
+    std::string pushServer {};
+    std::string persistStatePath {};
+    dht::crypto::Identity identity {};
+};
+
 /**
 * Describes the REST API
 */
@@ -66,10 +82,9 @@ public:
     * @note if the server fails to start (if port is already used or reserved),
     * it will fails silently
     */
-    DhtProxyServer(
-       dht::crypto::Identity identity,
-       std::shared_ptr<DhtRunner> dht, in_port_t port = 8000, const std::string& pushServer = "",
-       std::shared_ptr<dht::Logger> logger = {});
+    DhtProxyServer(const std::shared_ptr<DhtRunner>& dht,
+        const ProxyServerConfig& config = {},
+        const std::shared_ptr<dht::Logger>& logger = {});

    virtual ~DhtProxyServer();

@@ -83,27 +98,28 @@ public:
    struct ServerStats {
        /** Current number of listen operations */
        size_t listenCount {0};
-        /** Current number of permanent put operations */
+        /** Current number of permanent put operations (hash used) */
        size_t putCount {0};
+        /** Current number of permanent put values */
+        size_t totalPermanentPuts {0};
        /** Current number of push tokens with at least one listen operation */
        size_t pushListenersCount {0};
        /** Average requests per second */
        double requestRate {0};
        /** Node Info **/
-        NodeInfo nodeInfo {};
+        std::shared_ptr<NodeInfo> nodeInfo {};

        std::string toString() const {
            std::ostringstream ss;
            ss << "Listens: " << listenCount << " Puts: " << putCount << " PushListeners: " << pushListenersCount << std::endl;
            ss << "Requests: " << requestRate << " per second." << std::endl;
-            auto& ni = nodeInfo;
-            auto& ipv4 = ni.ipv4;
-            if (ipv4.table_depth > 1) {
-                ss << "IPv4 Network estimation: " << ipv4.getNetworkSizeEstimation() << std::endl;;
-            }
-            auto& ipv6 = ni.ipv6;
-            if (ipv6.table_depth > 1) {
-                ss << "IPv6 Network estimation: " << ipv6.getNetworkSizeEstimation() << std::endl;;
+            if (nodeInfo) {
+                auto& ipv4 = nodeInfo->ipv4;
+                if (ipv4.table_depth > 1)
+                    ss << "IPv4 Network estimation: " << ipv4.getNetworkSizeEstimation() << std::endl;;
+                auto& ipv6 = nodeInfo->ipv6;
+                if (ipv6.table_depth > 1)
+                    ss << "IPv6 Network estimation: " << ipv6.getNetworkSizeEstimation() << std::endl;;
            }
            return ss.str();
        }
@@ -115,16 +131,18 @@ public:
            Json::Value result;
            result["listenCount"] = static_cast<Json::UInt64>(listenCount);
            result["putCount"] = static_cast<Json::UInt64>(putCount);
+            result["totalPermanentPuts"] = static_cast<Json::UInt64>(totalPermanentPuts);
            result["pushListenersCount"] = static_cast<Json::UInt64>(pushListenersCount);
            result["requestRate"] = requestRate;
-            result["nodeInfo"] = nodeInfo.toJson();
+            if (nodeInfo)
+                result["nodeInfo"] = nodeInfo->toJson();
            return result;
        }
    };

-    ServerStats stats() const { return stats_; }
+    std::shared_ptr<ServerStats> stats() const { return stats_; }

-    void updateStats() const;
+    std::shared_ptr<ServerStats> updateStats(std::shared_ptr<NodeInfo> info) const;

    std::shared_ptr<DhtRunner> getNode() const { return dht_; }

@@ -134,7 +152,8 @@ private:
    struct RestRouterTraits;

    template <typename HttpResponse>
-    HttpResponse initHttpResponse(HttpResponse response) const;
+    static HttpResponse initHttpResponse(HttpResponse response);
+    static restinio::request_handling_status_t serverError(restinio::request_t& request);

    template< typename ServerSettings >
    void addServerSettings(ServerSettings& serverSettings,
@@ -281,17 +300,17 @@ private:
     * @param key of the device
     * @param json, the content to send
     */
-    void sendPushNotification(const std::string& key, Json::Value&& json, bool isAndroid);
+    void sendPushNotification(const std::string& key, Json::Value&& json, PushType type, bool highPriority);

    /**
     * Send push notification with an expire timeout.
     * @param ec
     * @param pushToken
     * @param json
-     * @param isAndroid
+     * @param type
     */
    void handleNotifyPushListenExpire(const asio::error_code &ec, const std::string pushToken,
-                                      Json::Value json, const bool isAndroid);
+                                      std::function<Json::Value()> json, PushType type);

    /**
     * Remove a push listener between a client and a hash
@@ -306,13 +325,24 @@ private:
 #endif //OPENDHT_PUSH_NOTIFICATIONS

    void handlePrintStats(const asio::error_code &ec);
+    void updateStats();
+
+    template <typename Os>
+    void saveState(Os& stream);
+
+    template <typename Is>
+    void loadState(Is& is, size_t size);

    using clock = std::chrono::steady_clock;
    using time_point = clock::time_point;

+    std::shared_ptr<asio::io_context> ioContext_;
    std::shared_ptr<DhtRunner> dht_;
    Json::StreamWriterBuilder jsonBuilder_;
    Json::CharReaderBuilder jsonReaderBuilder_;
+    std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};
+
+    std::string persistPath_;

    // http server
    std::thread serverThread_;
@@ -321,15 +351,14 @@ private:

    // http client
    std::pair<std::string, std::string> pushHostPort_;
-    
+
    mutable std::mutex requestLock_;
    std::map<unsigned int /*id*/, std::shared_ptr<http::Request>> requests_;

    std::shared_ptr<dht::Logger> logger_;

-    mutable std::mutex statsMutex_;
-    mutable ServerStats stats_;
-    mutable NodeInfo nodeInfo_ {};
+    std::shared_ptr<ServerStats> stats_;
+    std::shared_ptr<NodeInfo> nodeInfo_ {};
    std::unique_ptr<asio::steady_timer> printStatsTimer_;

    // Thread-safe access to listeners map.
@@ -339,15 +368,45 @@ private:
    // Connection Listener observing conn state changes.
    std::shared_ptr<ConnectionListener> connListener_;

+    struct PushSessionContext {
+        std::mutex lock;
+        std::string sessionId;
+        PushSessionContext(const std::string& id) : sessionId(id) {}
+    };
    struct PermanentPut {
        time_point expiration;
        std::string pushToken;
        std::string clientId;
+        std::shared_ptr<PushSessionContext> sessionCtx;
        std::unique_ptr<asio::steady_timer> expireTimer;
        std::unique_ptr<asio::steady_timer> expireNotifyTimer;
+        Sp<Value> value;
+        PushType type;
+
+        template <typename Packer>
+        void msgpack_pack(Packer& p) const
+        {
+            p.pack_map(2 + (sessionCtx ? 1 : 0) + (clientId.empty() ? 0 : 1) + (type == PushType::None ? 0 : 2));
+            p.pack("value"); p.pack(value);
+            p.pack("exp"); p.pack(to_time_t(expiration));
+            if (not clientId.empty()) {
+                p.pack("cid"); p.pack(clientId);
+            }
+            if (sessionCtx) {
+                std::lock_guard<std::mutex> l(sessionCtx->lock);
+                p.pack("sid"); p.pack(sessionCtx->sessionId);
+            }
+            if (type != PushType::None) {
+                p.pack("t"); p.pack(type);
+                p.pack("token"); p.pack(pushToken);
+            }
+        }
+
+        void msgpack_unpack(const msgpack::object& o);
    };
    struct SearchPuts {
        std::map<dht::Value::Id, PermanentPut> puts;
+        MSGPACK_DEFINE_ARRAY(puts)
    };
    std::mutex lockSearchPuts_;
    std::map<InfoHash, SearchPuts> puts_;
@@ -359,13 +418,32 @@ private:

 #ifdef OPENDHT_PUSH_NOTIFICATIONS
    struct Listener {
+        time_point expiration;
        std::string clientId;
+        std::shared_ptr<PushSessionContext> sessionCtx;
        std::future<size_t> internalToken;
        std::unique_ptr<asio::steady_timer> expireTimer;
        std::unique_ptr<asio::steady_timer> expireNotifyTimer;
+        PushType type;
+
+        template <typename Packer>
+        void msgpack_pack(Packer& p) const
+        {
+            p.pack_map(sessionCtx ? 4 : 3);
+            p.pack("cid"); p.pack(clientId);
+            p.pack("exp"); p.pack(to_time_t(expiration));
+            if (sessionCtx) {
+                std::lock_guard<std::mutex> l(sessionCtx->lock);
+                p.pack("sid"); p.pack(sessionCtx->sessionId);
+            }
+            p.pack("t"); p.pack(type);
+        }
+
+        void msgpack_unpack(const msgpack::object& o);
    };
    struct PushListener {
        std::map<InfoHash, std::vector<Listener>> listeners;
+        MSGPACK_DEFINE_ARRAY(listeners)
    };
    std::mutex lockPushListeners_;
    std::map<std::string, PushListener> pushListeners_;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -327,7 +327,10 @@ public:

    std::vector<ValuesExport> exportValues() const;

-    void setLogger(const Logger& logger = {});
+    void setLogger(const Sp<Logger>& logger = {});
+    void setLogger(const Logger& logger) {
+        setLogger(std::make_shared<Logger>(logger));
+    }
    void setLoggers(LogMethod err = {}, LogMethod warn = {}, LogMethod debug = {});

    /**
@@ -346,6 +349,7 @@ public:
    NodeStats getNodesStats(sa_family_t af) const;
    unsigned getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return, unsigned *incoming_return) const;
    NodeInfo getNodeInfo() const;
+    void getNodeInfo(std::function<void(std::shared_ptr<NodeInfo>)>);

    std::vector<unsigned> getNodeMessageStats(bool in = false) const;
    std::string getStorageLog() const;
@@ -385,7 +389,7 @@ public:
     * @param threaded: If false, loop() must be called periodically. Otherwise a thread is launched.
     * @param cb: Optional callback to receive general state information.
     */
-    void run(const SockAddr& local4, const SockAddr& local6, const Config& config, Context&& context = {});
+    void run(SockAddr& local4, SockAddr& local6, const Config& config, Context&& context = {});

    /**
     * Same as @run(sockaddr_in, sockaddr_in6, Identity, bool, StatusCallback), but with string IP addresses and service (port).
@@ -455,14 +459,6 @@ private:
        Stopping
    };

-    /**
-     * Will try to resolve the list of hostnames `bootstrap_nodes` on seperate
-     * thread and then queue ping requests. This list should contain reliable
-     * nodes so that the DHT node can recover quickly from losing connection
-     * with the network.
-     */
-    void tryBootstrapContinuously();
-
    time_point loop_();

    NodeStatus getStatus() const {
@@ -506,16 +502,8 @@ private:
    std::thread dht_thread {};
    std::condition_variable cv {};
    std::mutex sock_mtx {};
-    std::queue<std::unique_ptr<net::ReceivedPacket>> rcv {};
-
-    /** true if currently actively boostraping */
-    std::atomic_bool bootstraping {false};
-    /* bootstrap nodes given as (host, service) pairs */
-    std::vector<std::pair<std::string,std::string>> bootstrap_nodes_all {};
-    std::vector<std::pair<std::string,std::string>> bootstrap_nodes {};
-    std::thread bootstrap_thread {};
-    std::mutex bootstrap_mtx {};
-    std::condition_variable bootstrap_cv {};
+    net::PacketList rcv {};
+    decltype(rcv) rcv_free {};

    std::queue<std::function<void(SecureDht&)>> pending_ops_prio {};
    std::queue<std::function<void(SecureDht&)>> pending_ops {};
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -35,6 +35,7 @@

 #include <memory>
 #include <queue>
+#include <mutex>

 namespace Json {
 class Value;
@@ -118,10 +119,12 @@ private:
    template<typename T>
    T wrapCallabck(T cb) const {
        return [t=shared_from_this(),cb=std::move(cb)](auto ...params) {
-            return cb(params...);
+            cb(params...);
        };
    }

+    mutable std::mutex mutex_;
+
    unsigned int id_;
    static std::atomic_uint ids_;

@@ -185,7 +188,7 @@ public:
 private:
    void resolve(const std::string& host, const std::string& service);

-    std::mutex mutex_;
+    mutable std::mutex mutex_;

    Url url_;
    asio::error_code ec_;
@@ -199,12 +202,15 @@ private:
    std::shared_ptr<dht::Logger> logger_;
 };

+class Request;
+
 struct Response
 {
-    unsigned int status_code {0};
+    unsigned status_code {0};
    std::map<std::string, std::string> headers;
    std::string body;
    bool aborted {false};
+    std::weak_ptr<Request> request;
 };

 class OPENDHT_PUBLIC Request : public std::enable_shared_from_this<Request>
@@ -217,10 +223,10 @@ public:
        RECEIVING,
        DONE
    };
-    using OnStatusCb = std::function<void(unsigned int status_code)>;
+    using OnStatusCb = std::function<void(unsigned status_code)>;
    using OnDataCb = std::function<void(const char* at, size_t length)>;
    using OnStateChangeCb = std::function<void(State state, const Response& response)>;
-    using OnJsonCb = std::function<void(Json::Value value, unsigned int status_code)>;
+    using OnJsonCb = std::function<void(Json::Value value, unsigned status_code)>;
    using OnDoneCb = std::function<void(const Response& response)>;

    // resolves implicitly
@@ -247,6 +253,12 @@ public:
    inline const Url& get_url() const {
        return resolver_->get_url();
    };
+
+    /** The previous request in case of redirect following */
+    std::shared_ptr<Request> getPrevious() const {
+        return prev_.lock();
+    }
+
    inline std::string& to_string() {
        return request_;
    }
@@ -312,6 +324,8 @@ private:
    void onBody(const char* at, size_t length);
    void onComplete();

+    mutable std::mutex mutex_;
+
    std::shared_ptr<dht::Logger> logger_;

    restinio::http_request_header_t header_;
@@ -339,6 +353,12 @@ private:
    std::atomic<bool> finishing_ {false};
    std::unique_ptr<http_parser> parser_;
    std::unique_ptr<http_parser_settings> parser_s_;
+
+    // Next request in case of redirect following
+    std::shared_ptr<Request> next_;
+    std::weak_ptr<Request> prev_;
+    unsigned num_redirect {0};
+    bool follow_redirect {true};
 };

 } // namespace http
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *              Nicolas Reynaud <nicolas.reynaud@savoirfairelinux.com>
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -50,7 +50,7 @@ namespace dht {
 using byte = uint8_t;

 namespace crypto {
-OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length);
+    OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length);
 }

 /**
@@ -237,6 +237,9 @@ public:

    static Hash getRandom();

+    template <typename Rd>
+    static Hash getRandom(Rd&);
+
    template <size_t M>
    OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Hash<M>& h);

@@ -324,6 +327,19 @@ Hash<N>::getRandom()
    return h;
 }

+template <size_t N>
+template <typename Rd>
+Hash<N>
+Hash<N>::getRandom(Rd& rdev)
+{
+    Hash h;
+    std::uniform_int_distribution<uint32_t> rand_int;
+    auto a = reinterpret_cast<uint32_t*>(h.data());
+    auto b = reinterpret_cast<uint32_t*>(h.data() + h.size());
+    std::generate(a, b, std::bind(rand_int, std::ref(rdev)));
+    return h;
+}
+
 struct HexMap : public std::array<std::array<char, 2>, 256> {
    HexMap() {
        for (size_t i=0; i<size(); i++) {
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -64,7 +64,7 @@ struct LogMethod {
        func(format, args);
        va_end(args);
    }
-    void log(char const* format, va_list args) const {
+    inline void log(char const* format, va_list args) const {
        func(format, args);
    }
    explicit operator bool() const {
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -180,7 +180,8 @@ private:
            const InfoHash&,
            const Blob&,
            Tid,
-            const Query&)> onListen {};
+            const Query&,
+            int)> onListen {};
    /**
     * Called on announce request.
     *
@@ -210,14 +211,16 @@ private:

 public:
    using RequestCb = std::function<void(const Request&, RequestAnswer&&)>;
+    using RequestErrorCb = std::function<bool(const Request&, DhtProtocolException&&)>;
    using RequestExpiredCb = std::function<void(const Request&, bool)>;

-    NetworkEngine(Logger& log, Scheduler& scheduler, std::unique_ptr<DatagramSocket>&& sock);
+    NetworkEngine(const Sp<Logger>& log, std::mt19937_64& rd, Scheduler& scheduler, std::unique_ptr<DatagramSocket>&& sock);
    NetworkEngine(
            InfoHash& myid,
            NetworkConfig config,
            std::unique_ptr<DatagramSocket>&& sock,
-            Logger& log,
+            const Sp<Logger>& log,
+            std::mt19937_64& rd,
            Scheduler& scheduler,
            decltype(NetworkEngine::onError)&& onError,
            decltype(NetworkEngine::onNewNode)&& onNewNode,
@@ -236,9 +239,10 @@ public:
    void clear();

    /**
-     * Sends values (with closest nodes) to a listenner.
+     * Sends values (with closest nodes) to a listener.
     *
-     * @param sa          The address of the listenner.
+     * @deprecated
+     * @param sa          The address of the listener.
     * @param sslen       The length of the sockaddr structure.
     * @param socket_id  The tid to use to write to the request socket.
     * @param hash        The hash key of the value.
@@ -247,13 +251,14 @@ public:
     * @param nodes       The ipv4 closest nodes.
     * @param nodes6      The ipv6 closest nodes.
     * @param values      The values to send.
+     * @param version     If version = 1, a request will be used to answer to the listener
     */
    void tellListener(Sp<Node> n, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken,
            std::vector<Sp<Node>>&& nodes, std::vector<Sp<Node>>&& nodes6,
-            std::vector<Sp<Value>>&& values, const Query& q);
+            std::vector<Sp<Value>>&& values, const Query& q, int version);

-    void tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
-    void tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
+    void tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values, int version);
+    void tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values, int version);

    bool isRunning(sa_family_t af) const;
    inline want_t want () const { return dht_socket->hasIPv4() and dht_socket->hasIPv6() ? (WANT4 | WANT6) : -1; }
@@ -274,7 +279,8 @@ public:
     * @return the request with information concerning its success.
     */
    Sp<Request>
-        sendPing(Sp<Node> n, RequestCb&& on_done, RequestExpiredCb&& on_expired);
+    sendPing(Sp<Node> n, RequestCb&& on_done, RequestExpiredCb&& on_expired);
+
    /**
     * Send a "ping" request to a given node.
     *
@@ -286,11 +292,12 @@ public:
     * @return the request with information concerning its success.
     */
    Sp<Request>
-        sendPing(SockAddr&& sa, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-            return sendPing(std::make_shared<Node>(zeroes, std::move(sa)),
-                    std::forward<RequestCb>(on_done),
-                    std::forward<RequestExpiredCb>(on_expired));
-        }
+    sendPing(SockAddr&& sa, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
+        return sendPing(std::make_shared<Node>(zeroes, std::move(sa), rd),
+                std::forward<RequestCb>(on_done),
+                std::forward<RequestExpiredCb>(on_expired));
+    }
+
    /**
     * Send a "find node" request to a given node.
     *
@@ -355,10 +362,9 @@ public:
                           const InfoHash& hash,
                           const Query& query,
                           const Blob& token,
-                           Sp<Request> previous,
+                           Tid socketId,
                           RequestCb&& on_done,
-                           RequestExpiredCb&& on_expired,
-                           SocketCb&& socket_cb);
+                           RequestExpiredCb&& on_expired);
    /**
     * Send a "announce" request to a given node.
     *
@@ -397,7 +403,26 @@ public:
                                 const Value::Id& vid,
                                 const Blob& token,
                                 RequestCb&& on_done,
+                                 RequestErrorCb&& on_error,
                                 RequestExpiredCb&& on_expired);
+    /**
+     * Send a "update" request to a given node. Used for Listen operations
+     *
+     * @param n           The node.
+     * @param hash        The target hash.
+     * @param values      The values.
+     * @param created     Time id.
+     * @param token       A security token.
+     * @param sid         The socket id.
+     *
+     * @return the request with information concerning its success.
+     */
+    Sp<Request> sendUpdateValues(Sp<Node> n,
+                                 const InfoHash& infohash,
+                                 const std::vector<Sp<Value>>& values,
+                                 time_point created,
+                                 const Blob& token,
+                                 const size_t& sid);

    /**
     * Parses a message and calls appropriate callbacks.
@@ -410,8 +435,8 @@ public:
     */
    void processMessage(const uint8_t *buf, size_t buflen, SockAddr addr);

-    Sp<Node> insertNode(const InfoHash& myid, const SockAddr& addr) {
-        auto n = cache.getNode(myid, addr, scheduler.time(), 0);
+    Sp<Node> insertNode(const InfoHash& id, const SockAddr& addr) {
+        auto n = cache.getNode(id, addr, scheduler.time(), 0);
        onNewNode(n, 0);
        return n;
    }
@@ -429,6 +454,21 @@ public:
        return cache.getCachedNodes(id, sa_f, count);
    }

+    size_t getNodeCacheSize() const {
+        return cache.size();
+    }
+    size_t getNodeCacheSize(sa_family_t af) const {
+        return cache.size(af);
+    }
+
+    size_t getRateLimiterSize() const {
+        return address_rate_limiter.size();
+    }
+
+    size_t getPartialCount() const {
+        return partial_messages.size();
+    }
+
 private:

    struct PartialMessage;
@@ -472,19 +512,20 @@ private:
    void sendRequest(const Sp<Request>& request);

    struct MessageStats {
-        unsigned ping    {0};
-        unsigned find    {0};
-        unsigned get     {0};
-        unsigned put     {0};
-        unsigned listen  {0};
-        unsigned refresh {0};
+        unsigned ping         {0};
+        unsigned find         {0};
+        unsigned get          {0};
+        unsigned put          {0};
+        unsigned listen       {0};
+        unsigned refresh      {0};
+        unsigned updateValue  {0};
    };


    // basic wrapper for socket sendto function
    int send(const SockAddr& addr, const char *buf, size_t len, bool confirmed = false);

-    void sendValueParts(const TransId& tid, const std::vector<Blob>& svals, const SockAddr& addr);
+    void sendValueParts(Tid tid, const std::vector<Blob>& svals, const SockAddr& addr);
    std::vector<Blob> packValueHeader(msgpack::sbuffer&, const std::vector<Sp<Value>>&);
    void maintainRxBuffer(Tid tid);

@@ -525,9 +566,10 @@ private:
    const InfoHash& myid;
    const NetworkConfig config {};
    const std::unique_ptr<DatagramSocket> dht_socket;
-    const Logger& DHT_LOG;
+    Sp<Logger> logger_;
+    std::mt19937_64& rd;

-    NodeCache cache {};
+    NodeCache cache;

    // global limiting should be triggered by at least 8 different IPs
    using IpLimiter = RateLimiter;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -36,11 +36,14 @@
 #include <thread>
 #include <atomic>
 #include <mutex>
+#include <list>

 namespace dht {
 namespace net {

 static const constexpr in_port_t DHT_DEFAULT_PORT = 4222;
+static const constexpr size_t RX_QUEUE_MAX_SIZE = 1024 * 16;
+static const constexpr std::chrono::milliseconds RX_QUEUE_MAX_DELAY(650);

 int bindSocket(const SockAddr& addr, SockAddr& bound);

@@ -54,10 +57,14 @@ struct ReceivedPacket {
    SockAddr from;
    time_point received;
 };
+using PacketList = std::list<ReceivedPacket>;

 class OPENDHT_PUBLIC DatagramSocket {
 public:
-    using OnReceive = std::function<void(std::unique_ptr<ReceivedPacket>&& packet)>;
+    /** A function that takes a list of new received packets and
+     *  optionally returns consumed packets for recycling.
+     **/
+    using OnReceive = std::function<PacketList(PacketList&& packets)>;
    virtual ~DatagramSocket() {};

    virtual int sendTo(const SockAddr& dest, const uint8_t* data, size_t size, bool replied) = 0;
@@ -81,24 +88,45 @@ public:

    virtual const SockAddr& getBoundRef(sa_family_t family = AF_UNSPEC) const = 0;

+    /** Virtual resolver mothod allows to implement custom resolver */
+    virtual std::vector<SockAddr> resolve(const std::string& host, const std::string& service = {}) {
+        return SockAddr::resolve(host, service);
+    }
+
    virtual void stop() = 0;
 protected:

-    inline void onReceived(std::unique_ptr<ReceivedPacket>&& packet) {
+    PacketList getNewPacket() {
+        PacketList pkts;
+        if (toRecycle_.empty()) {
+            pkts.emplace_back();
+        } else {
+            auto begIt = toRecycle_.begin();
+            auto begItNext = std::next(begIt);
+            pkts.splice(pkts.end(), toRecycle_, begIt, begItNext);
+        }
+        return pkts;
+    }
+
+    inline void onReceived(PacketList&& packets) {
        std::lock_guard<std::mutex> lk(lock);
-        if (rx_callback)
-            rx_callback(std::move(packet));
+        if (rx_callback) {
+            auto r = rx_callback(std::move(packets));
+            if (not r.empty() and toRecycle_.size() < RX_QUEUE_MAX_SIZE)
+                toRecycle_.splice(toRecycle_.end(), std::move(r));
+        }
    }
 protected:
    mutable std::mutex lock;
 private:
    OnReceive rx_callback;
+    PacketList toRecycle_;
 };

 class OPENDHT_PUBLIC UdpSocket : public DatagramSocket {
 public:
-    UdpSocket(in_port_t port, const Logger& l = {});
-    UdpSocket(const SockAddr& bind4, const SockAddr& bind6, const Logger& l = {});
+    UdpSocket(in_port_t port, const std::shared_ptr<Logger>& l = {});
+    UdpSocket(const SockAddr& bind4, const SockAddr& bind6, const std::shared_ptr<Logger>& l = {});
    ~UdpSocket();

    int sendTo(const SockAddr& dest, const uint8_t* data, size_t size, bool replied) override;
@@ -118,7 +146,7 @@ public:

    void stop() override;
 private:
-    Logger logger;
+    std::shared_ptr<Logger> logger;
    int s4 {-1};
    int s6 {-1};
    int stopfd {-1};
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -47,10 +47,10 @@ struct Socket {
 struct Node {
    const InfoHash id;

-    Node(const InfoHash& id, const SockAddr& addr, bool client=false);
-    Node(const InfoHash& id, SockAddr&& addr, bool client=false);
-    Node(const InfoHash& id, const sockaddr* sa, socklen_t salen)
-        : Node(id, SockAddr(sa, salen)) {}
+    Node(const InfoHash& id, const SockAddr& addr, std::mt19937_64& rd, bool client=false);
+    Node(const InfoHash& id, SockAddr&& addr, std::mt19937_64& rd, bool client=false);
+    Node(const InfoHash& id, const sockaddr* sa, socklen_t salen, std::mt19937_64& rd)
+        : Node(id, SockAddr(sa, salen), rd) {}

    InfoHash getId() const {
        return id;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -26,6 +26,13 @@
 namespace dht {

 struct NodeCache {
+    size_t size(sa_family_t family) const {
+        return cache(family).count();
+    }
+    size_t size() const {
+        return size(AF_INET) + size(AF_INET6);
+    }
+
    Sp<Node> getNode(const InfoHash& id, sa_family_t family);
    Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false);
    std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const;
@@ -37,17 +44,19 @@ struct NodeCache {
     */
    void clearBadNodes(sa_family_t family = 0);

+    NodeCache(std::mt19937_64& r) : rd(r) {};
    ~NodeCache();

 private:
    class NodeMap : private std::map<InfoHash, std::weak_ptr<Node>> {
    public:
        Sp<Node> getNode(const InfoHash& id);
-        Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client);
+        Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client, std::mt19937_64& rd);
        std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, size_t count) const;
        void clearBadNodes();
        void setExpired();
        void cleanup();
+        size_t count() const { return size(); }
    private:
        size_t cleanup_counter {0};
    };
@@ -56,6 +65,7 @@ private:
    NodeMap& cache(sa_family_t af) { return af == AF_INET ? cache_4 : cache_6; }
    NodeMap cache_4;
    NodeMap cache_6;
+    std::mt19937_64& rd;
 };

 }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Mingrui Zhang <mingrui.zhang@savoirfairelinux.com>
 *              Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *
@@ -82,6 +82,8 @@ public:
    bool stopPublish(const std::string &type);
    bool stopPublish(sa_family_t domain, const std::string &type);

+    void connectivityChanged();
+
 private:
    class DomainPeerDiscovery;
    std::unique_ptr<DomainPeerDiscovery> peerDiscovery4_;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2017-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -22,8 +22,8 @@
 namespace dht {
 namespace proxy {

-constexpr const std::chrono::seconds OP_TIMEOUT {1 * 60 * 60}; // one hour
-constexpr const std::chrono::seconds OP_MARGIN {5 * 60}; // 5 minutes
+constexpr const std::chrono::hours OP_TIMEOUT {24}; // one day
+constexpr const std::chrono::hours OP_MARGIN {2}; // two hours
 constexpr const char* const HTTP_PROTO {"http://"};
 using ListenToken = uint64_t;

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2017-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -21,6 +21,8 @@
 #include <random>
 #include <algorithm>
 #include <functional>
+#include <thread>
+#include <stdexcept>

 namespace dht {
 namespace crypto {
@@ -111,11 +113,18 @@ using random_device = std::random_device;
 template<class T = std::mt19937, std::size_t N = T::state_size>
 auto getSeededRandomEngine () -> typename std::enable_if<!!N, T>::type {
    typename T::result_type random_data[N];
-    random_device source;
-    std::generate(std::begin(random_data), std::end(random_data), std::ref(source));
-    std::seed_seq seeds(std::begin(random_data), std::end(random_data));
-    T seededEngine (seeds);
-    return seededEngine;
+    for (unsigned i=0; i<256; i++) {
+        try {
+            random_device source;
+            std::generate(std::begin(random_data), std::end(random_data), std::ref(source));
+            std::seed_seq seeds(std::begin(random_data), std::end(random_data));
+            T seededEngine (seeds);
+            return seededEngine;
+        } catch (...) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+    }
+    throw std::runtime_error("Can't seed random engine");
 }

 }} // dht::crypto
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -39,7 +39,7 @@ struct Bucket {
    Sp<Node> cached;                    /* the address of a likely candidate */

    /** Return a random node in a bucket. */
-    Sp<Node> randomNode();
+    Sp<Node> randomNode(std::mt19937_64& rd);

    void sendCachedPing(net::NetworkEngine& ne);
    void connectivityChanged() {
@@ -89,7 +89,7 @@ public:
    /**
     * Return a random id in the bucket's range.
     */
-    InfoHash randomId(const RoutingTable::const_iterator& bucket) const;
+    InfoHash randomId(const RoutingTable::const_iterator& bucket, std::mt19937_64& rd) const;

    unsigned depth(const RoutingTable::const_iterator& bucket) const;

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -84,7 +84,6 @@ public:
     * @return The time for the next job to run.
     */
    time_point run() {
-        syncTime();
        while (not timers.empty()) {
            auto timer = timers.begin();
            /*
@@ -114,6 +113,7 @@ public:
     */
    inline const time_point& time() const { return now; }
    inline time_point syncTime() { return (now = clock::now()); }
+    inline void syncTime(const time_point& n) { now = n; }

 private:
    time_point now {clock::now()};
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -195,11 +195,14 @@ public:
    std::vector<SockAddr> getPublicAddress(sa_family_t family = 0) override {
        return dht_->getPublicAddress(family);
    }
-    time_point periodic(const uint8_t *buf, size_t buflen, SockAddr sa) override {
-        return dht_->periodic(buf, buflen, std::move(sa));
+    time_point periodic(const uint8_t *buf, size_t buflen, SockAddr sa, const time_point& now) override {
+        return dht_->periodic(buf, buflen, std::move(sa), now);
    }
-    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) override {
-        return dht_->periodic(buf, buflen, from, fromlen);
+    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen, const time_point& now) override {
+        return dht_->periodic(buf, buflen, from, fromlen, now);
+    }
+    NodeStatus updateStatus(sa_family_t af) override  {
+        return dht_->updateStatus(af);
    }
    NodeStatus getStatus(sa_family_t af) const override {
        return dht_->getStatus(af);
@@ -216,6 +219,12 @@ public:
    const ValueType& getType(ValueType::Id type_id) const override {
        return dht_->getType(type_id);
    }
+    void addBootstrap(const std::string& host, const std::string& service) override {
+        dht_->addBootstrap(host, service);
+    }
+    void clearBootstrap() override {
+        dht_->clearBootstrap();
+    }
    void insertNode(const InfoHash& id, const SockAddr& sa) override {
        dht_->insertNode(id, sa);
    }
@@ -316,11 +325,16 @@ public:
        dht_->setLogger(logger);
    }

+    void setLogger(const std::shared_ptr<Logger>& logger) override {
+        DhtInterface::setLogger(logger);
+        dht_->setLogger(logger);
+    }
+
    /**
     * Only print logs related to the given InfoHash (if given), or disable filter (if zeroes).
     */
    void setLogFilter(const InfoHash& f) override {
-        DHT_LOG.setFilter(f);
+        DhtInterface::setLogFilter(f);
        dht_->setLogFilter(f);
    }

@@ -345,6 +359,7 @@ private:
    std::map<InfoHash, Sp<const crypto::PublicKey>> nodesPubKeys_ {};

    std::atomic_bool forward_all_ {false};
+    bool enableCache_ {false};
 };

 const ValueType CERTIFICATE_TYPE = {
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -77,12 +77,20 @@ public:
 // Time related definitions and utility functions

 using clock = std::chrono::steady_clock;
+using system_clock = std::chrono::system_clock;
 using time_point = clock::time_point;
 using duration = clock::duration;

 time_point from_time_t(std::time_t t);
 std::time_t to_time_t(time_point t);

+inline std::string
+to_str(double d) {
+    char buf[16];
+    auto ret = snprintf(buf, sizeof(buf), "%.3g", d);
+    return (ret < 0) ? std::to_string(d) : std::string(buf, ret);
+}
+
 /**
 * Converts std::chrono::duration to floating-point seconds.
 */
@@ -97,16 +105,28 @@ static std::string
 print_duration(DT d) {
    if (d < std::chrono::seconds(0)) {
        return "-" + print_duration(-d);
-    }
-    if (d < std::chrono::milliseconds(1)) {
-        return std::to_string(std::chrono::duration_cast<std::chrono::duration<double, std::micro>>(d).count()) +  " us";
+    } else if (d < std::chrono::milliseconds(1)) {
+        return to_str(std::chrono::duration_cast<std::chrono::duration<double, std::micro>>(d).count()) +  " us";
    } else if (d < std::chrono::seconds(1)) {
-        return std::to_string(std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(d).count()) +  " ms";
+        return to_str(std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(d).count()) +  " ms";
+    } else if (d < std::chrono::minutes(1)) {
+        return to_str(std::chrono::duration_cast<std::chrono::duration<double>>(d).count()) +  " s";
+    } else if (d < std::chrono::hours(1)) {
+        return to_str(std::chrono::duration_cast<std::chrono::duration<double, std::ratio<60>>>(d).count()) +  " min";
    } else {
-        return std::to_string(print_dt(d)) + " s";
+        return to_str(std::chrono::duration_cast<std::chrono::duration<double, std::ratio<3600>>>(d).count()) +  " h";
    }
 }

+template <class TimePoint>
+static std::string
+print_time_relative(TimePoint now, TimePoint d) {
+    if (d == TimePoint::min()) return "never";
+    if (d == now)              return "now";
+    return (d > now) ? std::string("in ") + print_duration(d - now)
+                     : print_duration(now - d) + std::string(" ago");
+}
+
 template <typename Duration = duration>
 class uniform_duration_distribution : public std::uniform_int_distribution<typename Duration::rep> {
    using Base = std::uniform_int_distribution<typename Duration::rep>;
@@ -133,7 +153,7 @@ using Blob = std::vector<uint8_t>;
 /**
 * Provides backward compatibility with msgpack 1.0
 */
-OPENDHT_PUBLIC Blob unpackBlob(msgpack::object& o);
+OPENDHT_PUBLIC Blob unpackBlob(const msgpack::object& o);

 template <typename Type>
 Blob
@@ -153,6 +173,13 @@ unpackMsg(Blob b) {

 msgpack::unpacked unpackMsg(Blob b);

-msgpack::object* findMapValue(msgpack::object& map, const std::string& key);
+msgpack::object* findMapValue(const msgpack::object& map, const char* key, size_t length);
+
+inline msgpack::object* findMapValue(const msgpack::object& map, const char* key) {
+    return findMapValue(map, key, strlen(key));
+}
+inline msgpack::object* findMapValue(const msgpack::object& map, const std::string& key) {
+    return findMapValue(map, key.c_str(), key.size());
+}

 } // namespace dht
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -20,7 +20,7 @@
 #pragma once

 #include "infohash.h"
-//#include "crypto.h"
+#include "crypto.h"
 #include "utils.h"
 #include "sockaddr.h"

@@ -43,15 +43,22 @@

 namespace dht {

+static const std::string VALUE_KEY_ID("id");
+static const std::string VALUE_KEY_DAT("dat");
+static const std::string VALUE_KEY_PRIO("p");
+static const std::string VALUE_KEY_SIGNATURE("sig");
+
+static const std::string VALUE_KEY_SEQ("seq");
+static const std::string VALUE_KEY_DATA("data");
+static const std::string VALUE_KEY_OWNER("owner");
+static const std::string VALUE_KEY_TYPE("type");
+static const std::string VALUE_KEY_TO("to");
+static const std::string VALUE_KEY_BODY("body");
+static const std::string VALUE_KEY_USERTYPE("utype");
+
 struct Value;
 struct Query;

-namespace crypto {
-class PublicKey;
-class PrivateKey;
-class Certificate;
-}
-
 /**
 * A storage policy is applied once to every incoming value storage requests.
 * If the policy returns false, the value is dropped.
@@ -247,9 +254,15 @@ struct OPENDHT_PUBLIC Value
        };
    }

-    static Filter OwnerFilter(const crypto::PublicKey& pk);
+    static Filter OwnerFilter(const crypto::PublicKey& pk) {
+        return OwnerFilter(pk.getId());
+    }

-    static Filter OwnerFilter(const InfoHash& pkh);
+    static Filter OwnerFilter(const InfoHash& pkh) {
+        return [pkh](const Value& v) {
+            return v.owner and v.owner->getId() == pkh;
+        };
+    }

    static Filter SeqNumFilter(uint16_t seq_no) {
        return [seq_no](const Value& v) {
@@ -341,13 +354,20 @@ struct OPENDHT_PUBLIC Value
     * Afterward, checkSignature() will return true and owner will
     * be set to the corresponding public key.
     */
-    void sign(const crypto::PrivateKey& key);
+    void sign(const crypto::PrivateKey& key) {
+        if (isEncrypted())
+            throw DhtException("Can't sign encrypted data.");
+        owner = std::make_shared<const crypto::PublicKey>(key.getPublicKey());
+        signature = key.sign(getToSign());
+    }

    /**
     * Check that the value is signed and that the signature matches.
     * If true, the owner field will contain the signer public key.
     */
-    bool checkSignature() const;
+    bool checkSignature() const {
+        return isSigned() and owner->checkSignature(getToSign(), signature);
+    }

    std::shared_ptr<const crypto::PublicKey> getOwner() const {
        return std::static_pointer_cast<const crypto::PublicKey>(owner);
@@ -356,7 +376,15 @@ struct OPENDHT_PUBLIC Value
    /**
     * Sign the value with from and returns the encrypted version for to.
     */
-    Value encrypt(const crypto::PrivateKey& from, const crypto::PublicKey& to);
+    Value encrypt(const crypto::PrivateKey& from, const crypto::PublicKey& to) {
+        if (isEncrypted())
+            throw DhtException("Data is already encrypted.");
+        setRecipient(to.getId());
+        sign(from);
+        Value nv {id};
+        nv.setCypher(to.encrypt(getToEncrypt()));
+        return nv;
+    }

    Value() {}

@@ -375,7 +403,7 @@ struct OPENDHT_PUBLIC Value
     * Build a value from a json object
     * @param json
     */
-    Value(Json::Value& json);
+    Value(const Json::Value& json);
 #endif

    template <typename Type>
@@ -393,7 +421,9 @@ struct OPENDHT_PUBLIC Value

    Value(Value&& o) noexcept
     : id(o.id), owner(std::move(o.owner)), recipient(o.recipient),
-     type(o.type), data(std::move(o.data)), user_type(std::move(o.user_type)), seq(o.seq), signature(std::move(o.signature)), cypher(std::move(o.cypher)) {}
+     type(o.type), data(std::move(o.data)), user_type(std::move(o.user_type)), seq(o.seq)
+     , signature(std::move(o.signature)), cypher(std::move(o.cypher))
+     , priority(o.priority) {}

    template <typename Type>
    Value(const Type& vs)
@@ -406,14 +436,20 @@ struct OPENDHT_PUBLIC Value
        msgpack_unpack(o);
    }

+    /**
+     * Returns true if value contents are equals (not considering the value ID)
+     */
+    inline bool contentEquals(const Value& o) {
+        return isEncrypted() ? cypher == o.cypher :
+            ((owner == o.owner || (owner and o.owner and *owner == *o.owner))
+                && type == o.type
+                && data == o.data
+                && user_type == o.user_type
+                && signature == o.signature);
+    }
+
    inline bool operator== (const Value& o) {
-        return id == o.id &&
-        (isEncrypted() ? cypher == o.cypher :
-        (
-#ifndef OPENDHT_LIGHT
-            (owner == o.owner || *owner == *o.owner) &&
-#endif
-            type == o.type && data == o.data && user_type == o.user_type && signature == o.signature));
+        return id == o.id and contentEquals(o);
    }

    void setRecipient(const InfoHash& r) {
@@ -471,24 +507,20 @@ struct OPENDHT_PUBLIC Value
    template <typename Packer>
    void msgpack_pack_to_sign(Packer& pk) const
    {
-#ifndef OPENDHT_LIGHT
-        bool has_owner = owner && *owner
+        bool has_owner = owner && *owner;
        pk.pack_map((user_type.empty()?0:1) + (has_owner?(recipient ? 5 : 4):2));
        if (has_owner) { // isSigned
-            pk.pack(std::string("seq"));   pk.pack(seq);
-            pk.pack(std::string("owner")); owner->msgpack_pack(pk);
+            pk.pack(VALUE_KEY_SEQ);   pk.pack(seq);
+            pk.pack(VALUE_KEY_OWNER); owner->msgpack_pack(pk);
            if (recipient) {
-                pk.pack(std::string("to")); pk.pack(recipient);
+                pk.pack(VALUE_KEY_TO); pk.pack(recipient);
            }
        }
-#else
-        pk.pack_map((user_type.empty()?0:1) + 2);
-#endif
-        pk.pack(std::string("type"));  pk.pack(type);
-        pk.pack(std::string("data"));  pk.pack_bin(data.size());
-                                       pk.pack_bin_body((const char*)data.data(), data.size());
+        pk.pack(VALUE_KEY_TYPE);  pk.pack(type);
+        pk.pack(VALUE_KEY_DATA);  pk.pack_bin(data.size());
+                                  pk.pack_bin_body((const char*)data.data(), data.size());
        if (not user_type.empty()) {
-            pk.pack(std::string("utype")); pk.pack(user_type);
+            pk.pack(VALUE_KEY_USERTYPE); pk.pack(user_type);
        }
    }

@@ -500,10 +532,10 @@ struct OPENDHT_PUBLIC Value
            pk.pack_bin_body((const char*)cypher.data(), cypher.size());
        } else {
            pk.pack_map(isSigned() ? 2 : 1);
-            pk.pack(std::string("body")); msgpack_pack_to_sign(pk);
+            pk.pack(VALUE_KEY_BODY); msgpack_pack_to_sign(pk);
            if (isSigned()) {
-                pk.pack(std::string("sig")); pk.pack_bin(signature.size());
-                                             pk.pack_bin_body((const char*)signature.data(), signature.size());
+                pk.pack(VALUE_KEY_SIGNATURE); pk.pack_bin(signature.size());
+                                              pk.pack_bin_body((const char*)signature.data(), signature.size());
            }
        }
    }
@@ -511,9 +543,12 @@ struct OPENDHT_PUBLIC Value
    template <typename Packer>
    void msgpack_pack(Packer& pk) const
    {
-        pk.pack_map(2);
-        pk.pack(std::string("id"));  pk.pack(id);
-        pk.pack(std::string("dat")); msgpack_pack_to_encrypt(pk);
+        pk.pack_map(2 + (priority?1:0));
+        pk.pack(VALUE_KEY_ID);  pk.pack(id);
+        pk.pack(VALUE_KEY_DAT); msgpack_pack_to_encrypt(pk);
+        if (priority) {
+            pk.pack(VALUE_KEY_PRIO);  pk.pack(priority);
+        }
    }

    template <typename Packer>
@@ -528,11 +563,9 @@ struct OPENDHT_PUBLIC Value
                    pk.pack(static_cast<uint64_t>(type));
                    break;
                case Value::Field::OwnerPk:
-#ifndef OPENDHT_LIGHT
                    if (owner)
                        owner->msgpack_pack(pk);
                    else
-#endif
                        InfoHash().msgpack_pack(pk);
                    break;
                case Value::Field::SeqNum:
@@ -546,7 +579,7 @@ struct OPENDHT_PUBLIC Value
            }
    }

-    void msgpack_unpack(msgpack::object o);
+    void msgpack_unpack(const msgpack::object& o);
    void msgpack_unpack_body(const msgpack::object& o);
    Blob getPacked() const {
        msgpack::sbuffer buffer;
@@ -597,6 +630,13 @@ struct OPENDHT_PUBLIC Value
     */
    Blob cypher {};

+    /**
+     * Priority of this value (used for push notifications)
+     * 0 = high priority
+     * 1 = normal priority
+     */
+    unsigned priority {0};
+
 private:
    friend class SecureDht;
    /* Cache for crypto ops */
@@ -653,7 +693,7 @@ struct OPENDHT_PUBLIC FieldValue
        }
    }

-    void msgpack_unpack(msgpack::object msg) {
+    void msgpack_unpack(const msgpack::object& msg) {
        hashValue = {};
        blobValue.clear();

@@ -740,6 +780,8 @@ struct OPENDHT_PUBLIC Select
        return ss.str();
    }

+    bool empty() const { return fieldSelection_.empty(); }
+
    OPENDHT_PUBLIC friend std::ostream& operator<<(std::ostream& s, const dht::Select& q);
 private:
    std::vector<Value::Field> fieldSelection_ {};
@@ -3,9 +3,9 @@ exec_prefix=@exec_prefix@
 libdir=@libdir@
 includedir=@includedir@
 Name: OpenDHT
-Description: C++ Distributed Hash Table library
+Description: C++14 Distributed Hash Table library
 Version: @VERSION@
 Libs: -L${libdir} -lopendht
-Libs.private: -lpthread
-Requires.private: gnutls >= 3.1@argon2_lib@
+Libs.private: @http_parser_lib@ -pthread
+Requires.private: gnutls >= 3.3, nettle >= 2.4@argon2_lib@@jsoncpp_lib@@openssl_lib@
 Cflags: -I${includedir}
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 Savoir-faire Linux Inc.
+# Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 # Author: Guillaume Roguez <guillaume.roguez@savoirfairelinux.com>
 # Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 #
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Copyright (C) 2015-2019 Savoir-faire Linux Inc.
+# Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 # Author(s): Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 #            Simon Désaulniers <sim.desaulniers@gmail.com>
 #
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Copyright (C) 2015-2019 Savoir-faire Linux Inc.
+# Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 # Author(s): Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 #
 # This program is free software; you can redistribute it and/or modify
@@ -35,13 +35,21 @@ pong_node.ping(ping_node.getBound())
 loc_ping = dht.InfoHash.get("toto99")
 loc_pong = dht.InfoHash.get(str(loc_ping))

+net = []
+for i in range(1,10):
+	node = dht.DhtRunner()
+	node.run(config=config)
+	node.ping(ping_node.getBound())
+	net.append(node)
+
 i = 0
 MAX = 2048

 loop = asyncio.get_event_loop()

 def done(h, ok):
-	print(h, "over", ok)
+	pass
+	#print(h, "over", ok)

 def ping(node, h):
 	global i
@@ -52,13 +60,20 @@ def ping(node, h):
 		loop.stop()

 def pong(node, h):
-	print(node.getNodeId().decode(), "got ping", h, i)
+	#print(node.getNodeId().decode(), "got ping", h, i)
 	loop.call_soon_threadsafe(ping, node, h)
 	return True

+t1 = time.time()
+
 ping_node.listen(loc_ping, lambda v, e: pong(pong_node, loc_pong) if not e else True)
 pong_node.listen(loc_pong, lambda v, e: pong(ping_node, loc_ping) if not e else True)

 ping(pong_node, loc_ping)

 loop.run_forever()
+
+t2 = time.time()
+
+print(MAX, "ping-pong done, took", t2 - t1, "s")
+print(1000 * (t2 - t1)/MAX, "ms per rt", MAX/(t2 - t1), "rt per s")
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
@@ -131,7 +131,7 @@ base64_encode(const std::vector<unsigned char>& str)
    return base64_encode(str.cbegin(), str.cend());
 }

-std::string
+std::vector<unsigned char>
 base64_decode(const std::string& str)
 {
    size_t output_length = str.length() / 4 * 3 + 2;
@@ -139,5 +139,5 @@ base64_decode(const std::string& str)
    output.resize(output_length);
    base64_decode(str.data(), str.size(), output.data(), &output_length);
    output.resize(output_length);
-    return std::string(output.begin(), output.end());
+    return output;
 }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
@@ -33,4 +33,4 @@ std::string base64_encode(const std::vector<unsigned char>& str);
 * @param str the input buffer
 * @return a base64-decoded buffer
 */
-std::string base64_decode(const std::string& str);
+std::vector<unsigned char> base64_decode(const std::string& str);
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -94,6 +94,7 @@ NodeStats::toString() const
 {
    std::stringstream ss;
    ss << "Known nodes: " << good_nodes << " good, " << dubious_nodes << " dubious, " << incoming_nodes << " incoming." << std::endl;
+    ss << searches << " searches, " << node_cache_size << " total cached nodes" << std::endl;
    if (table_depth > 1) {
        ss << "Routing table depth: " << table_depth << std::endl;
        ss << "Network size estimation: " << getNetworkSizeEstimation() << " nodes" << std::endl;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -412,12 +412,18 @@ PublicKey::pack(Blob& b) const
        throw CryptoException(std::string("Could not export public key: null key"));
    std::vector<uint8_t> tmp(2048);
    size_t sz = tmp.size();
-    if (int err = gnutls_pubkey_export(pk, GNUTLS_X509_FMT_DER, tmp.data(), &sz))
+    if (int err = pack(tmp.data(), &sz))
        throw CryptoException(std::string("Could not export public key: ") + gnutls_strerror(err));
    tmp.resize(sz);
    b.insert(b.end(), tmp.begin(), tmp.end());
 }

+int
+PublicKey::pack(uint8_t* out, size_t* out_len) const
+{
+    return gnutls_pubkey_export(pk, GNUTLS_X509_FMT_DER, out, out_len);
+}
+
 void
 PublicKey::unpack(const uint8_t* data, size_t data_size)
 {
@@ -447,7 +453,7 @@ PublicKey::toString() const
 }

 void
-PublicKey::msgpack_unpack(msgpack::object o)
+PublicKey::msgpack_unpack(const msgpack::object& o)
 {
    if (o.type == msgpack::type::BIN)
        unpack((const uint8_t*)o.via.bin.ptr, o.via.bin.size);
@@ -792,7 +798,7 @@ Certificate::unpack(const uint8_t* dat, size_t dat_size)
 }

 void
-Certificate::msgpack_unpack(msgpack::object o)
+Certificate::msgpack_unpack(const msgpack::object& o)
 {
    if (o.type == msgpack::type::BIN)
        unpack((const uint8_t*)o.via.bin.ptr, o.via.bin.size);
@@ -1252,7 +1258,7 @@ RevocationList::unpack(const uint8_t* dat, size_t dat_size)
 }

 void
-RevocationList::msgpack_unpack(msgpack::object o)
+RevocationList::msgpack_unpack(const msgpack::object& o)
 {
    try {
        if (o.type == msgpack::type::BIN)
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -15,9 +15,6 @@
 *  You should have received a copy of the GNU General Public License
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif

 #include "default_types.h"

@@ -86,8 +83,6 @@ IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr<Value>& v, const

 const ValueType DhtMessage::TYPE(1, "DHT message", std::chrono::minutes(5), DhtMessage::storePolicy);
 const ValueType IpServiceAnnouncement::TYPE(2, "Internet Service Announcement", std::chrono::minutes(15), IpServiceAnnouncement::storePolicy);
-
-#ifndef OPENDHT_LIGHT
 const ValueType ImMessage::TYPE = {3, "IM message", std::chrono::minutes(5)};
 const ValueType TrustRequest::TYPE = {4, "Certificate trust request", std::chrono::hours(24*7)};
 const ValueType IceCandidates::TYPE = {5, "ICE candidates", std::chrono::minutes(1)};
@@ -101,13 +96,6 @@ DEFAULT_TYPES
    IceCandidates::TYPE,
    TrustRequest::TYPE
 }};
-#else
-const std::array<std::reference_wrapper<const ValueType>, 1>
-DEFAULT_TYPES
-{{
-    ValueType::USER_DATA
-}};
-#endif

 const std::array<std::reference_wrapper<const ValueType>, 1>
 DEFAULT_INSECURE_TYPES
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *          Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *          Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
@@ -46,7 +46,7 @@ struct DhtProxyClient::Listener

    unsigned callbackId;
    OpValueCache cache;
-    ValueCallback cb;
+    CacheValueCallback cb;
    Sp<OperationState> opstate;
    std::shared_ptr<http::Request> request;
    std::unique_ptr<asio::steady_timer> refreshSubscriberTimer;
@@ -68,6 +68,7 @@ struct DhtProxyClient::ProxySearch {
    std::unique_ptr<asio::steady_timer> opExpirationTimer;
    std::map<size_t, Listener> listeners {};
    std::map<Value::Id, PermanentPut> puts {};
+    std::set<Sp<Value>> pendingPuts  {};
 };

 struct LineSplit {
@@ -77,13 +78,12 @@ struct LineSplit {
    bool getLine(char c) {
        auto it = buf_.begin();
        while (it != buf_.end()) {
-            if (*it == c) {
+            if (*(it++) == c) {
                line_.clear();
-                line_.insert(line_.end(), buf_.begin(), ++it);
+                line_.insert(line_.end(), buf_.begin(), it);
                buf_.erase(buf_.begin(), it);
                return true;
            }
-            it++;
        }
        return false;
    }
@@ -93,17 +93,28 @@ private:
    std::string line_ {};
 };

+std::string
+getRandomSessionId(size_t length = 8) {
+    static constexpr const char chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,./:;<=>?@[]^_`{|}~";
+    std::string str(length, 0);
+    crypto::random_device rdev;
+    std::uniform_int_distribution<> dist(0, (sizeof(chars)/sizeof(char)) - 2);
+    std::generate_n( str.begin(), length, [&]{ return chars[dist(rdev)]; } );
+    return str;
+}
+
 DhtProxyClient::DhtProxyClient() {}

 DhtProxyClient::DhtProxyClient(
        std::shared_ptr<dht::crypto::Certificate> serverCA, dht::crypto::Identity clientIdentity,
        std::function<void()> signal, const std::string& serverHost,
        const std::string& pushClientId, std::shared_ptr<dht::Logger> logger)
-    : proxyUrl_(serverHost)
+    : DhtInterface(logger)
+    , proxyUrl_(serverHost)
    , clientIdentity_(clientIdentity), serverCertificate_(serverCA)
-    , pushClientId_(pushClientId), loopSignal_(signal)
+    , pushClientId_(pushClientId), pushSessionId_(getRandomSessionId())
+    , loopSignal_(signal)
    , jsonReader_(Json::CharReaderBuilder{}.newCharReader())
-    , logger_(logger)
 {
    jsonBuilder_["commentStyle"] = "None";
    jsonBuilder_["indentation"] = "";
@@ -269,7 +280,7 @@ DhtProxyClient::isRunning(sa_family_t af) const
 }

 time_point
-DhtProxyClient::periodic(const uint8_t*, size_t, SockAddr)
+DhtProxyClient::periodic(const uint8_t*, size_t, SockAddr, const time_point& /*now*/)
 {
    // Exec all currently stored callbacks
    decltype(callbacks_) callbacks;
@@ -315,7 +326,8 @@ DhtProxyClient::get(const InfoHash& key, GetCallback cb, DoneCallback donecb, Va
                auto& b = *rxBuf;
                b.append(at, length);
                // one value per body line
-                while (b.getLine('\n') and !opstate->stop){
+                std::vector<Sp<Value>> values;
+                while (b.getLine('\n') and !opstate->stop) {
                    std::string err;
                    Json::Value json;
                    const auto& line = b.line();
@@ -324,17 +336,19 @@ DhtProxyClient::get(const InfoHash& key, GetCallback cb, DoneCallback donecb, Va
                        return;
                    }
                    auto value = std::make_shared<Value>(json);
-                    if ((not filter or filter(*value)) and cb){
-                        {
-                            std::lock_guard<std::mutex> lock(lockCallbacks_);
-                            callbacks_.emplace_back([opstate, cb, value](){
-                                if (not opstate->stop.load() and not cb({value})){
-                                    opstate->stop.store(true);
-                                }
-                            });
-                        }
-                        loopSignal_();
+                    if ((not filter or filter(*value)) and cb)
+                        values.emplace_back(std::move(value));
+                }
+                if (not values.empty() and cb) {
+                    {
+                        std::lock_guard<std::mutex> lock(lockCallbacks_);
+                        callbacks_.emplace_back([opstate, cb, values = std::move(values)](){
+                            if (not opstate->stop.load() and not cb(values)){
+                                opstate->stop.store(true);
+                            }
+                        });
                    }
+                    loopSignal_();
                }
            } catch(const std::exception& e) {
                if (logger_)
@@ -387,23 +401,22 @@ DhtProxyClient::put(const InfoHash& key, Sp<Value> val, DoneCallback cb, time_po
    if (logger_)
        logger_->d("[proxy:client] [put] [search %s]", key.to_c_str());

-    if (val->id == Value::INVALID_ID) {
-        crypto::random_device rdev;
-        val->id = std::uniform_int_distribution<Value::Id>{1}(rdev);
-    }
    std::shared_ptr<std::atomic_bool> ok;
    if (permanent) {
        std::lock_guard<std::mutex> lock(searchLock_);
-        auto id = val->id;
-        auto& search = searches_[key];
-        auto refreshPutTimer = std::make_unique<asio::steady_timer>(httpContext_, std::chrono::steady_clock::now() +
-                                                                    proxy::OP_TIMEOUT - proxy::OP_MARGIN);
-        refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this, std::placeholders::_1, key, id));
-        search.puts.erase(id);
        ok = std::make_shared<std::atomic_bool>(true);
-        search.puts.emplace(std::piecewise_construct,
-            std::forward_as_tuple(id),
-            std::forward_as_tuple(val, std::move(refreshPutTimer), ok));
+        auto& search = searches_[key];
+        if (val->id) {
+            auto id = val->id;
+            auto refreshPutTimer = std::make_unique<asio::steady_timer>(httpContext_, proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+            refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this, std::placeholders::_1, key, id));
+            search.puts.erase(id);
+            search.puts.emplace(std::piecewise_construct,
+                std::forward_as_tuple(id),
+                std::forward_as_tuple(val, std::move(refreshPutTimer), ok));
+        } else {
+            search.pendingPuts.emplace(val);
+        }
    }
    doPut(key, val, [this, cb, ok](bool result){
        if (ok)
@@ -438,7 +451,7 @@ DhtProxyClient::handleRefreshPut(const asio::error_code &ec, InfoHash key, Value
            doPut(key, p->second.value, [ok = p->second.ok](bool result){
                *ok = result;
            }, time_point::max(), true);
-            p->second.refreshPutTimer->expires_at(std::chrono::steady_clock::now() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+            p->second.refreshPutTimer->expires_after(proxy::OP_TIMEOUT - proxy::OP_MARGIN);
            p->second.refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this, std::placeholders::_1, key, id));
        }
    }
@@ -450,7 +463,7 @@ DhtProxyClient::buildRequest(const std::string& target)
    auto resolver = resolver_;
    if (not resolver)
        resolver = std::make_shared<http::Resolver>(httpContext_, proxyUrl_, logger_);
-    auto request = target.empty() 
+    auto request = target.empty()
        ? std::make_shared<http::Request>(httpContext_, resolver)
        : std::make_shared<http::Request>(httpContext_, resolver, target);
    if (serverCertificate_)
@@ -488,9 +501,35 @@ DhtProxyClient::doPut(const InfoHash& key, Sp<Value> val, DoneCallbackSimple cb,
            }
        }
        request->set_body(Json::writeString(jsonBuilder_, json));
-        request->add_on_done_callback([this, reqid, cb] (const http::Response& response){
+        request->add_on_done_callback([this, reqid, cb, val, key, permanent] (const http::Response& response){
            bool ok = response.status_code == 200;
-            if (not ok) {
+            if (ok) {
+                if (val->id == Value::INVALID_ID) {
+                    std::string err;
+                    Json::Value parsedValue;
+                    if (jsonReader_->parse(response.body.data(), response.body.data() + response.body.size(), &parsedValue, &err)){
+                        auto id = dht::Value(parsedValue).id;
+                        val->id = id;
+                        if (permanent) {
+                            std::lock_guard<std::mutex> lock(searchLock_);
+                            auto& search = searches_[key];
+                            auto it = search.pendingPuts.find(val);
+                            if (it != search.pendingPuts.end()) {
+                                auto sok = std::make_shared<std::atomic_bool>(ok);
+                                auto refreshPutTimer = std::make_unique<asio::steady_timer>(httpContext_, proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+                                refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this, std::placeholders::_1, key, id));
+                                search.puts.emplace(std::piecewise_construct,
+                                    std::forward_as_tuple(id),
+                                    std::forward_as_tuple(val, std::move(refreshPutTimer), sok));
+                                search.pendingPuts.erase(it);
+                            }
+                        }
+                    } else {
+                        if (logger_)
+                            logger_->e("[proxy:client] [status] failed to parse value from  server", response.status_code);
+                    }
+                }
+            } else {
                if (logger_)
                    logger_->e("[proxy:client] [status] failed with code=%i", response.status_code);
                if (not response.aborted and response.status_code == 0)
@@ -585,20 +624,20 @@ DhtProxyClient::getProxyInfos()
        logger_->d("[proxy:client] [status] sending request");

    auto resolver = std::make_shared<http::Resolver>(httpContext_, proxyUrl_, logger_);
-    queryProxyInfo(infoState, AF_INET, resolver);
-    queryProxyInfo(infoState, AF_INET6, resolver);
+    queryProxyInfo(infoState, resolver, AF_INET);
+    queryProxyInfo(infoState, resolver, AF_INET6);
    resolver_ = resolver;
 }

 void
-DhtProxyClient::queryProxyInfo(std::shared_ptr<InfoState> infoState, sa_family_t family, std::shared_ptr<http::Resolver> resolver)
+DhtProxyClient::queryProxyInfo(const Sp<InfoState>& infoState, const Sp<http::Resolver>& resolver, sa_family_t family)
 {
    if (logger_)
        logger_->d("[proxy:client] [status] query ipv%i info", family == AF_INET ? 4 : 6);
    try {
        auto request = std::make_shared<http::Request>(httpContext_, resolver, family);
        auto reqid = request->id();
-        request->set_method(restinio::http_method_get()); 
+        request->set_method(restinio::http_method_get());
        setHeaderFields(*request);
        request->add_on_done_callback([this, reqid, family, infoState] (const http::Response& response){
            if (infoState->cancel.load())
@@ -615,10 +654,9 @@ DhtProxyClient::queryProxyInfo(std::shared_ptr<InfoState> infoState, sa_family_t
                Json::Value proxyInfos;
                if (!jsonReader_->parse(response.body.data(), response.body.data() + response.body.size(), &proxyInfos, &err)){
                    onProxyInfos(Json::Value{}, family);
-                    return;
-                }
-                if (not infoState->cancel)
+                } else if (not infoState->cancel) {
                    onProxyInfos(proxyInfos, family);
+                }
            }
            if (not isDestroying_) {
                std::lock_guard<std::mutex> l(requestLock_);
@@ -750,7 +788,7 @@ DhtProxyClient::listen(const InfoHash& key, ValueCallback cb, Value::Filter filt
        // Add cache callback
        auto opstate = std::make_shared<OperationState>();
        l->second.opstate = opstate;
-        l->second.cb = [this,key,token,opstate](const std::vector<Sp<Value>>& values, bool expired){
+        l->second.cb = [this,key,token,opstate](const std::vector<Sp<Value>>& values, bool expired, system_clock::time_point t){
            if (opstate->stop)
                return false;
            std::lock_guard<std::mutex> lock(searchLock_);
@@ -758,7 +796,7 @@ DhtProxyClient::listen(const InfoHash& key, ValueCallback cb, Value::Filter filt
            if (s != searches_.end()) {
                auto l = s->second.listeners.find(token);
                if (l != s->second.listeners.end()) {
-                    return l->second.cache.onValue(values, expired);
+                    return l->second.cache.onValue(values, expired, t);
                }
            }
            return false;
@@ -929,8 +967,8 @@ DhtProxyClient::handleExpireListener(const asio::error_code &ec, const InfoHash&
 }

 void
-DhtProxyClient::sendListen(const restinio::http_request_header_t header,
-                           const ValueCallback& cb,
+DhtProxyClient::sendListen(const restinio::http_request_header_t& header,
+                           const CacheValueCallback& cb,
                           const Sp<OperationState>& opstate,
                           Listener& listener, ListenMethod method)
 {
@@ -957,7 +995,7 @@ DhtProxyClient::sendListen(const restinio::http_request_header_t header,
                b.append(at, length);

                // one value per body line
-                while (b.getLine('\n') and !opstate->stop){
+                while (b.getLine('\n') and !opstate->stop) {
                    std::string err;
                    Json::Value json;
                    const auto& line = b.line();
@@ -975,7 +1013,7 @@ DhtProxyClient::sendListen(const restinio::http_request_header_t header,
                        {
                            std::lock_guard<std::mutex> lock(lockCallbacks_);
                            callbacks_.emplace_back([cb, value, opstate, expired]() {
-                                if (not opstate->stop.load() and not cb({value}, expired))
+                                if (not opstate->stop.load() and not cb({value}, expired, system_clock::time_point::min()))
                                    opstate->stop.store(true);
                            });
                        }
@@ -1051,17 +1089,15 @@ DhtProxyClient::restartListeners()
    for (auto& search : searches_) {
        auto key = search.first;
        for (auto& put : search.second.puts) {
-            if (!*put.second.ok) {
-                doPut(key, put.second.value, [ok = put.second.ok](bool result){
-                    *ok = result;
-                }, time_point::max(), true);
-                if (!put.second.refreshPutTimer){
-                    put.second.refreshPutTimer = std::make_unique<asio::steady_timer>(httpContext_);
-                }
-                put.second.refreshPutTimer->expires_at(std::chrono::steady_clock::now() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
-                put.second.refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this,
-                                                        std::placeholders::_1, key, put.first));
+            doPut(key, put.second.value, [ok = put.second.ok](bool result){
+                *ok = result;
+            }, time_point::max(), true);
+            if (!put.second.refreshPutTimer) {
+                put.second.refreshPutTimer = std::make_unique<asio::steady_timer>(httpContext_);
            }
+            put.second.refreshPutTimer->expires_at(std::chrono::steady_clock::now() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+            put.second.refreshPutTimer->async_wait(std::bind(&DhtProxyClient::handleRefreshPut, this,
+                                                   std::placeholders::_1, key, put.first));
        }
    }
    if (not deviceKey_.empty()) {
@@ -1118,6 +1154,12 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
        statusIpv6_ = NodeStatus::Connected;
    }
    try {
+        auto sessionId = notification.find("s");
+        if (sessionId != notification.end() and sessionId->second != pushSessionId_) {
+            if (logger_)
+                logger_->d("[proxy:client] [push] ignoring push for other session");
+            return;
+        }
        std::lock_guard<std::mutex> lock(searchLock_);
        auto timeout = notification.find("timeout");
        if (timeout != notification.cend()) {
@@ -1140,6 +1182,10 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
            }
        } else {
            auto key = InfoHash(notification.at("key"));
+            system_clock::time_point sendTime = system_clock::time_point::min();
+            try {
+                sendTime = system_clock::time_point(std::chrono::milliseconds(std::stoull(notification.at("t"))));
+            } catch (...) {}
            auto& search = searches_.at(key);
            for (auto& list : search.listeners) {
                if (list.second.opstate->stop)
@@ -1152,24 +1198,24 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
                if (expired == notification.end()) {
                    auto cb = list.second.cb;
                    auto oldValues = list.second.cache.getValues();
-                    get(key, [cb](const std::vector<Sp<Value>>& vals) {
-                        return cb(vals, false);
-                    }, [cb, oldValues](bool /*ok*/) {
+                    get(key, [cb, sendTime](const std::vector<Sp<Value>>& vals) {
+                        return cb(vals, false, sendTime);
+                    }, [cb, oldValues, sendTime](bool /*ok*/) {
                        // Decrement old values refcount to expire values not
                        // present in the new list
-                        cb(oldValues, true);
+                        cb(oldValues, true, sendTime);
                    });
                } else {
                    std::stringstream ss(expired->second);
                    std::vector<Value::Id> ids;
-                    while(ss.good()){
+                    while(ss.good()) {
                        std::string substr;
                        getline(ss, substr, ',');
                        ids.emplace_back(std::stoull(substr));
                    }
                    {
                        std::lock_guard<std::mutex> lock(lockCallbacks_);
-                        callbacks_.emplace_back([this, key, token, opstate, ids]() {
+                        callbacks_.emplace_back([this, key, token, opstate, ids, sendTime]() {
                            if (opstate->stop)
                                return;
                            std::lock_guard<std::mutex> lock(searchLock_);
@@ -1179,7 +1225,7 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
                            auto l = s->second.listeners.find(token);
                            if (l == s->second.listeners.end())
                                return;
-                            if (not opstate->stop and not l->second.cache.onValuesExpired(ids))
+                            if (not opstate->stop and not l->second.cache.onValuesExpired(ids, sendTime))
                                opstate->stop = true;
                        });
                    }
@@ -1237,6 +1283,7 @@ DhtProxyClient::getPushRequest(Json::Value& body) const
 {
    body["key"] = deviceKey_;
    body["client_id"] = pushClientId_;
+    body["session_id"] = pushSessionId_;
 #ifdef __ANDROID__
    body["platform"] = "android";
 #endif
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -32,17 +32,11 @@
 #include "dht_proxy_client.h"
 #endif

-#ifdef _WIN32
-#include <cstring>
-#define close(x) closesocket(x)
-#define write(s, b, f) send(s, b, (int)strlen(b), 0)
-#endif
+#include <fstream>

 namespace dht {

 constexpr std::chrono::seconds DhtRunner::BOOTSTRAP_PERIOD;
-static constexpr size_t RX_QUEUE_MAX_SIZE = 1024 * 16;
-static constexpr std::chrono::milliseconds RX_QUEUE_MAX_DELAY(500);
 static const std::string PEER_DISCOVERY_DHT_SERVICE = "dht";

 struct DhtRunner::Listener {
@@ -98,16 +92,46 @@ DhtRunner::run(const char* ip4, const char* ip6, const char* service, const Conf
 {
    auto res4 = SockAddr::resolve(ip4, service);
    auto res6 = SockAddr::resolve(ip6, service);
-    run(res4.empty() ? SockAddr() : res4.front(),
-        res6.empty() ? SockAddr() : res6.front(), config, std::move(context));
+    if (res4.empty())
+        res4.emplace_back();
+    if (res6.empty())
+        res6.emplace_back();
+    run(res4.front(), res6.front(), config, std::move(context));
 }

 void
-DhtRunner::run(const SockAddr& local4, const SockAddr& local6, const Config& config, Context&& context)
+DhtRunner::run(SockAddr& local4, SockAddr& local6, const Config& config, Context&& context)
 {
    if (running == State::Idle) {
+        auto state_path = config.dht_config.node_config.persist_path;
+        if (not state_path.empty()) {
+            state_path += "_port.txt";
+            std::ifstream inConfig(state_path);
+            if (inConfig.is_open()) {
+                in_port_t port;
+                if (inConfig >> port) {
+                    if (context.logger)
+                        context.logger->d("[runner %p] Using IPv4 port %hu from saved configuration", this, port);
+                    if (local4.getPort() == 0)
+                        local4.setPort(port);
+                }
+                if (inConfig >> port) {
+                    if (context.logger)
+                        context.logger->d("[runner %p] Using IPv6 port %hu from saved configuration", this, port);
+                    if (local6.getPort() == 0)
+                        local6.setPort(port);
+                }
+            }
+        }
+
        if (not context.sock)
-            context.sock.reset(new net::UdpSocket(local4, local6, context.logger ? *context.logger : Logger{}));
+            context.sock.reset(new net::UdpSocket(local4, local6, context.logger));
+
+        if (not state_path.empty()) {
+            std::ofstream outConfig(state_path);
+            outConfig << context.sock->getBoundRef(AF_INET).getPort() << std::endl;
+            outConfig << context.sock->getBoundRef(AF_INET6).getPort() << std::endl;
+        }
        run(config, std::move(context));
    }
 }
@@ -125,20 +149,25 @@ DhtRunner::run(const Config& config, Context&& context)
        logger_->d("[runner %p] state changed to Running", this);
    }

-    context.sock->setOnReceive([&] (std::unique_ptr<net::ReceivedPacket>&& pkt) {
+    context.sock->setOnReceive([&] (net::PacketList&& pkts) {
+        net::PacketList ret;
        {
            std::lock_guard<std::mutex> lck(sock_mtx);
-            if (rcv.size() >= RX_QUEUE_MAX_SIZE) {
+            auto maxSize = net::RX_QUEUE_MAX_SIZE - pkts.size();
+            while (rcv.size() > maxSize) {
                if (logger_)
                    logger_->e("Dropping packet: queue is full!");
-                rcv.pop();
+                rcv.pop_front();
            }
-            rcv.emplace(std::move(pkt));
+
+            rcv.splice(rcv.end(), std::move(pkts));
+            ret = std::move(rcv_free);
        }
        cv.notify_all();
+        return ret;
    });

-    auto dht = std::unique_ptr<DhtInterface>(new Dht(std::move(context.sock), SecureDht::getConfig(config.dht_config), context.logger ? *context.logger : Logger{}));
+    auto dht = std::unique_ptr<DhtInterface>(new Dht(std::move(context.sock), SecureDht::getConfig(config.dht_config), context.logger));
    dht_ = std::unique_ptr<SecureDht>(new SecureDht(std::move(dht), config.dht_config));

 #ifdef OPENDHT_PROXY_CLIENT
@@ -146,7 +175,7 @@ DhtRunner::run(const Config& config, Context&& context)
 #endif
    enableProxy(not config.proxy_server.empty());
    if (context.logger and dht_via_proxy_) {
-        dht_via_proxy_->setLogger(*context.logger);
+        dht_via_proxy_->setLogger(context.logger);
    }
    if (context.statusChangedCallback) {
        statusCb = std::move(context.statusChangedCallback);
@@ -177,7 +206,7 @@ DhtRunner::run(const Config& config, Context&& context)
                    if (not pending_ops_prio.empty())
                        return true;
                    auto s = getStatus();
-                    if (not pending_ops.empty() and (s == NodeStatus::Connected or (s == NodeStatus::Disconnected and not bootstraping)))
+                    if (not pending_ops.empty() and (s == NodeStatus::Connected or s == NodeStatus::Disconnected))
                        return true;
                }
                return false;
@@ -215,13 +244,13 @@ DhtRunner::run(const Config& config, Context&& context)
        adc.net = netId;
        adc.nodeId = dht_->getNodeId();
        // IPv4
-        if (auto bound4 = dht_->getSocket()->getBound(AF_INET)) {
+        if (const auto& bound4 = dht_->getSocket()->getBoundRef(AF_INET)) {
            adc.port = bound4.getPort();
            msgpack::pack(sbuf_node, adc);
            peerDiscovery_->startPublish(AF_INET, PEER_DISCOVERY_DHT_SERVICE, sbuf_node);
        }
        // IPv6
-        if (auto bound6 = dht_->getSocket()->getBound(AF_INET6)) {
+        if (const auto& bound6 = dht_->getSocket()->getBoundRef(AF_INET6)) {
            adc.port = bound6.getPort();
            sbuf_node.clear();
            msgpack::pack(sbuf_node, adc);
@@ -303,7 +332,6 @@ DhtRunner::join()
        if (running.exchange(State::Idle) == State::Idle)
            return;
        cv.notify_all();
-        bootstrap_cv.notify_all();
 #ifdef OPENDHT_PEER_DISCOVERY
        if (peerDiscovery_)
            peerDiscovery_->stop();
@@ -318,9 +346,6 @@ DhtRunner::join()
    if (dht_thread.joinable())
        dht_thread.join();

-    if (bootstrap_thread.joinable())
-        bootstrap_thread.join();
-
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
        pending_ops = decltype(pending_ops)();
@@ -410,8 +435,9 @@ DhtRunner::exportValues() const {
 }

 void
-DhtRunner::setLogger(const Logger& logger) {
+DhtRunner::setLogger(const Sp<Logger>& logger) {
    std::lock_guard<std::mutex> lck(dht_mtx);
+    logger_ = logger;
    if (dht_)
        dht_->setLogger(logger);
 #ifdef OPENDHT_PROXY_CLIENT
@@ -429,7 +455,6 @@ DhtRunner::setLoggers(LogMethod error, LogMethod warn, LogMethod debug) {
 void
 DhtRunner::setLogFilter(const InfoHash& f) {
    std::lock_guard<std::mutex> lck(dht_mtx);
-    activeDht()->setLogFilter(f);
    if (dht_)
        dht_->setLogFilter(f);
 #ifdef OPENDHT_PROXY_CLIENT
@@ -482,11 +507,38 @@ DhtRunner::getNodeInfo() const {
        info.node_id = dht->getNodeId();
        info.ipv4 = dht->getNodesStats(AF_INET);
        info.ipv6 = dht->getNodesStats(AF_INET6);
+        if (auto sock = dht->getSocket()) {
+            info.bound4 = sock->getBoundRef(AF_INET).getPort();
+            info.bound6 = sock->getBoundRef(AF_INET6).getPort();
+        }
    }
    info.ongoing_ops = ongoing_ops;
    return info;
 }

+void
+DhtRunner::getNodeInfo(std::function<void(std::shared_ptr<NodeInfo>)> cb)
+{
+    std::lock_guard<std::mutex> lck(storage_mtx);
+    ongoing_ops++;
+    pending_ops_prio.emplace([cb = std::move(cb), this](SecureDht& dht){
+        auto sinfo = std::make_shared<NodeInfo>();
+        auto& info = *sinfo;
+        info.id = dht.getId();
+        info.node_id = dht.getNodeId();
+        info.ipv4 = dht.getNodesStats(AF_INET);
+        info.ipv6 = dht.getNodesStats(AF_INET6);
+        if (auto sock = dht.getSocket()) {
+            info.bound4 = sock->getBoundRef(AF_INET).getPort();
+            info.bound6 = sock->getBoundRef(AF_INET6).getPort();
+        }
+        info.ongoing_ops = ongoing_ops;
+        cb(std::move(sinfo));
+        opEnded();
+    });
+    cv.notify_all();
+}
+
 std::vector<unsigned>
 DhtRunner::getNodeMessageStats(bool in) const
 {
@@ -568,7 +620,7 @@ DhtRunner::loop_()
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
        auto s = getStatus();
-        ops = (pending_ops_prio.empty() && (s == NodeStatus::Connected or (s == NodeStatus::Disconnected and not bootstraping))) ?
+        ops = (pending_ops_prio.empty() && (s == NodeStatus::Connected or s == NodeStatus::Disconnected)) ?
               std::move(pending_ops) : std::move(pending_ops_prio);
    }
    while (not ops.empty()) {
@@ -578,6 +630,7 @@ DhtRunner::loop_()

    time_point wakeup {};
    decltype(rcv) received {};
+    decltype(rcv) received_treated {};
    {
        std::lock_guard<std::mutex> lck(sock_mtx);
        // move to stack
@@ -587,45 +640,46 @@ DhtRunner::loop_()
    // Discard old packets
    size_t dropped {0};
    if (not received.empty()) {
-        auto now = clock::now();
-        while (not received.empty() and now - received.front()->received > RX_QUEUE_MAX_DELAY) {
-            received.pop();
+        auto limit = clock::now() - net::RX_QUEUE_MAX_DELAY;
+        auto it = received.begin();
+        while (it != received.end() and it->received < limit) {
+            it->data.clear();
+            ++it;
            dropped++;
        }
+        received_treated.splice(received_treated.end(), received, received.begin(), it);
    }

    // Handle packets
    if (not received.empty()) {
-        while (not received.empty()) {
-            auto& pck = received.front();
-            if (clock::now() - pck->received > RX_QUEUE_MAX_DELAY)
+        for (auto& pkt : received) {
+            auto now = clock::now();
+            if (now - pkt.received > net::RX_QUEUE_MAX_DELAY)
                dropped++;
            else
-                wakeup = dht->periodic(pck->data.data(), pck->data.size(), std::move(pck->from));
-            received.pop();
+                wakeup = dht->periodic(pkt.data.data(), pkt.data.size(), std::move(pkt.from), now);
+            pkt.data.clear();
        }
+        received_treated.splice(received_treated.end(), std::move(received));
    } else {
        // Or just run the scheduler
-        wakeup = dht->periodic(nullptr, 0, nullptr, 0);
+        wakeup = dht->periodic(nullptr, 0, nullptr, 0, clock::now());
+    }
+
+    if (not received_treated.empty()) {
+        std::lock_guard<std::mutex> lck(sock_mtx);
+        if (rcv_free.size() < net::RX_QUEUE_MAX_SIZE)
+            rcv_free.splice(rcv_free.end(), std::move(received_treated));
    }

    if (dropped)
        std::cerr << "Dropped " << dropped << " packets with high delay" << std::endl;

-    NodeStatus nstatus4 = dht->getStatus(AF_INET);
-    NodeStatus nstatus6 = dht->getStatus(AF_INET6);
+    NodeStatus nstatus4 = dht->updateStatus(AF_INET);
+    NodeStatus nstatus6 = dht->updateStatus(AF_INET6);
    if (nstatus4 != status4 || nstatus6 != status6) {
        status4 = nstatus4;
        status6 = nstatus6;
-        if (status4 == NodeStatus::Disconnected and status6 == NodeStatus::Disconnected) {
-            // We have lost connection with the DHT.  Try to recover using bootstrap nodes.
-            std::unique_lock<std::mutex> lck(bootstrap_mtx);
-            bootstrap_nodes = bootstrap_nodes_all;
-            tryBootstrapContinuously();
-        } else {
-            std::unique_lock<std::mutex> lck(bootstrap_mtx);
-            bootstrap_nodes.clear();
-        }
        if (statusCb)
            statusCb(status4, status6);
    }
@@ -872,89 +926,34 @@ DhtRunner::putEncrypted(const std::string& key, InfoHash to, Value&& value, Done
    putEncrypted(InfoHash::get(key), to, std::forward<Value>(value), std::move(cb), permanent);
 }

-void
-DhtRunner::tryBootstrapContinuously()
-{
-    if (bootstrap_thread.joinable()) {
-        if (bootstraping)
-            return; // already running
-        else
-            bootstrap_thread.join();
-    }
-    bootstraping = true;
-    bootstrap_thread = std::thread([this]() {
-        auto next = clock::now();
-        do {
-            decltype(bootstrap_nodes) nodes;
-            {
-                std::lock_guard<std::mutex> lck(bootstrap_mtx);
-                nodes = bootstrap_nodes;
-            }
-
-            next += BOOTSTRAP_PERIOD;
-            {
-                std::mutex mtx;
-                std::unique_lock<std::mutex> blck(mtx);
-                unsigned ping_count(0);
-                // Reverse: try last inserted bootstrap nodes first
-                for (auto it = nodes.rbegin(); it != nodes.rend(); it++) {
-                    ++ping_count;
-                    try {
-                        bootstrap(SockAddr::resolve(it->first, it->second), [&](bool) {
-                            if (running != State::Running)
-                                return;
-                            {
-                                std::unique_lock<std::mutex> blck(mtx);
-                                --ping_count;
-                            }
-                            bootstrap_cv.notify_all();
-                        });
-                    } catch (std::invalid_argument& e) {
-                        --ping_count;
-                        std::cerr << e.what() << std::endl;
-                    }
-                }
-                // wait at least until the next BOOTSTRAP_PERIOD
-                bootstrap_cv.wait_until(blck, next, [&]() { return running != State::Running; });
-                // wait for bootstrap requests to end.
-                if (running != State::Running)
-                   bootstrap_cv.wait(blck, [&]() { return running != State::Running or ping_count == 0; });
-            }
-            // update state
-            {
-                std::lock_guard<std::mutex> lck(dht_mtx);
-                bootstraping = running == State::Running and
-                               status4 == NodeStatus::Disconnected and
-                               status6 == NodeStatus::Disconnected;
-            }
-        } while (bootstraping);
-    });
-}
-
 void
 DhtRunner::bootstrap(const std::string& host, const std::string& service)
 {
-    std::lock_guard<std::mutex> lck(bootstrap_mtx);
-    bootstrap_nodes_all.emplace_back(host, service);
-    bootstrap_nodes.emplace_back(host, service);
-    tryBootstrapContinuously();
+    std::lock_guard<std::mutex> lck(storage_mtx);
+    pending_ops_prio.emplace([host, service] (SecureDht& dht) mutable {
+        dht.addBootstrap(host, service);
+    });
+    cv.notify_all();
 }

 void
 DhtRunner::bootstrap(const std::string& hostService)
 {
-    std::lock_guard<std::mutex> lck(bootstrap_mtx);
-    auto host_service = splitPort(hostService);
-    bootstrap_nodes_all.emplace_back(host_service.first, host_service.second);
-    bootstrap_nodes.emplace_back(std::move(host_service.first), std::move(host_service.second));
-    tryBootstrapContinuously();
+    std::lock_guard<std::mutex> lck(storage_mtx);
+    pending_ops_prio.emplace([host_service = splitPort(hostService)] (SecureDht& dht) mutable {
+        dht.addBootstrap(host_service.first, host_service.second);
+    });
+    cv.notify_all();
 }

 void
 DhtRunner::clearBootstrap()
 {
-    std::lock_guard<std::mutex> lck(bootstrap_mtx);
-    bootstrap_nodes_all.clear();
+    std::lock_guard<std::mutex> lck(storage_mtx);
+    pending_ops_prio.emplace([] (SecureDht& dht) mutable {
+        dht.clearBootstrap();
+    });
+    cv.notify_all();
 }

 void
@@ -1033,6 +1032,10 @@ DhtRunner::connectivityChanged()
    std::lock_guard<std::mutex> lck(storage_mtx);
    pending_ops_prio.emplace([=](SecureDht& dht) {
        dht.connectivityChanged();
+#ifdef OPENDHT_PEER_DISCOVERY
+        if (peerDiscovery_)
+            peerDiscovery_->connectivityChanged();
+#endif
    });
    cv.notify_all();
 }
@@ -1115,11 +1118,11 @@ DhtRunner::enableProxy(bool proxify)
                },
                config_.proxy_server, config_.push_node_id, logger_)
        );
-        dht_via_proxy_ = std::unique_ptr<SecureDht>(new SecureDht(std::move(dht_via_proxy), config_.dht_config));
 #ifdef OPENDHT_PUSH_NOTIFICATIONS
        if (not config_.push_token.empty())
-            dht_via_proxy_->setPushNotificationToken(config_.push_token);
+            dht_via_proxy->setPushNotificationToken(config_.push_token);
 #endif
+        dht_via_proxy_ = std::unique_ptr<SecureDht>(new SecureDht(std::move(dht_via_proxy), config_.dht_config));
        // add current listeners
        for (auto& l: listeners_)
            l.second.tokenProxyDht = dht_via_proxy_->listen(l.second.hash, l.second.gcb, l.second.f, l.second.w);
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author: Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -31,6 +31,7 @@ namespace http {

 constexpr char HTTP_HEADER_CONTENT_TYPE_JSON[] = "application/json";
 constexpr char HTTP_HEADER_DELIM[] = "\r\n\r\n";
+constexpr unsigned MAX_REDIRECTS {5};

 Url::Url(const std::string& url): url(url)
 {
@@ -130,6 +131,7 @@ Connection::~Connection() {
 void
 Connection::close()
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    asio::error_code ec;
    if (ssl_socket_) {
        if (ssl_socket_->is_open())
@@ -160,6 +162,7 @@ Connection::is_ssl() const
 void
 Connection::set_ssl_verification(const asio::ip::tcp::endpoint& endpoint, const asio::ssl::verify_mode verify_mode)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (ssl_socket_ and verify_mode != asio::ssl::verify_none) {
        ssl_socket_->asio_ssl_stream().set_verify_mode(verify_mode);
        ssl_socket_->asio_ssl_stream().set_verify_callback([
@@ -192,6 +195,7 @@ Connection::input()
 std::string
 Connection::read_bytes(size_t bytes)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (bytes == 0)
        bytes = read_buf_.in_avail();
    std::string content;
@@ -212,6 +216,7 @@ Connection::read_until(const char delim)
 void
 Connection::async_connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, ConnectHandlerCb cb)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (ssl_socket_)
        asio::async_connect(ssl_socket_->lowest_layer(), std::move(endpoints), wrapCallabck(std::move(cb)));
    else if (socket_)
@@ -223,6 +228,7 @@ Connection::async_connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, Conn
 void
 Connection::async_handshake(HandlerCb cb)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (ssl_socket_) {
        std::weak_ptr<Connection> wthis = shared_from_this();
        ssl_socket_->async_handshake(asio::ssl::stream<asio::ip::tcp::socket>::client,
@@ -254,44 +260,61 @@ Connection::async_handshake(HandlerCb cb)
 void
 Connection::async_write(BytesHandlerCb cb)
 {
-    if (!is_open())   return;
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!is_open()) {
+        if (cb) ctx_.post([cb](){ cb(asio::error::broken_pipe, 0); });
+        return;
+    }
    if (ssl_socket_)  asio::async_write(*ssl_socket_, write_buf_, wrapCallabck(std::move(cb)));
    else if (socket_) asio::async_write(*socket_, write_buf_, wrapCallabck(std::move(cb)));
-    else if (cb)      cb(asio::error::operation_aborted, 0);
+    else if (cb)      ctx_.post([cb](){ cb(asio::error::operation_aborted, 0); });
 }

 void
 Connection::async_read_until(const char* delim, BytesHandlerCb cb)
 {
-    if (!is_open())   return;
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!is_open()) {
+        if (cb) ctx_.post([cb](){ cb(asio::error::broken_pipe, 0); });
+        return;
+    }
    if (ssl_socket_)  asio::async_read_until(*ssl_socket_, read_buf_, delim, wrapCallabck(std::move(cb)));
    else if (socket_) asio::async_read_until(*socket_, read_buf_, delim, wrapCallabck(std::move(cb)));
-    else if (cb)      cb(asio::error::operation_aborted, 0);
+    else if (cb)      ctx_.post([cb](){ cb(asio::error::operation_aborted, 0); });
 }

 void
 Connection::async_read_until(char delim, BytesHandlerCb cb)
 {
-    if (!is_open())   return;
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!is_open()) {
+        if (cb) ctx_.post([cb](){ cb(asio::error::broken_pipe, 0); });
+        return;
+    }
    if (ssl_socket_)  asio::async_read_until(*ssl_socket_, read_buf_, delim, wrapCallabck(std::move(cb)));
    else if (socket_) asio::async_read_until(*socket_, read_buf_, delim, wrapCallabck(std::move(cb)));
-    else if (cb)      cb(asio::error::operation_aborted, 0);
+    else if (cb)      ctx_.post([cb](){ cb(asio::error::operation_aborted, 0); });
 }

 void
 Connection::async_read(size_t bytes, BytesHandlerCb cb)
 {
-    if (!is_open())   return;
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!is_open()) {
+        if (cb) ctx_.post([cb](){ cb(asio::error::broken_pipe, 0); });
+        return;
+    }
    if (ssl_socket_)  asio::async_read(*ssl_socket_, read_buf_, asio::transfer_exactly(bytes), wrapCallabck(std::move(cb)));
    else if (socket_) asio::async_read(*socket_, read_buf_, asio::transfer_exactly(bytes), wrapCallabck(std::move(cb)));
-    else if (cb)      cb(asio::error::operation_aborted, 0);
+    else if (cb)      ctx_.post([cb](){ cb(asio::error::operation_aborted, 0); });
 }

 void
 Connection::async_read_some(size_t bytes, BytesHandlerCb cb)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (!is_open()) {
-        if (cb) cb(asio::error::operation_aborted, 0);
+        if (cb) ctx_.post([cb](){ cb(asio::error::broken_pipe, 0); });
        return;
    }
    auto buf = read_buf_.prepare(bytes);
@@ -504,7 +527,7 @@ Request::Request(asio::io_context& ctx, std::shared_ptr<Resolver> resolver, cons
    : logger_(resolver->getLogger()), id_(Request::ids_++), ctx_(ctx), family_(family), resolver_(resolver)
 {
    set_header_field(restinio::http_field_t::host, get_url().host + ":" + get_url().service);
-    set_target(target);
+    set_target(Url(target).target);
 }

 Request::~Request()
@@ -672,6 +695,8 @@ Request::notify_state_change(State state) {
 void
 Request::init_parser()
 {
+    response_.request = shared_from_this();
+
    if (!parser_)
        parser_ = std::make_unique<http_parser>();
    http_parser_init(parser_.get(), HTTP_RESPONSE);
@@ -754,6 +779,7 @@ Request::connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, HandlerCb cb)
        if (not sthis)
            return;
        auto& this_ = *sthis;
+        std::lock_guard<std::mutex> lock(this_.mutex_);
        if (ec == asio::error::operation_aborted){
            this_.terminate(ec);
            return;
@@ -810,6 +836,7 @@ Request::send()
                                   std::vector<asio::ip::tcp::endpoint> endpoints) {
        if (auto sthis = wthis.lock()) {
            auto& this_ = *sthis;
+            std::lock_guard<std::mutex> lock(this_.mutex_);
            if (ec){
                if (this_.logger_)
                    this_.logger_->e("[http:request:%i] resolve error: %s", this_.id_, ec.message().c_str());
@@ -882,6 +909,7 @@ Request::terminate(const asio::error_code& ec)
 void
 Request::handle_request(const asio::error_code& ec)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (ec and ec != asio::error::eof){
        terminate(ec);
        return;
@@ -905,6 +933,7 @@ Request::handle_request(const asio::error_code& ec)
 void
 Request::handle_response(const asio::error_code& ec, size_t n_bytes)
 {
+    std::lock_guard<std::mutex> lock(mutex_);
    if (ec && ec != asio::error::eof){
        terminate(ec);
        return;
@@ -948,18 +977,44 @@ void
 Request::onHeadersComplete() {
    notify_state_change(State::HEADER_RECEIVED);

-    auto expect_it = headers_.find(restinio::http_field_t::expect);
-    if (expect_it != headers_.end() and (expect_it->second == "100-continue") and response_.status_code != 200){
-        notify_state_change(State::SENDING);
-        request_.append(body_);
-        std::ostream request_stream(&conn_->input());
-        request_stream << body_ << "\r\n";
-        std::weak_ptr<Request> wthis = shared_from_this();
-        conn_->async_write([wthis](const asio::error_code& ec, size_t) {
-            if (auto sthis = wthis.lock())
-                sthis->handle_request(ec);
-        });
-        return;
+    if (response_.status_code == restinio::status_code::moved_permanently.raw_code() or
+        response_.status_code == restinio::status_code::found.raw_code())
+    {
+        auto location_it = response_.headers.find(restinio::field_to_string(restinio::http_field_t::location));
+        if (location_it == response_.headers.end()){
+            if (logger_)
+                logger_->e("[http:client] [request:%i] got redirect without location", id_);
+            terminate(asio::error::connection_aborted);
+        }
+
+        if (follow_redirect and num_redirect < MAX_REDIRECTS) {
+            auto next = std::make_shared<Request>(ctx_, location_it->second, logger_);
+            next->set_method(header_.method());
+            next->headers_ = std::move(headers_);
+            next->body_ = std::move(body_);
+            next->cbs_ = std::move(cbs_);
+            next->num_redirect = num_redirect + 1;
+            next_ = next;
+            next->prev_ = shared_from_this();
+            next->send();
+        } else {
+            if (logger_)
+                logger_->e("[http:client] [request:%i] got redirect without location", id_);
+            terminate(asio::error::connection_aborted);
+        }
+    } else {
+        auto expect_it = headers_.find(restinio::http_field_t::expect);
+        if (expect_it != headers_.end() and (expect_it->second == "100-continue") and response_.status_code != 200){
+            notify_state_change(State::SENDING);
+            request_.append(body_);
+            std::ostream request_stream(&conn_->input());
+            request_stream << body_ << "\r\n";
+            std::weak_ptr<Request> wthis = shared_from_this();
+            conn_->async_write([wthis](const asio::error_code& ec, size_t) {
+                if (auto sthis = wthis.lock())
+                    sthis->handle_request(ec);
+            });
+        }
    }
 }

@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *              Nicolas Reynaud <nicolas.reynaud@savoirfairelinux.com>
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -30,8 +30,9 @@ namespace dht {
 struct Listener {
    time_point time;
    Query query;
+    int version;

-    Listener(time_point t, Query&& q) : time(t), query(std::move(q)) {}
+    Listener(time_point t, Query&& q, int version = 0) : time(t), query(std::move(q)), version(version) {}

    void refresh(time_point t, Query&& q) {
        time = t;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -115,22 +115,22 @@ getSyslogLogger(const char* name) {

 void
 enableLogging(dht::DhtRunner &dht) {
-    dht.setLogger(*getStdLogger());
+    dht.setLogger(getStdLogger());
 }

 void
 enableFileLogging(dht::DhtRunner &dht, const std::string &path) {
-    dht.setLogger(*getFileLogger(path));
+    dht.setLogger(getFileLogger(path));
 }

 OPENDHT_PUBLIC void
 enableSyslog(dht::DhtRunner &dht, const char* name) {
-    dht.setLogger(*getSyslogLogger(name));
+    dht.setLogger(getSyslogLogger(name));
 }

 void
 disableLogging(dht::DhtRunner &dht) {
-    dht.setLogger({});
+    dht.setLogger();
 }

 }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -31,7 +31,8 @@ enum class MessageType {
    Refresh,
    Listen,
    ValueData,
-    ValueUpdate
+    ValueUpdate,
+    UpdateValue
 };

 } /* namespace net */
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -27,6 +27,7 @@

 namespace dht {
 namespace net {
+using namespace std::chrono_literals;

 const std::string DhtProtocolException::GET_NO_INFOHASH {"Get_values with no info_hash"};
 const std::string DhtProtocolException::LISTEN_NO_INFOHASH {"Listen with no info_hash"};
@@ -47,23 +48,6 @@ static constexpr uint8_t v4prefix[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xF
 constexpr unsigned SEND_NODES {8};


-/* Transaction-ids are 4-bytes long, with the first two bytes identifying
- * the kind of request, and the remaining two a sequence number in
- * host order.
- */
-struct TransId final : public std::array<uint8_t, 4> {
-    TransId() { std::fill(begin(), end(), 0); }
-    TransId(const std::array<char, 4>& o) { std::copy(o.begin(), o.end(), begin()); }
-    TransId(uint32_t id) {
-        *reinterpret_cast<uint32_t*>(data()) = htonl(id);
-    }
-
-    uint32_t toInt() const {
-        return ntohl(*reinterpret_cast<const uint32_t*>(&(*this)[0]));
-    }
-};
-
-
 struct NetworkEngine::PartialMessage {
    SockAddr from;
    time_point start;
@@ -98,11 +82,15 @@ RequestAnswer::RequestAnswer(ParsedMessage&& msg)
   nodes6(std::move(msg.nodes6))
 {}

-NetworkEngine::NetworkEngine(Logger& log, Scheduler& scheduler, std::unique_ptr<DatagramSocket>&& sock)
-    : myid(zeroes), dht_socket(std::move(sock)), DHT_LOG(log), rate_limiter((size_t)-1), scheduler(scheduler)
+NetworkEngine::NetworkEngine(const Sp<Logger>& log, std::mt19937_64& rand, Scheduler& scheduler, std::unique_ptr<DatagramSocket>&& sock)
+    : myid(zeroes), dht_socket(std::move(sock)), logger_(log), rd(rand), cache(rd), rate_limiter((size_t)-1), scheduler(scheduler)
 {}

-NetworkEngine::NetworkEngine(InfoHash& myid, NetworkConfig c, std::unique_ptr<DatagramSocket>&& sock, Logger& log, Scheduler& scheduler,
+NetworkEngine::NetworkEngine(InfoHash& myid, NetworkConfig c,
+        std::unique_ptr<DatagramSocket>&& sock,
+        const Sp<Logger>& log,
+        std::mt19937_64& rand,
+        Scheduler& scheduler,
        decltype(NetworkEngine::onError)&& onError,
        decltype(NetworkEngine::onNewNode)&& onNewNode,
        decltype(NetworkEngine::onReportedAddr)&& onReportedAddr,
@@ -121,7 +109,8 @@ NetworkEngine::NetworkEngine(InfoHash& myid, NetworkConfig c, std::unique_ptr<Da
    onListen(std::move(onListen)),
    onAnnounce(std::move(onAnnounce)),
    onRefresh(std::move(onRefresh)),
-    myid(myid), config(c), dht_socket(std::move(sock)), DHT_LOG(log),
+    myid(myid), config(c), dht_socket(std::move(sock)), logger_(log), rd(rand),
+    cache(rd),
    rate_limiter(config.max_req_per_sec),
    scheduler(scheduler)
 {}
@@ -134,72 +123,119 @@ void
 NetworkEngine::tellListener(Sp<Node> node, Tid socket_id, const InfoHash& hash, want_t want,
        const Blob& ntoken, std::vector<Sp<Node>>&& nodes,
        std::vector<Sp<Node>>&& nodes6, std::vector<Sp<Value>>&& values,
-        const Query& query)
+        const Query& query, int version)
 {
    auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
    try {
-        sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, values, query, ntoken);
+        if (version >= 1) {
+            sendUpdateValues(node, hash, values, scheduler.time(), ntoken, socket_id);
+        } else {
+            sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, values, query, ntoken);
+        }
    } catch (const std::overflow_error& e) {
-        DHT_LOG.e("Can't send value: buffer not large enough !");
+        if (logger_)
+            logger_->e("Can't send value: buffer not large enough !");
    }
 }

 void
-NetworkEngine::tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values)
+NetworkEngine::tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values, int version)
 {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4+(config.network?1:0));

-    pk.pack(KEY_U);
-    pk.pack_map(1 + (not values.empty()?1:0) + (not token.empty()?1:0));
-    pk.pack(KEY_REQ_ID); pk.pack(myid);
-    if (not token.empty()) {
-        pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
-    }
-    if (not values.empty()) {
-        pk.pack(KEY_REQ_REFRESHED);
-        pk.pack(values);
-        DHT_LOG.d(n->id, "[node %s] sending %zu refreshed values", n->toString().c_str(), values.size());
-    }
+    pk.pack_map(4 + (version >= 1 ? 1 : 0) + (config.network?1:0));

-    pk.pack(KEY_TID); pk.pack(socket_id);
-    pk.pack(KEY_Y); pk.pack(KEY_R);
+    pk.pack(version >= 1 ? KEY_A : KEY_U);
+        pk.pack_map(1 + (version >= 1 ? 1 : 0) + (not values.empty()?1:0) + (not token.empty()?1:0));
+        pk.pack(KEY_REQ_ID); pk.pack(myid);
+        if (version >= 1) {
+            pk.pack(KEY_REQ_SID);   pk.pack(socket_id);
+        }
+        if (not token.empty()) {
+            pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
+        }
+        if (not values.empty()) {
+            pk.pack(KEY_REQ_REFRESHED);
+            pk.pack(values);
+            if (logger_)
+                logger_->d(n->id, "[node %s] sending %zu refreshed values", n->toString().c_str(), values.size());
+        }
+
+    pk.pack(KEY_Y); pk.pack(version >= 1 ? KEY_Q : KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

+    if (version >= 1) {
+        Tid tid (n->getNewTid());
+
+        pk.pack(KEY_Q);   pk.pack(QUERY_UPDATE);
+        pk.pack(KEY_TID); pk.pack(tid);
+
+        auto req = std::make_shared<Request>(MessageType::UpdateValue, tid, n,
+            Blob(buffer.data(), buffer.data() + buffer.size()),
+            [=](const Request&, ParsedMessage&&) { /* on done */ },
+            [=](const Request&, bool) { /* on expired */ }
+        );
+        sendRequest(req);
+        ++out_stats.updateValue;
+        return;
+    }
+    pk.pack(KEY_TID); pk.pack(socket_id);
+
    // send response
    send(n->getAddr(), buffer.data(), buffer.size());
 }

 void
-NetworkEngine::tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values)
+NetworkEngine::tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values, int version)
 {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4+(config.network?1:0));

-    pk.pack(KEY_U);
-    pk.pack_map(1 + (not values.empty()?1:0) + (not token.empty()?1:0));
-    pk.pack(KEY_REQ_ID); pk.pack(myid);
-    if (not token.empty()) {
-        pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
-    }
-    if (not values.empty()) {
-        pk.pack(KEY_REQ_EXPIRED);
-        pk.pack(values);
-        DHT_LOG.d(n->id, "[node %s] sending %zu expired values", n->toString().c_str(), values.size());
-    }
+    pk.pack_map(4 + (version >= 1 ? 1 : 0) + (config.network?1:0));

-    pk.pack(KEY_TID); pk.pack(socket_id);
-    pk.pack(KEY_Y); pk.pack(KEY_R);
+    pk.pack(version >= 1 ? KEY_A : KEY_U);
+        pk.pack_map(1 + (version >= 1 ? 1 : 0) + (not values.empty()?1:0) + (not token.empty()?1:0));
+        pk.pack(KEY_REQ_ID); pk.pack(myid);
+        if (version >= 1) {
+            pk.pack(KEY_REQ_SID);   pk.pack(socket_id);
+        }
+        if (not token.empty()) {
+            pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
+        }
+        if (not values.empty()) {
+            pk.pack(KEY_REQ_EXPIRED);
+            pk.pack(values);
+            if (logger_)
+                logger_->d(n->id, "[node %s] sending %zu expired values", n->toString().c_str(), values.size());
+        }
+
+    pk.pack(KEY_Y); pk.pack(version >= 1 ? KEY_Q : KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

+    if (version >= 1) {
+        Tid tid (n->getNewTid());
+
+        pk.pack(KEY_Q);   pk.pack(QUERY_UPDATE);
+        pk.pack(KEY_TID);     pk.pack(tid);
+
+        auto req = std::make_shared<Request>(MessageType::UpdateValue, tid, n,
+            Blob(buffer.data(), buffer.data() + buffer.size()),
+            [=](const Request&, ParsedMessage&&) { /* on done */ },
+            [=](const Request&, bool) { /* on expired */ }
+        );
+        sendRequest(req);
+        ++out_stats.updateValue;
+        return;
+    }
+    pk.pack(KEY_TID); pk.pack(socket_id);
+
    // send response
    send(n->getAddr(), buffer.data(), buffer.size());
 }
@@ -246,7 +282,8 @@ NetworkEngine::requestStep(Sp<Request> sreq)
    auto now = scheduler.time();
    auto& node = *req.node;
    if (req.isExpired(now)) {
-        // DHT_LOG.d(node.id, "[node %s] expired !", node.toString().c_str());
+        // if (logger_)
+        //     logger_->d(node.id, "[node %s] expired !", node.toString().c_str());
        node.setExpired();
        if (not node.id)
            requests.erase(req.tid);
@@ -266,12 +303,17 @@ NetworkEngine::requestStep(Sp<Request> sreq)
        if (not node.id)
            requests.erase(req.tid);
    } else {
+        req.last_try = now;
        if (err != EAGAIN) {
            ++req.attempt_count;
+            req.attempt_duration +=
+                req.attempt_duration + uniform_duration_distribution<>(0ms, ((duration)Node::MAX_RESPONSE_TIME)/4)(rd);
+            if (not req.parts.empty()){
+                sendValueParts(req.tid, req.parts, node.getAddr());
+            }
        }
-        req.last_try = now;
        std::weak_ptr<Request> wreq = sreq;
-        scheduler.add(req.last_try + Node::MAX_RESPONSE_TIME, [this,wreq] {
+        scheduler.add(req.last_try + req.attempt_duration, [this,wreq] {
            if (auto req = wreq.lock())
                requestStep(req);
        });
@@ -366,12 +408,14 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
 {
    auto from = f.getMappedIPv4();
    if (isMartian(from)) {
-        DHT_LOG.w("Received packet from martian node %s", from.toString().c_str());
+        if (logger_)
+            logger_->w("Received packet from martian node %s", from.toString().c_str());
        return;
    }

    if (isNodeBlacklisted(from)) {
-        DHT_LOG.w("Received packet from blacklisted node %s", from.toString().c_str());
+        if (logger_)
+            logger_->w("Received packet from blacklisted node %s", from.toString().c_str());
        return;
    }

@@ -380,13 +424,16 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
        msgpack::unpacked msg_res = msgpack::unpack((const char*)buf, buflen);
        msg->msgpack_unpack(msg_res.get());
    } catch (const std::exception& e) {
-        DHT_LOG.w("Can't parse message of size %lu: %s", buflen, e.what());
-        //DHT_LOG.DBG.logPrintable(buf, buflen);
+        if (logger_)
+            logger_->w("Can't parse message of size %lu: %s", buflen, e.what());
+        // if (logger_)
+        //     logger_->DBG.logPrintable(buf, buflen);
        return;
    }

    if (msg->network != config.network) {
-        DHT_LOG.d("Received message from other config.network %u", msg->network);
+        if (logger_)
+            logger_->d("Received message from other config.network %u", msg->network);
        return;
    }

@@ -397,12 +444,14 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
        auto pmsg_it = partial_messages.find(msg->tid);
        if (pmsg_it == partial_messages.end()) {
            if (logIncoming_)
-                DHT_LOG.d("Can't find partial message");
+                if (logger_)
+                    logger_->d("Can't find partial message");
            rateLimit(from);
            return;
        }
        if (!pmsg_it->second.from.equals(from)) {
-            DHT_LOG.d("Received partial message data from unexpected IP address");
+            if (logger_)
+                logger_->d("Received partial message data from unexpected IP address");
            rateLimit(from);
            return;
        }
@@ -421,14 +470,16 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
    }

    if (msg->id == myid or not msg->id) {
-        DHT_LOG.d("Received message from self");
+        if (logger_)
+            logger_->d("Received message from self");
        return;
    }

    if (msg->type > MessageType::Reply) {
        /* Rate limit requests. */
        if (!rateLimit(from)) {
-            DHT_LOG.w("Dropping request due to rate limiting");
+            if (logger_)
+                logger_->w("Dropping request due to rate limiting");
            return;
        }
    }
@@ -447,7 +498,8 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, SockAddr f)
            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
        } else
-            DHT_LOG.e("Partial message with given TID already exists");
+            if (logger_)
+                logger_->e("Partial message with given TID already exists");
    }
 }

@@ -481,7 +533,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                node->received(now, req);
                if (not node->isClient())
                    onNewNode(node, 1);
-                DHT_LOG.d(node->id, "[node %s] can't find transaction with id %u", node->toString().c_str(), msg->tid);
+                if (logger_)
+                    logger_->d(node->id, "[node %s] can't find transaction with id %u", node->toString().c_str(), msg->tid);
                return;
            }
        }
@@ -493,7 +546,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
        onReportedAddr(msg->id, msg->addr);

        if (req and (req->cancelled() or req->expired() or req->completed())) {
-            DHT_LOG.w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str());
+            if (logger_)
+                logger_->w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str());
            return;
        }

@@ -506,10 +560,12 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
            {
                req->last_try = time_point::min();
                req->reply_time = time_point::min();
-                onError(req, DhtProtocolException {msg->error_code});
+                if (not req->setError(DhtProtocolException {msg->error_code}))
+                    onError(req, DhtProtocolException {msg->error_code});
            } else {
                if (logIncoming_)
-                    DHT_LOG.w(msg->id, "[node %s %s] received unknown error message %u",
+                    if (logger_)
+                        logger_->w(msg->id, "[node %s %s] received unknown error message %u",
                        msg->id.toString().c_str(), from.toString().c_str(), msg->error_code);
            }
            break;
@@ -517,8 +573,11 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
        case MessageType::Reply:
            if (req) { /* request reply */
                auto& r = *req;
-                if (r.getType() == MessageType::AnnounceValue or r.getType() == MessageType::Listen)
+                if (r.getType() == MessageType::AnnounceValue
+                 or r.getType() == MessageType::Listen
+                 or r.getType() == MessageType::Refresh) {
                    r.node->authSuccess();
+                }
                r.reply_time = scheduler.time();

                deserializeNodes(*msg, from);
@@ -541,12 +600,14 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
            case MessageType::Ping:
                ++in_stats.ping;
                if (logIncoming_)
-                    DHT_LOG.d(node->id, "[node %s] sending pong", node->toString().c_str());
+                    if (logger_)
+                        logger_->d(node->id, "[node %s] sending pong", node->toString().c_str());
                onPing(node);
                sendPong(from, msg->tid);
                break;
            case MessageType::FindNode: {
-                //DHT_LOG.d(msg->target, node->id, "[node %s] got 'find' request for %s (%d)", node->toString().c_str(), msg->target.toString().c_str(), msg->want);
+                // if (logger_)
+                //     logger_->d(msg->target, node->id, "[node %s] got 'find' request for %s (%d)", node->toString().c_str(), msg->target.toString().c_str(), msg->want);
                ++in_stats.find;
                RequestAnswer answer = onFindNode(node, msg->target, msg->want);
                auto nnodes = bufferNodes(from.getFamily(), msg->target, msg->want, answer.nodes4, answer.nodes6);
@@ -554,7 +615,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                break;
            }
            case MessageType::GetValues: {
-                //DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'get' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                // if (logger_)
+                //     logger_->d(msg->info_hash, node->id, "[node %s] got 'get' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.get;
                RequestAnswer answer = onGetValues(node, msg->info_hash, msg->want, msg->query);
                auto nnodes = bufferNodes(from.getFamily(), msg->info_hash, msg->want, answer.nodes4, answer.nodes6);
@@ -562,8 +624,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                break;
            }
            case MessageType::AnnounceValue: {
-                if (logIncoming_)
-                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'put' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_ and logger_)
+                    logger_->d(msg->info_hash, node->id, "[node %s] got 'put' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.put;
                onAnnounce(node, msg->info_hash, msg->token, msg->values, msg->created);

@@ -576,27 +638,39 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                break;
            }
            case MessageType::Refresh:
-                if (logIncoming_)
-                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'refresh' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_ and logger_)
+                    logger_->d(msg->info_hash, node->id, "[node %s] got 'refresh' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                onRefresh(node, msg->info_hash, msg->token, msg->value_id);
                /* Same note as above in MessageType::AnnounceValue applies. */
                sendValueAnnounced(from, msg->tid, msg->value_id);
                break;
            case MessageType::Listen: {
-                if (logIncoming_)
-                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_ and logger_)
+                    logger_->d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.listen;
-                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query));
+                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query), msg->version);
                auto nnodes = bufferNodes(from.getFamily(), msg->info_hash, msg->want, answer.nodes4, answer.nodes6);
                sendListenConfirmation(from, msg->tid);
                break;
            }
+            case MessageType::UpdateValue: {
+                if (logIncoming_ and logger_)
+                    logger_->d(msg->info_hash, node->id, "[node %s] got 'update' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                ++in_stats.updateValue;
+                if (auto rsocket = node->getSocket(msg->socket_id))
+                    rsocket->on_receive(node, std::move(*msg));
+                else if (logger_)
+                    logger_->e(msg->info_hash, node->id, "[node %s] 'update' request without socket for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                sendListenConfirmation(from, msg->tid);
+                break;
+            }
            default:
                break;
            }
        } catch (const std::overflow_error& e) {
-            DHT_LOG.e("Can't send value: buffer not large enough !");
-        } catch (DhtProtocolException& e) {
+            if (logger_)
+                logger_->e("Can't send value: buffer not large enough !");
+        } catch (const DhtProtocolException& e) {
            sendError(from, msg->tid, e.getCode(), e.getMsg().c_str(), true);
        }
    }
@@ -622,7 +696,7 @@ NetworkEngine::send(const SockAddr& addr, const char *buf, size_t len, bool conf

 Sp<Request>
 NetworkEngine::sendPing(Sp<Node> node, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    TransId tid (node->getNewTid());
+    Tid tid (node->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));
@@ -631,18 +705,18 @@ NetworkEngine::sendPing(Sp<Node> node, RequestCb&& on_done, RequestExpiredCb&& o
     pk.pack(KEY_REQ_ID); pk.pack(myid);

    pk.pack(KEY_Q); pk.pack(QUERY_PING);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                              pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_Q);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::Ping, tid.toInt(), node,
+    auto req = std::make_shared<Request>(MessageType::Ping, tid, node,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&&) {
-            DHT_LOG.d(req_status.node->id, "[node %s] got pong !", req_status.node->toString().c_str());
+            if (logger_)
+                logger_->d(req_status.node->id, "[node %s] got pong !", req_status.node->toString().c_str());
            if (on_done) {
                on_done(req_status, {});
            }
@@ -668,9 +742,7 @@ NetworkEngine::sendPong(const SockAddr& addr, Tid tid) {
      pk.pack(KEY_REQ_ID); pk.pack(myid);
      insertAddr(pk, addr);

-    TransId t (tid);
-    pk.pack(KEY_TID); pk.pack_bin(t.size());
-                               pk.pack_bin_body((const char*)t.data(), t.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
@@ -683,7 +755,7 @@ NetworkEngine::sendPong(const SockAddr& addr, Tid tid) {
 Sp<Request>
 NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
        RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    TransId tid (n->getNewTid());
+    Tid tid (n->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));
@@ -699,15 +771,14 @@ NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
    }

    pk.pack(KEY_Q); pk.pack(QUERY_FIND);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                               pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_Q);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::FindNode, tid.toInt(), n,
+    auto req = std::make_shared<Request>(MessageType::FindNode, tid, n,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&& msg) { /* on done */
            if (on_done) {
@@ -729,34 +800,38 @@ NetworkEngine::sendFindNode(Sp<Node> n, const InfoHash& target, want_t want,
 Sp<Request>
 NetworkEngine::sendGetValues(Sp<Node> n, const InfoHash& info_hash, const Query& query, want_t want,
        RequestCb&& on_done, RequestExpiredCb&& on_expired) {
-    TransId tid (n->getNewTid());
+    Tid tid (n->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));

-    pk.pack(KEY_A);  pk.pack_map(2 +
-                                (query.where.getFilter() or not query.select.getSelection().empty() ? 1:0) +
-                                (want>0?1:0));
+    unsigned sendQuery = (not query.where.empty() or not query.select.empty()) ? 1 : 0;
+    unsigned sendWant = (want > 0) ? 1 : 0;
+
+    pk.pack(KEY_A);  pk.pack_map(2 + sendQuery + sendWant);
      pk.pack(KEY_REQ_ID); pk.pack(myid);
      pk.pack(KEY_REQ_H);  pk.pack(info_hash);
-      pk.pack(KEY_Q); pk.pack(query);
-    if (want > 0) {
-      pk.pack(KEY_REQ_WANT);
-      pk.pack_array(((want & WANT4)?1:0) + ((want & WANT6)?1:0));
-      if (want & WANT4) pk.pack(AF_INET);
-      if (want & WANT6) pk.pack(AF_INET6);
-    }
+      if (sendQuery) {
+        pk.pack(KEY_Q); pk.pack(query);
+      }
+      if (sendWant) {
+        pk.pack(KEY_REQ_WANT);
+        unsigned sendWant4 = (want & WANT4) ? 1 : 0;
+        unsigned sendWant6 = (want & WANT6) ? 1 : 0;
+        pk.pack_array(sendWant4 + sendWant6);
+        if (sendWant4) pk.pack(AF_INET);
+        if (sendWant6) pk.pack(AF_INET6);
+      }

    pk.pack(KEY_Q); pk.pack(QUERY_GET);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                               pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_Q);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::GetValues, tid.toInt(), n,
+    auto req = std::make_shared<Request>(MessageType::GetValues, tid, n,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&& msg) { /* on done */
            if (on_done) {
@@ -847,7 +922,8 @@ NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Va
    if (svals.size() < 50 && total_size < MAX_PACKET_VALUE_SIZE) {
        for (const auto& b : svals)
            buffer.write((const char*)b.data(), b.size());
-        // DHT_LOG.d("sending %lu bytes of values", total_size);
+        // if (logger_)
+        //     logger_->d("sending %lu bytes of values", total_size);
        svals.clear();
    } else {
        for (const auto& b : svals)
@@ -857,7 +933,7 @@ NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector<Sp<Va
 }

 void
-NetworkEngine::sendValueParts(const TransId& tid, const std::vector<Blob>& svals, const SockAddr& addr)
+NetworkEngine::sendValueParts(Tid tid, const std::vector<Blob>& svals, const SockAddr& addr)
 {
    msgpack::sbuffer buffer;
    unsigned i=0;
@@ -872,8 +948,7 @@ NetworkEngine::sendValueParts(const TransId& tid, const std::vector<Blob>& svals
                pk.pack(KEY_NETID); pk.pack(config.network);
            }
            pk.pack(KEY_Y); pk.pack(KEY_V);
-            pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                                       pk.pack_bin_body((const char*)tid.data(), tid.size());
+            pk.pack(KEY_TID); pk.pack(tid);
            pk.pack(KEY_V); pk.pack_map(1);
                pk.pack(i); pk.pack_map(2);
                    pk.pack(std::string("o")); pk.pack(start);
@@ -913,10 +988,10 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, Tid tid, const Blob& nodes,
    }
    std::vector<Blob> svals {};
    if (not st.empty()) { /* pack complete values */
-        auto fields = query.select.getSelection();
-        if (fields.empty()) {
+        if (query.select.empty()) {
            svals = packValueHeader(buffer, st);
        } else { /* pack fields */
+            auto fields = query.select.getSelection();
            pk.pack(KEY_REQ_FIELDS);
            pk.pack_map(2);
            pk.pack(std::string("f")); pk.pack(fields);
@@ -928,9 +1003,7 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, Tid tid, const Blob& nodes,
        }
    }

-    TransId t (tid);
-    pk.pack(KEY_TID); pk.pack_bin(t.size());
-                      pk.pack_bin_body((const char*)t.data(), t.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
@@ -1000,52 +1073,35 @@ NetworkEngine::sendListen(Sp<Node> n,
        const InfoHash& hash,
        const Query& query,
        const Blob& token,
-        Sp<Request> previous,
+        Tid socketId,
        RequestCb&& on_done,
-        RequestExpiredCb&& on_expired,
-        SocketCb&& socket_cb)
+        RequestExpiredCb&& on_expired)
 {
-    Tid socket;
-    TransId tid (n->getNewTid());
-    if (previous and previous->node == n) {
-        socket = previous->getSocket();
-    } else {
-        if (previous)
-            DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str());
-        socket = n->openSocket(std::move(socket_cb));
-    }
-
-    if (not socket) {
-        DHT_LOG.e(hash, "[node %s] unable to get a valid socket for listen. Aborting listen", n->toString().c_str());
-        return {};
-    }
-    TransId sid(socket);
-
+    Tid tid (n->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));

-    auto has_query = query.where.getFilter() or not query.select.getSelection().empty();
-    pk.pack(KEY_A); pk.pack_map(4 + has_query);
+    auto has_query = not query.where.empty() or not query.select.empty();
+    pk.pack(KEY_A); pk.pack_map(5 + has_query);
      pk.pack(KEY_REQ_ID);    pk.pack(myid);
+      pk.pack(KEY_VERSION);   pk.pack(1);
      pk.pack(KEY_REQ_H);     pk.pack(hash);
      pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
-      pk.pack(KEY_REQ_SID);   pk.pack_bin(sid.size());
-                              pk.pack_bin_body((const char*)sid.data(), sid.size());
+      pk.pack(KEY_REQ_SID);   pk.pack(socketId);
      if (has_query) {
          pk.pack(KEY_REQ_QUERY); pk.pack(query);
      }

    pk.pack(KEY_Q); pk.pack(QUERY_LISTEN);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                               pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_Q);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::Listen, tid.toInt(), n,
+    auto req = std::make_shared<Request>(MessageType::Listen, tid, n,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&& msg) { /* on done */
            if (on_done)
@@ -1054,8 +1110,7 @@ NetworkEngine::sendListen(Sp<Node> n,
        [=](const Request& req_status, bool done) { /* on expired */
            if (on_expired)
                on_expired(req_status, done);
-        },
-        socket
+        }
    );
    sendRequest(req);
    ++out_stats.listen;
@@ -1072,9 +1127,7 @@ NetworkEngine::sendListenConfirmation(const SockAddr& addr, Tid tid) {
      pk.pack(KEY_REQ_ID); pk.pack(myid);
      insertAddr(pk, addr);

-    TransId t (tid);
-    pk.pack(KEY_TID); pk.pack_bin(t.size());
-                               pk.pack_bin_body((const char*)t.data(), t.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
@@ -1093,7 +1146,7 @@ NetworkEngine::sendAnnounceValue(Sp<Node> n,
        RequestCb&& on_done,
        RequestExpiredCb&& on_expired)
 {
-    TransId tid (n->getNewTid());
+    Tid tid (n->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));
@@ -1109,19 +1162,19 @@ NetworkEngine::sendAnnounceValue(Sp<Node> n,
      pk.pack(KEY_REQ_TOKEN);  pk.pack(token);

    pk.pack(KEY_Q);   pk.pack(QUERY_PUT);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                      pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y);   pk.pack(KEY_Q);
    pk.pack(KEY_UA);  pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::AnnounceValue, tid.toInt(), n,
+    auto req = std::make_shared<Request>(MessageType::AnnounceValue, tid, n,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&& msg) { /* on done */
            if (msg.value_id == Value::INVALID_ID) {
-                DHT_LOG.d(infohash, "Unknown search or announce!");
+                if (logger_)
+                    logger_->d(infohash, "Unknown search or announce!");
            } else {
                if (on_done) {
                    RequestAnswer answer {};
@@ -1136,22 +1189,68 @@ NetworkEngine::sendAnnounceValue(Sp<Node> n,
            }
        }
    );
+    req->parts = std::move(v);
    sendRequest(req);
-    if (not v.empty())
-        sendValueParts(tid, v, n->getAddr());
    ++out_stats.put;
    return req;
 }

+Sp<Request>
+NetworkEngine::sendUpdateValues(Sp<Node> n,
+                                const InfoHash& infohash,
+                                const std::vector<Sp<Value>>& values,
+                                time_point created,
+                                const Blob& token,
+                                const size_t& socket_id)
+{
+    Tid tid (n->getNewTid());
+    Tid sid (socket_id);
+
+    msgpack::sbuffer buffer;
+    msgpack::packer<msgpack::sbuffer> pk(&buffer);
+    pk.pack_map(5+(config.network?1:0));
+
+    pk.pack(KEY_A); pk.pack_map((created < scheduler.time() ? 7 : 6));
+      pk.pack(KEY_REQ_ID);     pk.pack(myid);
+      pk.pack(KEY_VERSION);    pk.pack(1);
+      pk.pack(KEY_REQ_H);      pk.pack(infohash);
+      pk.pack(KEY_REQ_SID);   pk.pack(sid);
+      auto v = packValueHeader(buffer, values);
+      if (created < scheduler.time()) {
+          pk.pack(KEY_REQ_CREATION);
+          pk.pack(to_time_t(created));
+      }
+      pk.pack(KEY_REQ_TOKEN);  pk.pack(token);
+
+    pk.pack(KEY_Q);   pk.pack(QUERY_UPDATE);
+    pk.pack(KEY_TID); pk.pack(tid);
+    pk.pack(KEY_Y);   pk.pack(KEY_Q);
+    pk.pack(KEY_UA);  pk.pack(my_v);
+    if (config.network) {
+        pk.pack(KEY_NETID); pk.pack(config.network);
+    }
+
+    auto req = std::make_shared<Request>(MessageType::UpdateValue, tid, n,
+        Blob(buffer.data(), buffer.data() + buffer.size()),
+        [=](const Request&, ParsedMessage&&) { /* on done */ },
+        [=](const Request&, bool) { /* on expired */ }
+    );
+    req->parts = std::move(v);
+    sendRequest(req);
+    ++out_stats.updateValue;
+    return req;
+}
+
 Sp<Request>
 NetworkEngine::sendRefreshValue(Sp<Node> n,
                const InfoHash& infohash,
                const Value::Id& vid,
                const Blob& token,
                RequestCb&& on_done,
+                RequestErrorCb&& on_error,
                RequestExpiredCb&& on_expired)
 {
-    TransId tid (n->getNewTid());
+    Tid tid (n->getNewTid());
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
    pk.pack_map(5+(config.network?1:0));
@@ -1163,19 +1262,19 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
      pk.pack(KEY_REQ_TOKEN);    pk.pack(token);

    pk.pack(KEY_Q); pk.pack(QUERY_REFRESH);
-    pk.pack(KEY_TID); pk.pack_bin(tid.size());
-                               pk.pack_bin_body((const char*)tid.data(), tid.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_Q);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
        pk.pack(KEY_NETID); pk.pack(config.network);
    }

-    auto req = std::make_shared<Request>(MessageType::Refresh, tid.toInt(), n,
+    auto req = std::make_shared<Request>(MessageType::Refresh, tid, n,
        Blob(buffer.data(), buffer.data() + buffer.size()),
        [=](const Request& req_status, ParsedMessage&& msg) { /* on done */
            if (msg.value_id == Value::INVALID_ID) {
-                DHT_LOG.d(infohash, "Unknown search or announce!");
+                if (logger_)
+                    logger_->d(infohash, "Unknown search or announce!");
            } else {
                if (on_done) {
                    RequestAnswer answer {};
@@ -1184,6 +1283,7 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
                }
            }
        },
+        on_error,
        [=](const Request& req_status, bool done) { /* on expired */
            if (on_expired) {
                on_expired(req_status, done);
@@ -1206,9 +1306,7 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, Tid tid, Value::Id vid)
      pk.pack(KEY_REQ_VALUE_ID); pk.pack(vid);
      insertAddr(pk, addr);

-    TransId t(tid);
-    pk.pack(KEY_TID); pk.pack_bin(t.size());
-                               pk.pack_bin_body((const char*)t.data(), t.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_R);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
@@ -1227,7 +1325,7 @@ NetworkEngine::sendError(const SockAddr& addr,
 {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4 + (include_id?1:0));
+    pk.pack_map(4 + (include_id?1:0) + (config.network?1:0));

    pk.pack(KEY_E); pk.pack_array(2);
      pk.pack(code);
@@ -1238,9 +1336,7 @@ NetworkEngine::sendError(const SockAddr& addr,
          pk.pack(KEY_REQ_ID); pk.pack(myid);
    }

-    TransId t(tid);
-    pk.pack(KEY_TID); pk.pack_bin(t.size());
-                               pk.pack_bin_body((const char*)t.data(), t.size());
+    pk.pack(KEY_TID); pk.pack(tid);
    pk.pack(KEY_Y); pk.pack(KEY_E);
    pk.pack(KEY_UA); pk.pack(my_v);
    if (config.network) {
@@ -1258,7 +1354,8 @@ NetworkEngine::maintainRxBuffer(Tid tid)
        const auto& now = scheduler.time();
        if (msg->second.start + RX_MAX_PACKET_TIME < now
         || msg->second.last_part + RX_TIMEOUT < now) {
-            DHT_LOG.w("Dropping expired partial message from %s", msg->second.from.toString().c_str());
+            if (logger_)
+                logger_->w("Dropping expired partial message from %s", msg->second.from.toString().c_str());
            partial_messages.erase(msg);
        }
    }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -106,7 +106,7 @@ void udpPipe(int fds[2])
 }
 #endif

-UdpSocket::UdpSocket(in_port_t port, const Logger& l) : logger(l) {
+UdpSocket::UdpSocket(in_port_t port, const std::shared_ptr<Logger>& l) : logger(l) {
    SockAddr bind4;
    bind4.setFamily(AF_INET);
    bind4.setPort(port);
@@ -117,7 +117,7 @@ UdpSocket::UdpSocket(in_port_t port, const Logger& l) : logger(l) {
    openSockets(bind4, bind6);
 }

-UdpSocket::UdpSocket(const SockAddr& bind4, const SockAddr& bind6, const Logger& l) : logger(l)
+UdpSocket::UdpSocket(const SockAddr& bind4, const SockAddr& bind6, const std::shared_ptr<Logger>& l) : logger(l)
 {
    std::lock_guard<std::mutex> lk(lock);
    openSockets(bind4, bind6);
@@ -155,7 +155,8 @@ UdpSocket::sendTo(const SockAddr& dest, const uint8_t* data, size_t size, bool r

    if (sendto(s, (const char*)data, size, flags, dest.get(), dest.getLength()) == -1) {
        int err = errno;
-        logger.d("Can't send message to %s: %s", dest.toString().c_str(), strerror(err));
+        if (logger)
+            logger->d("Can't send message to %s: %s", dest.toString().c_str(), strerror(err));
        if (err == EPIPE || err == ENOTCONN || err == ECONNRESET) {
            std::lock_guard<std::mutex> lk(lock);
            auto bind4 = std::move(bound4), bind6 = std::move(bound6);
@@ -194,7 +195,8 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
        try {
            s4 = bindSocket(bind4, bound4);
        } catch (const DhtException& e) {
-            logger.e("Can't bind inet socket: %s", e.what());
+            if (logger)
+                logger->e("Can't bind inet socket: %s", e.what());
        }
    }

@@ -209,7 +211,8 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
                try {
                    s6 = bindSocket(b6, bound6);
                } catch (const DhtException& e) {
-                    logger.e("Can't bind inet6 socket: %s", e.what());
+                    if (logger)
+                        logger->e("Can't bind inet6 socket: %s", e.what());
                }
            }
        }
@@ -217,7 +220,8 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
            try {
                s6 = bindSocket(bind6, bound6);
            } catch (const DhtException& e) {
-                logger.e("Can't bind inet6 socket: %s", e.what());
+                if (logger)
+                    logger->e("Can't bind inet6 socket: %s", e.what());
            }
        }
    }
@@ -244,7 +248,8 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
                int rc = select(selectFd, &readfds, nullptr, nullptr, nullptr);
                if (rc < 0) {
                    if (errno != EINTR) {
-                        logger.e("Select error: %s", strerror(errno));
+                        if (logger)
+                            logger->e("Select error: %s", strerror(errno));
                        std::this_thread::sleep_for(std::chrono::seconds(1));
                    }
                }
@@ -259,7 +264,8 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)

                    if (FD_ISSET(stop_readfd, &readfds)) {
                        if (recv(stop_readfd, (char*)buf.data(), buf.size(), 0) < 0) {
-                            logger.e("Got stop packet error: %s", strerror(errno));
+                            if (logger)
+                                logger->e("Got stop packet error: %s", strerror(errno));
                            break;
                        }
                    }
@@ -271,43 +277,54 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
                        continue;

                    if (rc > 0) {
-                        auto pkt = std::unique_ptr<ReceivedPacket>(new ReceivedPacket);
-                        pkt->data = {buf.begin(), buf.begin()+rc};
-                        pkt->from = {from, from_len};
-                        pkt->received = clock::now();
-                        onReceived(std::move(pkt));
+                        auto pkts = getNewPacket();
+                        auto& pkt = pkts.front();
+                        pkt.data.insert(pkt.data.end(), buf.begin(), buf.begin()+rc);
+                        pkt.from = {from, from_len};
+                        pkt.received = clock::now();
+                        onReceived(std::move(pkts));
                    } else if (rc == -1) {
-                        logger.e("Error receiving packet: %s", strerror(errno));
+                        if (logger)
+                            logger->e("Error receiving packet: %s", strerror(errno));
                        int err = errno;
                        if (err == EPIPE || err == ENOTCONN || err == ECONNRESET) {
-                            std::lock_guard<std::mutex> lk(lock);
-                            if (ls4 >= 0) {
-                                close(ls4);
-                                try {
-                                    ls4 = bindSocket(bound4, bound4);
-                                } catch (const DhtException& e) {
-                                    logger.e("Can't bind inet socket: %s", e.what());
+                            if (not running) break;
+                            std::unique_lock<std::mutex> lk(lock, std::try_to_lock);
+                            if (lk.owns_lock()) {
+                                if (not running) break;
+                                if (ls4 >= 0) {
+                                    close(ls4);
+                                    try {
+                                        ls4 = bindSocket(bound4, bound4);
+                                    } catch (const DhtException& e) {
+                                        if (logger)
+                                            logger->e("Can't bind inet socket: %s", e.what());
+                                    }
                                }
-                            }
-                            if (ls6 >= 0) {
-                                close(ls6);
-                                try {
-                                    ls6 = bindSocket(bound6, bound6);
-                                } catch (const DhtException& e) {
-                                    logger.e("Can't bind inet6 socket: %s", e.what());
+                                if (ls6 >= 0) {
+                                    close(ls6);
+                                    try {
+                                        ls6 = bindSocket(bound6, bound6);
+                                    } catch (const DhtException& e) {
+                                        if (logger)
+                                            logger->e("Can't bind inet6 socket: %s", e.what());
+                                    }
                                }
-                            }
-                            if (ls4 < 0 && ls6 < 0)
+                                if (ls4 < 0 && ls6 < 0)
+                                    break;
+                                s4 = ls4;
+                                s6 = ls6;
+                                selectFd = std::max({ls4, ls6, stop_readfd}) + 1;
+                            } else {
                                break;
-                            s4 = ls4;
-                            s6 = ls6;
-                            selectFd = std::max({ls4, ls6, stop_readfd}) + 1;
+                            }
                        }
                    }
                }
            }
        } catch (const std::exception& e) {
-            logger.e("Error in UdpSocket rx thread: %s", e.what());
+            if (logger)
+                logger->e("Error in UdpSocket rx thread: %s", e.what());
        }
        if (ls4 >= 0)
            close(ls4);
@@ -315,14 +332,16 @@ UdpSocket::openSockets(const SockAddr& bind4, const SockAddr& bind6)
            close(ls6);
        if (stop_readfd != -1)
            close(stop_readfd);
-        std::lock_guard<std::mutex> lk(lock);
-        s4 = -1;
-        s6 = -1;
-        bound4 = {};
-        bound6 = {};
        if (stopfd != -1)
            close(stopfd);
-        stopfd = -1;
+        std::unique_lock<std::mutex> lk(lock, std::try_to_lock);
+        if (lk.owns_lock()) {
+            s4 = -1;
+            s6 = -1;
+            bound4 = {};
+            bound6 = {};
+            stopfd = -1;
+        }
    });
 }

@@ -332,7 +351,8 @@ UdpSocket::stop()
    if (running.exchange(false)) {
        auto sfd = stopfd;
        if (sfd != -1 && write(sfd, "\0", 1) == -1) {
-            logger.e("Can't write to stop fd");
+            if (logger)
+                logger->e("Can't write to stop fd");
        }
    }
 }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -30,17 +30,15 @@ constexpr std::chrono::minutes Node::NODE_EXPIRE_TIME;
 constexpr std::chrono::minutes Node::NODE_GOOD_TIME;
 constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME;

-Node::Node(const InfoHash& id, const SockAddr& addr, bool client)
+Node::Node(const InfoHash& id, const SockAddr& addr, std::mt19937_64& rd, bool client)
 : id(id), addr(addr), is_client(client), sockets_()
 {
-    thread_local crypto::random_device rd {};
    transaction_id = std::uniform_int_distribution<Tid>{1}(rd);
 }

-Node::Node(const InfoHash& id, SockAddr&& addr, bool client)
+Node::Node(const InfoHash& id, SockAddr&& addr, std::mt19937_64& rd, bool client)
 : id(id), addr(std::move(addr)), is_client(client), sockets_()
 {
-    thread_local crypto::random_device rd {};
    transaction_id = std::uniform_int_distribution<Tid>{1}(rd);
 }

@@ -118,7 +116,6 @@ Node::cancelRequest(const Sp<net::Request>& req)
 {
    if (req) {
        req->cancel();
-        closeSocket(req->closeSocket());
        requests_.erase(req->getTid());
    }
 }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -37,8 +37,8 @@ NodeCache::getNode(const InfoHash& id, sa_family_t family) {
 Sp<Node>
 NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) {
    if (not id)
-        return std::make_shared<Node>(id, addr);
-    return cache(addr.getFamily()).getNode(id, addr, now, confirm, client);
+        return std::make_shared<Node>(id, addr, rd, client);
+    return cache(addr.getFamily()).getNode(id, addr, now, confirm, client, rd);
 }

 std::vector<Sp<Node>>
@@ -104,13 +104,13 @@ NodeCache::NodeMap::getNode(const InfoHash& id)
 }

 Sp<Node>
-NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client)
+NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client, std::mt19937_64& rd)
 {
-    auto it = emplace(id, std::weak_ptr<Node>{});
-    auto node = it.first->second.lock();
+    auto& nref = (*this)[id];
+    auto node = nref.lock();
    if (not node) {
-        node = std::make_shared<Node>(id, addr, client);
-        it.first->second = node;
+        node = std::make_shared<Node>(id, addr, rd, client);
+        nref = node;
        if (cleanup_counter++ == CLEANUP_FREQ) {
            cleanup();
            cleanup_counter = 0;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -22,7 +22,7 @@ namespace dht {
 constexpr const std::chrono::seconds OpCache::EXPIRATION;

 bool
-OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
+OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals, const system_clock::time_point& t) {
    std::vector<Sp<Value>> newValues;
    for (const auto& v : vals) {
        auto viop = values.emplace(v->id, v);
@@ -31,16 +31,21 @@ OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
        } else {
            viop.first->second.refCount++;
        }
+        viop.first->second.updated = t;
    }
    return newValues.empty() ? true : callback(newValues, false);
 }

 bool
-OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
+OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals, const system_clock::time_point& t) {
    std::vector<Sp<Value>> expiredValues;
    for (const auto& v : vals) {
        auto vit = values.find(v->id);
        if (vit != values.end()) {
+            if (vit->second.updated > t)
+                continue;
+            
+            vit->second.updated = t;
            vit->second.refCount--;
            if (not vit->second.refCount) {
                expiredValues.emplace_back(std::move(vit->second.data));
@@ -52,12 +57,16 @@ OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
 }

 bool
-OpValueCache::onValuesExpired(const std::vector<Value::Id>& vids)
+OpValueCache::onValuesExpired(const std::vector<Value::Id>& vids, const system_clock::time_point& t)
 {
    std::vector<Sp<Value>> expiredValues;
    for (const auto& vid : vids) {
        auto vit = values.find(vid);
        if (vit != values.end()) {
+            if (vit->second.updated > t)
+                continue;
+            
+            vit->second.updated = t;
            vit->second.refCount--;
            if (not vit->second.refCount) {
                expiredValues.emplace_back(std::move(vit->second.data));
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -27,6 +27,7 @@ struct OpCacheValueStorage
 {
    Sp<Value> data {};
    unsigned refCount {1};
+    system_clock::time_point updated {system_clock::time_point::min()};
    OpCacheValueStorage(Sp<Value> val) : data(val) {}
 };

@@ -47,16 +48,16 @@ public:
        };
    }

-    bool onValue(const std::vector<Sp<Value>>& vals, bool expired) {
+    bool onValue(const std::vector<Sp<Value>>& vals, bool expired, const system_clock::time_point& t = system_clock::time_point::min()) {
        if (expired)
-            return onValuesExpired(vals);
+            return onValuesExpired(vals, t);
        else
-            return onValuesAdded(vals);
+            return onValuesAdded(vals, t);
    }

-    bool onValuesAdded(const std::vector<Sp<Value>>& vals);
-    bool onValuesExpired(const std::vector<Sp<Value>>& vals);
-    bool onValuesExpired(const std::vector<Value::Id>& vals);
+    bool onValuesAdded(const std::vector<Sp<Value>>& vals, const system_clock::time_point& t = system_clock::time_point::min());
+    bool onValuesExpired(const std::vector<Sp<Value>>& vals, const system_clock::time_point& t = system_clock::time_point::min());
+    bool onValuesExpired(const std::vector<Value::Id>& vals, const system_clock::time_point& t = system_clock::time_point::min());

    void onNodeChanged(ListenSyncStatus status) {
        switch (status) {
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -54,10 +54,12 @@ static const std::string KEY_REQ_EXPIRED {"exp"};
 static const std::string KEY_REQ_REFRESHED {"re"};
 static const std::string KEY_REQ_FIELDS {"fileds"};
 static const std::string KEY_REQ_WANT {"w"};
+static const std::string KEY_VERSION {"ve"};

 static const std::string QUERY_PING {"ping"};
 static const std::string QUERY_FIND {"find"};
 static const std::string QUERY_GET {"get"};
+static const std::string QUERY_UPDATE {"update"};
 static const std::string QUERY_PUT {"put"};
 static const std::string QUERY_LISTEN {"listen"};
 static const std::string QUERY_REFRESH {"refresh"};
@@ -113,6 +115,7 @@ struct ParsedMessage {
    uint16_t error_code;
    /* reported address by the distant node */
    std::string ua;
+    int version {0};
    SockAddr addr;
    void msgpack_unpack(const msgpack::object& o);

@@ -224,6 +227,8 @@ ParsedMessage::msgpack_unpack(const msgpack::object& msg)
        type = MessageType::AnnounceValue;
    else if (parsed.q == QUERY_REFRESH)
        type = MessageType::Refresh;
+    else if (parsed.q == QUERY_UPDATE)
+        type = MessageType::UpdateValue;
    else
        throw msgpack::type_error();

@@ -295,6 +300,8 @@ ParsedMessage::msgpack_unpack(const msgpack::object& msg)
            parsedReq.fields = &o.val;
        else if (key == KEY_REQ_WANT)
            parsedReq.want = &o.val;
+        else if (key == KEY_VERSION)
+            version = o.val.as<int>();
    }

    if (parsedReq.sa) {
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Mingrui Zhang <mingrui.zhang@savoirfairelinux.com>
 *              Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *              Adrien Béraud <adrien.beraud@savoirfairelinux.com>
@@ -24,8 +24,6 @@

 #include <asio.hpp>

-using namespace std::chrono_literals;
-
 namespace dht {

 // Organization-local Scope multicast
@@ -46,6 +44,8 @@ public:
    bool stopDiscovery(const std::string &type);
    bool stopPublish(const std::string &type);

+    void connectivityChanged();
+
 private:
    Sp<Logger> logger_;
    //dmtx_ for callbackmap_ and drunning_ (write)
@@ -55,7 +55,6 @@ private:
    std::shared_ptr<asio::io_context> ioContext_;
    asio::ip::udp::socket sockFd_;
    asio::ip::udp::endpoint sockAddrSend_;
-    asio::steady_timer publishTimer_;

    std::array<char, 64 * 1024> receiveBuf_;
    asio::ip::udp::endpoint receiveFrom_;
@@ -67,11 +66,15 @@ private:
    bool drunning_ {false};

    void loopListener();
+    void query(const asio::ip::udp::endpoint& peer);
    void reloadMessages();

    void stopDiscovery();
    void stopPublish();
-    void publish();
+
+    void publish(const asio::ip::udp::endpoint& peer);
+
+    void reDiscover();
 };

 PeerDiscovery::DomainPeerDiscovery::DomainPeerDiscovery(asio::ip::udp domain, in_port_t port, Sp<asio::io_context> ioContext, Sp<Logger> logger)
@@ -80,11 +83,16 @@ PeerDiscovery::DomainPeerDiscovery::DomainPeerDiscovery(asio::ip::udp domain, in
    , sockFd_(*ioContext_, domain)
    , sockAddrSend_(asio::ip::address::from_string(domain.family() == AF_INET ? MULTICAST_ADDRESS_IPV4
                                                                              : MULTICAST_ADDRESS_IPV6), port)
-    , publishTimer_(*ioContext)
 {
-    sockFd_.set_option(asio::ip::multicast::join_group(sockAddrSend_.address()));
-    sockFd_.set_option(asio::ip::udp::socket::reuse_address(true));
-    sockFd_.bind({domain, port});
+    try {
+        sockFd_.set_option(asio::ip::multicast::join_group(sockAddrSend_.address()));
+        sockFd_.set_option(asio::ip::udp::socket::reuse_address(true));
+        sockFd_.bind({domain, port});
+    } catch (const std::exception& e) {
+        if (logger_)
+            logger_->e("Can't start peer discovery for %s: %s",
+                    domain.family() == AF_INET ? "IPv4" : "IPv6", e.what());
+    }
 }

 PeerDiscovery::DomainPeerDiscovery::~DomainPeerDiscovery()
@@ -100,7 +108,10 @@ PeerDiscovery::DomainPeerDiscovery::startDiscovery(const std::string &type, Serv
    callbackmap_[type] = callback;
    if (not drunning_) {
        drunning_ = true;
-        ioContext_->post(std::bind(&PeerDiscovery::DomainPeerDiscovery::loopListener, this));
+        ioContext_->post([this] () {
+                    loopListener();
+                    query(sockAddrSend_);
+                });
    }
 }

@@ -121,25 +132,30 @@ PeerDiscovery::DomainPeerDiscovery::loopListener()
            auto rcv = msgpack::unpack(receiveBuf_.data(), bytes);
            msgpack::object obj = rcv.get();

-            if (obj.type != msgpack::type::MAP)
-                throw msgpack::type_error{};
-            for (unsigned i = 0; i < obj.via.map.size; i++) {
-                auto& o = obj.via.map.ptr[i];
-                if (o.key.type != msgpack::type::STR)
-                    continue;
-                auto key = o.key.as<std::string>();
-                ServiceDiscoveredCallback cb;
-                {
-                    std::lock_guard<std::mutex> lck(dmtx_);
-                    if (drunning_) {
-                        auto callback = callbackmap_.find(key);
-                        if (callback != callbackmap_.end())
-                            cb = callback->second;
-                    } else
-                        return;
+            if (obj.type == msgpack::type::STR) {
+                if (lrunning_ and obj.as<std::string>() == "q")
+                    publish(receiveFrom_);
+            } else if (obj.type == msgpack::type::MAP) {
+                for (unsigned i = 0; i < obj.via.map.size; i++) {
+                    auto& o = obj.via.map.ptr[i];
+                    if (o.key.type != msgpack::type::STR)
+                        continue;
+                    auto key = o.key.as<std::string>();
+                    ServiceDiscoveredCallback cb;
+                    {
+                        std::lock_guard<std::mutex> lck(dmtx_);
+                        if (drunning_) {
+                            auto callback = callbackmap_.find(key);
+                            if (callback != callbackmap_.end())
+                                cb = callback->second;
+                        } else
+                            return;
+                    }
+                    if (cb)
+                        cb(std::move(o.val), SockAddr{ receiveFrom_.data(), (socklen_t)receiveFrom_.size() });
                }
-                if (cb)
-                    cb(std::move(o.val), SockAddr{ receiveFrom_.data(), (socklen_t)receiveFrom_.size() });
+            } else {
+                throw msgpack::type_error{};
            }
        } catch (const std::exception& e) {
            if (logger_)
@@ -150,21 +166,45 @@ PeerDiscovery::DomainPeerDiscovery::loopListener()
 }

 void
-PeerDiscovery::DomainPeerDiscovery::publish()
+PeerDiscovery::DomainPeerDiscovery::query(const asio::ip::udp::endpoint& peer)
+{
+    std::lock_guard<std::mutex> lck(dmtx_);
+    if (not drunning_)
+        return;
+
+    msgpack::sbuffer pbuf_request;
+    msgpack::pack(pbuf_request, "q");
+
+    sockFd_.async_send_to(asio::buffer(pbuf_request.data(), pbuf_request.size()), peer,
+            [logger=logger_, peer] (const asio::error_code& ec, size_t)
+            {
+                if (ec and (ec != asio::error::operation_aborted) and logger)
+                    logger->w("Error sending packet to: %s with err: %s",
+				    peer.address().to_string().c_str(),
+				    ec.message().c_str());
+            }
+    );
+}
+
+void
+PeerDiscovery::DomainPeerDiscovery::publish(const asio::ip::udp::endpoint& peer)
 {
    std::lock_guard<std::mutex> lck(mtx_);
-    if (lrunning_) {
-        sockFd_.async_send_to(asio::buffer(sbuf_.data(), sbuf_.size()), sockAddrSend_, [logger=logger_](const asio::error_code& ec, size_t){
-            if (ec and ec != asio::error::operation_aborted) {
-                if (logger)
-                    logger->w("Error sending packet: %s", ec.message().c_str());
+    if (not lrunning_)
+        return;
+
+    sockFd_.async_send_to(asio::buffer(sbuf_.data(), sbuf_.size()), peer,
+            [logger=logger_, peer] (const asio::error_code& ec, size_t)
+            {
+                if (ec and (ec != asio::error::operation_aborted) and logger)
+                    logger->w("Error sending packet to: %s with err: %s",
+				    peer.address().to_string().c_str(),
+				    ec.message().c_str());
            }
-        });
-        publishTimer_.expires_after(3s);
-        publishTimer_.async_wait(std::bind(&PeerDiscovery::DomainPeerDiscovery::publish, this));
-    }
+    );
 }

+
 void
 PeerDiscovery::DomainPeerDiscovery::startPublish(const std::string &type, const msgpack::sbuffer &pack_buf)
 {
@@ -174,10 +214,8 @@ PeerDiscovery::DomainPeerDiscovery::startPublish(const std::string &type, const
    std::lock_guard<std::mutex> lck(mtx_);
    messages_[type] = std::move(pack_buf_c);
    reloadMessages();
-    if (not lrunning_) {
-        lrunning_ = true;
-        ioContext_->post(std::bind(&PeerDiscovery::DomainPeerDiscovery::publish, this));
-    }
+    lrunning_ = true;
+    ioContext_->post([this] () { publish(sockAddrSend_); });
 }

 bool
@@ -209,17 +247,13 @@ PeerDiscovery::DomainPeerDiscovery::stopPublish(const std::string& type)
 void
 PeerDiscovery::DomainPeerDiscovery::stopDiscovery()
 {
-    if (drunning_)
-        drunning_ = false;
+    drunning_ = false;
 }

 void
 PeerDiscovery::DomainPeerDiscovery::stopPublish()
 {
-    if (lrunning_) {
-        lrunning_ = false;
-        publishTimer_.cancel();
-    }
+    lrunning_ = false;
 }

 void
@@ -247,6 +281,26 @@ PeerDiscovery::DomainPeerDiscovery::reloadMessages()
    }
 }

+void
+PeerDiscovery::DomainPeerDiscovery::reDiscover()
+{
+    asio::error_code ec;
+
+    sockFd_.set_option(asio::ip::multicast::join_group(sockAddrSend_.address()), ec);
+    if (ec and logger_)
+        logger_->w("can't multicast on %s: %s",
+                sockAddrSend_.address().to_string().c_str(),
+                ec.message().c_str());
+    query(sockAddrSend_);
+}
+
+void
+PeerDiscovery::DomainPeerDiscovery::connectivityChanged()
+{
+    reDiscover();
+    publish(sockAddrSend_);
+}
+
 PeerDiscovery::PeerDiscovery(in_port_t port, Sp<asio::io_context> ioContext, Sp<Logger> logger)
 {
    if (not ioContext) {
@@ -348,4 +402,13 @@ PeerDiscovery::stopPublish(sa_family_t domain, const std::string& type)
    return false;
 }

+void
+PeerDiscovery::connectivityChanged()
+{
+    if (peerDiscovery4_)
+        peerDiscovery4_->connectivityChanged();
+    if (peerDiscovery6_)
+        peerDiscovery6_->connectivityChanged();
 }
+
+} /* namespace dht */
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -27,6 +27,7 @@ struct Node;
 namespace net {

 class NetworkEngine;
+class DhtProtocolException;
 struct ParsedMessage;

 /*!
@@ -72,16 +73,19 @@ struct Request {
            Sp<Node> node,
            Blob&& msg,
            std::function<void(const Request&, ParsedMessage&&)> on_done,
-            std::function<void(const Request&, bool)> on_expired,
-            Tid socket = 0) :
-        node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)), socket(socket) { }
+            std::function<void(const Request&, bool)> on_expired) :
+        node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)) { }
+    Request(MessageType type, Tid tid,
+            Sp<Node> node,
+            Blob&& msg,
+            std::function<void(const Request&, ParsedMessage&&)> on_done,
+            std::function<bool(const Request&, DhtProtocolException&&)> on_error,
+            std::function<void(const Request&, bool)> on_expired) :
+        node(node), tid(tid), type(type), on_done(on_done), on_error(on_error), on_expired(on_expired), msg(std::move(msg)) { }

    Tid getTid() const { return tid; }
    MessageType getType() const { return type; }

-    Tid getSocket() { return socket; }
-    Tid closeSocket() { auto ret = socket; socket = 0; return ret; }
-
    void setExpired() {
        if (pending()) {
            state_ = Request::State::EXPIRED;
@@ -96,6 +100,15 @@ struct Request {
            clear();
        }
    }
+    bool setError(DhtProtocolException&& e) {
+        if (pending()) {
+            state_ = Request::State::EXPIRED;
+            bool handled = on_error and on_error(*this, std::forward<DhtProtocolException>(e));
+            clear();
+            return handled;
+        }
+        return true;
+    }

    void cancel() {
        if (pending()) {
@@ -108,13 +121,15 @@ private:
    static const constexpr size_t MAX_ATTEMPT_COUNT {3};

    bool isExpired(time_point now) const {
-        return pending() and now > last_try + Node::MAX_RESPONSE_TIME and attempt_count >= Request::MAX_ATTEMPT_COUNT;
+        return pending() and now > last_try + attempt_duration and attempt_count >= Request::MAX_ATTEMPT_COUNT;
    }

    void clear() {
        on_done = {};
+        on_error = {};
        on_expired = {};
-        msg.clear();
+        msg = {};
+        parts = {};
    }

    const Tid tid {0}; /* the request id. */
@@ -122,14 +137,16 @@ private:
    State state_ {State::PENDING};

    unsigned attempt_count {0};                /* number of attempt to process the request. */
+    duration attempt_duration {((duration)Node::MAX_RESPONSE_TIME)/2};
    time_point start {time_point::min()};      /* time when the request is created. */
    time_point last_try {time_point::min()};   /* time of the last attempt to process the request. */

    std::function<void(const Request&, ParsedMessage&&)> on_done {};
+    std::function<bool(const Request&, DhtProtocolException&&)> on_error {};
    std::function<void(const Request&, bool)> on_expired {};

    Blob msg {};                      /* the serialized message. */
-    Tid socket;   /* the socket used for further reponses. */
+    std::vector<Blob> parts;
 };

 } /* namespace net  */
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -25,7 +25,7 @@ namespace dht {
 namespace crypto {

 random_device::random_device() :
-   gen(std::chrono::system_clock::now().time_since_epoch().count())
+   gen(std::chrono::system_clock::now().time_since_epoch().count() ^ std::chrono::high_resolution_clock::now().count())
 {}

 random_device::result_type
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -25,15 +25,8 @@

 namespace dht {

-static std::mt19937 rd{ dht::crypto::random_device{}() };
-#ifdef _WIN32
-static std::uniform_int_distribution<int> rand_byte{ 0, std::numeric_limits<uint8_t>::max() };
-#else
-static std::uniform_int_distribution<uint8_t> rand_byte;
-#endif
-
 Sp<Node>
-Bucket::randomNode()
+Bucket::randomNode(std::mt19937_64& rd)
 {
    if (nodes.empty())
        return nullptr;
@@ -65,7 +58,7 @@ void Bucket::sendCachedPing(net::NetworkEngine& ne)
 }

 InfoHash
-RoutingTable::randomId(const RoutingTable::const_iterator& it) const
+RoutingTable::randomId(const RoutingTable::const_iterator& it, std::mt19937_64& rd) const
 {
    int bit1 = it->first.lowbit();
    int bit2 = std::next(it) != end() ? std::next(it)->first.lowbit() : -1;
@@ -74,6 +67,12 @@ RoutingTable::randomId(const RoutingTable::const_iterator& it) const
    if (bit >= 8*(int)HASH_LEN)
        return it->first;

+#ifdef _WIN32
+    std::uniform_int_distribution<int> rand_byte{ 0, std::numeric_limits<uint8_t>::max() };
+#else
+    std::uniform_int_distribution<uint8_t> rand_byte;
+#endif
+
    int b = bit/8;
    InfoHash id_return;
    std::copy_n(it->first.cbegin(), b, id_return.begin());
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -65,11 +65,16 @@ struct Dht::SearchNode {
        ValueCache cache;
        Sp<Scheduler::Job> cacheExpirationJob {};
        Sp<net::Request> req {};
-        CachedListenStatus(ValueStateCallback&& cb, SyncCallback&& scb)
-         : cache(std::forward<ValueStateCallback>(cb), std::forward<SyncCallback>(scb)) {}
-        CachedListenStatus(CachedListenStatus&&) = default;
+        Tid socketId {0};
+        CachedListenStatus(ValueStateCallback&& cb, SyncCallback scb, Tid sid)
+         : cache(std::forward<ValueStateCallback>(cb), std::forward<SyncCallback>(scb)), socketId(sid) {}
+        CachedListenStatus(CachedListenStatus&&) = delete;
        CachedListenStatus(const CachedListenStatus&) = delete;
-        CachedListenStatus& operator=(const CachedListenStatus&) = delete;
+        ~CachedListenStatus() {
+            if (socketId and req and req->node) {
+                req->node->closeSocket(socketId);
+            }
+        }
    };
    using NodeListenerStatus = std::map<Sp<Query>, CachedListenStatus>;

@@ -86,15 +91,15 @@ struct Dht::SearchNode {

    Blob token {};                                 /* last token the node sent to us after a get request */
    time_point last_get_reply {time_point::min()}; /* last time received valid token */
+    Sp<Scheduler::Job> syncJob {};
    bool candidate {false};                        /* A search node is candidate if the search is/was synced and this
                                                      node is a new candidate for inclusion. */
-    Sp<Scheduler::Job> syncJob {};

    SearchNode() : node() {}
    SearchNode(const SearchNode&) = delete;
-    SearchNode(SearchNode&&) = default;
+    SearchNode(SearchNode&&) = delete;
    SearchNode& operator=(const SearchNode&) = delete;
-    SearchNode& operator=(SearchNode&&) = default;
+    SearchNode& operator=(SearchNode&&) = delete;

    SearchNode(const Sp<Node>& node) : node(node) {}
    ~SearchNode() {
@@ -263,7 +268,7 @@ struct Dht::SearchNode {
     * @return true if there exists an expired request, else false.
     */
    static bool pending(const SyncStatus& status) {
-        return std::find_if(status.begin(), status.end(),
+        return std::find_if(status.cbegin(), status.cend(),
            [](const SyncStatus::value_type& r){
                return r.second and r.second->pending();
            }) != status.end();
@@ -293,26 +298,26 @@ struct Dht::SearchNode {
        acked.clear();
    }

-    bool isListening(time_point now) const {
+    inline bool isListening(time_point now, duration listen_expire) const {
        auto ls = listenStatus.begin();
        for ( ; ls != listenStatus.end() ; ++ls) {
-            if (isListening(now, ls)) {
+            if (isListening(now, ls, listen_expire)) {
                break;
            }
        }
        return ls != listenStatus.end();
    }
-    bool isListening(time_point now, const Sp<Query>& q) const {
+    inline bool isListening(time_point now, const Sp<Query>& q, duration listen_expire) const {
        const auto& ls = listenStatus.find(q);
        if (ls == listenStatus.end())
            return false;
        else
-            return isListening(now, ls);
+            return isListening(now, ls, listen_expire);
    }
-    bool isListening(time_point now, NodeListenerStatus::const_iterator listen_status) const {
+    inline bool isListening(time_point now, NodeListenerStatus::const_iterator listen_status, duration listen_expire) const {
        if (listen_status == listenStatus.end() or not listen_status->second.req)
            return false;
-        return listen_status->second.req->reply_time + LISTEN_EXPIRE_TIME > now;
+        return listen_status->second.req->reply_time + listen_expire > now;
    }
    void cancelListen() {
        for (const auto& status : listenStatus)
@@ -345,12 +350,14 @@ struct Dht::SearchNode {
     * Assuming the node is synced, should the "listen" request with Query q be
     * sent to this node now ?
     */
-    time_point getListenTime(const Sp<Query>& q) const {
-        auto listen_status = listenStatus.find(q);
-        if (listen_status == listenStatus.end() or not listen_status->second.req)
+    time_point getListenTime(const decltype(listenStatus)::const_iterator listen_status, duration listen_expire) const {
+        if (listen_status == listenStatus.cend() or not listen_status->second.req)
            return time_point::min();
        return listen_status->second.req->pending() ? time_point::max() :
-            listen_status->second.req->reply_time + LISTEN_EXPIRE_TIME - REANNOUNCE_MARGIN;
+            listen_status->second.req->reply_time + listen_expire - REANNOUNCE_MARGIN;
+    }
+    time_point getListenTime(const Sp<Query>& q, duration listen_expire) const {
+        return getListenTime(listenStatus.find(q), listen_expire);
    }

    /**
@@ -376,7 +383,7 @@ struct Dht::Search {

    bool expired {false};              /* no node, or all nodes expired */
    bool done {false};                 /* search is over, cached for later */
-    std::vector<SearchNode> nodes {};
+    std::vector<std::unique_ptr<SearchNode>> nodes {};

    /* pending puts */
    std::vector<Announce> announce {};
@@ -416,17 +423,17 @@ struct Dht::Search {
    bool insertNode(const Sp<Node>& n, time_point now, const Blob& token={});

    SearchNode* getNode(const Sp<Node>& n) {
-        auto srn = std::find_if(nodes.begin(), nodes.end(), [&](SearchNode& sn) {
-            return n == sn.node;
+        auto srn = std::find_if(nodes.begin(), nodes.end(), [&](std::unique_ptr<SearchNode>& sn) {
+            return n == sn->node;
        });
-        return (srn == nodes.end()) ? nullptr : &(*srn);
+        return (srn == nodes.end()) ? nullptr : (*srn).get();
    }

    /* number of concurrent sync requests */
    unsigned currentlySolicitedNodeCount() const {
        unsigned count = 0;
        for (const auto& n : nodes)
-            if (not n.isBad() and n.pendingGet())
+            if (not n->isBad() and n->pendingGet())
                count++;
        return count;
    }
@@ -460,12 +467,12 @@ struct Dht::Search {
     */
    void setDone(const Get& get) {
        for (auto& n : nodes) {
-            auto pqs = n.pagination_queries.find(get.query);
-            if (pqs != n.pagination_queries.cend()) {
+            auto pqs = n->pagination_queries.find(get.query);
+            if (pqs != n->pagination_queries.cend()) {
                for (auto& pq : pqs->second)
-                    n.getStatus.erase(pq);
+                    n->getStatus.erase(pq);
            }
-            n.getStatus.erase(get.query);
+            n->getStatus.erase(get.query);
        }
        if (get.done_cb)
            get.done_cb(true, getNodes());
@@ -478,15 +485,15 @@ struct Dht::Search {
     */
    void setDone() {
        for (auto& n : nodes) {
-            n.getStatus.clear();
-            n.listenStatus.clear();
-            n.acked.clear();
+            n->getStatus.clear();
+            n->listenStatus.clear();
+            n->acked.clear();
        }
        done = true;
    }

    bool isAnnounced(Value::Id id) const;
-    bool isListening(time_point now) const;
+    bool isListening(time_point now, duration exp) const;

    void get(const Value::Filter& f, const Sp<Query>& q, const QueryCallback& qcb, const GetCallback& gcb, const DoneCallback& dcb, Scheduler& scheduler) {
        if (gcb or qcb) {
@@ -522,9 +529,9 @@ struct Dht::Search {
                    }
                    for (auto& sn : nodes) {
                        if (listeners.empty())
-                            sn.cancelListen();
+                            sn->cancelListen();
                        else if (query)
-                            sn.cancelListen(query);
+                            sn->cancelListen(query);
                    }
                });
                scheduler.edit(opExpirationJob, nextExpire);
@@ -558,21 +565,66 @@ struct Dht::Search {
            else
                ++it;
        }
+        for (auto& n : nodes) {
+            auto ackIt = n->acked.find(vid);
+            if (ackIt != n->acked.end()) {
+                if (ackIt->second.first)
+                    ackIt->second.first->cancel();
+                n->acked.erase(ackIt);
+            }
+        }
        return canceled;
    }

+    void put(const Sp<Value>& value, DoneCallback callback, time_point created, bool permanent) {
+        done = false;
+        expired = false;
+        auto a_sr = std::find_if(announce.begin(), announce.end(), [&](const Announce& a){
+            return a.value->id == value->id;
+        });
+        if (a_sr == announce.end()) {
+            announce.emplace_back(Announce {permanent, value, created, callback});
+            for (auto& n : nodes) {
+                n->probe_query.reset();
+                n->acked[value->id].first.reset();
+            }
+        } else {
+            a_sr->permanent = permanent;
+            a_sr->created = created;
+            if (a_sr->value != value) {
+                a_sr->value = value;
+                for (auto& n : nodes) {
+                    n->acked[value->id].first.reset();
+                    n->probe_query.reset();
+                }
+            }
+            if (isAnnounced(value->id)) {
+                if (a_sr->callback)
+                    a_sr->callback(true, {});
+                a_sr->callback = {};
+                if (callback)
+                    callback(true, {});
+                return;
+            } else {
+                if (a_sr->callback)
+                    a_sr->callback(false, {});
+                a_sr->callback = callback;
+            }
+        }
+    }
+
    /**
     * @return The number of non-good search nodes.
     */
    unsigned getNumberOfBadNodes() const {
-        return std::count_if(nodes.begin(), nodes.end(), [](const SearchNode& sn) {
-            return sn.isBad();
+        return std::count_if(nodes.begin(), nodes.end(), [](const std::unique_ptr<SearchNode>& sn) {
+            return sn->isBad();
        });
    }
    unsigned getNumberOfConsecutiveBadNodes() const {
        unsigned count = 0;
-        std::find_if(nodes.begin(), nodes.end(), [&count](const SearchNode& sn) {
-            if (not sn.isBad())
+        std::find_if(nodes.begin(), nodes.end(), [&count](const std::unique_ptr<SearchNode>& sn) {
+            if (not sn->node->isExpired())
                return true;
            ++count;
            return false;
@@ -591,7 +643,7 @@ struct Dht::Search {
     */
    bool removeExpiredNode(const time_point& now) {
        for (auto e = nodes.cend(); e != nodes.cbegin();) {
-            const Node& n = *(--e)->node;
+            const Node& n = *(*(--e))->node;
            if (n.isRemovable(now)) {
                //std::cout << "Removing expired node " << n.id << " from IPv" << (af==AF_INET?'4':'6') << " search " << id << std::endl;
                nodes.erase(e);
@@ -663,8 +715,14 @@ struct Dht::Search {
        });
        // remove acked for cleared annouces
        for (auto it = announced; it != announce.end(); ++it) {
-            for (auto& n : nodes)
-                n.acked.erase(it->value->id);
+            for (auto& n : nodes) {
+                auto ackIt = n->acked.find(it->value->id);
+                if (ackIt != n->acked.end()) {
+                    if (ackIt->second.first)
+                        ackIt->second.first->cancel();
+                    n->acked.erase(ackIt);
+                }
+            }
        }
        announce.erase(announced, announce.end());
    }
@@ -697,13 +755,13 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
    auto n = nodes.end();
    while (n != nodes.begin()) {
        --n;
-        if (n->node == snode) {
+        if ((*n)->node == snode) {
            found = true;
            break;
        }

        /* Node not found. We could insert it after this one. */
-        if (id.xorCmp(nid, n->node->id) > 0) {
+        if (id.xorCmp(nid, (*n)->node->id) > 0) {
            ++n;
            break;
        }
@@ -727,7 +785,7 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
            full = nodes.size() - bad >=  SEARCH_NODES;
            while (std::distance(nodes.cbegin(), t) - bad >  SEARCH_NODES) {
                --t;
-                if (t->isBad())
+                if ((*t)->isBad())
                    bad--;
            }
        }
@@ -743,7 +801,7 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
        if (nodes.empty()) {
            step_time = time_point::min();
        }
-        n = nodes.insert(n, SearchNode(snode));
+        n = nodes.emplace(n, std::make_unique<SearchNode>(snode));
        node.setTime(now);
        new_search_node = true;
        if (node.isExpired()) {
@@ -755,16 +813,16 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
        }

        while (nodes.size() - bad >  SEARCH_NODES) {
-            if (not expired and nodes.back().isBad())
+            if (not expired and nodes.back()->isBad())
                bad--;
            nodes.pop_back();
        }
    }
    if (not token.empty()) {
-        n->candidate = false;
-        n->last_get_reply = now;
+        (*n)->candidate = false;
+        (*n)->last_get_reply = now;
        if (token.size() <= 64)
-            n->token = token;
+            (*n)->token = token;
        expired = false;
    }
    if (new_search_node)
@@ -778,7 +836,7 @@ Dht::Search::getNodes() const
    std::vector<Sp<Node>> ret {};
    ret.reserve(nodes.size());
    for (const auto& sn : nodes)
-        ret.emplace_back(sn.node);
+        ret.emplace_back(sn->node);
    return ret;
 }

@@ -787,9 +845,9 @@ Dht::Search::isSynced(time_point now) const
 {
    unsigned i = 0;
    for (const auto& n : nodes) {
-        if (n.isBad())
+        if (n->isBad())
            continue;
-        if (not n.isSynced(now))
+        if (not n->isSynced(now))
            return false;
        if (++i == TARGET_NODES)
            break;
@@ -820,9 +878,9 @@ Dht::Search::isDone(const Get& get) const
 {
    unsigned i = 0;
    for (const auto& sn : nodes) {
-        if (sn.isBad())
+        if (sn->isBad())
            continue;
-        if (not sn.isDone(get))
+        if (not sn->isDone(get))
            return false;
        if (++i == TARGET_NODES)
            break;
@@ -837,9 +895,9 @@ Dht::Search::isAnnounced(Value::Id id) const
        return false;
    unsigned i = 0;
    for (const auto& n : nodes) {
-        if (n.isBad())
+        if (n->isBad())
            continue;
-        if (not n.isAnnounced(id))
+        if (not n->isAnnounced(id))
            return false;
        if (++i == TARGET_NODES)
            return true;
@@ -848,20 +906,20 @@ Dht::Search::isAnnounced(Value::Id id) const
 }

 bool
-Dht::Search::isListening(time_point now) const
+Dht::Search::isListening(time_point now, duration listen_expire) const
 {
    if (nodes.empty() or listeners.empty())
        return false;
    unsigned i = 0;
    for (const auto& n : nodes) {
-        if (n.isBad())
+        if (n->isBad())
            continue;
        SearchNode::NodeListenerStatus::const_iterator ls {};
-        for (ls = n.listenStatus.begin(); ls != n.listenStatus.end() ; ++ls) {
-            if (n.isListening(now, ls))
+        for (ls = n->listenStatus.begin(); ls != n->listenStatus.end() ; ++ls) {
+            if (n->isListening(now, ls, listen_expire))
                break;
        }
-        if (ls == n.listenStatus.end())
+        if (ls == n->listenStatus.end())
            return false;
        if (++i == LISTEN_NODES)
            break;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *           Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *           Sébastien Blin <sebastien.blin@savoirfairelinux.com>
@@ -37,7 +37,7 @@ extern "C" {
 namespace dht {

 SecureDht::SecureDht(std::unique_ptr<DhtInterface> dht, SecureDht::Config conf)
-: dht_(std::move(dht)), key_(conf.id.first), certificate_(conf.id.second)
+: dht_(std::move(dht)), key_(conf.id.first), certificate_(conf.id.second), enableCache_(conf.cert_cache_all)
 {
    if (!dht_) return;
    for (const auto& type : DEFAULT_TYPES)
@@ -49,7 +49,7 @@ SecureDht::SecureDht(std::unique_ptr<DhtInterface> dht, SecureDht::Config conf)
    registerInsecureType(CERTIFICATE_TYPE);

    if (certificate_) {
-        auto certId = certificate_->getPublicKey().getId();
+        auto certId = certificate_->getId();
        if (key_ and certId != key_->getPublicKey().getId())
            throw DhtException("SecureDht: provided certificate doesn't match private key.");

@@ -59,7 +59,8 @@ SecureDht::SecureDht(std::unique_ptr<DhtInterface> dht, SecureDht::Config conf)
            1
        }, [this, certId](bool ok) {
            if (ok)
-                DHT_LOG.d(certId, "SecureDht: public key announced successfully");
+                if (logger_)
+                    logger_->d(certId, "SecureDht: public key announced successfully");
        }, {}, true);
    }
 }
@@ -76,7 +77,8 @@ SecureDht::secureType(ValueType&& type)
                v->signatureValid = v->owner and v->owner->checkSignature(v->getToSign(), v->signature);
            }
            if (!v->signatureValid) {
-                DHT_LOG.w("Signature verification failed");
+                if (logger_)
+                    logger_->w("Signature verification failed");
                return false;
            }
        }
@@ -86,7 +88,8 @@ SecureDht::secureType(ValueType&& type)
        if (!o->isSigned())
            return type.editPolicy(id, o, n, nid, a);
        if (o->owner != n->owner) {
-            DHT_LOG.w("Edition forbidden: owner changed.");
+            if (logger_)
+                logger_->w("Edition forbidden: owner changed.");
            return false;
        }
        if (!n->signatureChecked) {
@@ -94,14 +97,16 @@ SecureDht::secureType(ValueType&& type)
            n->signatureValid = o->owner and o->owner->checkSignature(n->getToSign(), n->signature);
        }
        if (!n->signatureValid) {
-            DHT_LOG.w("Edition forbidden: signature verification failed.");
+            if (logger_)
+                logger_->w("Edition forbidden: signature verification failed.");
            return false;
        }
        if (o->seq == n->seq) {
            // If the data is exactly the same,
            // it can be reannounced, possibly by someone else.
            if (o->getToSign() != n->getToSign()) {
-                DHT_LOG.w("Edition forbidden: sequence number must be increasing.");
+                if (logger_)
+                    logger_->w("Edition forbidden: sequence number must be increasing.");
                return false;
            }
        }
@@ -147,7 +152,8 @@ SecureDht::registerCertificate(const InfoHash& node, const Blob& data)
    }
    InfoHash h = crt->getPublicKey().getId();
    if (node == h) {
-        DHT_LOG.d("Registering certificate for %s", h.toString().c_str());
+        if (logger_)
+            logger_->d("Registering certificate for %s", h.toString().c_str());
        auto it = nodesCertificates_.find(h);
        if (it == nodesCertificates_.end())
            std::tie(it, std::ignore) = nodesCertificates_.emplace(h, std::move(crt));
@@ -155,7 +161,8 @@ SecureDht::registerCertificate(const InfoHash& node, const Blob& data)
            it->second = std::move(crt);
        return it->second;
    } else {
-        DHT_LOG.w("Certificate %s for node %s does not match node id !", h.toString().c_str(), node.toString().c_str());
+        if (logger_)
+            logger_->w("Certificate %s for node %s does not match node id !", h.toString().c_str(), node.toString().c_str());
        return nullptr;
    }
 }
@@ -172,7 +179,8 @@ SecureDht::findCertificate(const InfoHash& node, const std::function<void(const
 {
    Sp<crypto::Certificate> b = getCertificate(node);
    if (b && *b) {
-        DHT_LOG.d("Using certificate from cache for %s", node.toString().c_str());
+        if (logger_)
+            logger_->d("Using certificate from cache for %s", node.to_c_str());
        if (cb)
            cb(b);
        return;
@@ -180,7 +188,8 @@ SecureDht::findCertificate(const InfoHash& node, const std::function<void(const
    if (localQueryMethod_) {
        auto res = localQueryMethod_(node);
        if (not res.empty()) {
-            DHT_LOG.d("Registering certificate from local store for %s", node.toString().c_str());
+            if (logger_)
+                logger_->d("Registering certificate from local store for %s", node.to_c_str());
            nodesCertificates_.emplace(node, res.front());
            if (cb)
                cb(res.front());
@@ -195,7 +204,8 @@ SecureDht::findCertificate(const InfoHash& node, const std::function<void(const
        for (const auto& v : vals) {
            if (auto cert = registerCertificate(node, v->data)) {
                *found = true;
-                DHT_LOG.d("Found certificate for %s", node.toString().c_str());
+                if (logger_)
+                    logger_->d(node, "Found certificate for %s", node.to_c_str());
                if (cb)
                    cb(cert);
                return false;
@@ -213,7 +223,8 @@ SecureDht::findPublicKey(const InfoHash& node, const std::function<void(const Sp
 {
    auto pk = getPublicKey(node);
    if (pk && *pk) {
-        DHT_LOG.d("Found public key from cache for %s", node.toString().c_str());
+        if (logger_)
+            logger_->d(node, "Found public key from cache for %s", node.to_c_str());
        if (cb)
            cb(pk);
        return;
@@ -257,7 +268,8 @@ SecureDht::checkValue(const Sp<Value>& v)
            }
            // Ignore values belonging to other people
        } catch (const std::exception& e) {
-            DHT_LOG.w("Could not decrypt value %s : %s", v->toString().c_str(), e.what());
+            if (logger_)
+                logger_->w("Could not decrypt value %s : %s", v->toString().c_str(), e.what());
        }
    }
    // Check signed values
@@ -268,11 +280,12 @@ SecureDht::checkValue(const Sp<Value>& v)
        v->signatureChecked = true;
        if (v->owner and v->owner->checkSignature(v->getToSign(), v->signature)) {
            v->signatureValid = true;
-            nodesPubKeys_[v->owner->getId()] = v->owner;
+            if (enableCache_)
+                nodesPubKeys_[v->owner->getId()] = v->owner;
            return v;
        }
-        else
-            DHT_LOG.w("Signature verification failed for %s", v->toString().c_str());
+        else if (logger_)
+            logger_->w("Signature verification failed for %s", v->toString().c_str());
    }
    // Forward normal values
    else {
@@ -340,6 +353,11 @@ SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where w)
 void
 SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback, bool permanent)
 {
+    if (not key_)  {
+        if (callback)
+            callback(false, {});
+        return;
+    }
    if (val->id == Value::INVALID_ID) {
        crypto::random_device rdev;
        std::uniform_int_distribution<Value::Id> rand_id;
@@ -355,13 +373,16 @@ SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback,
    // Check if data already exists on the dht
    get(hash,
        [val,this] (const std::vector<Sp<Value>>& vals) {
-            DHT_LOG.d("Found online previous value being announced.");
+            if (logger_)
+                logger_->d("Found online previous value being announced.");
            for (const auto& v : vals) {
-                if (!v->isSigned())
-                    DHT_LOG.e("Existing non-signed value seems to exists at this location.");
-                else if (not v->owner or v->owner->getId() != getId())
-                    DHT_LOG.e("Existing signed value belonging to someone else seems to exists at this location.");
-                else if (val->seq <= v->seq)
+                if (!v->isSigned()) {
+                    if (logger_)
+                        logger_->e("Existing non-signed value seems to exists at this location.");
+                } else if (not v->owner or v->owner->getId() != getId()) {
+                    if (logger_)
+                        logger_->e("Existing signed value belonging to someone else seems to exists at this location.");
+                } else if (val->seq <= v->seq)
                    val->seq = v->seq + 1;
            }
            return true;
@@ -378,17 +399,24 @@ SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback,
 void
 SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, Sp<Value> val, DoneCallback callback, bool permanent)
 {
+    if (not key_)  {
+        if (callback)
+            callback(false, {});
+        return;
+    }
    findPublicKey(to, [=](const Sp<const crypto::PublicKey>& pk) {
        if(!pk || !*pk) {
            if (callback)
                callback(false, {});
            return;
        }
-        DHT_LOG.w("Encrypting data for PK: %s", pk->getId().toString().c_str());
+        if (logger_)
+            logger_->w("Encrypting data for PK: %s", pk->getId().toString().c_str());
        try {
            dht_->put(hash, encrypt(*val, *pk), callback, time_point::max(), permanent);
        } catch (const std::exception& e) {
-            DHT_LOG.e("Error putting encrypted data: %s", e.what());
+            if (logger_)
+                logger_->e("Error putting encrypted data: %s", e.what());
            if (callback)
                callback(false, {});
        }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -76,11 +76,8 @@ SockAddr::resolve(const std::string& host, const std::string& service)
    if(rc != 0)
        throw std::invalid_argument(std::string("Error: `") + host + ":" + service + "`: " + gai_strerror(rc));

-    addrinfo* infop = info;
-    while (infop) {
+    for (addrinfo* infop = info; infop; infop = infop->ai_next)
        ips.emplace_back(infop->ai_addr, infop->ai_addrlen);
-        infop = infop->ai_next;
-    }
    freeaddrinfo(info);
    return ips;
 }
@@ -236,17 +233,27 @@ bool operator==(const SockAddr& a, const SockAddr& b) {
 }

 time_point from_time_t(std::time_t t) {
-    return clock::now() + (std::chrono::system_clock::from_time_t(t) - std::chrono::system_clock::now());
+    auto dt = system_clock::from_time_t(t) - system_clock::now();
+    auto now = clock::now();
+    if (dt > system_clock::duration(0) and now > time_point::max() - dt)
+        return time_point::max();
+    else if (dt < system_clock::duration(0) and now < time_point::min() - dt)
+        return time_point::min();
+    return now + dt;
 }

 std::time_t to_time_t(time_point t) {
-    return std::chrono::system_clock::to_time_t(
-            std::chrono::system_clock::now() +
-            std::chrono::duration_cast<std::chrono::system_clock::duration>(t - clock::now()));
+    auto dt = t - clock::now();
+    auto now = system_clock::now();
+    if (dt > duration(0) and now >= system_clock::time_point::max() - dt)
+        return system_clock::to_time_t(system_clock::time_point::max());
+    else if (dt < duration(0) and now <= system_clock::time_point::min() - dt)
+        return system_clock::to_time_t(system_clock::time_point::min());
+    return system_clock::to_time_t(now + std::chrono::duration_cast<system_clock::duration>(dt));
 }

 Blob
-unpackBlob(msgpack::object& o) {
+unpackBlob(const msgpack::object& o) {
    switch (o.type) {
    case msgpack::type::BIN:
        return {o.via.bin.ptr, o.via.bin.ptr+o.via.bin.size};
@@ -270,11 +277,13 @@ unpackMsg(Blob b) {
 }

 msgpack::object*
-findMapValue(msgpack::object& map, const std::string& key) {
+findMapValue(const msgpack::object& map, const char* key, size_t key_length) {
    if (map.type != msgpack::type::MAP) throw msgpack::type_error();
    for (unsigned i = 0; i < map.via.map.size; i++) {
        auto& o = map.via.map.ptr[i];
-        if (o.key.type == msgpack::type::STR && o.key.as<std::string>() == key)
+        if (o.key.type == msgpack::type::STR
+            && key_length == o.key.via.str.size
+            && std::strncmp(o.key.via.str.ptr, key, o.key.via.str.size) == 0)
            return &o.val;
    }
    return nullptr;
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -22,12 +22,9 @@
 #endif

 #include "value.h"
-#include "default_types.h"
-#include "callbacks.h"

-#ifndef OPENDHT_LIGHT
+#include "default_types.h"
 #include "securedht.h" // print certificate ID
-#endif

 #ifdef OPENDHT_JSONCPP
 #include "base64.h"
@@ -45,50 +42,10 @@ Value::Filter bindFilterRaw(FilterRaw raw_filter, void* user_data) {
    };
 }

-#ifndef OPENDHT_LIGHT
-Value::Filter
-Value::OwnerFilter(const crypto::PublicKey& pk) {
-    return OwnerFilter(pk.getId());
-}
-
-
-Value::Filter
-Value::OwnerFilter(const InfoHash& pkh) {
-    return [pkh](const Value& v) {
-        return v.owner and v.owner->getId() == pkh;
-    };
-}
-
-void
-Value::sign(const crypto::PrivateKey& key) {
-    if (isEncrypted())
-        throw DhtException("Can't sign encrypted data.");
-    owner = std::make_shared<const crypto::PublicKey>(key.getPublicKey());
-    signature = key.sign(getToSign());
-}
-
-bool
-Value::checkSignature() const {
-    return isSigned() and owner->checkSignature(getToSign(), signature);
-}
-
-Value
-Value::encrypt(const crypto::PrivateKey& from, const crypto::PublicKey& to) {
-    if (isEncrypted())
-        throw DhtException("Data is already encrypted.");
-    setRecipient(to.getId());
-    sign(from);
-    Value nv {id};
-    nv.setCypher(to.encrypt(getToEncrypt()));
-    return nv;
-}
-
-#endif
-
 std::ostream& operator<< (std::ostream& s, const Value& v)
 {
    auto flags(s.flags());
-    s << "Value[id:" << std::hex << v.id << std::dec << " ";
+    s << "Value[id:" << std::hex << v.id << std::dec << ' ';
    if (v.isEncrypted())
        s << "encrypted ";
    else if (v.isSigned()) {
@@ -99,9 +56,7 @@ std::ostream& operator<< (std::ostream& s, const Value& v)
    if (not v.isEncrypted()) {
        if (v.type == IpServiceAnnouncement::TYPE.id) {
            s << IpServiceAnnouncement(v.data);
-        }
-#ifndef OPENDHT_LIGHT
-        else if (v.type == CERTIFICATE_TYPE.id) {
+        } else if (v.type == CERTIFICATE_TYPE.id) {
            s << "Certificate";
 #ifdef OPENDHT_LOG_CRT_ID
            try {
@@ -111,17 +66,26 @@ std::ostream& operator<< (std::ostream& s, const Value& v)
                s << " (invalid)";
            }
 #endif
-        }
-#endif
-        else {
-            s << "Data (type: " << v.type << " ): ";
-            s << std::hex;
-            for (auto i : v.data)
-                s << std::setfill('0') << std::setw(2) << (unsigned)i;
-            s << std::dec;
+        } else {
+            if (v.user_type.empty())
+                s << "data:";
+            else
+                s << "data(" << v.user_type << "):";
+            if (v.user_type == "text/plain") {
+                s << '"';
+                s.write((const char*)v.data.data(), v.data.size());
+                s << '"';
+            } else if (v.data.size() < 1024) {
+                s << std::hex;
+                for (auto i : v.data)
+                    s << std::setfill('0') << std::setw(2) << (unsigned)i;
+                s << std::dec;
+            } else {
+                s << v.data.size() << " bytes";
+            }
        }
    }
-    s << "]";
+    s << ']';
    s.flags(flags);
    return s;
 }
@@ -134,20 +98,6 @@ ValueType::DEFAULT_STORE_POLICY(InfoHash, const std::shared_ptr<Value>& v, const
    return v->size() <= MAX_VALUE_SIZE;
 }

-msgpack::object*
-findMapValue(const msgpack::object& map, const std::string& key) {
-    if (map.type != msgpack::type::MAP) throw msgpack::type_error();
-    for (unsigned i = 0; i < map.via.map.size; i++) {
-        auto& o = map.via.map.ptr[i];
-        if(o.key.type != msgpack::type::STR)
-            continue;
-        if (o.key.as<std::string>() == key) {
-            return &o.val;
-        }
-    }
-    return nullptr;
-}
-
 size_t
 Value::size() const
 {
@@ -155,20 +105,24 @@ Value::size() const
 }

 void
-Value::msgpack_unpack(msgpack::object o)
+Value::msgpack_unpack(const msgpack::object& o)
 {
-    if (o.type != msgpack::type::MAP) throw msgpack::type_error();
-    if (o.via.map.size < 2) throw msgpack::type_error();
+    if (o.type != msgpack::type::MAP or o.via.map.size < 2)
+        throw msgpack::type_error();

-    if (auto rid = findMapValue(o, "id")) {
+    if (auto rid = findMapValue(o, VALUE_KEY_ID)) {
        id = rid->as<Id>();
    } else
        throw msgpack::type_error();

-    if (auto rdat = findMapValue(o, "dat")) {
+    if (auto rdat = findMapValue(o, VALUE_KEY_DAT)) {
        msgpack_unpack_body(*rdat);
    } else
        throw msgpack::type_error();
+
+    if (auto rprio = findMapValue(o, VALUE_KEY_PRIO)) {
+        priority = rprio->as<unsigned>();
+    }
 }

 void
@@ -187,40 +141,37 @@ Value::msgpack_unpack_body(const msgpack::object& o)
    } else {
        if (o.type != msgpack::type::MAP)
            throw msgpack::type_error();
-        auto rbody = findMapValue(o, "body");
+        auto rbody = findMapValue(o, VALUE_KEY_BODY);
        if (not rbody)
            throw msgpack::type_error();

-        if (auto rdata = findMapValue(*rbody, "data")) {
+        if (auto rdata = findMapValue(*rbody, VALUE_KEY_DATA)) {
            data = unpackBlob(*rdata);
        } else
            throw msgpack::type_error();

-        if (auto rtype = findMapValue(*rbody, "type")) {
+        if (auto rtype = findMapValue(*rbody, VALUE_KEY_TYPE)) {
            type = rtype->as<ValueType::Id>();
        } else
            throw msgpack::type_error();

-        if (auto rutype = findMapValue(*rbody, "utype")) {
+        if (auto rutype = findMapValue(*rbody, VALUE_KEY_USERTYPE)) {
            user_type = rutype->as<std::string>();
        }

-        if (auto rowner = findMapValue(*rbody, "owner")) {
-            if (auto rseq = findMapValue(*rbody, "seq"))
+        if (auto rowner = findMapValue(*rbody, VALUE_KEY_OWNER)) {
+            if (auto rseq = findMapValue(*rbody, VALUE_KEY_SEQ))
                seq = rseq->as<decltype(seq)>();
            else
                throw msgpack::type_error();
-
-#ifndef OPENDHT_LIGHT
            crypto::PublicKey new_owner;
            new_owner.msgpack_unpack(*rowner);
            owner = std::make_shared<const crypto::PublicKey>(std::move(new_owner));
-#endif
-            if (auto rrecipient = findMapValue(*rbody, "to")) {
+            if (auto rrecipient = findMapValue(*rbody, VALUE_KEY_TO)) {
                recipient = rrecipient->as<InfoHash>();
            }

-            if (auto rsig = findMapValue(o, "sig")) {
+            if (auto rsig = findMapValue(o, VALUE_KEY_SIGNATURE)) {
                signature = unpackBlob(*rsig);
            } else
                throw msgpack::type_error();
@@ -229,65 +180,65 @@ Value::msgpack_unpack_body(const msgpack::object& o)
 }

 #ifdef OPENDHT_JSONCPP
-Value::Value(Json::Value& json)
+Value::Value(const Json::Value& json)
 {
-    id = Value::Id(unpackId(json, "id"));
-    if (json.isMember("cypher")) {
-        auto cypherStr = json["cypher"].asString();
-        cypherStr = base64_decode(cypherStr);
-        cypher = std::vector<unsigned char>(cypherStr.begin(), cypherStr.end());
-    }
-    if (json.isMember("sig")) {
-        auto sigStr = json["sig"].asString();
-        sigStr = base64_decode(sigStr);
-        signature = std::vector<unsigned char>(sigStr.begin(), sigStr.end());
-    }
-    if (json.isMember("seq"))
-        seq = json["seq"].asInt();
-#ifndef OPENDHT_LIGHT
-    if (json.isMember("owner")) {
-        auto ownerStr = json["owner"].asString();
+    id = Value::Id(unpackId(json, VALUE_KEY_ID));
+    const auto& jcypher = json["cypher"];
+    if (jcypher.isString())
+        cypher = base64_decode(jcypher.asString());
+    const auto& jsig = json[VALUE_KEY_SIGNATURE];
+    if (jsig.isString())
+        signature = base64_decode(jsig.asString());
+    const auto& jseq = json[VALUE_KEY_SEQ];
+    if (!jseq.isNull())
+        seq = jseq.asInt();
+    const auto& jowner = json[VALUE_KEY_OWNER];
+    if (jowner.isString()) {
+        auto ownerStr = jowner.asString();
        auto ownerBlob = std::vector<unsigned char>(ownerStr.begin(), ownerStr.end());
        owner = std::make_shared<const crypto::PublicKey>(ownerBlob);
    }
-#endif
-    if (json.isMember("to")) {
-        auto toStr = json["to"].asString();
-        recipient = InfoHash(toStr);
-    }
-    if (json.isMember("type"))
-        type = json["type"].asInt();
-    if (json.isMember("data")){
-        auto dataStr = json["data"].asString();
-        dataStr = base64_decode(dataStr);
-        data = std::vector<unsigned char>(dataStr.begin(), dataStr.end());
-    }
-    if (json.isMember("utype"))
-        user_type = json["utype"].asString();
+    const auto& jto = json[VALUE_KEY_TO];
+    if (jto.isString())
+        recipient = InfoHash(jto.asString());
+    const auto& jtype = json[VALUE_KEY_TYPE];
+    if (!jtype.isNull())
+        type = jtype.asInt();
+    const auto& jdata = json[VALUE_KEY_DATA];
+    if (jdata.isString())
+        data = base64_decode(jdata.asString());
+    const auto& jutype = json[VALUE_KEY_USERTYPE];
+    if (jutype.isString())
+        user_type = jutype.asString();
+    const auto& jprio = json["prio"];
+    if (jprio.isIntegral())
+        priority = jprio.asUInt();
 }

 Json::Value
 Value::toJson() const
 {
    Json::Value val;
-    val["id"] = std::to_string(id);
+    val[VALUE_KEY_ID] = std::to_string(id);
    if (isEncrypted()) {
        val["cypher"] = base64_encode(cypher);
    } else {
        if (isSigned())
-            val["sig"] = base64_encode(signature);
+            val[VALUE_KEY_SIGNATURE] = base64_encode(signature);
        bool has_owner = owner && *owner;
        if (has_owner) { // isSigned
-            val["seq"] = seq;
-            val["owner"] = owner->toString();
+            val[VALUE_KEY_SEQ] = seq;
+            val[VALUE_KEY_OWNER] = owner->toString();
            if (recipient)
-                val["to"] = recipient.toString();
+                val[VALUE_KEY_TO] = recipient.toString();
        }
-        val["type"] = type;
-        val["data"] = base64_encode(data);
+        val[VALUE_KEY_TYPE] = type;
+        val[VALUE_KEY_DATA] = base64_encode(data);
        if (not user_type.empty())
-            val["utype"] = user_type;
+            val[VALUE_KEY_USERTYPE] = user_type;
    }
+    if (priority)
+        val["prio"] = priority;
    return val;
 }

@@ -295,13 +246,11 @@ uint64_t
 unpackId(const Json::Value& json, const std::string& key) {
    uint64_t ret = 0;
    try {
-        if (json.isMember(key)) {
-            const auto& t = json[key];
-            if (t.isString()) {
-                ret = std::stoull(t.asString());
-            } else {
-                ret = t.asLargestUInt();
-            }
+        const auto& t = json[key];
+        if (t.isString()) {
+            ret = std::stoull(t.asString());
+        } else {
+            ret = t.asLargestUInt();
        }
    } catch (...) {}
    return ret;
@@ -314,18 +263,18 @@ FieldValue::operator==(const FieldValue& vfd) const
    if (field != vfd.field)
        return false;
    switch (field) {
-        case Value::Field::Id:
-        case Value::Field::ValueType:
-        case Value::Field::SeqNum:
-            return intValue == vfd.intValue;
-        case Value::Field::OwnerPk:
-            return hashValue == vfd.hashValue;
-        case Value::Field::UserType:
-            return blobValue == vfd.blobValue;
-        case Value::Field::None:
-            return true;
-        default:
-            return false;
+    case Value::Field::Id:
+    case Value::Field::ValueType:
+    case Value::Field::SeqNum:
+        return intValue == vfd.intValue;
+    case Value::Field::OwnerPk:
+        return hashValue == vfd.hashValue;
+    case Value::Field::UserType:
+        return blobValue == vfd.blobValue;
+    case Value::Field::None:
+        return true;
+    default:
+        return false;
    }
 }

@@ -333,25 +282,25 @@ Value::Filter
 FieldValue::getLocalFilter() const
 {
    switch (field) {
-        case Value::Field::Id:
-            return Value::IdFilter(intValue);
-        case Value::Field::ValueType:
-            return Value::TypeFilter(intValue);
-        case Value::Field::OwnerPk:
-            return Value::OwnerFilter(hashValue);
-        case Value::Field::SeqNum:
-            return Value::SeqNumFilter(intValue);
-        case Value::Field::UserType:
-            return Value::UserTypeFilter(std::string {blobValue.begin(), blobValue.end()});
-        default:
-            return {};
+    case Value::Field::Id:
+        return Value::IdFilter(intValue);
+    case Value::Field::ValueType:
+        return Value::TypeFilter(intValue);
+    case Value::Field::OwnerPk:
+        return Value::OwnerFilter(hashValue);
+    case Value::Field::SeqNum:
+        return Value::SeqNumFilter(intValue);
+    case Value::Field::UserType:
+        return Value::UserTypeFilter(std::string {blobValue.begin(), blobValue.end()});
+    default:
+        return {};
    }
 }

 FieldValueIndex::FieldValueIndex(const Value& v, const Select& s)
 {
-    auto selection = s.getSelection();
-    if (not selection.empty()) {
+    if (not s.empty()) {
+        auto selection = s.getSelection();
        std::transform(selection.begin(), selection.end(), std::inserter(index, index.end()),
            [](const std::set<Value::Field>::value_type& f) {
                return std::make_pair(f, FieldValue {});
@@ -364,25 +313,23 @@ FieldValueIndex::FieldValueIndex(const Value& v, const Select& s)
    for (const auto& fvp : index) {
        const auto& f = fvp.first;
        switch (f) {
-            case Value::Field::Id:
-                index[f] = {f, v.id};
-                break;
-            case Value::Field::ValueType:
-                index[f] = {f, v.type};
-                break;
-#ifndef OPENDHT_LIGHT
-            case Value::Field::OwnerPk:
-                index[f] = {f, v.owner ? v.owner->getId() : InfoHash() };
-                break;
-#endif
-            case Value::Field::SeqNum:
-                index[f] = {f, v.seq};
-                break;
-            case Value::Field::UserType:
-                index[f] = {f, Blob {v.user_type.begin(), v.user_type.end()}};
-                break;
-            default:
-                break;
+        case Value::Field::Id:
+            index[f] = {f, v.id};
+            break;
+        case Value::Field::ValueType:
+            index[f] = {f, v.type};
+            break;
+        case Value::Field::OwnerPk:
+            index[f] = {f, v.owner ? v.owner->getId() : InfoHash() };
+            break;
+        case Value::Field::SeqNum:
+            index[f] = {f, v.seq};
+            break;
+        case Value::Field::UserType:
+            index[f] = {f, Blob {v.user_type.begin(), v.user_type.end()}};
+            break;
+        default:
+            break;
        }
    }
 }
@@ -402,28 +349,28 @@ std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) {
    os << "Index[";
    for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) {
        switch (v->first) {
-            case Value::Field::Id: {
-                auto flags(os.flags());
-                os << "Id:" << std::hex << v->second.getInt();
-                os.flags(flags);
-                break;
-            }
-            case Value::Field::ValueType:
-                os << "ValueType:" << v->second.getInt();
-                break;
-            case Value::Field::OwnerPk:
-                os << "Owner:" << v->second.getHash().toString();
-                break;
-            case Value::Field::SeqNum:
-                os << "Seq:" << v->second.getInt();
-                break;
-            case Value::Field::UserType: {
-                auto ut = v->second.getBlob();
-                os << "UserType:" << std::string(ut.begin(), ut.end());
-                break;
-            }
-            default:
-                break;
+        case Value::Field::Id: {
+            auto flags(os.flags());
+            os << "Id:" << std::hex << v->second.getInt();
+            os.flags(flags);
+            break;
+        }
+        case Value::Field::ValueType:
+            os << "ValueType:" << v->second.getInt();
+            break;
+        case Value::Field::OwnerPk:
+            os << "Owner:" << v->second.getHash();
+            break;
+        case Value::Field::SeqNum:
+            os << "Seq:" << v->second.getInt();
+            break;
+        case Value::Field::UserType: {
+            auto ut = v->second.getBlob();
+            os << "UserType:" << std::string(ut.begin(), ut.end());
+            break;
+        }
+        default:
+            break;
        }
        os << (std::next(v) != fvi.index.end() ? "," : "");
    }
@@ -439,19 +386,19 @@ FieldValueIndex::msgpack_unpack_fields(const std::set<Value::Field>& fields, con
    for (const auto& field : fields) {
        auto& field_value = o.via.array.ptr[offset+(j++)];
        switch (field) {
-            case Value::Field::Id:
-            case Value::Field::ValueType:
-            case Value::Field::SeqNum:
-                index[field] = FieldValue(field, field_value.as<uint64_t>());
-                break;
-            case Value::Field::OwnerPk:
-                index[field] = FieldValue(field, field_value.as<InfoHash>());
-                break;
-            case Value::Field::UserType:
-                index[field] = FieldValue(field, field_value.as<Blob>());
-                break;
-            default:
-                throw msgpack::type_error();
+        case Value::Field::Id:
+        case Value::Field::ValueType:
+        case Value::Field::SeqNum:
+            index[field] = FieldValue(field, field_value.as<uint64_t>());
+            break;
+        case Value::Field::OwnerPk:
+            index[field] = FieldValue(field, field_value.as<InfoHash>());
+            break;
+        case Value::Field::UserType:
+            index[field] = FieldValue(field, field_value.as<Blob>());
+            break;
+        default:
+            throw msgpack::type_error();
        }
    }
 }
@@ -473,13 +420,13 @@ Select::Select(const std::string& q_str) {

        while (std::getline(fields, token, ',')) {
            trim_str(token);
-            if (token == "id")
+            if (token == VALUE_KEY_ID)
                field(Value::Field::Id);
            else if (token == "value_type")
                field(Value::Field::ValueType);
            else if (token == "owner_pk")
                field(Value::Field::OwnerPk);
-            if (token == "seq")
+            if (token == VALUE_KEY_SEQ)
                field(Value::Field::SeqNum);
            else if (token == "user_type")
                field(Value::Field::UserType);
@@ -515,13 +462,13 @@ Where::Where(const std::string& q_str) {
                    s = value_str.substr(1, value_str.size()-2);
                else
                    s = value_str;
-                if (field_str == "id")
+                if (field_str == VALUE_KEY_ID)
                    id(v);
                else if (field_str == "value_type")
                    valueType(v);
                else if (field_str == "owner_pk")
                    owner(InfoHash(s));
-                else if (field_str == "seq")
+                else if (field_str == VALUE_KEY_SEQ)
                    seq(v);
                else if (field_str == "user_type")
                    userType(s);
@@ -577,11 +524,14 @@ bool Query::isSatisfiedBy(const Query& q) const {
 }

 std::ostream& operator<<(std::ostream& s, const dht::Select& select) {
-    s << "SELECT " << (select.fieldSelection_.empty() ? "*" : "");
-    for (auto fs = select.fieldSelection_.begin() ; fs != select.fieldSelection_.end() ; ++fs) {
-        switch (*fs) {
+    s << "SELECT ";
+    if (select.fieldSelection_.empty())
+        s << '*';
+    else
+        for (auto fs = select.fieldSelection_.begin(); fs != select.fieldSelection_.end();) {
+            switch (*fs) {
            case Value::Field::Id:
-                s << "id";
+                s << VALUE_KEY_ID;
                break;
            case Value::Field::ValueType:
                s << "value_type";
@@ -593,13 +543,14 @@ std::ostream& operator<<(std::ostream& s, const dht::Select& select) {
                s << "owner_public_key";
                break;
            case Value::Field::SeqNum:
-                s << "seq";
+                s << VALUE_KEY_SEQ;
                break;
            default:
                break;
+            }
+            if (++fs != select.fieldSelection_.end())
+                s << ',';
        }
-        s << (std::next(fs) != select.fieldSelection_.end() ? "," : "");
-    }
    return s;
 }

@@ -608,25 +559,25 @@ std::ostream& operator<<(std::ostream& s, const dht::Where& where) {
        s << "WHERE ";
        for (auto f = where.filters_.begin() ; f != where.filters_.end() ; ++f) {
            switch (f->getField()) {
-                case Value::Field::Id:
-                    s << "id=" << f->getInt();
-                    break;
-                case Value::Field::ValueType:
-                    s << "value_type=" << f->getInt();
-                    break;
-                case Value::Field::OwnerPk:
-                    s << "owner_pk_hash=" << f->getHash().toString();
-                    break;
-                case Value::Field::SeqNum:
-                    s << "seq=" << f->getInt();
-                    break;
-                case Value::Field::UserType: {
-                    auto b = f->getBlob();
-                    s << "user_type=" << std::string {b.begin(), b.end()};
-                    break;
-                }
-                default:
-                    break;
+            case Value::Field::Id:
+                s << VALUE_KEY_ID << '=' << f->getInt();
+                break;
+            case Value::Field::ValueType:
+                s << "value_type=" << f->getInt();
+                break;
+            case Value::Field::OwnerPk:
+                s << "owner_pk_hash=" << f->getHash();
+                break;
+            case Value::Field::SeqNum:
+                s << VALUE_KEY_SEQ << '=' << f->getInt();
+                break;
+            case Value::Field::UserType: {
+                auto b = f->getBlob();
+                s << "user_type=" << std::string {b.begin(), b.end()};
+                break;
+            }
+            default:
+                break;
            }
            s << (std::next(f) != where.filters_.end() ? "," : "");
        }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *          Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
@@ -34,24 +34,25 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DhtProxyTester);

 void
 DhtProxyTester::setUp() {
-    nodePeer.run(0);
+    clientConfig.dht_config.node_config.max_peer_req_per_sec = -1;
+    clientConfig.dht_config.node_config.max_req_per_sec = -1;
+
+    nodePeer.run(0, clientConfig);

    nodeProxy = std::make_shared<dht::DhtRunner>();
-    nodeProxy->run(0);
+    nodeProxy->run(0, clientConfig);
    nodeProxy->bootstrap(nodePeer.getBound());

    auto serverCAIdentity = dht::crypto::generateEcIdentity("DHT Node CA");
-    auto serverIdentity = dht::crypto::generateIdentity("DHT Node", serverCAIdentity);

-    serverProxy = std::make_unique<dht::DhtProxyServer>(
-            //dht::crypto::Identity{}, // http
-            serverIdentity,            // https
-            nodeProxy, 8080, /*pushServer*/"127.0.0.1:8090");
+    dht::ProxyServerConfig serverConfig;
+    serverConfig.identity = dht::crypto::generateIdentity("DHT Node", serverCAIdentity);
+    serverConfig.port = 8080;
+    serverConfig.pushServer = "127.0.0.1:8090";
+    serverProxy = std::make_unique<dht::DhtProxyServer>(nodeProxy, serverConfig);

    clientConfig.server_ca = serverCAIdentity.second;
    clientConfig.client_identity = dht::crypto::generateIdentity("DhtProxyTester");
-    clientConfig.dht_config.node_config.maintain_storage = false;
-    clientConfig.threaded = true;
    clientConfig.push_node_id = "dhtnode";
    clientConfig.proxy_server = "https://127.0.0.1:8080";
 }
@@ -280,7 +281,7 @@ DhtProxyTester::testFuzzy()
        },[&](bool ok){
            CPPUNIT_ASSERT(ok);
        });
-        std::this_thread::sleep_for(10ms);
+        std::this_thread::sleep_for(5ms);
    }

    // Assert
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -29,8 +29,12 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DhtRunnerTester);

 void
 DhtRunnerTester::setUp() {
-    node1.run(42222);
-    node2.run(42232);
+    dht::DhtRunner::Config config;
+    config.dht_config.node_config.max_peer_req_per_sec = -1;
+    config.dht_config.node_config.max_req_per_sec = -1;
+
+    node1.run(42222, config);
+    node2.run(42232, config);
    node2.bootstrap(node1.getBound());
 }

@@ -84,7 +88,9 @@ DhtRunnerTester::testListen() {
    auto a = dht::InfoHash::get("234");
    auto b = dht::InfoHash::get("2345");
    auto c = dht::InfoHash::get("23456");
-    constexpr unsigned N = 32;
+    auto d = dht::InfoHash::get("234567");
+    constexpr unsigned N = 256;
+    constexpr unsigned SZ = 56 * 1024;

    auto ftokena = node1.listen(a, [&](const std::shared_ptr<dht::Value>&) {
        valueCount++;
@@ -101,6 +107,16 @@ DhtRunnerTester::testListen() {
        return true;
    });

+    auto ftokend = node1.listen(d, [&](const std::shared_ptr<dht::Value>&) {
+        valueCount++;
+        return true;
+    });
+
+    std::vector<uint8_t> mtu;
+    mtu.reserve(SZ);
+    for (size_t i = 0; i < SZ; i++)
+        mtu.emplace_back((i % 2) ? 'T' : 'M');
+
    for (unsigned i=0; i<N; i++) {
        node2.put(a, dht::Value("v1"), [&](bool ok) {
            std::lock_guard<std::mutex> lock(mutex);
@@ -114,29 +130,88 @@ DhtRunnerTester::testListen() {
            if (ok) putOkCount++;
            cv.notify_all();
        });
+        auto bigVal = std::make_shared<dht::Value>();
+        bigVal->data = mtu;
+        node2.put(c, bigVal, [&](bool ok) {
+            std::lock_guard<std::mutex> lock(mutex);
+            putCount++;
+            if (ok) putOkCount++;
+            cv.notify_all();
+        });
    }

    {
        std::unique_lock<std::mutex> lk(mutex);
-        CPPUNIT_ASSERT(cv.wait_for(lk, 30s, [&]{ return putCount == N * 2u; }));
-        CPPUNIT_ASSERT_EQUAL(N * 2u, putOkCount);
+        CPPUNIT_ASSERT(cv.wait_for(lk, 30s, [&]{ return putCount == N * 3u; }));
+        CPPUNIT_ASSERT_EQUAL(N * 3u, putOkCount);
    }

    CPPUNIT_ASSERT(ftokena.valid());
    CPPUNIT_ASSERT(ftokenb.valid());
    CPPUNIT_ASSERT(ftokenc.valid());
+    CPPUNIT_ASSERT(ftokend.valid());

    auto tokena = ftokena.get();
    auto tokenc = ftokenc.get();
+    auto tokend = ftokend.get();
    // tokenb might be 0 since the callback returns false.

    CPPUNIT_ASSERT(tokena);
    CPPUNIT_ASSERT(tokenc);
-    CPPUNIT_ASSERT_EQUAL(N + 1u, valueCount.load());
+    CPPUNIT_ASSERT(tokend);
+    CPPUNIT_ASSERT_EQUAL(N * 2u + 1u, valueCount.load());

    node1.cancelListen(a, tokena);
    node1.cancelListen(b, std::move(ftokenb));
    node1.cancelListen(c, tokenc);
+    node1.cancelListen(d, tokend);
+}
+
+void
+DhtRunnerTester::testListenLotOfBytes() {
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::atomic_uint valueCount(0);
+    unsigned putCount(0);
+    unsigned putOkCount(0);
+
+    std::string data(10000, 'a');
+
+    auto foo = dht::InfoHash::get("foo");
+    constexpr unsigned N = 50;
+
+    for (unsigned i=0; i<N; i++) {
+        node2.put(foo, data, [&](bool ok) {
+            std::lock_guard<std::mutex> lock(mutex);
+            putCount++;
+            if (ok) putOkCount++;
+            cv.notify_all();
+        });
+    }
+    {
+        std::unique_lock<std::mutex> lk(mutex);
+        CPPUNIT_ASSERT(cv.wait_for(lk, 30s, [&]{ return putCount == N; }));
+    }
+
+    dht::DhtRunner node3 {};
+    dht::DhtRunner::Config config;
+    config.dht_config.node_config.max_peer_req_per_sec = -1;
+    config.dht_config.node_config.max_req_per_sec = -1;
+    node3.run(42242, config);
+    node3.bootstrap(node1.getBound());
+
+    auto ftokenfoo = node3.listen(foo, [&](const std::shared_ptr<dht::Value>&) {
+        valueCount++;
+        cv.notify_all();
+        return true;
+    });
+
+    {
+        std::unique_lock<std::mutex> lk(mutex);
+        CPPUNIT_ASSERT(cv.wait_for(lk, 30s, [&]{ return valueCount == N; }));
+    }
+
+    node3.cancelListen(foo, ftokenfoo.get());
 }

 }  // namespace test
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
@@ -32,6 +32,7 @@ class DhtRunnerTester : public CppUnit::TestFixture {
    CPPUNIT_TEST(testConstructors);
    CPPUNIT_TEST(testGetPut);
    CPPUNIT_TEST(testListen);
+    CPPUNIT_TEST(testListenLotOfBytes);
    CPPUNIT_TEST_SUITE_END();

    dht::DhtRunner node1 {};
@@ -57,6 +58,10 @@ class DhtRunnerTester : public CppUnit::TestFixture {
     * Test listen method
     */
    void testListen();
+    /**
+     * Test listen method with lot of datas
+     */
+    void testListenLotOfBytes();
 };

 }  // namespace test
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *
@@ -39,9 +39,10 @@ HttpTester::setUp() {
    nodeProxy->run(0, /*identity*/{}, /*threaded*/true);
    nodeProxy->bootstrap(nodePeer->getBound());

-    serverProxy = std::unique_ptr<dht::DhtProxyServer>(
-        new dht::DhtProxyServer(
-            /*http*/dht::crypto::Identity{}, nodeProxy, 8080, /*pushServer*/"127.0.0.1:8090"));
+    dht::ProxyServerConfig config;
+    config.port = 8080;
+    config.pushServer = "127.0.0.1:8090";
+    serverProxy = std::make_unique<dht::DhtProxyServer>(nodeProxy, config);
 }

 void
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
 *
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2020 Savoir-faire Linux Inc.
 *
 *  Author: Mingrui Zhang <mingrui.zhang@savoirfairelinux.com>
 *
Author	SHA1	Message	Date
Adrien Béraud	08c7356481	dht: move socket to SearchNode	2020-06-12 02:51:08 -04:00
Adrien Béraud	e35f551c38	dht: add error handler for sendRefreshValue	2020-06-12 02:51:08 -04:00
Adrien Béraud	bd8c9d05d8	network engine: complete request in case of error	2020-06-12 02:51:08 -04:00
Adrien Béraud	6158751a32	network engine: auth success in case of refresh	2020-06-12 02:51:08 -04:00
Adrien Béraud	4c7d126782	network engine: fix sendError with non-default netId	2020-06-11 15:10:48 -04:00
Sébastien Blin	3e24ea1212	rng.h: fix build	2020-05-25 18:50:35 -04:00
Adrien Béraud	a7ac979da6	update to version 2.1.3	2020-05-25 15:18:51 -04:00
Adrien Béraud	87f48f0a28	getSeededRandomEngine: retry in case of exception	2020-05-25 15:18:51 -04:00
Adrien Béraud	15586a3cb1	travis: update configuration	2020-05-24 14:24:30 -04:00
Adrien Béraud	2195c0ed3e	proxy server: allow build without push notification support	2020-05-24 14:11:13 -04:00
Adrien Béraud	af1e904895	dockerfile: add pkg-config	2020-05-24 11:52:56 -04:00
Adrien Béraud	a4fbf7ced0	docker: update	2020-05-24 02:04:45 -04:00
Adrien Béraud	1b7e0679d5	proxy server: check push token when loading state	2020-05-24 02:04:45 -04:00
Sébastien Blin	9292c1116c	proxy: count permanent put values by hash in stats	2020-05-24 01:25:31 -04:00
Adrien Béraud	a2afb5bb31	network engine: send confirmation for value update	2020-05-24 00:20:27 -04:00
Adrien Béraud	15fbfdebcc	dht: only load id if set in configuration	2020-05-22 12:28:25 -04:00
Adrien Béraud	15eb221cd1	dht: save id, port when saving state	2020-05-22 12:15:10 -04:00
Adrien Béraud	7d3781acc8	dht: use pseudo-rng for ID generation	2020-05-22 12:14:47 -04:00
Sébastien Blin	f3100a6752	proxyclient: avoid to generate tokens with \0	2020-05-20 12:23:59 -04:00
Adrien Béraud	e3cd2a9d02	proxy server: add missing listener type	2020-05-19 19:52:51 -04:00
Adrien Béraud	19ccc3a327	proxy server: restart timers on resubscribe	2020-05-19 19:50:20 -04:00
Adrien Béraud	4587c0925f	utils: always check length in findMapValue	2020-05-19 19:50:20 -04:00
Adrien Béraud	fc96245891	cmake: cleanup	2020-05-19 19:50:20 -04:00
Adrien Béraud	a27815b4a9	proxy server: persist ops on disk	2020-05-19 19:50:20 -04:00
Adrien Béraud	5daaf372b4	proxyserver: generate id before put	2020-05-11 12:31:07 -04:00
Sébastien Blin	cc51d2684c	proxy: only keep last total stat The rest is not needed.	2020-05-05 17:00:07 -04:00
Sébastien Blin	28b5893239	listener: fully migrate to requests This patch migrates tellListenerRefreshed and tellListenerExpired to the new behavior	2020-05-05 10:38:03 -04:00
Adrien Béraud	c754b5cfce	update to version 2.1.0	2020-04-30 14:58:25 -04:00
Adrien Béraud	b0c1249710	network engine: check for logger	2020-04-29 20:23:19 -04:00
Adrien Béraud	c41ee417fb	tools: cleanup	2020-04-29 19:22:24 -04:00
Adrien Béraud	47ba0415ca	tools: use bootstrap split/default port from dht	2020-04-29 17:12:11 -04:00
Adrien Béraud	99857e25cc	dhtnode: use async version of getNodeInfo	2020-04-29 16:13:36 -04:00
Adrien Béraud	7f832ba9d5	NodeInfo: add bound ports	2020-04-29 16:00:02 -04:00
Sébastien Blin	8c4a894f93	misc: replace TransId to Tid	2020-04-28 13:55:29 -04:00
Sébastien Blin	dd3de412fb	dht: move updateValues logic into network_engine	2020-04-28 13:55:29 -04:00
Sébastien Blin	4ad5e6c9ba	misc: clean	2020-04-28 13:55:29 -04:00
Sébastien Blin	b445818521	tests: enable test for listening with lot of values	2020-04-28 13:55:29 -04:00
Sébastien Blin	7c14ea1730	network_engine: fix pack for sendUpdateValues	2020-04-28 13:55:29 -04:00
Sébastien Blin	7bf9f861a5	dht: send update values for listeners	2020-04-28 13:55:29 -04:00
Sébastien Blin	4cf5c87761	network_engin: move to pack(socket)	2020-04-28 13:55:29 -04:00
Sébastien Blin	e1c6fec581	dht: starts to add update and version support	2020-04-28 13:55:29 -04:00
Sébastien Blin	31da5b91fa	[WIP]net: add updateValues to replace Listen operations Actually, packets are not retransmitted for listen operations. This leads to some problems when they are already several values present at one hash. Ideally, packets should be transmitted inside a request to avoid packet loss and get all datas. This patch breaks the compatibility with previous versions	2020-04-28 13:55:29 -04:00
Adrien Béraud	01bcb1a750	network engine: completely remove TransId	2020-04-24 18:46:46 -04:00
Adrien Béraud	35348bf77f	dht: use default port for bootstraping	2020-04-24 17:45:40 -04:00
Adrien Béraud	ae58ea8312	search: expire only after candidate expiration	2020-04-24 16:32:07 -04:00
Adrien Béraud	f30d21e96f	move bootstrap logic from DhtRunner to Dht	2020-04-24 16:32:07 -04:00
Vladimir Stoiakin	7628aab230	opendht.pc: change -lpthread to -pthread	2020-04-19 14:13:56 -04:00
Adrien Béraud	f4cba56b3c	dht: optimize getStatus()	2020-04-17 02:09:57 -04:00
Adrien Béraud	7155dc4fb7	stats: add search, cached node count	2020-04-16 02:08:00 -04:00
Adrien Béraud	c961e2e99f	dht: allow to configure max searches & store size	2020-04-16 02:08:00 -04:00
Adrien Béraud	2fb8e1c7ee	Update README.md	2020-04-12 15:56:23 -04:00
Vladimir Stoiakin	4d689ef51a	update opendht.pc	2020-04-10 09:24:31 -04:00
Adrien Béraud	face9f12b2	request: cleanup	2020-03-30 18:39:28 -04:00
Adrien Béraud	c92f553a24	network engine: retransmit value parts	2020-03-30 17:54:50 -04:00
Adrien Béraud	8564cf3b86	tests/dhtrunner: test put/listen for large values	2020-03-30 17:54:47 -04:00
Adrien Béraud	e7c810d5d6	value: don't print large values	2020-03-30 17:54:42 -04:00
Adrien Béraud	e76c80b8be	tests: disable rate limiting	2020-03-30 17:54:34 -04:00
Adrien Béraud	69f8969883	utils: fix build in 32 bit	2020-03-25 23:39:54 -04:00
Adrien Béraud	bc200cd76d	utils: fix indentation	2020-03-25 23:08:39 -04:00
Adrien Béraud	187c7dbce3	http: preserve headers during redirect	2020-03-25 23:00:16 -04:00
Adrien Béraud	81131f1c4e	utils: check time bounds	2020-03-25 23:00:16 -04:00
Adrien Béraud	ce9445a93f	pseudo-rng: improve fallback seed	2020-03-25 23:00:16 -04:00
Adrien Béraud	16372cba0c	Update README.md	2020-03-21 12:45:11 -04:00
Adrien Béraud	a2cbef315a	dht: avoid crash in persistent put when logging	2020-02-22 19:59:36 -05:00
Adrien Béraud	bc7c43dfd1	value: cleanup	2020-02-22 19:59:36 -05:00
Adrien Béraud	771b300622	durl: cleanup, fix on macOS	2020-02-22 19:59:36 -05:00
Adrien Béraud	5ffb5c628d	dht: avoid use-after-free when sending probe query	2020-02-14 13:52:55 -05:00
Adrien Béraud	cfa4ce61b4	cmake: add option to build http without proxy	2020-02-13 16:55:59 -05:00
Adrien Béraud	5f15043f03	tools: add durl for http client testing	2020-02-12 16:46:06 -05:00
Adrien Béraud	f445b2536d	http: handle redirects	2020-02-12 16:45:25 -05:00
Adrien Béraud	6c2f791c4d	network utils: don't block on mutex in rx thread (2)	2020-02-11 17:07:53 -05:00
Adrien Béraud	780726c66d	network utils: don't block on mutex in rx thread	2020-02-11 16:20:30 -05:00
Adrien Béraud	8703cfdaee	dht: erase announce form search nodes on cancelPut	2020-02-09 15:15:47 -05:00
Adrien Béraud	1c536cbf5e	permanent put: set clientId	2020-02-09 10:34:42 -05:00
Adrien Béraud	a39e2443a0	dht: reuse probe query, avoid leak	2020-02-09 00:51:58 -05:00
Adrien Béraud	57528e7aab	scheduler: don't sync time on run	2020-02-08 22:04:50 -05:00
Adrien Béraud	e9d70f204a	tools: add perftest	2020-02-08 22:04:41 -05:00
Adrien Béraud	ceb35c05c1	securedht: add option to disable cache	2020-02-08 20:27:44 -05:00
Adrien Béraud	55ccec5871	securedht: putEncrypted, putSigned fail if no key	2020-02-07 00:54:18 -05:00
Adrien Béraud	95028e3770	dht: always use pseudo-RNG for non-critical random numbers	2020-02-07 00:36:42 -05:00
Adrien Béraud	abede1e72a	pingpong: print execution time	2020-02-07 00:21:13 -05:00
Adrien Béraud	859d8dc9cf	proxy server: check clientId for merging put	2020-02-07 00:21:13 -05:00
Sébastien Blin	1a67140463	proxyserver: enable keep alive packets	2020-02-06 14:15:47 -05:00
Adrien Béraud	9ae3d3c4d4	proxy: move info if possible	2020-02-05 14:25:03 -05:00
Adrien Béraud	d3508c80f5	dhtrunner: take bound address by reference	2020-02-04 16:21:24 -05:00
Adrien Béraud	6c43efa4da	http: use proper target in requests	2020-02-04 16:21:24 -05:00
Sébastien Blin	c5c4c217dd	tools: add script for following dhtproxy's stats	2020-01-28 16:27:20 -05:00
Sébastien Blin	5cfcf4861d	proxyserver: set permanent put pushToken	2020-01-28 16:27:20 -05:00
Adrien Béraud	8597267c63	dht: add 'public stable' node configuration	2020-01-27 16:27:21 -05:00
Adrien Béraud	1b81d707f3	value: fix potential crash in contentEquals	2020-01-27 12:13:17 -05:00
Adrien Béraud	7791163d49	proxy: receive values in blocks	2020-01-22 16:02:26 -05:00
Adrien Béraud	56db1bba19	proxy server: merge identical put requests	2020-01-22 15:56:48 -05:00
Adrien Béraud	a7092c2dc7	dhtnode: show value id in put response	2020-01-22 13:15:37 -05:00
Adrien Béraud	fc95c85983	proxy server: send values in batches	2020-01-22 13:15:37 -05:00
Adrien Béraud	3490b19fe3	proxy client: cleanup	2020-01-22 13:15:37 -05:00
Adrien Béraud	4a20814d59	proxy client: let the server choose the value id	2020-01-22 13:15:37 -05:00
Adrien Béraud	707fb39975	net: fix sending query in sendGetValues	2020-01-20 12:15:33 -05:00
Adrien Béraud	cbb2406a12	utils: improve duration display	2020-01-19 17:47:54 -05:00
Adrien Béraud	7764620460	proxy client: fix warning	2020-01-19 14:32:57 -05:00
Adrien Béraud	5af0f0c3d0	proxy server: flush response after all values are sent	2020-01-19 14:32:57 -05:00
Adrien Béraud	ec8beec427	proxy server: update stats on startup	2020-01-19 14:32:57 -05:00
Adrien Béraud	7b8ebbc54d	proxy server: catch more exceptions, cleanup	2020-01-19 14:32:57 -05:00
Adrien Béraud	dc46c2274c	net: exponential backoff, randomize packet expiration time	2020-01-19 14:32:57 -05:00
Adrien Béraud	052ed1f58c	proxy server: handlePrintStats on startup	2020-01-18 22:20:01 -05:00
Adrien Béraud	9096f650f5	dht: increase default max request rate to 8k req/s	2020-01-18 19:54:54 -05:00
Adrien Béraud	900655b11f	dht: give current time through periodic	2020-01-18 13:58:22 -05:00
Adrien Béraud	2900b1d17f	net: limit recycled packet queue max size	2020-01-18 13:56:50 -05:00
Adrien Béraud	69e71dbeaa	SecureDht: cleanup	2020-01-17 15:58:22 -05:00
Adrien Béraud	3793423b2f	dht: log cleanup	2020-01-17 15:57:32 -05:00
Adrien Béraud	b012878488	dhtrunner: set logger in setLogger	2020-01-17 15:57:32 -05:00
Adrien Béraud	c1005fa334	request: free message memory on clear	2020-01-17 15:17:35 -05:00
Adrien Béraud	0f9f623b31	dht: use unique_ptr for SearchNode	2020-01-17 12:47:14 -05:00
Adrien Béraud	2cfebdf9fa	net: use packet lists and recycle old packet structures	2020-01-17 12:26:03 -05:00
Adrien Béraud	481b1cb3b4	dht: use pseudo-random generator for value ids	2020-01-16 13:51:51 -05:00
Adrien Béraud	7f1d960a56	dhtrunner: add async/non-blocking version of getNodeInfo	2020-01-16 13:24:37 -05:00
Adrien Béraud	91e1e942ab	Update peer_discovery.cpp	2020-01-15 23:16:36 -05:00
Adrien Béraud	ae89365f23	Update peer_discovery.cpp	2020-01-15 23:16:36 -05:00
Adrien Béraud	93cd8d541c	Update peer_discovery.cpp	2020-01-15 23:16:36 -05:00
Paymon	9487bc5e42	implement connectivity change in peer discovery this makes peer discovery pub/sub on connectivity change, it aims to make peer discovery more energy efficient. also, stop echo'ing. TODO: in its current implementation however, peer_discovery operates in the dark; it doesn't know routes available to it (v4 vs v6); nor does it know of the state of the link(s) (UP vs. DOWN) and therefore it merely tries to, on connectivity change, (re)pub/sub on previous ip version routes. IDEA: open an interface to kernel so that peer discovery, and perhaps others, can learn about the routes available to them. don't leave add query() and direct publish(endpoint) check if peer discovery was request (run and compile time) add try/catch routine to DomainPeerDiscovery's constructor	2020-01-15 23:16:36 -05:00
Adrien Béraud	10d48137a9	proxy server: update stats on node info	2020-01-15 17:04:12 -05:00
Adrien Béraud	5b15fdeeef	proxy server: use shared_ptr for stats, remove statsMutex_	2020-01-15 13:54:24 -05:00
Adrien Béraud	3ba43c80a2	http: add weak_ptr to Request in Response	2020-01-15 11:45:26 -05:00
Adrien Béraud	16333f4224	http: add mutex to Request, Connection	2020-01-15 11:45:03 -05:00
Adrien Béraud	0864ac9a39	proxy server: handle put session id provided after first request	2020-01-10 12:26:57 -05:00
Adrien Béraud	616d6fe4b8	update copyright date	2020-01-03 15:14:32 -05:00
Adrien Béraud	dea4b202bf	proxy server: make json type explicit	2020-01-03 15:10:22 -05:00
Adrien Béraud	dc2f354652	proxy: use timestamps to handle reordered notifications	2020-01-03 14:47:22 -05:00
Sébastien Blin	87041e301f	proxyclient: refresh put on restartListeners There is no need to check if the put is ok or not before refreshing it. Indeed, the first successful put will make the ok flag to true. But, if an expiration is missed (when not connected for example), the put will not be refreshed when the connectivity will change (moreover, server and client can have different timeouts).	2020-01-03 11:39:05 -05:00
Sébastien Blin	189f94442e	dhtproxyclient: setPushNotification before first permanent put When SecureDHT is initialized, a permanent put is done to announce the node and then the push token is set. This means the the first permanent put will not get expirations notification because it's considered as a non push device's put. We should inject the push token as soon as possible.	2020-01-03 11:39:05 -05:00
Adrien Béraud	c5c3695612	logger: use shared_ptr	2019-12-19 15:12:23 -05:00
Adrien Béraud	6a7ad871e2	value: toString prints raw data if text/plain	2019-12-19 15:12:23 -05:00
Adrien Béraud	99890fca64	proxy server: update sessionId when necessary	2019-12-19 14:20:03 -05:00
Adrien Béraud	2dbf57d36b	Update README.md	2019-12-16 23:42:31 -05:00
Adrien Béraud	5438d50578	proxy: add session token	2019-12-16 16:37:07 -05:00
Adrien Béraud	548255073f	docker: build c bindings, use asio 1.12	2019-12-16 12:22:05 -05:00
Adrien Béraud	abaf091772	c binding: fix build	2019-12-16 11:41:49 -05:00
Adrien Béraud	7094b79714	crypto: add PublicKey::pack(uint8_t* out, size_t* out_len)	2019-12-14 15:29:42 -05:00
Adrien Béraud	1f672773ba	proxy server: respect value priority for push notifications	2019-12-14 13:02:07 -05:00
Adrien Béraud	374fdd6cc9	proxy: send normal priority push message for value expiration	2019-12-14 13:02:07 -05:00
Adrien Béraud	54b1314ab8	value: optimize json parsing	2019-12-14 13:02:07 -05:00
Adrien Béraud	51c35d8b54	value: add priority	2019-12-14 13:02:07 -05:00
Adrien Béraud	fea736428e	msgpack: take const reference in unpack	2019-12-14 13:01:24 -05:00
Adrien Béraud	2d358adf0e	utils: add findMapValue(string)	2019-12-14 13:00:13 -05:00
Adrien Béraud	26c2e16a42	value: optimize json parsing	2019-12-14 00:41:41 -05:00
Adrien Béraud	e8b385c220	utils: optimize findMapValue	2019-12-13 23:57:22 -05:00