sockaddr: use more precise exception

securedht: move crypto ops to Value
proxy client: fix race condition
2026-11-08 20:37:23 +00:00 · 2018-12-17 16:41:00 -05:00 · 2018-12-17 16:41:00 -05:00 · 2018-12-17 16:41:00 -05:00 · 2018-12-11 14:52:07 -05:00 · 2018-12-11 14:48:26 -05:00
69 changed files with 4334 additions and 1603 deletions
@@ -48,6 +48,7 @@ Makefile.in

 # Python
 *.pyc
+*.stamp
 python/opendht.cpp
 python/setup.py

@@ -60,3 +61,6 @@ doc/Doxyfile
 # vim swap files
 *.swp
 *.swo
+
+# build dir
+build
@@ -1,3 +1,4 @@
+dist: xenial
 sudo: required

 services:
@@ -11,18 +12,13 @@ env:
    - OPENDHT_TEST_JOB="opendht.llvm"
    - OPENDHT_TEST_JOB="opendht.proxyserver"
    - OPENDHT_TEST_JOB="opendht.proxyclient"
-    - OPENDHT_TEST_JOB="opendht.proxyserverpush"
-    - OPENDHT_TEST_JOB="opendht.proxyclientpush"
-    - OPENDHT_TEST_JOB="opendht.all"
+    - OPENDHT_TEST_JOB="opendht.push"

 before_install:
  - |
    # non llvm builds
    if [[ "$OPENDHT_TEST_JOB" != *"opendht.llvm"* ]]; then
      docker pull aberaud/opendht-deps;
-      if [[ "$OPENDHT_TEST_JOB" != *"opendht.classic"* ]]; then
-        docker build -t opendht-deps-proxy -f docker/DockerfileDepsProxy .;
-      fi
    fi

  - |
@@ -42,23 +38,23 @@ script:
    # proxy builds
    if [[ "$OPENDHT_TEST_JOB" != *"opendht.llvm"* ]] && [[ "$OPENDHT_TEST_JOB" != *"opendht.classic"* ]]; then
      docker build -t opendht-proxy -f docker/DockerfileTravisProxy .;
-      options='';
-      if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyserver"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyserverpush"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.all"* ]]; then
+      options='-DOPENDHT_SANITIZE=On ';
+      if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyserver"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.push"* ]]; then
        options+='-DOPENDHT_PROXY_SERVER=ON ';
      else
        options+='-DOPENDHT_PROXY_SERVER=OFF ';
      fi
-      if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyclient"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyclientpush"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.all"* ]]; then
+      if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyclient"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.push"* ]]; then
        options+='-DOPENDHT_PROXY_CLIENT=ON ';
      else
        options+='-DOPENDHT_PROXY_CLIENT=OFF ';
      fi
-      if [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyserverpush"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.proxyclientpush"* ]] || [[ "$OPENDHT_TEST_JOB" == *"opendht.all"* ]]; then
+      if [[ "$OPENDHT_TEST_JOB" == *"opendht.push"* ]]; then
        options+='-DOPENDHT_PUSH_NOTIFICATIONS=ON ';
      else
        options+='-DOPENDHT_PUSH_NOTIFICATIONS=OFF ';
      fi
-      docker run opendht-proxy /bin/sh -c "cd /root/opendht && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=ON -DOPENDHT_LTO=ON $options .. && make -j8 && make install";
+      docker run opendht-proxy /bin/sh -c "cd /root/opendht && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=ON -DOPENDHT_LTO=ON -DOPENDHT_TESTS=ON $options .. && make -j8 && ./opendht_unit_tests && make install";
    fi

  - |
@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 3.1)
 project (opendht)
 set (opendht_VERSION_MAJOR 1)
-set (opendht_VERSION_MINOR 6.0)
+set (opendht_VERSION_MINOR 8.0)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -21,6 +21,7 @@ option (OPENDHT_PUSH_NOTIFICATIONS "Enable push notifications support" OFF)
 option (OPENDHT_PROXY_SERVER_IDENTITY "Allow clients to use the node identity" OFF)
 option (OPENDHT_PROXY_CLIENT "Enable DHT proxy client, use Restbed and jsoncpp" OFF)
 option (OPENDHT_INDEX "Build DHT indexation feature" OFF)
+option (OPENDHT_TESTS "Add unit tests executable" OFF)

 find_package(Doxygen)
 option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND})
@@ -46,9 +47,21 @@ if (NOT OPENDHT_ARGON2)
        set(OPENDHT_ARGON2 ON)
    endif()
 endif ()
+
+pkg_search_module(Jsoncpp jsoncpp)
+if (Jsoncpp_FOUND)
+    add_definitions(-DOPENDHT_JSONCPP)
+    list (APPEND opendht_SOURCES
+      src/base64.h
+      src/base64.cpp
+    )
+endif()
+
 if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT)
    find_package(Restbed REQUIRED)
-    pkg_search_module(Jsoncpp jsoncpp)
+    if (NOT Jsoncpp_FOUND)
+        message(SEND_ERROR "Jsoncpp is required for DHT proxy support")
+    endif()
 endif()

 # Build flags
@@ -86,8 +99,8 @@ endif ()
 if (Nettle_INCLUDE_DIRS)
    include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}")
 endif ()
-if (RESTBED_INCLUDE)
-    include_directories (SYSTEM "${RESTBED_INCLUDE}")
+if (Restbed_INCLUDE_DIR)
+    include_directories (SYSTEM "${Restbed_INCLUDE_DIR}")
 endif ()
 if (Jsoncpp_INCLUDE_DIRS)
    include_directories (SYSTEM "${Jsoncpp_INCLUDE_DIRS}")
@@ -120,9 +133,12 @@ list (APPEND opendht_SOURCES
    src/node.cpp
    src/value.cpp
    src/dht.cpp
+    src/op_cache.cpp
    src/storage.h
    src/listener.h
    src/search.h
+    src/value_cache.h
+    src/op_cache.h
    src/net.h
    src/parsed_message.h
    src/request.h
@@ -178,11 +194,6 @@ if (OPENDHT_PROXY_SERVER)
  list (APPEND opendht_SOURCES
    src/dht_proxy_server.cpp
  )
-  if (OPENDHT_PUSH_NOTIFICATIONS)
-    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=true)
-  else ()
-    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=false)
-  endif ()
 else ()
    add_definitions(-DENABLE_PROXY_SERVER=false)
 endif ()
@@ -200,13 +211,17 @@ else ()
 endif ()

 if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT)
-  list (APPEND opendht_SOURCES
-    src/base64.h
-    src/base64.cpp
+  if (OPENDHT_PUSH_NOTIFICATIONS)
+    message("Using push notification")
+    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=true)
+  else ()
+    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=false)
+  endif ()
+  list (APPEND opendht_HEADERS
+    include/opendht/proxy.h
  )
 endif ()

-
 if(OPENDHT_ARGON2)
    # make sure argon2 submodule is up to date and initialized
    message("Initializing Argon2 submodule")
@@ -231,16 +246,14 @@ if (OPENDHT_STATIC)
        ${opendht_HEADERS}
    )
    set_target_properties (opendht-static PROPERTIES OUTPUT_NAME "opendht")
-    if (OPENDHT_LTO)
-        target_link_libraries(opendht-static -flto -fuse-linker-plugin)
-    endif ()
    if (OPENDHT_ARGON2)
        target_include_directories(opendht-static SYSTEM PRIVATE argon2)
    else ()
-        target_link_libraries(opendht-static ${argon2_LIBRARIES})
        target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
    endif ()
-    target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES})
+    target_link_libraries(opendht-static
+        PRIVATE ${Restbed_LIBRARY} ${argon2_LIBRARIES}
+        PUBLIC ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Jsoncpp_LIBRARIES})
    install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()

@@ -252,16 +265,13 @@ if (OPENDHT_SHARED)
    set_target_properties (opendht PROPERTIES IMPORT_SUFFIX "_import.lib")
    set_target_properties (opendht PROPERTIES SOVERSION ${opendht_VERSION_MAJOR} VERSION ${opendht_VERSION})
    target_compile_definitions(opendht PRIVATE OPENDHT_BUILD)
-    if (OPENDHT_LTO)
-        target_link_libraries(opendht PRIVATE -flto -fuse-linker-plugin)
-    endif ()
    if (OPENDHT_ARGON2)
        target_include_directories(opendht SYSTEM PRIVATE argon2)
    else ()
        target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES})
        target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS})
    endif ()
-    target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES})
+    target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARY} ${Jsoncpp_LIBRARIES})
    install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht)
 endif ()

@@ -292,3 +302,39 @@ install (DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX})
 install (FILES ${CMAKE_CURRENT_BINARY_DIR}/opendht.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 install (EXPORT opendht DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/opendht FILE opendhtConfig.cmake)
 install (FILES ${CMAKE_CURRENT_BINARY_DIR}/opendhtConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/opendht)
+
+# Unit tests
+IF(OPENDHT_TESTS)
+  FIND_PACKAGE(Cppunit REQUIRED)
+  # unit testing
+    list (APPEND test_FILES
+      tests/infohashtester.h
+      tests/infohashtester.cpp
+      tests/cryptotester.h
+      tests/cryptotester.cpp
+      tests/dhtrunnertester.h
+      tests/dhtrunnertester.cpp
+    )
+    if (OPENDHT_PROXY_SERVER AND OPENDHT_PROXY_CLIENT)
+      list (APPEND test_FILES
+        tests/dhtproxytester.h
+        tests/dhtproxytester.cpp
+      )
+    endif()
+    add_executable(opendht_unit_tests
+      tests/tests_runner.cpp
+      ${test_FILES}
+    )
+    if (OPENDHT_SHARED)
+      TARGET_LINK_LIBRARIES(opendht_unit_tests opendht)
+    else ()
+      TARGET_LINK_LIBRARIES(opendht_unit_tests opendht-static)
+    endif ()
+    TARGET_LINK_LIBRARIES(opendht_unit_tests
+       ${CMAKE_THREAD_LIBS_INIT}
+       ${CPPUNIT_LIBRARIES}
+       ${GNUTLS_LIBRARIES}
+    )
+    enable_testing()
+    add_test(TEST opendht_unit_tests)
+ENDIF()
@@ -146,7 +146,7 @@
      <Optimization>Disabled</Optimization>
      <SDLCheck>true</SDLCheck>
      <AdditionalIncludeDirectories>..\..\include;..\include;..\include\opendht;..\..\msgpack-c\include;</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_NATIVE;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <DisableSpecificWarnings>4804;4800;4101;4267;4244;4503;</DisableSpecificWarnings>
      <AdditionalOptions>-D_SCL_SECURE_NO_WARNINGS %(AdditionalOptions)</AdditionalOptions>
      <SuppressStartupBanner>false</SuppressStartupBanner>
@@ -169,7 +169,7 @@ copy ..\include\opendht.h $(OutDir)\include\</Command>
      <Optimization>Disabled</Optimization>
      <SDLCheck>true</SDLCheck>
      <AdditionalIncludeDirectories>..\..\include;..\include;..\include\opendht;..\..\msgpack-c\include;</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_NATIVE;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <DisableSpecificWarnings>4804;4800;4101;4267;4244;4503;</DisableSpecificWarnings>
      <AdditionalOptions>-D_SCL_SECURE_NO_WARNINGS %(AdditionalOptions)</AdditionalOptions>
    </ClCompile>
@@ -193,7 +193,7 @@ copy ..\include\opendht.h $(OutDir)\include\</Command>
      <IntrinsicFunctions>true</IntrinsicFunctions>
      <SDLCheck>true</SDLCheck>
      <AdditionalIncludeDirectories>..\..\include;..\include;..\include\opendht;..\..\msgpack-c\include;</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_NATIVE;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <DisableSpecificWarnings>4804;4800;4101;4267;4244;4503;</DisableSpecificWarnings>
      <AdditionalOptions>-D_SCL_SECURE_NO_WARNINGS %(AdditionalOptions)</AdditionalOptions>
      <ProgramDataBaseFileName>$(OutDir)\lib\x86\$(TargetName).pdb</ProgramDataBaseFileName>
@@ -223,7 +223,7 @@ copy ..\include\opendht.h $(OutDir)\include\</Command>
      <IntrinsicFunctions>true</IntrinsicFunctions>
      <SDLCheck>true</SDLCheck>
      <AdditionalIncludeDirectories>$(ProjectDir)contrib\build\include;$(ProjectDir)..\include;$(ProjectDir)..\include\opendht;$(ProjectDir)contrib\build\msgpack-c\include;$(ProjectDir)contrib\build\argon2\include;$(ProjectDir)..\..\include;$(ProjectDir)..\..\argon2\include</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_NATIVE;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <DisableSpecificWarnings>4804;4800;4101;4267;4244;4503;4273;</DisableSpecificWarnings>
      <AdditionalOptions>-D_SCL_SECURE_NO_WARNINGS %(AdditionalOptions)</AdditionalOptions>
      <ProgramDataBaseFileName>$(OutDir)\lib\x64\$(TargetName).pdb</ProgramDataBaseFileName>
@@ -12,7 +12,13 @@ if USE_CYTHON
 SUBDIRS += python
 endif

+if ENABLE_TESTS
+SUBDIRS += tests
+endif
+
+if HAVE_DOXYGEN
 SUBDIRS += doc
+endif

 ACLOCAL_AMFLAGS = -I m4

@@ -4,14 +4,19 @@
    <a id="user-content-opendht-" class="anchor" href="/savoirfairelinux/opendht/blob/master/README.md#opendht-" aria-hidden="true"></a>OpenDHT
 </h1>

-A lightweight C++11 Distributed Hash Table implementation originally based on https://github.com/jech/dht by Juliusz Chroboczek.
+A lightweight C++11 Distributed Hash Table implementation.

- * Light and fast C++11 Kademlia DHT library.
- * Distributed shared key->value data-store
- * Clean and powerful distributed map API with storage of arbitrary binary values of up to 56 KB.
- * Optional public key cryptography layer providing data signature and encryption (using GnuTLS).
- * IPv4 and IPv6 support.
- * Python binding.
+OpenDHT provides an easy to use distributed in-memory data store.
+Every node in the network can read and write values to the store.
+Values are distributed over the network, with redundancy.
+
+ * Lightweight and scalable, designed for large networks and small devices
+ * High resilience to network disruption
+ * Public key cryptography layer providing optional data signature and encryption (using GnuTLS)
+ * IPv4 and IPv6 support
+ * Clean and powerful C++11 map API
+ * Python 3 bindings
+ * REST API

 ## Documentation
 See the wiki: <https://github.com/savoirfairelinux/opendht/wiki>
@@ -93,23 +98,25 @@ for r in results:
 - Nettle 2.4+, a GnuTLS dependency for crypto.
 - (optional) restbed used for the REST API. commit fb84213e170bc171fecd825a8e47ed9f881a12cd (https://github.com/AmarOk1412/restbed/tree/async_read_until)
 - (optional) jsoncpp 1.7.4-3+, used for the REST API.
- Build tested with GCC 5.2+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, macOS).
+- Build tested with GCC 5.2+ (GNU/Linux, Windows with MinGW), Clang/LLVM (GNU/Linux, Android, macOS, iOS).
 - Build tested with Microsoft Visual Studio 2015

 ## Contact

 IRC: join us on Freenode at [`#opendht`](https://webchat.freenode.net/?channels=%23opendht).

-## Licence
-Copyright (C) 2014-2017 Savoir-faire Linux Inc.
+## License
+Copyright (C) 2014-2018 Savoir-faire Linux Inc.

 OpenDHT is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.

 This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.

-See COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html for the full GPLv3 licence.
+See COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html for the full GPLv3 license.

-This project is independent from another project called OpenDHT (Sean Rhea. Ph.D. Thesis, 2005), now extinct.
+## Acknowledgements
+This project was originally based on https://github.com/jech/dht by Juliusz Chroboczek.
+It is independent from another project called OpenDHT (Sean Rhea. Ph.D. Thesis, 2005), now extinct.

 ## Donations
 We gratefully accept Bitcoin donations to support OpenDHT development at: `bitcoin:3EykSd1An888efq4Bq3KaV3hJ3JQ4FPnwm`.
@@ -0,0 +1,28 @@
+INCLUDE(FindPkgConfig)
+PKG_CHECK_MODULES(PC_CPPUNIT "cppunit")
+
+FIND_PATH(CPPUNIT_INCLUDE_DIRS
+    NAMES cppunit/TestCase.h
+    HINTS ${PC_CPPUNIT_INCLUDE_DIR}
+    ${CMAKE_INSTALL_PREFIX}/include
+    PATHS
+    /usr/local/include
+    /usr/include
+)
+
+FIND_LIBRARY(CPPUNIT_LIBRARIES
+    NAMES cppunit
+    HINTS ${PC_CPPUNIT_LIBDIR}
+    ${CMAKE_INSTALL_PREFIX}/lib
+    ${CMAKE_INSTALL_PREFIX}/lib64
+    PATHS
+    ${CPPUNIT_INCLUDE_DIRS}/../lib
+    /usr/local/lib
+    /usr/lib
+)
+
+LIST(APPEND CPPUNIT_LIBRARIES ${CMAKE_DL_LIBS})
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(CPPUNIT DEFAULT_MSG CPPUNIT_LIBRARIES CPPUNIT_INCLUDE_DIRS)
+MARK_AS_ADVANCED(CPPUNIT_LIBRARIES CPPUNIT_INCLUDE_DIRS)
@@ -1,17 +1,16 @@
-find_path (RESTBED_INCLUDE restbed
-           HINTS
-           "/usr/include"
-           "/usr/local/include"
-           "/opt/local/include"
-)
-
-if (RESTBED_INCLUDE)
-    message(STATUS "${green}Found Restbed: ${RESTBED_INCLUDE}")
-else()
-    message(FATAL_ERROR "${red}Failed to locate Restbed.}")
-endif()
-
-if (RESTBED_INCLUDE)
-    set(RESTBED_FOUND TRUE)
-    set(Restbed_LIBRARIES restbed)
+if(NOT Restbed_FOUND)
+    find_path (Restbed_INCLUDE_DIR restbed
+               HINTS
+               "/usr/include"
+               "/usr/local/include"
+               "/opt/local/include")
+    find_library(Restbed_LIBRARY restbed
+                 HINTS ${Restbed_ROOT_DIR} PATH_SUFFIXES lib)
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(Restbed DEFAULT_MSG Restbed_LIBRARY Restbed_INCLUDE_DIR)
+    if (Restbed_INCLUDE_DIR)
+        set(Restbed_FOUND TRUE)
+        set(Restbed_LIBRARIES ${Restbed_LIBRARY})
+        set(Restbed_INCLUDE_DIRS ${Restbed_INCLUDE_DIR})
+    endif()
 endif()
@@ -1,6 +1,6 @@
 dnl define macros
 m4_define([opendht_major_version], 1)
-m4_define([opendht_minor_version], 6)
+m4_define([opendht_minor_version], 8)
 m4_define([opendht_patch_version], 0)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
@@ -25,7 +25,7 @@ AC_PROG_CXX
 AM_PROG_AR

 dnl Check for logs
-AC_ARG_ENABLE([logs], [AS_HELP_STRING([--disable-logs], [Disble DHT logs])])
+AC_ARG_ENABLE([logs], [AS_HELP_STRING([--disable-logs], [Disable DHT logs])])
 AS_IF([test "x$enable_logs" != "xno"], [
 	AC_DEFINE([OPENDHT_LOG], [true], [Define if DHT logs are enabled])
 ], [
@@ -33,9 +33,9 @@ AS_IF([test "x$enable_logs" != "xno"], [
 ])

 dnl Check for indexation
-AC_ARG_ENABLE([indexation], [AS_HELP_STRING([--enable-indexation], [Enable DHT indexation])])
-AM_CONDITIONAL(ENABLE_INDEXATION, test x$enable_indexation == xyes)
-AS_IF([test "x$enable_indexation" = "xyes"], [
+AC_ARG_ENABLE([indexation], [AS_HELP_STRING([--disable-indexation], [Disable DHT indexation])])
+AM_CONDITIONAL(ENABLE_INDEXATION, test x$enable_indexation != "xno")
+AS_IF([test "x$enable_indexation" != "xno"], [
 	AC_DEFINE([OPENDHT_INDEXATION], [1], [Define if DHT indexation is enabled])
 ])

@@ -101,9 +101,51 @@ LT_LANG(C++)
 AX_CXX_COMPILE_STDCXX(11,[noext],[mandatory])

 PKG_PROG_PKG_CONFIG()
+
+AC_ARG_ENABLE([proxy_server], AS_HELP_STRING([--enable-proxy-server], [Enable proxy server ability]), proxy_server=yes, proxy_server=no)
+AM_CONDITIONAL(ENABLE_PROXY_SERVER, test x$proxy_server == xyes)
+
+AC_ARG_ENABLE([push_notifications], AS_HELP_STRING([--enable-push-notifications], [Enable push notifications support]), push_notifications=yes, push_notifications=no)
+AM_CONDITIONAL(ENABLE_PUSH_NOTIFICATIONS, test x$push_notifications == xyes)
+
+AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity],
+	[Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no)
+AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes -a x$proxy_server == xyes)
+AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity],
+	[Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no)
+
+AC_ARG_ENABLE([proxy_client], AS_HELP_STRING([--enable-proxy-client], [Enable proxy client ability]), proxy_client=yes, proxy_client=no)
+AM_CONDITIONAL(ENABLE_PROXY_CLIENT, test x$proxy_client == xyes)
+
+AC_ARG_ENABLE([tests], AS_HELP_STRING([--enable-tests], [Enable tests]), build_tests=yes, build_tests=no)
+AM_CONDITIONAL(ENABLE_TESTS, test x$build_tests == xyes)
+AM_COND_IF([ENABLE_TESTS], [
+	PKG_CHECK_MODULES([CppUnit], [cppunit >= 1.12])
+])
+
+AM_CONDITIONAL(PROXY_CLIENT_OR_SERVER, test x$proxy_client == xyes | test x$proxy_server == xyes)
+
 PKG_CHECK_MODULES([Nettle], [nettle >= 2.4])
 PKG_CHECK_MODULES([GnuTLS], [gnutls >= 3.3])
 PKG_CHECK_MODULES([MsgPack], [msgpack >= 1.2])
+
+AC_ARG_WITH([jsoncpp], AS_HELP_STRING([--without-jsoncpp], [Build without JsonCpp support]))
+AS_IF([test "x$with_jsoncpp" != "xno"],
+      [PKG_CHECK_MODULES([JsonCpp], [jsoncpp >= 1.7.2], [have_jsoncpp=yes], [have_jsoncpp=no])],
+      [have_jsoncpp=no])
+AS_IF([test "x$have_jsoncpp" = "xyes"], [
+    AC_MSG_NOTICE([Using JsonCpp])
+    CPPFLAGS+=" -DOPENDHT_JSONCPP=1"
+], [
+    AC_MSG_NOTICE([Not using JsonCpp])
+    AM_COND_IF(PROXY_CLIENT_OR_SERVER, AC_MSG_ERROR(["JsonCpp is required for proxy/push notification support"]))
+])
+
+AM_COND_IF([PROXY_CLIENT_OR_SERVER], [
+	AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files]))
+	LDFLAGS="${LDFLAGS} -lrestbed"
+])
+
 CXXFLAGS="${CXXFLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT"

 dnl Check for Argon2
@@ -135,56 +177,26 @@ AM_COND_IF([ENABLE_TOOLS], [
  ])
 ])

-AC_ARG_ENABLE([proxy_server], AS_HELP_STRING([--enable-proxy-server], [Enable proxy server ability]), proxy_server=yes, proxy_server=no)
-AM_CONDITIONAL(ENABLE_PROXY_SERVER, test x$proxy_server == xyes)
-AC_ARG_ENABLE([push_notifications], AS_HELP_STRING([--enable-push-notifications], [Enable push notifications support]), push_notifications=yes, push_notifications=no)
-AM_CONDITIONAL(ENABLE_PUSH_NOTIFICATIONS, test x$push_notifications == xyes)
-AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity],
-	[Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no)
-AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes)
-AC_ARG_ENABLE([proxy_client], AS_HELP_STRING([--enable-proxy-client], [Enable proxy client ability]), proxy_client=yes, proxy_client=no)
-AM_CONDITIONAL(ENABLE_PROXY_CLIENT, test x$proxy_client == xyes)
+AM_COND_IF(ENABLE_PROXY_SERVER,
+    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true"],
+    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false"])

-AM_COND_IF([ENABLE_PROXY_SERVER], [
-	AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files]))
-	PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.2])
-	CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true -ljsoncpp -lrestbed"
-], [
-	CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false"
+AM_COND_IF(ENABLE_PROXY_CLIENT,
+    [CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=true"],
+    [CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=false"])
+
+AM_COND_IF(ENABLE_PUSH_NOTIFICATIONS,
+    [CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=true"],
+    [CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=false"])
+
+AM_COND_IF(ENABLE_PROXY_SERVER_IDENTITY,
+    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=true"],
+    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=false"])
+
+AM_COND_IF([HAVE_DOXYGEN], [
+  AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])
 ])

-AM_COND_IF([ENABLE_PUSH_NOTIFICATIONS], [
-	CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=true"
-], [
-	CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=false"
-])
-
-AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity],
-    [Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no)
-AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes -a x$proxy_server == xyes)
-AM_COND_IF([ENABLE_PROXY_SERVER_IDENTITY], [
-    CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=true"
-], [
-    CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=false"
-])
-
-AM_COND_IF([ENABLE_PROXY_CLIENT], [
-	CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=true"
-], [
-	CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=false"
-])
-
-AM_CONDITIONAL(PROXY_CLIENT_OR_SERVER, test x$proxy_client == xyes | test x$proxy_server == xyes)
-AM_COND_IF([PROXY_CLIENT_OR_SERVER], [
-	AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files]))
-	PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.4])
-	CPPFLAGS="${CPPFLAGS} ${Jsoncpp_CFLAGS}"
-	LDFLAGS="${LDFLAGS} ${Jsoncpp_LIBS} -lrestbed"
-], [])
-
-
-AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])
-
 dnl Configure setup.py if we build the python module
 AM_COND_IF([USE_CYTHON], [
    AC_SUBST(CURRENT_SOURCE_DIR, ".")
@@ -197,5 +209,6 @@ AM_COND_IF([USE_CYTHON], [
 AC_CONFIG_FILES([Makefile
                 src/Makefile
                 tools/Makefile
+                 tests/Makefile
                 opendht.pc])
 AC_OUTPUT
@@ -1,3 +1,18 @@
-FROM ubuntu:17.04
+FROM ubuntu:16.04
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
-RUN apt-get update && apt-get install -y build-essential cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls28-dev libuv1-dev libmsgpack-dev libargon2-0-dev cython3 python3-dev python3-setuptools && apt-get clean
+RUN apt-get update && apt-get install -y build-essential cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls28-dev libuv1-dev  cython3 python3-dev libcppunit-dev libjsoncpp-dev libasio-dev libssl-dev python3-setuptools && apt-get clean
+
+# build restbed from sources
+RUN git clone --recursive https://github.com/corvusoft/restbed.git \
+	&& cd restbed && mkdir build && cd build \
+	&& cmake -DBUILD_TESTS=NO -DBUILD_EXAMPLES=NO -DBUILD_SSL=NO -DBUILD_SHARED=YES -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib .. \
+	&& make -j8 install \
+	&& cd .. && rm -rf restbed
+
+#build msgpack from source
+RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.1.5/msgpack-2.1.5.tar.gz \
+	&& tar -xzf msgpack-2.1.5.tar.gz \
+	&& cd msgpack-2.1.5 && mkdir build && cd build \
+	&& cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
+	&& make -j8 && make install \
+	&& cd ../.. && rm -rf msgpack-2.1.5 msgpack-2.1.5.tar.gz
@@ -1,7 +1,23 @@
-FROM ubuntu:17.04
+FROM ubuntu:16.04
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 RUN apt-get update \
-	&& apt-get install -y llvm llvm-dev clang make cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls28-dev libuv1-dev libmsgpack-dev libargon2-0-dev cython3 python3-dev python3-setuptools \
+	&& apt-get install -y llvm llvm-dev clang make cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls28-dev libuv1-dev libmsgpack-dev libjsoncpp-dev libasio-dev cython3 python3-dev python3-setuptools libcppunit-dev \
 	&& apt-get remove -y gcc g++ && apt-get autoremove -y && apt-get clean
+
 ENV CC cc
-ENV CXX c++
+ENV CXX c++
+
+# build restbed from sources
+RUN git clone --recursive https://github.com/corvusoft/restbed.git \
+	&& cd restbed && mkdir build && cd build \
+	&& cmake -DBUILD_TESTS=NO -DBUILD_EXAMPLES=NO -DBUILD_SSL=NO -DBUILD_SHARED=YES -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib .. \
+	&& make -j8 install \
+	&& cd .. && rm -rf restbed
+
+#build msgpack from source
+RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.1.5/msgpack-2.1.5.tar.gz \
+	&& tar -xzf msgpack-2.1.5.tar.gz \
+	&& cd msgpack-2.1.5 && mkdir build && cd build \
+	&& cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
+	&& make -j8 && make install \
+	&& cd ../.. && rm -rf msgpack-2.1.5 msgpack-2.1.5.tar.gz
@@ -1,16 +0,0 @@
-FROM aberaud/opendht-deps
-
-# install jsoncpp
-RUN apt-get install libjsoncpp-dev -y
-
-# install restbed dependencies
-RUN sed -i -e 's/archive.ubuntu.com\|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list # LTS... avoid 404 for libasio-dev
-RUN apt-get update -y
-RUN apt-get install libasio-dev -y
-
-# build restbed from sources
-RUN git clone --recursive https://github.com/corvusoft/restbed.git
-WORKDIR restbed/build
-RUN cmake -DBUILD_TESTS=NO -DBUILD_EXAMPLES=NO -DBUILD_SSL=NO -DBUILD_SHARED=YES -DCMAKE_INSTALL_PREFIX=/usr ..
-RUN make -j 8 install
-RUN mv /usr/library/librestbed* /usr/lib/
@@ -1,5 +1,7 @@
 FROM aberaud/opendht-deps
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
 COPY . /root/opendht
 RUN cd /root/opendht && mkdir build && cd build \
-	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On .. && make -j8 && make install
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On -DOPENDHT_TESTS=ON ..  \
+	&& make -j8 && ./opendht_unit_tests && make install
@@ -1,5 +1,7 @@
 FROM aberaud/opendht-deps-llvm
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
 COPY . /root/opendht
 RUN cd /root/opendht && mkdir build && cd build \
-	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On .. && make -j8 && make install
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_TESTS=ON .. \
+	&& make -j8 && ./opendht_unit_tests && make install
@@ -1,3 +1,4 @@
-FROM opendht-deps-proxy
+FROM aberaud/opendht-deps
 MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
 COPY . /root/opendht
@@ -27,9 +27,9 @@
 #include <memory>
 #include <functional>

-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_JSONCPP
 #include <json/json.h>
-#endif //OPENDHT_PROXY_SERVER
+#endif

 namespace dht {

@@ -51,15 +51,37 @@ struct OPENDHT_PUBLIC NodeStats {
             incoming_nodes {0};
    unsigned table_depth {0};
    unsigned getKnownNodes() const { return good_nodes + dubious_nodes; }
+    unsigned long getNetworkSizeEstimation() const { return 8 * std::exp2(table_depth); }
    std::string toString() const;
-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+
+#ifdef OPENDHT_JSONCPP
    /**
     * Build a json object from a NodeStats
     */
    Json::Value toJson() const;
    NodeStats() {};
    explicit NodeStats(const Json::Value& v);
-#endif //OPENDHT_PROXY_SERVER
+#endif
+
+    MSGPACK_DEFINE_MAP(good_nodes, dubious_nodes, cached_nodes, incoming_nodes, table_depth)
+};
+
+struct OPENDHT_PUBLIC NodeInfo {
+    InfoHash id;
+    InfoHash node_id;
+    NodeStats ipv4;
+    NodeStats ipv6;
+
+#ifdef OPENDHT_JSONCPP
+    /**
+     * Build a json object from a NodeStats
+     */
+    Json::Value toJson() const;
+    NodeInfo() {};
+    explicit NodeInfo(const Json::Value& v);
+#endif
+
+    MSGPACK_DEFINE_MAP(id, node_id, ipv4, ipv6)
 };

 /**
@@ -98,6 +120,7 @@ using ValuesExport = std::pair<InfoHash, Blob>;

 using QueryCallback = std::function<bool(const std::vector<std::shared_ptr<FieldValueIndex>>& fields)>;
 using GetCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values)>;
+using ValueCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values, bool expired)>;
 using GetCallbackSimple = std::function<bool(std::shared_ptr<Value> value)>;
 using ShutdownCallback = std::function<void()>;

@@ -86,8 +86,16 @@ struct OPENDHT_PUBLIC PublicKey

    PublicKey& operator=(PublicKey&& o) noexcept;

+    /**
+     * Get public key fingerprint
+     */
    InfoHash getId() const;
+
+    /**
+     * Get public key long fingerprint
+     */
    PkId getLongId() const;
+
    bool checkSignature(const Blob& data, const Blob& signature) const;
    Blob encrypt(const Blob&) const;

@@ -133,6 +141,7 @@ struct OPENDHT_PUBLIC PrivateKey
    PrivateKey(const Blob& import, const std::string& password = {});
    ~PrivateKey();
    explicit operator bool() const { return key; }
+
    PublicKey getPublicKey() const;
    Blob serialize(const std::string& password = {}) const;

@@ -164,8 +173,6 @@ private:
    PrivateKey(const PrivateKey&) = delete;
    PrivateKey& operator=(const PrivateKey&) = delete;
    Blob decryptBloc(const uint8_t* src, size_t src_size) const;
-
-    //friend dht::crypto::Identity dht::crypto::generateIdentity(const std::string&, dht::crypto::Identity, unsigned key_length);
 };


@@ -368,6 +375,7 @@ struct OPENDHT_PUBLIC Certificate {

    /** Same as getPublicKey().getId() */
    InfoHash getId() const;
+    /** Same as getPublicKey().getLongId() */
    PkId getLongId() const;

    /** Read certificate Common Name (CN) */
@@ -404,8 +412,20 @@ struct OPENDHT_PUBLIC Certificate {

    std::string print() const;

+    /**
+     * As a CA, revoke a certificate, adding it to
+     * the attached Certificate Revocation List (CRL)
+     */
    void revoke(const PrivateKey&, const Certificate&);
+
+    /**
+     * Get the list of certificates revoked as as CA.
+     */
    std::vector<std::shared_ptr<RevocationList>> getRevocationLists() const;
+
+    /**
+     * Attach existing revocation list.
+     */
    void addRevocationList(RevocationList&&);
    void addRevocationList(std::shared_ptr<RevocationList>);

@@ -508,9 +528,13 @@ public:
    ~secure_vector() { clean(); }

    static secure_vector<T> getRandom(size_t size) {
-        secure_vector<T> ret(size/sizeof(T));
+        secure_vector<T> ret(size);
        crypto::random_device rdev;
+#ifdef _WIN32
+        std::uniform_int_distribution<int> rand_byte{ 0, std::numeric_limits<uint8_t>::max() };
+#else
        std::uniform_int_distribution<uint8_t> rand_byte;
+#endif
        std::generate_n((uint8_t*)ret.data_.data(), ret.size()*sizeof(T), std::bind(rand_byte, std::ref(rdev)));
        return ret;
    }
@@ -114,6 +114,8 @@ public:
    ImMessage() {}
    ImMessage(dht::Value::Id id, std::string&& m, long d = 0)
        : id(id), msg(std::move(m)), date(d) {}
+    ImMessage(dht::Value::Id id, std::string&& dt, std::string&& m, long d = 0)
+        : id(id), msg(std::move(m)), datatype(std::move(dt)), date(d) {}

    virtual void unpackValue(const Value& v) override {
        to = v.recipient;
@@ -123,10 +125,11 @@ public:
    dht::InfoHash to;
    dht::Value::Id id {0};
    std::string msg;
+    std::string datatype;
    long date {0};
    ImStatus status {ImStatus::NONE};

-    MSGPACK_DEFINE_MAP(id, msg, date, status)
+    MSGPACK_DEFINE_MAP(id, msg, date, status, datatype)
 };

 class OPENDHT_PUBLIC TrustRequest : public EncryptedValue<TrustRequest>
@@ -99,11 +99,10 @@ public:
    bool isRunning(sa_family_t af = 0) const;

    virtual void registerType(const ValueType& type) {
-        types[type.id] = type;
+        types.registerType(type);
    }
    const ValueType& getType(ValueType::Id type_id) const {
-        const auto& t_it = types.find(type_id);
-        return (t_it == types.end()) ? ValueType::USER_DATA : t_it->second;
+        return types.getType(type_id);
    }

    /**
@@ -231,8 +230,16 @@ public:
     *
     * @return a token to cancel the listener later.
     */
-    virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where w = {});
-    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) {
+    virtual size_t listen(const InfoHash&, ValueCallback, Value::Filter={}, Where={});
+
+    virtual size_t listen(const InfoHash& key, GetCallback cb, Value::Filter f={}, Where w={}) {
+        return listen(key, [cb](const std::vector<Sp<Value>>& vals, bool expired){
+            if (not expired)
+                return cb(vals);
+            return true;
+        }, std::forward<Value::Filter>(f), std::forward<Where>(w));
+    }
+    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w={}) {
        return listen(key, bindGetCb(cb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }

@@ -350,7 +357,7 @@ private:
    uint64_t oldsecret {};

    // registred types
-    std::map<ValueType::Id, ValueType> types;
+    TypeStore types;

    // are we a bootstrap node ?
    // note: Any running node can be used as a bootstrap node.
@@ -402,12 +409,14 @@ private:

    // Storage
    void storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t tid, Query&& = {});
-    bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa = {});
+    bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa = {}, bool permanent = false);
+    bool storageErase(const InfoHash& id, Value::Id vid);
+    bool storageRefresh(const InfoHash& id, Value::Id vid);
    void expireStore();
    void expireStorage(InfoHash h);
    void expireStore(decltype(store)::iterator);

-    void storageChanged(const InfoHash& id, Storage& st, ValueStorage&);
+    void storageChanged(const InfoHash& id, Storage& st, ValueStorage&, bool newValue);
    std::string printStorageLog(const decltype(store)::value_type&) const;

    /**
@@ -457,10 +466,10 @@ private:
     * Low-level method that will perform a search on the DHT for the specified
     * infohash (id), using the specified IP version (IPv4 or IPv6).
     */
-    Sp<Search> search(const InfoHash& id, sa_family_t af, GetCallback = {}, QueryCallback = {}, DoneCallback = {}, Value::Filter = {}, Query q = {});
+    Sp<Search> search(const InfoHash& id, sa_family_t af, GetCallback = {}, QueryCallback = {}, DoneCallback = {}, Value::Filter = {}, const Sp<Query>& q = {});

    void announce(const InfoHash& id, sa_family_t af, Sp<Value> value, DoneCallback callback, time_point created=time_point::max(), bool permanent = false);
-    size_t listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter f = Value::AllFilter(), const Sp<Query>& q = {});
+    size_t listenTo(const InfoHash& id, sa_family_t af, ValueCallback cb, Value::Filter f = Value::AllFilter(), const Sp<Query>& q = {});

    /**
     * Refill the search with good nodes if possible.
@@ -528,7 +537,9 @@ private:
     *
     * @param sr  The search to execute its operations.
     */
-    void searchStep(Sp<Search> sr);
+    void searchStep(Sp<Search>);
+    void searchSynchedNodeListen(const Sp<Search>&, SearchNode&);
+
    void dumpSearch(const Search& sr, std::ostream& out) const;

    bool neighbourhoodMaintenance(RoutingTable&);
@@ -40,7 +40,7 @@ public:
    virtual NodeStatus getStatus() const = 0;

    /**
-     * Get the ID of the node.
+     * Get the ID of the DHT node.
     */
    virtual const InfoHash& getNodeId() const = 0;

@@ -165,6 +165,7 @@ public:
     */
    virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where w = {}) = 0;
    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) = 0;
+    virtual size_t listen(const InfoHash&, ValueCallback, Value::Filter={}, Where w = {}) = 0;

    virtual bool cancelListen(const InfoHash&, size_t token) = 0;

@@ -215,7 +216,7 @@ public:
     */
    virtual void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG)
    {
-        DHT_LOG.DEBUG = debug;
+        DHT_LOG.DBG = debug;
        DHT_LOG.WARN = warn;
        DHT_LOG.ERR = error;
    }
@@ -235,11 +236,6 @@ public:
     * @param notification to process
     */
    virtual void pushNotificationReceived(const std::map<std::string, std::string>& data) = 0;
-    /**
-     * Refresh a listen via a token
-     * @param token
-     */
-    virtual void resubscribe(unsigned token) = 0;

 protected:
    bool logFilerEnable_ {};
@@ -29,6 +29,7 @@
 #include "def.h"
 #include "dht_interface.h"
 #include "scheduler.h"
+#include "proxy.h"

 namespace restbed {
    class Request;
@@ -40,10 +41,12 @@ namespace Json {

 namespace dht {

+class SearchCache;
+
 class OPENDHT_PUBLIC DhtProxyClient final : public DhtInterface {
 public:

-    DhtProxyClient() : scheduler(DHT_LOG) {}
+    DhtProxyClient();

    explicit DhtProxyClient(std::function<void()> loopSignal, const std::string& serverHost, const std::string& pushClientId = "");

@@ -102,8 +105,6 @@ public:
        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }

-    void get(const InfoHash& key, const GetCallback& cb, DoneCallback donecb, const Value::Filter& filterChain);
-
    /**
     * Announce a value on all available protocols (IPv4, IPv6).
     *
@@ -161,22 +162,25 @@ public:
     *
     * @return a token to cancel the listener later.
     */
-    virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where={});
-    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) {
+    virtual size_t listen(const InfoHash&, ValueCallback, Value::Filter={}, Where={});
+
+    virtual size_t listen(const InfoHash& key, GetCallback cb, Value::Filter f={}, Where w={}) {
+        return listen(key, [cb](const std::vector<Sp<Value>>& vals, bool expired){
+            if (not expired)
+                return cb(vals);
+            return true;
+        }, std::forward<Value::Filter>(f), std::forward<Where>(w));
+    }
+    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w={}) {
        return listen(key, bindGetCb(cb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    virtual bool cancelListen(const InfoHash&, size_t token);
+    virtual bool cancelListen(const InfoHash& key, size_t token);

    /**
     * Call linked callback with a push notification
     * @param notification to process
     */
    void pushNotificationReceived(const std::map<std::string, std::string>& notification);
-    /**
-     * Refresh a listen via a token
-     * @param token
-     */
-    void resubscribe(const unsigned token);

    time_point periodic(const uint8_t*, size_t, const SockAddr&);
    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) {
@@ -202,21 +206,31 @@ public:
    /**
     * Get data currently being put at the given hash.
     */
-    std::vector<Sp<Value>> getPut(const InfoHash&) { return {}; }
+    std::vector<Sp<Value>> getPut(const InfoHash&);

    /**
     * Get data currently being put at the given hash with the given id.
     */
-    Sp<Value> getPut(const InfoHash&, const Value::Id&) { return {}; }
+    Sp<Value> getPut(const InfoHash&, const Value::Id&);

    /**
     * Stop any put/announce operation at the given location,
     * for the value with the given id.
     */
-    bool cancelPut(const InfoHash&, const Value::Id&) { return false; }
+    bool cancelPut(const InfoHash&, const Value::Id&);

    void pingNode(const sockaddr*, socklen_t, DoneCallbackSimple&& /*cb*/={}) { }

+    virtual void registerType(const ValueType& type) {
+        types.registerType(type);
+    }
+    const ValueType& getType(ValueType::Id type_id) const {
+        return types.getType(type_id);
+    }
+
+    std::vector<Sp<Value>> getLocal(const InfoHash& k, Value::Filter filter) const;
+    Sp<Value> getLocalById(const InfoHash& k, Value::Id id) const;
+
    /**
     * NOTE: The following methods will not be implemented because the
     * DhtProxyClient doesn't have any storage nor synchronization process
@@ -225,10 +239,6 @@ public:
    void insertNode(const InfoHash&, const sockaddr*, socklen_t) { }
    void insertNode(const NodeExport&) { }
    std::pair<size_t, size_t> getStoreSize() const { return {}; }
-    virtual void registerType(const ValueType&) { }
-    const ValueType& getType(ValueType::Id) const { return NO_VALUE; }
-    std::vector<Sp<Value>> getLocal(const InfoHash&, Value::Filter) const { return {}; }
-    Sp<Value> getLocalById(const InfoHash&, Value::Id) const { return {}; }
    std::vector<NodeExport> exportNodes() { return {}; }
    std::vector<ValuesExport> exportValues() const { return {}; }
    void importValues(const std::vector<ValuesExport>&) {}
@@ -240,12 +250,16 @@ public:
    void dumpTables() const {}
    std::vector<unsigned> getNodeMessageStats(bool) { return {}; }
    void setStorageLimit(size_t) {}
-    void connectivityChanged(sa_family_t) {}
-    void connectivityChanged() { }
+    void connectivityChanged(sa_family_t) {
+        restartListeners();
+    }
+    void connectivityChanged() {
+        getProxyInfos();
+        restartListeners();
+        loopSignal_();
+    }

 private:
-    const ValueType NO_VALUE;
-
    /**
     * Start the connection with a server.
     */
@@ -255,12 +269,22 @@ private:
     * Get informations from the proxy node
     * @return the JSON returned by the proxy
     */
+    struct InfoState;
    void getProxyInfos();
-    void onProxyInfos(const Json::Value& val);
+    void onProxyInfos(const Json::Value& val, sa_family_t family);
    SockAddr parsePublicAddress(const Json::Value& val);

    void opFailed();

+    size_t doListen(const InfoHash& key, ValueCallback, Value::Filter);
+    bool doCancelListen(const InfoHash& key, size_t token);
+
+    struct ListenState;
+    void sendListen(const std::shared_ptr<restbed::Request>& request, const ValueCallback&, const Value::Filter& filter, const Sp<ListenState>& state);
+    void sendSubscribe(const std::shared_ptr<restbed::Request>& request, const Sp<proxy::ListenToken>&, const Sp<ListenState>& state);
+
+    void doPut(const InfoHash&, Sp<Value>, DoneCallback, time_point created, bool permanent);
+
    /**
     * Initialize statusIpvX_
     */
@@ -273,35 +297,32 @@ private:
     * cancel all Operations
     */
    void cancelAllOperations();
+
    std::string serverHost_;
    std::string pushClientId_;

-    std::atomic_flag ongoingStatusUpdate_ = ATOMIC_FLAG_INIT;
+    mutable std::mutex lockCurrentProxyInfos_;
    NodeStatus statusIpv4_ {NodeStatus::Disconnected};
    NodeStatus statusIpv6_ {NodeStatus::Disconnected};
    NodeStats stats4_ {};
    NodeStats stats6_ {};
-    SockAddr publicAddress_;
+    SockAddr publicAddressV4_;
+    SockAddr publicAddressV6_;

    InfoHash myid {};

+    // registred types
+    TypeStore types;
+
    /**
     * Store listen requests.
     */
-    struct Listener
-    {
-        size_t token;
-        std::shared_ptr<restbed::Request> req;
-        std::string key;
-        GetCallback cb;
-        Value::Filter filterChain;
-        std::thread thread;
-        unsigned callbackId;
-        std::shared_ptr<unsigned> pushNotifToken; // NOTE: unused if not using push notifications
-    };
-    std::vector<Listener> listeners_;
-    size_t listener_token_ {0};
-    std::mutex lockListener_;
+    struct Listener;
+    struct ProxySearch;
+
+    size_t listenerToken_ {0};
+    std::map<InfoHash, ProxySearch> searches_;
+    mutable std::mutex searchLock_;

    /**
     * Store current put and get requests.
@@ -320,8 +341,9 @@ private:
    std::vector<std::function<void()>> callbacks_;
    std::mutex lockCallbacks;

+    Sp<InfoState> infoState_;
    std::thread statusThread_;
-    mutable std::mutex lockCurrentProxyInfos_;
+    mutable std::mutex statusLock_;

    Scheduler scheduler;
    /**
@@ -334,20 +356,26 @@ private:
     */
    void restartListeners();

+    /**
+     * Refresh a listen via a token
+     * @param token
+     */
+    void resubscribe(const InfoHash& key, Listener& listener);
+
    /**
     * If we want to use push notifications by default.
     * NOTE: empty by default to avoid to use services like FCM or APN.
     */
    std::string deviceKey_ {};
-    unsigned callbackId_ {0};
-    std::mutex lockCallback_;

    const std::function<void()> loopSignal_;

 #if OPENDHT_PUSH_NOTIFICATIONS
-    void fillBodyToGetToken(std::shared_ptr<restbed::Request> request, unsigned callbackId);
+    void fillBodyToGetToken(std::shared_ptr<restbed::Request> request, unsigned token = 0);
+    void getPushRequest(Json::Value&) const;
 #endif // OPENDHT_PUSH_NOTIFICATIONS

+    std::atomic_bool isDestroying_ {false};
 };

 }
@@ -1,6 +1,7 @@
 /*
 *  Copyright (C) 2017-2018 Savoir-faire Linux Inc.
 *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *          Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
@@ -20,15 +21,23 @@

 #pragma once

+#include "callbacks.h"
 #include "def.h"
-#include "sockaddr.h"
 #include "infohash.h"
+#include "proxy.h"
+#include "scheduler.h"
+#include "sockaddr.h"
+#include "value.h"

 #include <thread>
 #include <memory>
 #include <mutex>
 #include <restbed>

+#ifdef OPENDHT_JSONCPP
+#include <json/json.h>
+#endif
+
 namespace Json {
    class Value;
 }
@@ -59,6 +68,56 @@ public:
    DhtProxyServer& operator=(const DhtProxyServer& other) = delete;
    DhtProxyServer& operator=(DhtProxyServer&& other) = delete;

+    struct ServerStats {
+        /** Current number of listen operations */
+        size_t listenCount;
+        /** Current number of permanent put operations */
+        size_t putCount;
+        /** Current number of push tokens with at least one listen operation */
+        size_t pushListenersCount;
+        /** Average requests per second */
+        double requestRate;
+        /** Node Info **/
+        NodeInfo nodeInfo;
+
+        std::string toString() const {
+            std::ostringstream ss;
+            ss << "Listens: " << listenCount << " Puts: " << putCount << " PushListeners: " << pushListenersCount << std::endl;
+            ss << "Requests: " << requestRate << " per second." << std::endl;
+            auto& ni = nodeInfo;
+            auto& ipv4 = ni.ipv4;
+            if (ipv4.table_depth > 1) {
+                ss << "IPv4 Network estimation: " << ipv4.getNetworkSizeEstimation() << std::endl;;
+            }
+            auto& ipv6 = ni.ipv6;
+            if (ipv6.table_depth > 1) {
+                ss << "IPv6 Network estimation: " << ipv6.getNetworkSizeEstimation() << std::endl;;
+            }
+            return ss.str();
+        }
+
+#ifdef OPENDHT_JSONCPP
+        /**
+         * Build a json object from a NodeStats
+         */
+        Json::Value toJson() const {
+            Json::Value result;
+            result["listenCount"] = static_cast<Json::UInt64>(listenCount);
+            result["putCount"] = static_cast<Json::UInt64>(putCount);
+            result["pushListenersCount"] = static_cast<Json::UInt64>(pushListenersCount);
+            result["requestRate"] = requestRate;
+            result["nodeInfo"] = nodeInfo.toJson();
+            return result;
+        }
+#endif
+    };
+
+    ServerStats stats() const { return stats_; }
+
+    void updateStats() const;
+
+    std::shared_ptr<DhtRunner> getNode() const { return dht_; }
+
    /**
     * Stop the DhtProxyServer
     */
@@ -74,6 +133,14 @@ private:
     */
    void getNodeInfo(const std::shared_ptr<restbed::Session>& session) const;

+    /**
+     * Return ServerStats in JSON format
+     * Method: STATS "/"
+     * Result: HTTP 200, body: Node infos in JSON format
+     * @param session
+     */
+    void getStats(const std::shared_ptr<restbed::Session>& session) const;
+
    /**
     * Return Values of an infoHash
     * Method: GET "/{InfoHash: .*}"
@@ -96,7 +163,7 @@ private:
     * On error: HTTP 503, body: {"err":"xxxx"}
     * @param session
     */
-    void listen(const std::shared_ptr<restbed::Session>& session) const;
+    void listen(const std::shared_ptr<restbed::Session>& session);

    /**
     * Put a value on the DHT
@@ -107,7 +174,9 @@ private:
     * HTTP 400, body: {"err":"xxxx"} if bad json or HTTP 502 if put fails
     * @param session
     */
-    void put(const std::shared_ptr<restbed::Session>& session) const;
+    void put(const std::shared_ptr<restbed::Session>& session);
+
+    void cancelPut(const InfoHash& key, Value::Id vid);

 #if OPENDHT_PROXY_SERVER_IDENTITY
    /**
@@ -154,43 +223,65 @@ private:
     */
    void handleOptionsMethod(const std::shared_ptr<restbed::Session>& session) const;

+    /**
+     * Remove finished listeners
+     * @param testSession if we remove the listener only if the session is closed
+     */
+    void removeClosedListeners(bool testSession = true);
+
 #if OPENDHT_PUSH_NOTIFICATIONS
    /**
     * Subscribe to push notifications for an iOS or Android device.
     * Method: SUBSCRIBE "/{InfoHash: .*}"
-     * Body: {"key": "device_key", (optional)"callback_id":y,
-     * (optional)"isAndroid":false (default true)}"
+     * Body: {"key": "device_key", (optional)"isAndroid":false (default true)}"
     * Return: {"token": x}" where x if a token to save
     * @note: the listen will timeout after six hours (and send a push notification).
     * so you need to refresh the operation each six hours.
-     * @note: callback_id is used to add the possibility to have multiple listen
-     * on same hash for same device and must be > 0
     * @param session
     */
-    void subscribe(const std::shared_ptr<restbed::Session>& session) const;
+    void subscribe(const std::shared_ptr<restbed::Session>& session);
    /**
     * Unsubscribe to push notifications for an iOS or Android device.
     * Method: UNSUBSCRIBE "/{InfoHash: .*}"
-     * Body: {"key": "device_key", "token": x, (optional)"callback_id":y"
-     * where x if the token to cancel
+     * Body: {"key": "device_key", "token": x} where x if the token to cancel
     * Return: nothing
-     * @note: callback id is used to add the possibility to have multiple listen
-     * on same hash for same device
     * @param session
     */
-    void unsubscribe(const std::shared_ptr<restbed::Session>& session) const;
+    void unsubscribe(const std::shared_ptr<restbed::Session>& session);
    /**
     * Send a push notification via a gorush push gateway
     * @param key of the device
     * @param json, the content to send
     */
    void sendPushNotification(const std::string& key, const Json::Value& json, bool isAndroid) const;
+
+    /**
+     * Remove a push listener between a client and a hash
+     * @param pushToken
+     * @param key
+     * @param clientId
+     */
+    void cancelPushListen(const std::string& pushToken, const InfoHash& key, const std::string& clientId);
+
+
 #endif //OPENDHT_PUSH_NOTIFICATIONS

+    using clock = std::chrono::steady_clock;
+    using time_point = clock::time_point;
+
    std::thread server_thread {};
    std::unique_ptr<restbed::Service> service_;
    std::shared_ptr<DhtRunner> dht_;

+    std::mutex schedulerLock_;
+    std::condition_variable schedulerCv_;
+    Scheduler scheduler_;
+    std::thread schedulerThread_;
+
+    Sp<Scheduler::Job> printStatsJob_;
+    mutable std::mutex statsMutex_;
+    mutable NodeInfo nodeInfo_ {};
+
    // Handle client quit for listen.
    // NOTE: can be simplified when we will supports restbed 5.0
    std::thread listenThread_;
@@ -199,37 +290,28 @@ private:
        InfoHash hash;
        std::future<size_t> token;
    };
-    mutable std::vector<SessionToHashToken> currentListeners_;
-    mutable std::mutex lockListener_;
+    std::vector<SessionToHashToken> currentListeners_;
+    std::mutex lockListener_;
    std::atomic_bool stopListeners {false};

-#if OPENDHT_PUSH_NOTIFICATIONS
-    struct PushListener {
-        std::string pushToken;
-        InfoHash hash;
-        unsigned token;
-        std::future<size_t> internalToken;
-        std::chrono::steady_clock::time_point deadline;
-        bool started {false};
-        unsigned callbackId {0};
-        std::string clientId {};
-        bool isAndroid {true};
-    };
-    mutable std::mutex lockPushListeners_;
-    mutable std::vector<PushListener> pushListeners_;
-    mutable unsigned tokenPushNotif_ {0};
-#endif //OPENDHT_PUSH_NOTIFICATIONS
+    struct PermanentPut;
+    struct SearchPuts;
+    std::map<InfoHash, SearchPuts> puts_;
+
+    mutable std::atomic<size_t> requestNum_ {0};
+    mutable std::atomic<time_point> lastStatsReset_ {time_point::min()};
+
    const std::string pushServer_;

-    /**
-     * Remove finished listeners
-     * @param testSession if we remove the listener only if the session is closed
-     */
-    void removeClosedListeners(bool testSession = true);
-    /**
-     * Launch or remove push listeners if needed
-     */
-    void handlePushListeners();
+    mutable ServerStats stats_;
+
+#if OPENDHT_PUSH_NOTIFICATIONS
+    struct Listener;
+    struct PushListener;
+    std::mutex lockPushListeners_;
+    std::map<std::string, PushListener> pushListeners_;
+    proxy::ListenToken tokenPushNotif_ {0};
+#endif //OPENDHT_PUSH_NOTIFICATIONS
 };

 }
@@ -136,7 +136,15 @@ public:
        query(hash, cb, bindDoneCb(done_cb), q);
    }

-    std::future<size_t> listen(InfoHash key, GetCallback vcb, Value::Filter f = Value::AllFilter(), Where w = {});
+    std::future<size_t> listen(InfoHash key, ValueCallback vcb, Value::Filter f = Value::AllFilter(), Where w = {});
+
+    std::future<size_t> listen(InfoHash key, GetCallback cb, Value::Filter f={}, Where w={}) {
+        return listen(key, [cb](const std::vector<Sp<Value>>& vals, bool expired){
+            if (not expired)
+                return cb(vals);
+            return true;
+        }, std::forward<Value::Filter>(f), std::forward<Where>(w));
+    }
    std::future<size_t> listen(const std::string& key, GetCallback vcb, Value::Filter f = Value::AllFilter(), Where w = {});
    std::future<size_t> listen(InfoHash key, GetCallbackSimple cb, Value::Filter f = Value::AllFilter(), Where w = {}) {
        return listen(key, bindGetCb(cb), f, w);
@@ -150,6 +158,15 @@ public:
        },
        getFilterSet<T>());
    }
+    template <class T>
+    std::future<size_t> listen(InfoHash hash, std::function<bool(std::vector<T>&&, bool)> cb)
+    {
+        return listen(hash, [=](const std::vector<std::shared_ptr<Value>>& vals, bool expired) {
+            return cb(unpackVector<T>(vals), expired);
+        },
+        getFilterSet<T>());
+    }
+
    template <typename T>
    std::future<size_t> listen(InfoHash hash, std::function<bool(T&&)> cb, Value::Filter f = Value::AllFilter(), Where w = {})
    {
@@ -166,6 +183,22 @@ public:
        },
        getFilterSet<T>(f), w);
    }
+    template <typename T>
+    std::future<size_t> listen(InfoHash hash, std::function<bool(T&&, bool)> cb, Value::Filter f = Value::AllFilter(), Where w = {})
+    {
+        return listen(hash, [=](const std::vector<std::shared_ptr<Value>>& vals, bool expired) {
+            for (const auto& v : vals) {
+                try {
+                    if (not cb(Value::unpack<T>(*v), expired))
+                        return false;
+                } catch (const std::exception&) {
+                    continue;
+                }
+            }
+            return true;
+        },
+        getFilterSet<T>(f), w);
+    }

    void cancelListen(InfoHash h, size_t token);
    void cancelListen(InfoHash h, std::shared_future<size_t> token);
@@ -240,8 +273,14 @@ public:

    void dumpTables() const;

+    /**
+     * Get the public key fingerprint if an identity is used with this node, 0 otherwise.
+     */
    InfoHash getId() const;

+    /**
+     * Get the ID of the DHT node.
+     */
    InfoHash getNodeId() const;

    /**
@@ -285,6 +324,7 @@ public:

    NodeStats getNodesStats(sa_family_t af) const;
    unsigned getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return, unsigned *incoming_return) const;
+    NodeInfo getNodeInfo() const;

    std::vector<unsigned> getNodeMessageStats(bool in = false) const;
    std::string getStorageLog() const;
@@ -307,7 +347,7 @@ public:
     * @param threaded: If false, ::loop() must be called periodically. Otherwise a thread is launched.
     * @param cb: Optional callback to receive general state information.
     */
-    void run(in_port_t port, const crypto::Identity identity, bool threaded = false, NetId network = 0) {
+    void run(in_port_t port = 4222, const crypto::Identity identity = {}, bool threaded = false, NetId network = 0) {
        run(port, {
            /*.dht_config = */{
                /*.node_config = */{
@@ -372,15 +412,7 @@ public:
     */
    void join();

-    void setProxyServer(const std::string& proxy, const std::string& pushNodeId = "") {
-#if OPENDHT_PROXY_CLIENT
-        if (config_.proxy_server == proxy and config_.push_node_id == pushNodeId)
-            return;
-        config_.proxy_server = proxy;
-        config_.push_node_id = pushNodeId;
-        enableProxy(use_proxy and not config_.proxy_server.empty());
-#endif
-    }
+    void setProxyServer(const std::string& proxy, const std::string& pushNodeId = "");

    /**
     * Start or stop the proxy
@@ -399,12 +431,7 @@ public:
    /**
     * Insert a push notification to process for OpenDHT
     */
-    void pushNotificationReceived(const std::map<std::string, std::string>& data) const;
-    /**
-     * Refresh a listen via a token
-     * @param token
-     */
-    void resubscribe(unsigned token);
+    void pushNotificationReceived(const std::map<std::string, std::string>& data);

    /* Proxy server mothods */
    void forwardAllMessages(bool forward);
@@ -420,6 +447,7 @@ private:
     */
    void tryBootstrapContinuously();

+    void stopNetwork();
    void startNetwork(const SockAddr sin4, const SockAddr sin6);
    time_point loop_();

@@ -451,15 +479,8 @@ private:
    /**
     * Store current listeners and translates global tokens for each client.
     */
-    struct Listener {
-        size_t tokenClassicDht;
-        size_t tokenProxyDht;
-        GetCallback gcb;
-        InfoHash hash;
-        Value::Filter f;
-        Where w;
-    };
-    std::map<size_t, Listener> listeners_ {};
+    struct Listener;
+    std::map<size_t, Listener> listeners_;
    size_t listener_token_ {1};

    mutable std::mutex dht_mtx {};
@@ -468,7 +489,13 @@ private:

    std::thread rcv_thread {};
    std::mutex sock_mtx {};
-    std::vector<std::pair<Blob, SockAddr>> rcv {};
+
+    struct ReceivedPacket {
+        Blob data;
+        SockAddr from;
+        time_point received;
+    };
+    std::queue<ReceivedPacket> rcv {};

    /** true if currently actively boostraping */
    std::atomic_bool bootstraping {false};
@@ -491,6 +518,7 @@ private:
               status6 {NodeStatus::Disconnected};
    StatusCallback statusCb {nullptr};

+    int stop_writefd {-1};
    int s4 {-1}, s6 {-1};
    SockAddr bound4 {};
    SockAddr bound6 {};
@@ -129,7 +129,7 @@ public:
     * Find the lowest 1 bit in an id.
     * Result will allways be lower than 8*N
     */
-    inline unsigned lowbit() const {
+    inline int lowbit() const {
        int i, j;
        for(i = N-1; i >= 0; i--)
            if(data_[i] != 0)
@@ -43,8 +43,23 @@ inline void NOLOG(char const*, va_list) {}
 struct LogMethod {
    LogMethod() = default;

+    LogMethod(LogMethod&& l) : func(std::move(l.func)) {}
+    LogMethod(const LogMethod& l) : func(l.func) {}
+
+    LogMethod& operator=(dht::LogMethod&& l) {
+        func = std::forward<LogMethod>(l.func);
+        return *this;
+    }
+    LogMethod& operator=(const dht::LogMethod& l) {
+        func = l.func;
+        return *this;
+    }
+
    template<typename T>
-    LogMethod(T&& t) : func(std::forward<T>(t)) {}
+    explicit LogMethod(T&& t) : func(std::forward<T>(t)) {}
+
+    template<typename T>
+    LogMethod(const T& t) : func(t) {}

    void operator()(char const* format, ...) const {
        va_list args;
@@ -70,7 +85,7 @@ private:
 };

 struct Logger {
-    LogMethod DEBUG = NOLOG;
+    LogMethod DBG = NOLOG;
    LogMethod WARN = NOLOG;
    LogMethod ERR = NOLOG;
    void setFilter(const InfoHash& f) {
@@ -99,7 +114,7 @@ struct Logger {
 #if OPENDHT_LOG
        va_list args;
        va_start(args, format);
-        log0(DEBUG, format, args);
+        log0(DBG, format, args);
        va_end(args);
 #endif
    }
@@ -107,7 +122,7 @@ struct Logger {
 #if OPENDHT_LOG
        va_list args;
        va_start(args, format);
-        log1(DEBUG, f, format, args);
+        log1(DBG, f, format, args);
        va_end(args);
 #endif
    }
@@ -115,7 +130,7 @@ struct Logger {
 #if OPENDHT_LOG
        va_list args;
        va_start(args, format);
-        log2(DEBUG, f1, f2, format, args);
+        log2(DBG, f1, f2, format, args);
        va_end(args);
 #endif
    }
@@ -1,5 +1,5 @@
 /*
- *  Copyright (C) 2014-2017 Savoir-faire Linux Inc.
+ *  Copyright (C) 2014-2018 Savoir-faire Linux Inc.
 *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 *              Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
 *
@@ -87,6 +87,8 @@ struct RequestAnswer {
    Blob ntoken {};
    Value::Id vid {};
    std::vector<Sp<Value>> values {};
+    std::vector<Value::Id> refreshed_values {};
+    std::vector<Value::Id> expired_values {};
    std::vector<Sp<FieldValueIndex>> fields {};
    std::vector<Sp<Node>> nodes4 {};
    std::vector<Sp<Node>> nodes6 {};
@@ -235,6 +237,9 @@ public:
            std::vector<Sp<Node>>&& nodes, std::vector<Sp<Node>>&& nodes6,
            std::vector<Sp<Value>>&& values, const Query& q);

+    void tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
+    void tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
+
    bool isRunning(sa_family_t af) const;
    inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; }

@@ -40,12 +40,16 @@ struct NodeCache {
    ~NodeCache();

 private:
-    class NodeMap : public std::map<InfoHash, std::weak_ptr<Node>> {
+    class NodeMap : private std::map<InfoHash, std::weak_ptr<Node>> {
    public:
        Sp<Node> getNode(const InfoHash& id);
        Sp<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client);
+        std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, size_t count) const;
        void clearBadNodes();
        void setExpired();
+        void cleanup();
+    private:
+        size_t cleanup_counter {0};
    };

    const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; }
@@ -0,0 +1,31 @@
+/*
+ *  Copyright (C) 2017-2018 Savoir-faire Linux Inc.
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <chrono>
+
+namespace dht {
+namespace proxy {
+
+constexpr const std::chrono::seconds OP_TIMEOUT {1 * 60 * 60}; // one hour
+constexpr const std::chrono::seconds OP_MARGIN {5 * 60}; // 5 minutes
+constexpr const char* const HTTP_PROTO {"http://"};
+using ListenToken = uint64_t;
+
+}
+}
@@ -25,7 +25,7 @@
 namespace dht {
 namespace crypto {

-#ifndef WIN32_NATIVE
+#ifndef _MSC_VER
 #ifdef _WIN32

 /**
@@ -36,11 +36,10 @@ namespace dht {
 */
 class Scheduler {
 public:
-    Scheduler(const Logger& l) : DHT_LOG(l) {}
-
    struct Job {
        Job(std::function<void()>&& f) : do_(std::move(f)) {}
        std::function<void()> do_;
+        void cancel() { do_ = {}; }
    };

    /**
@@ -58,17 +57,19 @@ public:
        return job;
    }

+    void add(const Sp<Scheduler::Job>& job, time_point t) {
+        if (t != time_point::max())
+            timers.emplace(std::move(t), job);
+    }
+
    /**
     * Reschedules a job.
     *
-     * @param time  The time at which the job shall be rescheduled.
     * @param job  The job to edit.
-     *
-     * @return pointer to the newly scheduled job.
+     * @param t  The time at which the job shall be rescheduled.
     */
    void edit(Sp<Scheduler::Job>& job, time_point t) {
        if (not job) {
-            DHT_LOG.ERR("editing an empty job");
            return;
        }
        // std::function move doesn't garantee to leave the object empty.
@@ -118,7 +119,6 @@ public:
 private:
    time_point now {clock::now()};
    std::multimap<time_point, Sp<Job>> timers {}; /* the jobs ordered by time */
-    const Logger& DHT_LOG;
 };

 }
@@ -71,7 +71,7 @@ public:
        return secureType(std::move(tmp_type));
    }

-    void registerType(const ValueType& type) {
+    void registerType(const ValueType& type) override {
        if (dht_)
            dht_->registerType(secureType(type));
    }
@@ -89,19 +89,17 @@ public:
     * If the signature can't be checked, or if the data can't be decrypted, it is not returned.
     * Public, non-signed & non-encrypted data is retransmitted as-is.
     */
-    void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}, Where&& w = {});
-    void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}, Where&& w = {}) {
+    void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}, Where&& w = {}) override;
+    void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}, Where&& w = {}) override {
        get(id, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) {
+    void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) override {
        get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) {
+    void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) override {
        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }

-    size_t listen(const InfoHash& id, GetCallback cb, Value::Filter = {}, Where w = {});
-
    /**
     * Will take ownership of the value, sign it using our private key and put it in the DHT.
     */
@@ -150,99 +148,100 @@ public:
    /**
     * SecureDht to Dht proxy
     */
-    void shutdown(ShutdownCallback cb) {
+    void shutdown(ShutdownCallback cb) override {
        dht_->shutdown(cb);
    }
-    void dumpTables() const {
+    void dumpTables() const override {
        dht_->dumpTables();
    }
-    inline const InfoHash& getNodeId() const { return dht_->getNodeId(); }
-    std::pair<size_t, size_t> getStoreSize() const {
+    inline const InfoHash& getNodeId() const override { return dht_->getNodeId(); }
+
+    std::pair<size_t, size_t> getStoreSize() const override {
        return dht_->getStoreSize();
    }
-    std::string getStorageLog() const {
+    std::string getStorageLog() const override {
        return dht_->getStorageLog();
    }
-    std::string getStorageLog(const InfoHash& h) const {
+    std::string getStorageLog(const InfoHash& h) const override {
        return dht_->getStorageLog(h);
    }
-    void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) {
+    void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) override {
        dht_->setStorageLimit(limit);
    }
-    std::vector<NodeExport> exportNodes() {
+    std::vector<NodeExport> exportNodes() override {
        return dht_->exportNodes();
    }
-    std::vector<ValuesExport> exportValues() const {
+    std::vector<ValuesExport> exportValues() const override {
        return dht_->exportValues();
    }
-    void importValues(const std::vector<ValuesExport>& v) {
+    void importValues(const std::vector<ValuesExport>& v) override {
        dht_->importValues(v);
    }
-    NodeStats getNodesStats(sa_family_t af) const {
+    NodeStats getNodesStats(sa_family_t af) const override {
        return dht_->getNodesStats(af);
    }
-    std::vector<unsigned> getNodeMessageStats(bool in = false) {
+    std::vector<unsigned> getNodeMessageStats(bool in = false) override {
        return dht_->getNodeMessageStats(in);
    }
-    std::string getRoutingTablesLog(sa_family_t af) const {
+    std::string getRoutingTablesLog(sa_family_t af) const override {
        return dht_->getRoutingTablesLog(af);
    }
-    std::string getSearchesLog(sa_family_t af) const {
+    std::string getSearchesLog(sa_family_t af) const override {
        return dht_->getSearchesLog(af);
    }
-    std::string getSearchLog(const InfoHash& h, sa_family_t af = AF_UNSPEC) const {
+    std::string getSearchLog(const InfoHash& h, sa_family_t af = AF_UNSPEC) const override {
        return dht_->getSearchLog(h, af);
    }
-    std::vector<SockAddr> getPublicAddress(sa_family_t family = 0) {
+    std::vector<SockAddr> getPublicAddress(sa_family_t family = 0) override {
        return dht_->getPublicAddress(family);
    }
-    time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr& sa) {
+    time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr& sa) override {
        return dht_->periodic(buf, buflen, sa);
    }
-    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) {
+    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) override {
        return dht_->periodic(buf, buflen, from, fromlen);
    }
-    NodeStatus getStatus(sa_family_t af) const {
+    NodeStatus getStatus(sa_family_t af) const override {
        return dht_->getStatus(af);
    }
-    NodeStatus getStatus() const {
+    NodeStatus getStatus() const override {
        return dht_->getStatus();
    }
-    bool isRunning(sa_family_t af = 0) const {
+    bool isRunning(sa_family_t af = 0) const override {
        return dht_->isRunning(af);
    }
-    const ValueType& getType(ValueType::Id type_id) const {
+    const ValueType& getType(ValueType::Id type_id) const override {
        return dht_->getType(type_id);
    }
-    void insertNode(const InfoHash& id, const SockAddr& sa) {
+    void insertNode(const InfoHash& id, const SockAddr& sa) override {
        dht_->insertNode(id, sa);
    }
-    void insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) {
+    void insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) override {
        dht_->insertNode(id, sa, salen);
    }
-    void insertNode(const NodeExport& n) {
+    void insertNode(const NodeExport& n) override {
        dht_->insertNode(n);
    }
-    void pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb={}) {
+    void pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb={}) override {
        dht_->pingNode(sa, salen, std::move(cb));
    }
-    void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {}) {
+    void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {}) override {
        dht_->query(key, cb, done_cb, std::move(q));
    }
-    void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) {
+    void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) override {
        dht_->query(key, cb, done_cb, std::move(q));
    }
-    std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const {
+    std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const override {
        return dht_->getLocal(key, f);
    }
-    Sp<Value> getLocalById(const InfoHash& key, Value::Id vid) const {
+    Sp<Value> getLocalById(const InfoHash& key, Value::Id vid) const override {
        return dht_->getLocalById(key, vid);
    }
    void put(const InfoHash& key,
            Sp<Value> v,
            DoneCallback cb=nullptr,
            time_point created=time_point::max(),
-            bool permanent = false)
+            bool permanent = false) override
    {
        dht_->put(key, v, cb, created, permanent);
    }
@@ -250,7 +249,7 @@ public:
            const Sp<Value>& v,
            DoneCallbackSimple cb,
            time_point created=time_point::max(),
-            bool permanent = false)
+            bool permanent = false) override
    {
        dht_->put(key, v, cb, created, permanent);
    }
@@ -259,7 +258,7 @@ public:
            Value&& v,
            DoneCallback cb=nullptr,
            time_point created=time_point::max(),
-            bool permanent = false)
+            bool permanent = false) override
    {
        dht_->put(key, std::move(v), cb, created, permanent);
    }
@@ -267,29 +266,32 @@ public:
            Value&& v,
            DoneCallbackSimple cb,
            time_point created=time_point::max(),
-            bool permanent = false)
+            bool permanent = false) override
    {
        dht_->put(key, std::move(v), cb, created, permanent);
    }
-    std::vector<Sp<Value>> getPut(const InfoHash& h) {
+    std::vector<Sp<Value>> getPut(const InfoHash& h) override {
        return dht_->getPut(h);
    }
-    Sp<Value> getPut(const InfoHash& h, const Value::Id& vid) {
+    Sp<Value> getPut(const InfoHash& h, const Value::Id& vid) override {
        return dht_->getPut(h, vid);
    }
-    bool cancelPut(const InfoHash& h, const Value::Id& vid) {
+    bool cancelPut(const InfoHash& h, const Value::Id& vid) override {
        return dht_->cancelPut(h, vid);
    }
-    size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) {
-        return dht_->listen(key, cb, f, w);
+
+    size_t listen(const InfoHash& key, ValueCallback, Value::Filter={}, Where={}) override;
+    size_t listen(const InfoHash& key, GetCallback cb, Value::Filter = {}, Where w = {}) override;
+    size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) override {
+        return listen(key, bindGetCb(cb), f, w);
    }
-    bool cancelListen(const InfoHash& h, size_t token) {
+    bool cancelListen(const InfoHash& h, size_t token) override {
        return dht_->cancelListen(h, token);
    }
-    void connectivityChanged(sa_family_t af) {
+    void connectivityChanged(sa_family_t af) override {
        dht_->connectivityChanged(af);
    }
-    void connectivityChanged() {
+    void connectivityChanged() override {
        dht_->connectivityChanged();
    }

@@ -297,7 +299,7 @@ public:
        forward_all_ = forward;
    }

-    void setPushNotificationToken(const std::string& token = "") {
+    void setPushNotificationToken(const std::string& token = "") override {
        dht_->setPushNotificationToken(token);
    }

@@ -305,29 +307,20 @@ public:
     * Call linked callback with push_notification
     * @param notification to process
     */
-    void pushNotificationReceived(const std::map<std::string, std::string>& notification) {
+    void pushNotificationReceived(const std::map<std::string, std::string>& notification) override {
        dht_->pushNotificationReceived(notification);
    }
-    /**
-     * Refresh a listen via a token
-     * @param token
-     */
-    void resubscribe(const unsigned token) {
-        dht_->resubscribe(token);
-    }

-    void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG)
+    void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG) override
    {
-        DHT_LOG.DEBUG = debug;
-        DHT_LOG.WARN = warn;
-        DHT_LOG.ERR = error;
-        dht_->setLoggers(std::forward<LogMethod>(error), std::forward<LogMethod>(warn), std::forward<LogMethod>(debug));
+        DhtInterface::setLoggers(error, warn, debug);
+        dht_->setLoggers(error, warn, debug);
    }

    /**
     * Only print logs related to the given InfoHash (if given), or disable filter (if zeroes).
     */
-    void setLogFilter(const InfoHash& f) {
+    void setLogFilter(const InfoHash& f) override {
        DHT_LOG.setFilter(f);
        dht_->setLogFilter(f);
    }
@@ -338,6 +331,8 @@ private:
    SecureDht(const SecureDht&) = delete;
    SecureDht& operator=(const SecureDht&) = delete;

+    Sp<Value> checkValue(const Sp<Value>& v);
+    ValueCallback getCallbackFilter(ValueCallback, Value::Filter&&);
    GetCallback getCallbackFilter(GetCallback, Value::Filter&&);

    Sp<crypto::PrivateKey> key_ {};
@@ -350,8 +345,6 @@ private:
    std::map<InfoHash, Sp<crypto::Certificate>> nodesCertificates_ {};
    std::map<InfoHash, Sp<const crypto::PublicKey>> nodesPubKeys_ {};

-    std::uniform_int_distribution<Value::Id> rand_id {};
-
    std::atomic_bool forward_all_ {false};
 };

@@ -67,7 +67,7 @@ public:
     */
    SockAddr(const sockaddr* sa, socklen_t length) {
        if (length > sizeof(sockaddr_storage))
-            throw std::runtime_error("Socket address length is too large");
+            throw std::invalid_argument("Socket address length is too large");
        set(sa, length);
    }
    SockAddr(const sockaddr* sa) {
@@ -78,7 +78,7 @@ public:
            else if(sa->sa_family == AF_INET6)
                len = sizeof(sockaddr_in6);
            else
-                throw std::runtime_error("Unknown address family");
+                throw std::invalid_argument("Unknown address family");
        }
        set(sa, len);
    }
@@ -54,6 +54,12 @@ void erase_if(std::map<Key, Item>& map, const Condition& condition)
    }
 }

+/**
+ * Split "[host]:port" or "host:port" to pair<"host", "port">.
+ */
+OPENDHT_PUBLIC std::pair<std::string, std::string>
+splitPort(const std::string& s);
+
 class OPENDHT_PUBLIC DhtException : public std::runtime_error {
 public:
    DhtException(const std::string &str = "") :
@@ -37,9 +37,9 @@
 #include <chrono>
 #include <set>

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
 #include <json/json.h>
-#endif //OPENDHT_PROXY_SERVER
+#endif

 namespace dht {

@@ -109,6 +109,21 @@ struct OPENDHT_PUBLIC ValueType {
    EditPolicy editPolicy {DEFAULT_EDIT_POLICY};
 };

+class TypeStore {
+public:
+    void registerType(const ValueType& type) {
+        types[type.id] = type;
+    }
+    const ValueType& getType(ValueType::Id type_id) const {
+        const auto& t_it = types.find(type_id);
+        return (t_it == types.end()) ? ValueType::USER_DATA : t_it->second;
+    }
+private:
+    std::map<ValueType::Id, ValueType> types {};
+};
+
+struct CryptoValueCache;
+
 /**
 * A "value" is data potentially stored on the Dht, with some metadata.
 *
@@ -174,6 +189,15 @@ struct OPENDHT_PUBLIC Value
                return f1(v) or f2(v);
            };
        }
+        std::vector<Sp<Value>> filter(const std::vector<Sp<Value>>& values) {
+            if (not (*this))
+                return values;
+            std::vector<Sp<Value>> ret;
+            for (const auto& v : values)
+                if ((*this)(v))
+                    ret.emplace_back(v);
+            return ret;
+        }
    };

    /* Sneaky functions disguised in classes */
@@ -350,13 +374,13 @@ struct OPENDHT_PUBLIC Value
    Value(ValueType::Id t, const uint8_t* dat_ptr, size_t dat_len, Id id = INVALID_ID)
     : id(id), type(t), data(dat_ptr, dat_ptr+dat_len) {}

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
    /**
     * Build a value from a json object
     * @param json
     */
    Value(Json::Value& json);
-#endif //OPENDHT_PROXY_SERVER
+#endif

    template <typename Type>
    Value(ValueType::Id t, const Type& d, Id id = INVALID_ID)
@@ -429,7 +453,7 @@ struct OPENDHT_PUBLIC Value
        return ss.str();
    }

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
    /**
     * Build a json object from a value
     * Example:
@@ -439,7 +463,7 @@ struct OPENDHT_PUBLIC Value
     * }
     */
    Json::Value toJson() const;
-#endif //OPENDHT_PROXY_SERVER
+#endif

    /** Return the size in bytes used by this value in memory (minimum). */
    size_t size() const;
@@ -553,7 +577,7 @@ struct OPENDHT_PUBLIC Value
    std::string user_type {};

    /**
-     * Sequence number to avoid replay attacks
+     * Sequence number
     */
    uint16_t seq {0};

@@ -566,6 +590,16 @@ struct OPENDHT_PUBLIC Value
     * Hold encrypted version of the data.
     */
    Blob cypher {};
+
+    bool checkSignature();
+    Sp<Value> decrypt(const crypto::PrivateKey& key);
+
+private:
+    /* Cache for crypto ops */
+    bool signatureChecked {false};
+    bool signatureValid {false};
+    bool decrypted {false};
+    Sp<Value> decryptedValue {};
 };

 using ValuesExport = std::pair<InfoHash, Blob>;
@@ -820,6 +854,10 @@ struct OPENDHT_PUBLIC Where
        return ss.str();
    }

+    bool empty() const {
+        return filters_.empty();
+    }
+
    OPENDHT_PUBLIC friend std::ostream& operator<<(std::ostream& s, const dht::Where& q);

 private:
@@ -970,6 +1008,10 @@ unpackVector(const std::vector<std::shared_ptr<Value>>& vals) {
    return ret;
 }

+#ifdef OPENDHT_JSONCPP
+uint64_t unpackId(const Json::Value& json, const std::string& key);
+#endif
+
 }

 MSGPACK_ADD_ENUM(dht::Value::Field)
@@ -1,7 +1,7 @@
 lib_LTLIBRARIES = libopendht.la

-libopendht_la_CPPFLAGS = @CPPFLAGS@ -I$(top_srcdir)/include/opendht @Argon2_CFLAGS@
-libopendht_la_LIBADD   = @Argon2_LIBS@ @GnuTLS_LIBS@ @Nettle_LIBS@
+libopendht_la_CPPFLAGS = @CPPFLAGS@ -I$(top_srcdir)/include/opendht @Argon2_CFLAGS@ @JsonCpp_CFLAGS@ @MsgPack_CFLAGS@
+libopendht_la_LIBADD   = @Argon2_LIBS@ @JsonCpp_LIBS@ @GnuTLS_LIBS@ @Nettle_LIBS@
 libopendht_la_LDFLAGS  = @LDFLAGS@ @Argon2_LDFLAGS@
 libopendht_la_SOURCES  = \
        dht.cpp \
@@ -9,6 +9,9 @@ libopendht_la_SOURCES  = \
        listener.h \
        request.h \
        search.h \
+        value_cache.h \
+        op_cache.h \
+        op_cache.cpp \
        net.h \
        parsed_message.h \
        node_cache.cpp \
@@ -53,7 +56,7 @@ nobase_include_HEADERS = \
        ../include/opendht/rng.h

 if ENABLE_PROXY_SERVER
-libopendht_la_SOURCES += base64.h base64.cpp dht_proxy_server.cpp
+libopendht_la_SOURCES += dht_proxy_server.cpp
 nobase_include_HEADERS += ../include/opendht/dht_proxy_server.h
 endif

@@ -62,6 +65,11 @@ libopendht_la_SOURCES += dht_proxy_client.cpp
 nobase_include_HEADERS += ../include/opendht/dht_proxy_client.h ../include/opendht/dht_interface.h
 endif

+libopendht_la_SOURCES += base64.h base64.cpp
+if PROXY_CLIENT_OR_SERVER
+nobase_include_HEADERS += ../include/opendht/proxy.h
+endif
+
 if ENABLE_INDEXATION
 libopendht_la_SOURCES += indexation/pht.cpp
 nobase_include_HEADERS += ../include/opendht/indexation/pht.h
@@ -62,13 +62,12 @@ NodeStats::toString() const
    ss << "Known nodes: " << good_nodes << " good, " << dubious_nodes << " dubious, " << incoming_nodes << " incoming." << std::endl;
    if (table_depth > 1) {
        ss << "Routing table depth: " << table_depth << std::endl;
-        unsigned long tot_nodes = 8 * std::exp2(table_depth);
-        ss << "Network size estimation: " << tot_nodes << " nodes" << std::endl;
+        ss << "Network size estimation: " << getNetworkSizeEstimation() << " nodes" << std::endl;
    }
    return ss.str();
 }

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
 /**
 * Build a json object from a NodeStats
 */
@@ -81,8 +80,7 @@ NodeStats::toJson() const
    val["incoming"] = static_cast<Json::LargestUInt>(incoming_nodes);
    if (table_depth > 1) {
        val["table_depth"] = static_cast<Json::LargestUInt>(table_depth);
-        unsigned long tot_nodes = 8 * std::exp2(table_depth);
-        val["network_size_estimation"] = static_cast<Json::LargestUInt>(tot_nodes);
+        val["network_size_estimation"] = static_cast<Json::LargestUInt>(getNetworkSizeEstimation());
    }
    return val;
 }
@@ -98,6 +96,32 @@ NodeStats::NodeStats(const Json::Value& val)
    if (val.isMember("table_depth"))
        table_depth = static_cast<unsigned>(val["table_depth"].asLargestUInt());
 }
-#endif //OPENDHT_PROXY_SERVER
+
+/**
+ * Build a json object from a NodeStats
+ */
+Json::Value
+NodeInfo::toJson() const
+{
+    Json::Value val;
+    if (id)
+        val["id"] = id.toString();
+    val["node_id"] = node_id.toString();
+    val["ipv4"] = ipv4.toJson();
+    val["ipv6"] = ipv6.toJson();
+    return val;
+}
+
+NodeInfo::NodeInfo(const Json::Value& v)
+{
+    if (v.isMember("id"))
+        id = InfoHash(v["id"].asString());
+    node_id = InfoHash(v["node_id"].asString());
+    ipv4 = NodeStats(v["ipv4"]);
+    ipv6 = NodeStats(v["ipv6"]);
+}
+
+#endif
+

 }
@@ -411,6 +411,8 @@ PublicKey::operator=(PublicKey&& o) noexcept
 void
 PublicKey::pack(Blob& b) const
 {
+    if (not pk)
+        throw CryptoException(std::string("Could not export public key: null key"));
    std::vector<uint8_t> tmp(2048);
    size_t sz = tmp.size();
    int err = gnutls_pubkey_export(pk, GNUTLS_X509_FMT_DER, tmp.data(), &sz);
@@ -437,6 +439,8 @@ PublicKey::unpack(const uint8_t* data, size_t data_size)
 std::string
 PublicKey::toString() const
 {
+    if (not pk)
+        throw CryptoException(std::string("Could not print public key: null key"));
    std::string ret;
    size_t sz = ret.size();
    int err = gnutls_pubkey_export(pk, GNUTLS_X509_FMT_PEM, (void*)ret.data(), &sz);
@@ -180,7 +180,7 @@ Dht::expireBuckets(RoutingTable& list)
 {
    for (auto& b : list) {
        bool changed = false;
-        b.nodes.remove_if([this,&changed](const Sp<Node>& n) {
+        b.nodes.remove_if([&changed](const Sp<Node>& n) {
            if (n->isExpired()) {
                changed = true;
                return true;
@@ -261,11 +261,11 @@ void Dht::paginate(std::weak_ptr<Search> ws, Sp<Query> query, SearchNode* n) {
    auto select_q = std::make_shared<Query>(Select {}.field(Value::Field::Id), query ? query->where : Where {});
    auto onSelectDone = [this,ws,query](const net::Request& status,
                                        net::RequestAnswer&& answer) mutable {
-        // retreive search
+        // Retrieve search
        auto sr = ws.lock();
        if (not sr) return;
        const auto& id = sr->id;
-        // retreive search node
+        // Retrieve search node
        auto sn = sr->getNode(status.node);
        if (not sn) return;
        // backward compatibility
@@ -319,11 +319,12 @@ Dht::searchSendGetValues(Sp<Search> sr, SearchNode* pn, bool update)

    std::weak_ptr<Search> ws = sr;
    auto cb = sr->callbacks.begin();
+    static const auto ANY_QUERY = std::make_shared<Query>(Select {}, Where {}, true);
    do { /* for all requests to send */
        SearchNode* n = nullptr;
-        auto query = not sr->callbacks.empty() ? cb->second.query : std::make_shared<Query>(Select {}, Where {}, true);
-        const time_point up = not sr->callbacks.empty() and update
-                                ? sr->getLastGetTime(query)
+        auto& query = sr->callbacks.empty() ? ANY_QUERY : cb->second.query;
+        const time_point up = (not sr->callbacks.empty() and update)
+                                ? sr->getLastGetTime(*query)
                                : time_point::min();

        if (pn and pn->canGet(now, up, query)) {
@@ -380,108 +381,212 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
    if (sr->announce.empty())
        return;
    unsigned i = 0;
-    auto probe_query = std::make_shared<Query>(Select {}.field(Value::Field::Id).field(Value::Field::SeqNum));
    std::weak_ptr<Search> ws = sr;
+
+    auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer)
+    { /* when put done */
+        if (auto sr = ws.lock()) {
+            onAnnounceDone(req.node, answer, sr);
+            searchStep(sr);
+        }
+    };
+
+    auto onExpired = [this,ws](const net::Request&, bool over)
+    { /* when put expired */
+        if (over)
+            if (auto sr = ws.lock())
+                scheduler.edit(sr->nextSearchStep, scheduler.time());
+    };
+
+    auto onSelectDone =
+    [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable
+    { /* on probing done */
+        auto sr = ws.lock();
+        if (not sr) return;
+        const auto& now = scheduler.time();
+        sr->insertNode(req.node, scheduler.time(), answer.ntoken);
+        auto sn = sr->getNode(req.node);
+        if (not sn) return;
+
+        if (not sn->isSynced(now)) {
+            /* Search is now unsynced. Let's call searchStep to sync again. */
+            scheduler.edit(sr->nextSearchStep, now);
+            return;
+        }
+        for (auto& a : sr->announce) {
+            if (sn->getAnnounceTime(a.value->id) > now)
+                continue;
+            bool hasValue {false};
+            uint16_t seq_no = 0;
+            try {
+                const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(),
+                        [&a](const Sp<FieldValueIndex>& i){
+                            return i->index.at(Value::Field::Id).getInt() == a.value->id;
+                        });
+                if (f != answer.fields.cend() and *f) {
+                    hasValue = true;
+                    seq_no = static_cast<uint16_t>((*f)->index.at(Value::Field::SeqNum).getInt());
+                }
+            } catch (std::out_of_range&) { }
+
+            auto next_refresh_time = now + getType(a.value->type).expiration;
+            /* only put the value if the node doesn't already have it */
+            if (not hasValue or seq_no < a.value->seq) {
+                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)",
+                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
+                sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node,
+                                                sr->id,
+                                                a.value,
+                                                a.permanent ? time_point::max() : a.created,
+                                                sn->token,
+                                                onDone,
+                                                onExpired), next_refresh_time);
+            } else if (hasValue and a.permanent) {
+                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)",
+                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
+                sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node,
+                                                sr->id,
+                                                a.value->id,
+                                                sn->token,
+                                                onDone,
+                                                onExpired), next_refresh_time);
+            } else {
+                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.",
+                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
+                auto ack_req = std::make_shared<net::Request>(net::Request::State::COMPLETED);
+                ack_req->reply_time = now;
+                sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time);
+
+                /* step to clear announces */
+                scheduler.edit(sr->nextSearchStep, now);
+            }
+            if (a.permanent) {
+                scheduler.add(next_refresh_time - REANNOUNCE_MARGIN, [this,ws] {
+                    if (auto sr = ws.lock()) {
+                        searchStep(sr);
+                    }
+                });
+            }
+        }
+    };
+
+    Sp<Query> probe_query {};
    const auto& now = scheduler.time();
    for (auto& n : sr->nodes) {
        if (not n.isSynced(now))
            continue;
-        if (std::find_if(sr->announce.cbegin(), sr->announce.cend(),
-            [this,&now,&n](const Announce& a) {
-                return n.getAnnounceTime(a.value->id) <= now;
-            }) == sr->announce.cend())
+
+        const auto& gs = n.probe_query ? n.getStatus.find(n.probe_query) : n.getStatus.cend();
+        if (gs != n.getStatus.cend() and gs->second and gs->second->pending()) {
            continue;
+        }

-        auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer)
-        { /* when put done */
-            if (auto sr = ws.lock()) {
-                onAnnounceDone(req.node, answer, sr);
-                searchStep(sr);
+        bool sendQuery = false;
+        for (auto& a : sr->announce) {
+            if (n.getAnnounceTime(a.value->id) <= now) {
+                if (a.permanent) {
+                    sendQuery = true;
+                } else {
+                    DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'put' (vid: %d)",
+                            sr->id.toString().c_str(), n.node->toString().c_str(), a.value->id);
+                    n.acked[a.value->id] = {
+                        network_engine.sendAnnounceValue(n.node, sr->id, a.value, a.created, n.token, onDone, onExpired),
+                        now + getType(a.value->type).expiration
+                    };
+                }
            }
-        };
-        auto onExpired = [this,ws](const net::Request&, bool over)
-        { /* when put expired */
-            if (over)
-                if (auto sr = ws.lock())
-                    scheduler.edit(sr->nextSearchStep, scheduler.time());
-        };
-        auto onSelectDone =
-            [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable
-            { /* on probing done */
-                auto sr = ws.lock();
-                if (not sr) return;
-                auto sn = sr->getNode(req.node);
-                if (not sn) return;
+        }

-                const auto& now = scheduler.time();
-                if (not sn->isSynced(now)) {
-                    /* Search is now unsynced. Let's call searchStep to sync again. */
-                    scheduler.edit(sr->nextSearchStep, now);
-                    return;
-                }
-                for (auto& a : sr->announce) {
-                    if (sn->getAnnounceTime(a.value->id) > now)
-                        continue;
-                    bool hasValue {false};
-                    uint16_t seq_no = 0;
-                    try {
-                        const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(),
-                                [&a](const Sp<FieldValueIndex>& i){
-                                    return i->index.at(Value::Field::Id).getInt() == a.value->id;
-                                });
-                        if (f != answer.fields.cend() and *f) {
-                            hasValue = true;
-                            seq_no = static_cast<uint16_t>((*f)->index.at(Value::Field::SeqNum).getInt());
-                        }
-                    } catch (std::out_of_range&) { }
-
-                    auto next_refresh_time = now + getType(a.value->type).expiration;
-                    /* only put the value if the node doesn't already have it */
-                    if (not hasValue or seq_no < a.value->seq) {
-                        DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)",
-                                sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
-                        sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node,
-                                                        sr->id,
-                                                        a.value,
-                                                        a.permanent ? time_point::max() : a.created,
-                                                        sn->token,
-                                                        onDone,
-                                                        onExpired), next_refresh_time);
-                    } else if (hasValue and a.permanent) {
-                        DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)",
-                                sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
-                        sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node,
-                                                        sr->id,
-                                                        a.value->id,
-                                                        sn->token,
-                                                        onDone,
-                                                        onExpired), next_refresh_time);
-                    } else {
-                        DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.",
-                                sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
-                        auto ack_req = std::make_shared<net::Request>(net::Request::State::COMPLETED);
-                        ack_req->reply_time = now;
-                        sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time);
-
-                        /* step to clear announces */
-                        scheduler.edit(sr->nextSearchStep, now);
-                    }
-                }
-            };
-        DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending %s",
-                sr->id.toString().c_str(), n.node->toString().c_str(), probe_query->toString().c_str());
-        n.probe_query = probe_query;
-        n.getStatus[probe_query] = network_engine.sendGetValues(n.node,
-                sr->id,
-                *probe_query,
-                -1,
-                onSelectDone,
-                std::bind(&Dht::searchNodeGetExpired, this, _1, _2, ws, probe_query));
+        if (sendQuery) {
+            if (not probe_query)
+                probe_query = std::make_shared<Query>(Select {}.field(Value::Field::Id).field(Value::Field::SeqNum));
+            DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending %s",
+                    sr->id.toString().c_str(), n.node->toString().c_str(), probe_query->toString().c_str());
+            n.probe_query = probe_query;
+            n.getStatus[probe_query] = network_engine.sendGetValues(n.node,
+                    sr->id,
+                    *probe_query,
+                    -1,
+                    onSelectDone,
+                    std::bind(&Dht::searchNodeGetExpired, this, _1, _2, ws, probe_query));
+        }
        if (not n.candidate and ++i == TARGET_NODES)
            break;
    }
 }

+void
+Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
+{
+    std::weak_ptr<Search> ws = sr;
+    for (const auto& l : sr->listeners) {
+        const auto& query = l.second.query;
+        auto list_token = l.first;
+        if (n.getListenTime(query) > scheduler.time())
+            continue;
+        DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
+                sr->id.toString().c_str(), n.node->toString().c_str());
+
+        auto r = n.listenStatus.find(query);
+        if (r == n.listenStatus.end()) {
+            r = n.listenStatus.emplace(query, SearchNode::CachedListenStatus{
+                [ws,list_token](const std::vector<Sp<Value>>& values, bool expired){
+                    if (auto sr = ws.lock()) {
+                        auto l = sr->listeners.find(list_token);
+                        if (l != sr->listeners.end()) {
+                            l->second.get_cb(l->second.filter.filter(values), expired);
+                        }
+                    }
+                }
+            }).first;
+            auto node = n.node;
+            r->second.cacheExpirationJob = scheduler.add(time_point::max(), [this,ws,query,node]{
+                if (auto sr = ws.lock()) {
+                    if (auto sn = sr->getNode(node)) {
+                        sn->expireValues(query, scheduler);
+                    }
+                }
+            });
+        }
+        auto prev_req = r != n.listenStatus.end() ? r->second.req : nullptr;
+        auto new_req = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req,
+            [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable
+            { /* on done */
+                if (auto sr = ws.lock()) {
+                    scheduler.edit(sr->nextSearchStep, scheduler.time());
+                    if (auto sn = sr->getNode(req.node))
+                        scheduler.add(sn->getListenTime(query), std::bind(&Dht::searchStep, this, sr));
+                    onListenDone(req.node, answer, sr);
+                }
+            },
+            [this,ws,query](const net::Request& req, bool over) mutable
+            { /* on request expired */
+                if (auto sr = ws.lock()) {
+                    scheduler.edit(sr->nextSearchStep, scheduler.time());
+                    if (over)
+                        if (auto sn = sr->getNode(req.node))
+                            sn->listenStatus.erase(query);
+                }
+            },
+            [this,ws,query](const Sp<Node>& node, net::RequestAnswer&& answer) mutable
+            { /* on new values */
+                if (auto sr = ws.lock()) {
+                    scheduler.edit(sr->nextSearchStep, scheduler.time());
+                    sr->insertNode(node, scheduler.time(), answer.ntoken);
+                    if (auto sn = sr->getNode(node)) {
+                        sn->onValues(query, std::move(answer), types, scheduler);
+                    }
+                }
+            }
+        );
+        // Here the request may have failed and the CachedListenStatus removed
+        r = n.listenStatus.find(query);
+        if (r != n.listenStatus.end()) {
+            r->second.req = new_req;
+        }
+    }
+}
+
 /* When a search is in progress, we periodically call search_step to send
   further requests. */
 void
@@ -530,7 +635,7 @@ Dht::searchStep(Sp<Search> sr)
        // true if this node is part of the target nodes cluter.
        /*bool in = sr->id.xorCmp(myid, sr->nodes.back().node->id) < 0;

-        DHT_LOG_DEBUG("[search %s IPv%c] synced%s",
+        DHT_LOG_DBG("[search %s IPv%c] synced%s",
                sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', in ? ", in" : "");*/

        if (not sr->listeners.empty()) {
@@ -538,45 +643,7 @@ Dht::searchStep(Sp<Search> sr)
            for (auto& n : sr->nodes) {
                if (not n.isSynced(now))
                    continue;
-                for (const auto& l : sr->listeners) {
-                    const auto& query = l.second.query;
-                    if (n.getListenTime(query) > now)
-                        continue;
-                    DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
-                            sr->id.toString().c_str(), n.node->toString().c_str());
-
-                    const auto& r = n.listenStatus.find(query);
-                    auto prev_req = r != n.listenStatus.end() ? r->second : nullptr;
-
-                    std::weak_ptr<Search> ws = sr;
-                    n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req,
-                        [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable
-                        { /* on done */
-                            if (auto sr = ws.lock()) {
-                                scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                if (auto sn = sr->getNode(req.node))
-                                    scheduler.add(sn->getListenTime(query), std::bind(&Dht::searchStep, this, sr));
-                                onListenDone(req.node, answer, sr);
-                            }
-                        },
-                        [this,ws,query](const net::Request& req, bool over) mutable
-                        { /* on expired */
-                            if (auto sr = ws.lock()) {
-                                scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                if (over)
-                                    if (auto sn = sr->getNode(req.node))
-                                        sn->listenStatus.erase(query);
-                            }
-                        },
-                        [this,ws,query](const Sp<Node>& node, net::RequestAnswer&& answer) mutable
-                        { /* on new values */
-                            if (auto sr = ws.lock()) {
-                                scheduler.edit(sr->nextSearchStep, scheduler.time());
-                                onGetValuesDone(node, answer, sr, query);
-                            }
-                        }
-                    );
-                }
+                searchSynchedNodeListen(sr, n);
                if (not n.candidate and ++i == LISTEN_NODES)
                    break;
            }
@@ -598,7 +665,7 @@ Dht::searchStep(Sp<Search> sr)
                i++;
        }
        while (sent and sr->currentlySolicitedNodeCount() < MAX_REQUESTED_SEARCH_NODES);
-        /*DHT_LOG_DEBUG("[search %s IPv%c] step: sent %u requests (total %u).",
+        /*DHT_LOG_DBG("[search %s IPv%c] step: sent %u requests (total %u).",
            sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', i, sr->currentlySolicitedNodeCount());*/
    }

@@ -643,7 +710,7 @@ unsigned Dht::refill(Dht::Search& sr) {

 /* Start a search. */
 Sp<Dht::Search>
-Dht::search(const InfoHash& id, sa_family_t af, GetCallback gcb, QueryCallback qcb, DoneCallback dcb, Value::Filter f, Query q)
+Dht::search(const InfoHash& id, sa_family_t af, GetCallback gcb, QueryCallback qcb, DoneCallback dcb, Value::Filter f, const Sp<Query>& q)
 {
    if (!isRunning(af)) {
        DHT_LOG.e(id, "[search %s IPv%c] unsupported protocol", id.toString().c_str(), (af == AF_INET) ? '4' : '6');
@@ -685,26 +752,14 @@ Dht::search(const InfoHash& id, sa_family_t af, GetCallback gcb, QueryCallback q
        sr->expired = false;
        sr->nodes.clear();
        sr->nodes.reserve(SEARCH_NODES+1);
+        sr->nextSearchStep = scheduler.add(time_point::max(), std::bind(&Dht::searchStep, this, sr));
        DHT_LOG.w(id, "[search %s IPv%c] new search", id.toString().c_str(), (af == AF_INET) ? '4' : '6');
        if (search_id == 0)
            search_id++;
    }

-    if (gcb or qcb) {
-        auto now = scheduler.time();
-        sr->callbacks.insert(std::make_pair<time_point, Get>(
-            std::move(now),
-            Get { scheduler.time(), f, std::make_shared<Query>(q),
-                qcb ? qcb : QueryCallback {}, gcb ? gcb : GetCallback {}, dcb
-            }
-        ));
-    }
-
+    sr->get(f, q, qcb, gcb, dcb, scheduler);
    refill(*sr);
-    if (sr->nextSearchStep)
-        scheduler.edit(sr->nextSearchStep, scheduler.time());
-    else
-        sr->nextSearchStep = scheduler.add(scheduler.time(), std::bind(&Dht::searchStep, this, sr));

    return sr;
 }
@@ -717,15 +772,6 @@ Dht::announce(const InfoHash& id,
        time_point created,
        bool permanent)
 {
-    const auto& now = scheduler.time();
-    if (!value) {
-        if (callback)
-            callback(false, {});
-        return;
-    }
-    created = std::min(now, created);
-    storageStore(id, value, created);
-
    auto& srs = searches(af);
    auto srp = srs.find(id);
    auto sr = srp == srs.end() ? search(id, af) : srp->second;
@@ -772,9 +818,8 @@ Dht::announce(const InfoHash& id,
 }

 size_t
-Dht::listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter f, const Sp<Query>& q)
+Dht::listenTo(const InfoHash& id, sa_family_t af, ValueCallback cb, Value::Filter f, const Sp<Query>& q)
 {
-    const auto& now = scheduler.time();
    if (!isRunning(af))
        return 0;
       // DHT_LOG_ERR("[search %s IPv%c] search_time is now in %lfs", sr->id.toString().c_str(), (sr->af == AF_INET) ? '4' : '6', print_dt(tm-clock::now()));
@@ -786,15 +831,11 @@ Dht::listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter
    if (!sr)
        throw DhtException("Can't create search");
    DHT_LOG.e(id, "[search %s IPv%c] listen", id.toString().c_str(), (af == AF_INET) ? '4' : '6');
-    sr->done = false;
-    auto token = ++sr->listener_token;
-    sr->listeners.emplace(token, LocalListener{q, f, cb});
-    scheduler.edit(sr->nextSearchStep, now);
-    return token;
+    return sr->listen(cb, f, q, scheduler);
 }

 size_t
-Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where where)
+Dht::listen(const InfoHash& id, ValueCallback cb, Value::Filter f, Where where)
 {
    scheduler.syncTime();

@@ -802,26 +843,7 @@ Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where where)
    auto vals = std::make_shared<std::map<Value::Id, Sp<Value>>>();
    auto token = ++listener_token;

-    auto gcb = [=](const std::vector<Sp<Value>>& values) {
-        std::vector<Sp<Value>> newvals;
-        for (const auto& v : values) {
-            auto it = vals->find(v->id);
-            if (it == vals->cend() || !(*it->second == *v))
-                newvals.push_back(v);
-        }
-        if (!newvals.empty()) {
-            if (!cb(newvals)) {
-                cancelListen(id, token);
-                return false;
-            }
-            for (const auto& v : newvals) {
-                auto it = vals->emplace(v->id, v);
-                if (not it.second)
-                    it.first->second = v;
-            }
-        }
-        return true;
-    };
+    auto gcb = OpValueCache::cacheCallback(std::move(cb));

    auto query = std::make_shared<Query>(q);
    auto filter = f.chain(q.where.getFilter());
@@ -833,7 +855,7 @@ Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where where)
        if (not st->second.empty()) {
            std::vector<Sp<Value>> newvals = st->second.get(filter);
            if (not newvals.empty()) {
-                if (!cb(newvals))
+                if (!gcb(newvals, false))
                    return 0;
                for (const auto& v : newvals) {
                    auto it = vals->emplace(v->id, v);
@@ -870,10 +892,10 @@ Dht::cancelListen(const InfoHash& id, size_t token)
    if (st != store.end() && tokenlocal)
        st->second.local_listeners.erase(tokenlocal);

-    auto searches_cancel_listen = [&id](std::map<InfoHash, Sp<Search>>& srs, size_t token) {
+    auto searches_cancel_listen = [this,&id](std::map<InfoHash, Sp<Search>>& srs, size_t token) {
        auto srp = srs.find(id);
        if (srp != srs.end() and token)
-            srp->second->cancelListen(token);
+            srp->second->cancelListen(token, scheduler);
    };
    searches_cancel_listen(searches4, std::get<1>(it->second));
    searches_cancel_listen(searches6, std::get<2>(it->second));
@@ -901,13 +923,20 @@ struct GetStatus : public OpStatus {
 void
 Dht::put(const InfoHash& id, Sp<Value> val, DoneCallback callback, time_point created, bool permanent)
 {
-    scheduler.syncTime();
-
+    if (not val) {
+        if (callback)
+            callback(false, {});
+        return;
+    }
    if (val->id == Value::INVALID_ID) {
        crypto::random_device rdev;
        std::uniform_int_distribution<Value::Id> rand_id {};
        val->id = rand_id(rdev);
    }
+    scheduler.syncTime();
+    const auto& now = scheduler.time();
+    created = std::min(now, created);
+    storageStore(id, val, created, {}, permanent);

    DHT_LOG.d(id, "put: adding %s -> %s", id.toString().c_str(), val->toString().c_str());

@@ -970,10 +999,10 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
 {
    scheduler.syncTime();

-    Query q {{}, where};
+    auto q = std::make_shared<Query>(Select {}, std::move(where));
    auto op = std::make_shared<GetStatus<Value>>();
+    auto f = filter.chain(q->where.getFilter());

-    auto f = filter.chain(q.where.getFilter());
    auto add_values = [op,f](const std::vector<Sp<Value>>& values) {
        std::vector<Sp<Value>> newvals {};
        for (const auto& v : values) {
@@ -1039,16 +1068,17 @@ void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Quer
    /* Try to answer this search locally. */
    qcb(local_fields);

+    auto sq = std::make_shared<Query>(std::move(q));
    Dht::search(id, AF_INET, {}, qcb, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
        //DHT_LOG_WARN("DHT done IPv4");
        op->status4 = {true, ok};
        doneCallbackWrapper(done_cb, nodes, *op);
-    }, f, q);
+    }, f, sq);
    Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, const std::vector<Sp<Node>>& nodes) {
        //DHT_LOG_WARN("DHT done IPv6");
        op->status6 = {true, ok};
        doneCallbackWrapper(done_cb, nodes, *op);
-    }, f, q);
+    }, f, sq);
 }

 std::vector<Sp<Value>>
@@ -1111,6 +1141,8 @@ bool
 Dht::cancelPut(const InfoHash& id, const Value::Id& vid)
 {
    bool canceled {false};
+    if (storageErase(id, vid))
+        canceled = true;
    auto sr_cancel_put = [&](std::map<InfoHash, Sp<Search>> srs) {
        auto srp = srs.find(id);
        if (srp == srs.end())
@@ -1135,25 +1167,27 @@ Dht::cancelPut(const InfoHash& id, const Value::Id& vid)
 // Storage

 void
-Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v)
+Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v, bool newValue)
 {
-    if (not st.local_listeners.empty()) {
-        DHT_LOG.d(id, "[store %s] %lu local listeners", id.toString().c_str(), st.local_listeners.size());
-        std::vector<std::pair<GetCallback, std::vector<Sp<Value>>>> cbs;
-        for (const auto& l : st.local_listeners) {
-            std::vector<Sp<Value>> vals;
-            if (not l.second.filter or l.second.filter(*v.data))
-                vals.push_back(v.data);
-            if (not vals.empty()) {
-                DHT_LOG.d(id, "[store %s] sending update local listener with token %lu",
-                        id.toString().c_str(),
-                        l.first);
-                cbs.emplace_back(l.second.get_cb, std::move(vals));
+    if (newValue) {
+        if (not st.local_listeners.empty()) {
+            DHT_LOG.d(id, "[store %s] %lu local listeners", id.toString().c_str(), st.local_listeners.size());
+            std::vector<std::pair<ValueCallback, std::vector<Sp<Value>>>> cbs;
+            for (const auto& l : st.local_listeners) {
+                std::vector<Sp<Value>> vals;
+                if (not l.second.filter or l.second.filter(*v.data))
+                    vals.push_back(v.data);
+                if (not vals.empty()) {
+                    DHT_LOG.d(id, "[store %s] sending update local listener with token %lu",
+                            id.toString().c_str(),
+                            l.first);
+                    cbs.emplace_back(l.second.get_cb, std::move(vals));
+                }
            }
+            // listeners are copied: they may be deleted by the callback
+            for (auto& cb : cbs)
+                cb.first(cb.second, false);
        }
-        // listeners are copied: they may be deleted by the callback
-        for (auto& cb : cbs)
-            cb.first(cb.second);
    }

    if (not st.listeners.empty()) {
@@ -1177,11 +1211,11 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v)
 }

 bool
-Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa)
+Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa, bool permanent)
 {
    const auto& now = scheduler.time();
    created = std::min(created, now);
-    auto expiration = created + getType(value->type).expiration;
+    auto expiration = permanent ? time_point::max() : created + getType(value->type).expiration;
    if (expiration < now)
        return false;

@@ -1203,16 +1237,30 @@ Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created
    if (auto vs = store.first) {
        total_store_size += store.second.size_diff;
        total_values += store.second.values_diff;
-        scheduler.add(expiration, std::bind(&Dht::expireStorage, this, id));
+        if (not permanent) {
+            scheduler.add(expiration, std::bind(&Dht::expireStorage, this, id));
+        }
        if (total_store_size > max_store_size) {
            expireStore();
        }
-        storageChanged(id, st->second, *vs);
+        storageChanged(id, st->second, *vs, store.second.values_diff > 0);
    }

    return std::get<0>(store);
 }

+bool
+Dht::storageErase(const InfoHash& id, Value::Id vid)
+{
+    auto st = store.find(id);
+    if (st == store.end())
+        return false;
+    auto ret = st->second.remove(id, vid);
+    total_store_size += ret.size_diff;
+    total_values += ret.values_diff;
+    return ret.values_diff;
+}
+
 void
 Dht::storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t socket_id, Query&& query)
 {
@@ -1241,11 +1289,36 @@ Dht::storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t socket_
 void
 Dht::expireStore(decltype(store)::iterator i)
 {
-    auto stats = i->second.expire(i->first, scheduler.time());
-    total_store_size += stats.size_diff;
-    total_values += stats.values_diff;
-    if (stats.values_diff) {
-        DHT_LOG.d(i->first, "[store %s] discarded %ld expired values (%ld bytes)", i->first.toString().c_str(), -stats.values_diff, -stats.size_diff);
+    const auto& id = i->first;
+    auto& st = i->second;
+    auto stats = st.expire(id, scheduler.time());
+    total_store_size += stats.first;
+    total_values -= stats.second.size();
+    if (not stats.second.empty()) {
+        DHT_LOG.d(id, "[store %s] discarded %ld expired values (%ld bytes)",
+            id.toString().c_str(), stats.second.size(), -stats.first);
+
+        if (not st.listeners.empty()) {
+            DHT_LOG.d(id, "[store %s] %lu remote listeners", id.toString().c_str(), st.listeners.size());
+
+            std::vector<Value::Id> ids;
+            ids.reserve(stats.second.size());
+            for (const auto& v : stats.second)
+                ids.emplace_back(v->id);
+
+            for (const auto& node_listeners : st.listeners) {
+                for (const auto& l : node_listeners.second) {
+                    DHT_LOG.w(id, node_listeners.first->id, "[store %s] [node %s] sending expired",
+                            id.toString().c_str(),
+                            node_listeners.first->toString().c_str());
+                    Blob ntoken = makeToken(node_listeners.first->getAddr(), false);
+                    network_engine.tellListenerExpired(node_listeners.first, l.first, id, ntoken, ids);
+                }
+            }
+        }
+        for (const auto& local_listeners : st.local_listeners) {
+            local_listeners.second.get_cb(stats.second, true);
+        }
    }
 }

@@ -1318,7 +1391,7 @@ Dht::connectivityChanged(sa_family_t af)
    for (auto& sp : searches(af))
        for (auto& sn : sp.second->nodes) {
            for (auto& ls : sn.listenStatus)
-                sn.node->cancelRequest(ls.second);
+                sn.node->cancelRequest(ls.second.req);
            sn.listenStatus.clear();
        }
    reported_addr.erase(std::remove_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& addr){
@@ -1504,10 +1577,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const
                    if (ack == n.acked.end() or not ack->second.first) {
                        out << ' ';
                    } else {
-                        if (ack->second.first->completed())
-                            out << 'a';
-                        else if (ack->second.first->pending())
-                            out << 'f';
+                        out << ack->second.first->getStateChar();
                    }
                }
                out << "] ";
@@ -1549,11 +1619,6 @@ Dht::getStorageLog() const
    std::stringstream out;
    for (const auto& s : store)
        out << printStorageLog(s);
-    out << "Total " << store.size() << " storages, " << total_values << " values (";
-    if (total_store_size < 1024)
-        out << total_store_size << " bytes)";
-    else
-        out << (total_store_size/1024) << " KB)";
    out << std::endl << std::endl;
    std::multimap<size_t, const SockAddr*> q_map;
    for (const auto& ip : store_quota)
@@ -1562,6 +1627,12 @@ Dht::getStorageLog() const
    for (auto ip = q_map.rbegin(); ip != q_map.rend(); ++ip)
        out << "IP " << ip->second->toString() << " uses " << ip->first << " bytes" << std::endl;
    out << std::endl;
+    out << "Total " << store.size() << " storages, " << total_values << " values (";
+    if (total_store_size < 1024)
+        out << total_store_size << " bytes)";
+    else
+        out << (total_store_size/1024) << " / " << (max_store_size/1024) << " KB)";
+    out << std::endl;
    return out.str();
 }

@@ -1653,13 +1724,12 @@ Dht::~Dht()
        s.second->clear();
 }

-Dht::Dht() : store(), scheduler(DHT_LOG), network_engine(DHT_LOG, scheduler) {}
+Dht::Dht() : store(), network_engine(DHT_LOG, scheduler) {}

 Dht::Dht(const int& s, const int& s6, Config config)
    : myid(config.node_id ? config.node_id : InfoHash::getRandom()),
    is_bootstrap(config.is_bootstrap),
    maintain_storage(config.maintain_storage), store(), store_quota(),
-    scheduler(DHT_LOG),
    network_engine(myid, config.network, s, s6, DHT_LOG, scheduler,
            std::bind(&Dht::onError, this, _1, _2),
            std::bind(&Dht::onNewNode, this, _1, _2),
@@ -1706,7 +1776,7 @@ Dht::Dht(const int& s, const int& s6, Config config)
 bool
 Dht::neighbourhoodMaintenance(RoutingTable& list)
 {
-    //DHT_LOG_DEBUG("neighbourhoodMaintenance");
+    //DHT_LOG_DBG("neighbourhoodMaintenance");
    auto b = list.findBucket(myid);
    if (b == list.end())
        return false;
@@ -1871,7 +1941,7 @@ Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from)
        try {
            network_engine.processMessage(buf, buflen, from);
        } catch (const std::exception& e) {
-            DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what());
+            DHT_LOG.e("Can't process message from %s: %s", from.toString().c_str(), e.what());
        }
    }
    return scheduler.run();
@@ -2168,7 +2238,7 @@ void Dht::onGetValuesDone(const Sp<Node>& node,
            }

            /* callbacks for local search listeners */
-            std::vector<std::pair<GetCallback, std::vector<Sp<Value>>>> tmp_lists;
+            /*std::vector<std::pair<ValueCallback, std::vector<Sp<Value>>>> tmp_lists;
            for (auto& l : sr->listeners) {
                if (!l.second.get_cb or (orig_query and l.second.query and not l.second.query->isSatisfiedBy(*orig_query)))
                    continue;
@@ -2180,7 +2250,10 @@ void Dht::onGetValuesDone(const Sp<Node>& node,
                    tmp_lists.emplace_back(l.second.get_cb, std::move(tmp));
            }
            for (auto& l : tmp_lists)
-                l.first(l.second);
+                l.first(l.second, false);*/
+        } else if (not a.expired_values.empty()) {
+            DHT_LOG.w(sr->id, node->id, "[search %s] [node %s] %u expired values",
+                      sr->id.toString().c_str(), node->toString().c_str(), a.expired_values.size());
        }
    } else {
        DHT_LOG.w(sr->id, "[node %s] no token provided. Ignoring response content.", node->toString().c_str());
@@ -2271,7 +2344,8 @@ Dht::onAnnounce(Sp<Node> n,
        Sp<Value> vc = v;
        if (lv) {
            if (*lv == *vc) {
-                DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str());
+                storageRefresh(hash, v->id);
+                DHT_LOG.d(hash, node.id, "[store %s] [node %s] refreshed value %s", hash.toString().c_str(), node.toString().c_str(), std::to_string(v->id).c_str());
            } else {
                const auto& type = getType(lv->type);
                if (type.editPolicy(hash, lv, vc, node.id, node.getAddr())) {
@@ -2303,14 +2377,11 @@ Dht::onRefresh(Sp<Node> node, const InfoHash& hash, const Blob& token, const Val
 {
    using namespace net;

-    const auto& now = scheduler.time();
    if (not tokenMatch(token, node->getAddr())) {
        DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str());
        throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::PUT_WRONG_TOKEN};
    }
-
-    auto s = store.find(hash);
-    if (s != store.end() and s->second.refresh(now, vid)) {
+    if (storageRefresh(hash, vid)) {
        DHT_LOG.d(hash, node->id, "[store %s] [node %s] refreshed value %s", hash.toString().c_str(), node->toString().c_str(), std::to_string(vid).c_str());
    } else {
        DHT_LOG.d(hash, node->id, "[store %s] [node %s] got refresh for unknown value",
@@ -2320,15 +2391,26 @@ Dht::onRefresh(Sp<Node> node, const InfoHash& hash, const Blob& token, const Val
    return {};
 }

+bool
+Dht::storageRefresh(const InfoHash& id, Value::Id vid)
+{
+    const auto& now = scheduler.time();
+    auto s = store.find(id);
+    if (s != store.end()) {
+        auto expiration = s->second.refresh(now, vid, types);
+        if (expiration != time_point::max())
+            scheduler.add(expiration, std::bind(&Dht::expireStorage, this, id));
+        return true;
+    }
+    return false;
+}
+
 void
 Dht::onAnnounceDone(const Sp<Node>& node, net::RequestAnswer& answer, Sp<Search>& sr)
 {
    DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got reply to put!",
            sr->id.toString().c_str(), node->toString().c_str());
    searchSendGetValues(sr);
-    /* if (auto sn = sr->getNode(req->node)) { */
-    /*     sn->setRefreshTime(answer.vid, now + answer) */
-    /* } */
    sr->checkAnnounced(answer.vid);
 }

@@ -37,11 +37,22 @@
 #include <winsock2.h>
 #include <ws2tcpip.h>
 #define close(x) closesocket(x)
+#define write(s, b, f) send(s, b, (int)strlen(b), 0)
 #endif

 namespace dht {

 constexpr std::chrono::seconds DhtRunner::BOOTSTRAP_PERIOD;
+static constexpr size_t RX_QUEUE_MAX_SIZE = 1024 * 16;
+
+struct DhtRunner::Listener {
+    size_t tokenClassicDht;
+    size_t tokenProxyDht;
+    ValueCallback gcb;
+    InfoHash hash;
+    Value::Filter f;
+    Where w;
+};

 DhtRunner::DhtRunner() : dht_()
 #if OPENDHT_PROXY_CLIENT
@@ -111,7 +122,8 @@ DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config
            } catch (const dht::SocketException& e) {
                startNetwork(local4, local6);
            }
-            cv.wait_until(lk, wakeup, [this]() {
+
+            auto hasJobToDo = [this]() {
                if (not running)
                    return true;
                {
@@ -128,7 +140,11 @@ DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config
                        return true;
                }
                return false;
-            });
+            };
+            if (wakeup == time_point::max())
+                cv.wait(lk, hasJobToDo);
+            else
+                cv.wait_until(lk, wakeup, hasJobToDo);
        }
    });
 }
@@ -149,7 +165,7 @@ DhtRunner::shutdown(ShutdownCallback cb) {
 void
 DhtRunner::join()
 {
-    running_network = false;
+    stopNetwork();
    running = false;
    cv.notify_all();
    bootstrap_cv.notify_all();
@@ -233,10 +249,10 @@ void
 DhtRunner::setLoggers(LogMethod error, LogMethod warn, LogMethod debug) {
    std::lock_guard<std::mutex> lck(dht_mtx);
    if (dht_)
-        dht_->setLoggers(std::forward<LogMethod>(error), std::forward<LogMethod>(warn), std::forward<LogMethod>(debug));
+        dht_->setLoggers(error, warn, debug);
 #if OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
-        dht_via_proxy_->setLoggers(std::forward<LogMethod>(error), std::forward<LogMethod>(warn), std::forward<LogMethod>(debug));
+        dht_via_proxy_->setLoggers(error, warn, debug);
 #endif
 }

@@ -287,6 +303,17 @@ DhtRunner::getNodesStats(sa_family_t af) const
    return activeDht()->getNodesStats(af);
 }

+NodeInfo
+DhtRunner::getNodeInfo() const {
+    std::lock_guard<std::mutex> lck(dht_mtx);
+    NodeInfo info;
+    info.id = getId();
+    info.node_id = getNodeId();
+    info.ipv4 = dht_->getNodesStats(AF_INET);
+    info.ipv6 = dht_->getNodesStats(AF_INET6);
+    return info;
+}
+
 std::vector<unsigned>
 DhtRunner::getNodeMessageStats(bool in) const
 {
@@ -328,7 +355,9 @@ std::vector<SockAddr>
 DhtRunner::getPublicAddress(sa_family_t af)
 {
    std::lock_guard<std::mutex> lck(dht_mtx);
-    return activeDht()->getPublicAddress(af);
+    if (auto dht = activeDht())
+        return dht->getPublicAddress(af);
+    return {};
 }
 std::vector<std::string>
 DhtRunner::getPublicAddressStr(sa_family_t af)
@@ -382,12 +411,15 @@ DhtRunner::loop_()
        received = std::move(rcv);
    }
    if (not received.empty()) {
-        for (const auto& pck : received) {
-            auto& buf = pck.first;
-            auto& from = pck.second;
-            wakeup = dht->periodic(buf.data(), buf.size()-1, from);
+        while (not received.empty()) {
+            auto& pck = received.front();
+            auto delay = clock::now() - pck.received;
+            if (delay > std::chrono::milliseconds(500))
+                std::cerr << "Dropping packet with high delay: " << print_dt(delay) << std::endl;
+            else
+                wakeup = dht->periodic(pck.data.data(), pck.data.size()-1, pck.from);
+            received.pop();
        }
-        received.clear();
    } else {
        wakeup = dht->periodic(nullptr, 0, nullptr, 0);
    }
@@ -427,8 +459,10 @@ int bindSocket(const SockAddr& addr, SockAddr& bound)
    if (is_ipv6)
        setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&set, sizeof(set));
    int rc = bind(sock, addr.get(), addr.getLength());
-    if(rc < 0)
+    if(rc < 0) {
+        close(sock);
        throw DhtException("Can't bind socket on " + addr.toString() + " " + strerror(rc));
+    }
    sockaddr_storage ss;
    socklen_t ss_len = sizeof(ss);
    getsockname(sock, (sockaddr*)&ss, &ss_len);
@@ -436,12 +470,64 @@ int bindSocket(const SockAddr& addr, SockAddr& bound)
    return sock;
 }

+#ifdef _WIN32
+inline void udpPipe(int fds[2])
+{
+    int lst = socket(AF_INET, SOCK_DGRAM, 0);
+    if (lst < 0)
+        throw DhtException(std::string("Can't open socket: ") + strerror(lst));
+    sockaddr_in inaddr;
+    sockaddr addr;
+    memset(&inaddr, 0, sizeof(inaddr));
+    memset(&addr, 0, sizeof(addr));
+    inaddr.sin_family = AF_INET;
+    inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+    inaddr.sin_port = 0;
+    int yes=1;
+    setsockopt(lst, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes));
+    int rc = bind(lst, (sockaddr*)&inaddr, sizeof(inaddr));
+    if (rc < 0) {
+        close(lst);
+        throw DhtException("Can't bind socket on " + print_addr((sockaddr*)&addr, sizeof(inaddr)) + " " + strerror(rc));
+    }
+    socklen_t len = sizeof(inaddr);
+    getsockname(lst, &addr, &len);
+    fds[0] = lst;
+    fds[1] = socket(AF_INET, SOCK_DGRAM, 0);
+    connect(fds[1], &addr, len);
+}
+#endif
+
+void
+DhtRunner::stopNetwork()
+{
+    running_network = false;
+    if (stop_writefd != -1) {
+        if (write(stop_writefd, "\0", 1) == -1) {
+            perror("write");
+        }
+    }
+}
+
 void
 DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
 {
-    running_network = false;
+    stopNetwork();
    if (rcv_thread.joinable())
        rcv_thread.join();
+
+    int stopfds[2];
+#ifndef _WIN32
+    auto status = pipe(stopfds);
+    if (status == -1) {
+        throw DhtException("Can't open pipe");
+    }
+#else
+    udpPipe(stopfds);
+#endif
+    int stop_readfd = stopfds[0];
+    stop_writefd = stopfds[1];
+
    s4 = -1;
    s6 = -1;

@@ -456,19 +542,19 @@ DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
 #endif

    running_network = true;
-    rcv_thread = std::thread([this]() {
+    rcv_thread = std::thread([this, stop_readfd]() {
        try {
            while (running_network) {
-                struct timeval tv {/*.tv_sec = */0, /*.tv_usec = */250000};
                fd_set readfds;

                FD_ZERO(&readfds);
+                FD_SET(stop_readfd, &readfds);
                if(s4 >= 0)
                    FD_SET(s4, &readfds);
                if(s6 >= 0)
                    FD_SET(s6, &readfds);

-                int rc = select(s4 > s6 ? s4 + 1 : s6 + 1, &readfds, nullptr, nullptr, &tv);
+                int rc = select(s4 > s6 ? s4 + 1 : s6 + 1, &readfds, nullptr, nullptr, nullptr);
                if(rc < 0) {
                    if(errno != EINTR) {
                        perror("select");
@@ -489,11 +575,15 @@ DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
                    else if(s6 >= 0 && FD_ISSET(s6, &readfds))
                        rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len);
                    else
-                        break;
+                        continue;
                    if (rc > 0) {
                        {
                            std::lock_guard<std::mutex> lck(sock_mtx);
-                            rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, SockAddr(from, from_len));
+                            if (rcv.size() >= RX_QUEUE_MAX_SIZE) {
+                                std::cerr << "Dropping packet: queue is full!" << std::endl;
+                                rcv.pop();
+                            }
+                            rcv.emplace(ReceivedPacket {Blob {buf.begin(), buf.begin()+rc+1}, SockAddr(from, from_len), clock::now()});
                        }
                        cv.notify_all();
                    }
@@ -510,6 +600,11 @@ DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
        s6 = -1;
        bound4 = {};
        bound6 = {};
+        if (stop_readfd != -1)
+            close(stop_readfd);
+        if (stop_writefd != -1)
+            close(stop_writefd);
+        stop_writefd = -1;
    });
 }

@@ -542,34 +637,34 @@ DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Q
 }

 std::future<size_t>
-DhtRunner::listen(InfoHash hash, GetCallback vcb, Value::Filter f, Where w)
+DhtRunner::listen(InfoHash hash, ValueCallback vcb, Value::Filter f, Where w)
 {
    auto ret_token = std::make_shared<std::promise<size_t>>();
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
        pending_ops.emplace([=](SecureDht& dht) mutable {
+#if OPENDHT_PROXY_CLIENT
            auto tokenbGlobal = listener_token_++;
            Listener listener {};
            listener.hash = hash;
            listener.f = std::move(f);
            listener.w = std::move(w);
-            listener.gcb = [hash,vcb,tokenbGlobal,this](const std::vector<Sp<Value>>& vals){
-                if (not vcb(vals)) {
-#if OPENDHT_PROXY_CLIENT
+            listener.gcb = [hash,vcb,tokenbGlobal,this](const std::vector<Sp<Value>>& vals, bool expired){
+                if (not vcb(vals, expired)) {
                    cancelListen(hash, tokenbGlobal);
-#endif
                    return false;
                }
                return true;
            };
-#if OPENDHT_PROXY_CLIENT
            if (use_proxy)
                listener.tokenProxyDht = dht.listen(hash, listener.gcb, listener.f, listener.w);
            else
-#endif
                listener.tokenClassicDht = dht.listen(hash, listener.gcb, listener.f, listener.w);
            listeners_.emplace(tokenbGlobal, std::move(listener));
            ret_token->set_value(tokenbGlobal);
+#else
+            ret_token->set_value(dht.listen(hash, vcb, f, w));
+#endif
        });
    }
    cv.notify_all();
@@ -585,34 +680,47 @@ DhtRunner::listen(const std::string& key, GetCallback vcb, Value::Filter f, Wher
 void
 DhtRunner::cancelListen(InfoHash h, size_t token)
 {
-    std::lock_guard<std::mutex> lck(storage_mtx);
+    {
+        std::lock_guard<std::mutex> lck(storage_mtx);
 #if OPENDHT_PROXY_CLIENT
-    pending_ops.emplace([=](SecureDht&) {
-        auto it = listeners_.find(token);
-        if (it == listeners_.end()) return;
-        if (it->second.tokenClassicDht)
-            dht_->cancelListen(h, it->second.tokenClassicDht);
-        if (it->second.tokenProxyDht and dht_via_proxy_)
-            dht_via_proxy_->cancelListen(h, it->second.tokenProxyDht);
-        listeners_.erase(it);
-    });
+        pending_ops.emplace([=](SecureDht&) {
+            auto it = listeners_.find(token);
+            if (it == listeners_.end()) return;
+            if (it->second.tokenClassicDht)
+                dht_->cancelListen(h, it->second.tokenClassicDht);
+            if (it->second.tokenProxyDht and dht_via_proxy_)
+                dht_via_proxy_->cancelListen(h, it->second.tokenProxyDht);
+            listeners_.erase(it);
+        });
 #else
-    pending_ops.emplace([=](SecureDht& dht) {
-        dht.cancelListen(h, token);
-    });
+        pending_ops.emplace([=](SecureDht& dht) {
+            dht.cancelListen(h, token);
+        });
 #endif // OPENDHT_PROXY_CLIENT
+    }
    cv.notify_all();
 }

 void
-DhtRunner::cancelListen(InfoHash h, std::shared_future<size_t> token)
+DhtRunner::cancelListen(InfoHash h, std::shared_future<size_t> ftoken)
 {
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
-        pending_ops.emplace([=](SecureDht& dht) {
-            auto tk = token.get();
-            dht.cancelListen(h, tk);
+#if OPENDHT_PROXY_CLIENT
+        pending_ops.emplace([=](SecureDht&) {
+            auto it = listeners_.find(ftoken.get());
+            if (it == listeners_.end()) return;
+            if (it->second.tokenClassicDht)
+                dht_->cancelListen(h, it->second.tokenClassicDht);
+            if (it->second.tokenProxyDht and dht_via_proxy_)
+                dht_via_proxy_->cancelListen(h, it->second.tokenProxyDht);
+            listeners_.erase(it);
        });
+#else
+        pending_ops.emplace([=](SecureDht& dht) {
+            dht.cancelListen(h, ftoken.get());
+        });
+#endif // OPENDHT_PROXY_CLIENT
    }
    cv.notify_all();
 }
@@ -829,7 +937,7 @@ DhtRunner::connectivityChanged()
 {
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
-        pending_ops.emplace([=](SecureDht& dht) {
+        pending_ops_prio.emplace([=](SecureDht& dht) {
            dht.connectivityChanged();
        });
    }
@@ -850,8 +958,8 @@ DhtRunner::findCertificate(InfoHash hash, std::function<void(const std::shared_p
 void
 DhtRunner::resetDht()
 {
-    listeners_.clear();
 #if OPENDHT_PROXY_CLIENT
+    listeners_.clear();
    dht_via_proxy_.reset();
 #endif // OPENDHT_PROXY_CLIENT
    dht_.reset();
@@ -867,6 +975,21 @@ DhtRunner::activeDht() const
 #endif // OPENDHT_PROXY_CLIENT
 }

+void
+DhtRunner::setProxyServer(const std::string& proxy, const std::string& pushNodeId)
+{
+#if OPENDHT_PROXY_CLIENT
+    if (config_.proxy_server == proxy and config_.push_node_id == pushNodeId)
+        return;
+    config_.proxy_server = proxy;
+    config_.push_node_id = pushNodeId;
+    enableProxy(use_proxy and not config_.proxy_server.empty());
+#else
+    if (not proxy.empty())
+        std::cerr << "DHT proxy requested but OpenDHT built without proxy support." << std::endl;
+#endif
+}
+
 void
 DhtRunner::enableProxy(bool proxify)
 {
@@ -944,20 +1067,17 @@ DhtRunner::setPushNotificationToken(const std::string& token) {
 }

 void
-DhtRunner::pushNotificationReceived(const std::map<std::string, std::string>& data) const
+DhtRunner::pushNotificationReceived(const std::map<std::string, std::string>& data)
 {
 #if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS
-    if (dht_via_proxy_)
-        dht_via_proxy_->pushNotificationReceived(data);
-#endif
-}
-
-void
-DhtRunner::resubscribe(unsigned token)
-{
-#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS
-    if (dht_via_proxy_)
-        dht_via_proxy_->resubscribe(token);
+    {
+        std::lock_guard<std::mutex> lck(storage_mtx);
+        pending_ops_prio.emplace([=](SecureDht&) {
+            if (dht_via_proxy_)
+                dht_via_proxy_->pushNotificationReceived(data);
+        });
+    }
+    cv.notify_all();
 #endif
 }

@@ -20,6 +20,7 @@

 #include "value.h"
 #include "utils.h"
+#include "callbacks.h"

 namespace dht {

@@ -44,7 +45,14 @@ struct Listener {
 struct LocalListener {
    Sp<Query> query;
    Value::Filter filter;
-    GetCallback get_cb;
+    ValueCallback get_cb;
+};
+
+
+struct SearchListener {
+    Sp<Query> query;
+    Value::Filter filter;
+    ValueCallback get_cb;
 };

 }
@@ -122,12 +122,25 @@ serializeValues(const std::vector<Sp<Value>>& st)
    return svals;
 }

+void
+packToken(msgpack::packer<msgpack::sbuffer>& pk, const Blob& token)
+{
+    pk.pack_bin(token.size());
+    pk.pack_bin_body((char*)token.data(), token.size());
+}
+
 RequestAnswer::RequestAnswer(ParsedMessage&& msg)
- : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)),
-    nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {}
+ : ntoken(std::move(msg.token)),
+   values(std::move(msg.values)),
+   refreshed_values(std::move(msg.refreshed_values)),
+   expired_values(std::move(msg.expired_values)),
+   fields(std::move(msg.fields)),
+   nodes4(std::move(msg.nodes4)),
+   nodes6(std::move(msg.nodes6))
+{}

 NetworkEngine::NetworkEngine(Logger& log, Scheduler& scheduler, const int& s, const int& s6)
-    : myid(zeroes), DHT_LOG(log), scheduler(scheduler), dht_socket(s), dht_socket6(s6)
+    : myid(zeroes), dht_socket(s), dht_socket6(s6), DHT_LOG(log), scheduler(scheduler)
 {}
 NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, const int& s, const int& s6, Logger& log, Scheduler& scheduler,
        decltype(NetworkEngine::onError) onError,
@@ -171,6 +184,67 @@ NetworkEngine::tellListener(Sp<Node> node, Tid socket_id, const InfoHash& hash,
    }
 }

+void
+NetworkEngine::tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values)
+{
+    msgpack::sbuffer buffer;
+    msgpack::packer<msgpack::sbuffer> pk(&buffer);
+    pk.pack_map(4+(network?1:0));
+
+    pk.pack(std::string("u"));
+    pk.pack_map(1 + (not values.empty()?1:0) + (not token.empty()?1:0));
+    pk.pack(std::string("id")); pk.pack(myid);
+    if (not token.empty()) {
+        pk.pack(std::string("token")); packToken(pk, token);
+    }
+    if (not values.empty()) {
+        pk.pack(std::string("re"));
+        pk.pack(values);
+        DHT_LOG.d(n->id, "[node %s] sending %zu refreshed values", n->toString().c_str(), values.size());
+    }
+
+    pk.pack(std::string("t")); pk.pack(socket_id);
+    pk.pack(std::string("y")); pk.pack(std::string("r"));
+    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
+
+    // send response
+    send(buffer.data(), buffer.size(), 0, n->getAddr());
+}
+
+void
+NetworkEngine::tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash&, const Blob& token, const std::vector<Value::Id>& values)
+{
+    msgpack::sbuffer buffer;
+    msgpack::packer<msgpack::sbuffer> pk(&buffer);
+    pk.pack_map(4+(network?1:0));
+
+    pk.pack(std::string("u"));
+    pk.pack_map(1 + (not values.empty()?1:0) + (not token.empty()?1:0));
+    pk.pack(std::string("id")); pk.pack(myid);
+    if (not token.empty()) {
+        pk.pack(std::string("token")); packToken(pk, token);
+    }
+    if (not values.empty()) {
+        pk.pack(std::string("exp"));
+        pk.pack(values);
+        DHT_LOG.d(n->id, "[node %s] sending %zu expired values", n->toString().c_str(), values.size());
+    }
+
+    pk.pack(std::string("t")); pk.pack(socket_id);
+    pk.pack(std::string("y")); pk.pack(std::string("r"));
+    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
+
+    // send response
+    send(buffer.data(), buffer.size(), 0, n->getAddr());
+}
+
+
 bool
 NetworkEngine::isRunning(sa_family_t af) const
 {
@@ -188,7 +262,13 @@ NetworkEngine::isRunning(sa_family_t af) const

 void
 NetworkEngine::clear()
-{}
+{
+    for (auto& request : requests) {
+        request.second->cancel();
+        request.second->node->setExpired();
+    }
+    requests.clear();
+}

 void
 NetworkEngine::connectivityChanged(sa_family_t af)
@@ -206,7 +286,7 @@ NetworkEngine::requestStep(Sp<Request> sreq)
    auto now = scheduler.time();
    auto& node = *req.node;
    if (req.isExpired(now)) {
-        DHT_LOG.e(node.id, "[node %s] expired !", node.toString().c_str());
+        DHT_LOG.d(node.id, "[node %s] expired !", node.toString().c_str());
        node.setExpired();
        if (not node.id)
            requests.erase(req.tid);
@@ -221,7 +301,8 @@ NetworkEngine::requestStep(Sp<Request> sreq)
    if (err == ENETUNREACH  ||
        err == EHOSTUNREACH ||
        err == EAFNOSUPPORT ||
-        err == EPIPE)
+        err == EPIPE        ||
+        err == EPERM)
    {
        node.setExpired();
        if (not node.id)
@@ -337,8 +418,8 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
        msgpack::unpacked msg_res = msgpack::unpack((const char*)buf, buflen);
        msg->msgpack_unpack(msg_res.get());
    } catch (const std::exception& e) {
-        DHT_LOG.w("Can't process message of size %lu: %s", buflen, e.what());
-        DHT_LOG.DEBUG.logPrintable(buf, buflen);
+        DHT_LOG.w("Can't parse message of size %lu: %s", buflen, e.what());
+        //DHT_LOG.DBG.logPrintable(buf, buflen);
        return;
    }

@@ -470,7 +551,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
        }
        case MessageType::Reply:
            if (req) { /* request reply */
-                if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen)
+                if (req->getType() == MessageType::AnnounceValue or req->getType() == MessageType::Listen)
                    req->node->authSuccess();
                req->reply_time = scheduler.time();

@@ -536,6 +617,7 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.listen;
                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query));
+                auto nnodes = bufferNodes(from.getFamily(), msg->info_hash, msg->want, answer.nodes4, answer.nodes6);
                sendListenConfirmation(from, msg->tid);
                break;
            }
@@ -550,13 +632,6 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
    }
 }

-void
-packToken(msgpack::packer<msgpack::sbuffer>& pk, const Blob& token)
-{
-    pk.pack_bin(token.size());
-    pk.pack_bin_body((char*)token.data(), token.size());
-}
-
 void
 insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const SockAddr& addr)
 {
@@ -586,7 +661,7 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr
    if (s < 0)
        return EAFNOSUPPORT;
 #ifdef MSG_NOSIGNAL
-    flags &= MSG_NOSIGNAL;
+    flags |= MSG_NOSIGNAL;
 #endif
    if (sendto(s, buf, len, flags, addr.get(), addr.getLength()) == -1) {
        int err = errno;
@@ -902,7 +977,7 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, Tid tid, const Blob& nodes,
            pk.pack(std::string("v")); pk.pack_array(st.size()*fields.size());
            for (const auto& v : st)
                v->msgpack_pack_fields(fields, pk);
-            //DHT_LOG_DEBUG("sending closest nodes (%d+%d nodes.), %u value headers containing %u fields",
+            //DHT_LOG_DBG("sending closest nodes (%d+%d nodes.), %u value headers containing %u fields",
            //        nodes.size(), nodes6.size(), st.size(), fields.size());
        }
    }
@@ -20,6 +20,9 @@

 namespace dht {

+constexpr size_t CLEANUP_MAX_NODES {1024};
+constexpr size_t CLEANUP_FREQ {1024};
+
 NodeCache::~NodeCache()
 {
    cache_4.setExpired();
@@ -41,29 +44,33 @@ NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, boo
 std::vector<Sp<Node>>
 NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const
 {
-    const auto& c = cache(sa_f);
+    return cache(sa_f).getCachedNodes(id, count);
+}

+std::vector<Sp<Node>>
+NodeCache::NodeMap::getCachedNodes(const InfoHash& id, size_t count) const
+{
    std::vector<Sp<Node>> nodes;
-    nodes.reserve(std::min(c.size(), count));
-    NodeMap::const_iterator it;
-    auto dec_it = [&c](NodeMap::const_iterator& it) {
+    nodes.reserve(std::min(size(), count));
+    const_iterator it;
+    auto dec_it = [this](const_iterator& it) {
        auto ret = it;
-        it = (it == c.cbegin()) ? c.cend() : std::prev(it);
+        it = (it == cbegin()) ? cend() : std::prev(it);
        return ret;
    };

-    auto it_p = c.lower_bound(id),
+    auto it_p = lower_bound(id),
        it_n = it_p;
-    if (not c.empty())
+    if (not empty())
        dec_it(it_p); /* Create 2 separate iterator if we could */

-    while (nodes.size() < count and (it_n != c.cend() or it_p != c.cend())) {
+    while (nodes.size() < count and (it_n != cend() or it_p != cend())) {
        /* If one of the iterator is at the end, then take the other one
           If they are both in middle of somewhere comapre both and take
           the closest to the id. */
-        if (it_p == c.cend())       it = it_n++;
-        else if (it_n == c.cend())  it = dec_it(it_p);
-        else                        it = id.xorCmp(it_p->first, it_n->first) < 0 ? dec_it(it_p) : it_n++;
+        if (it_p == cend())      it = it_n++;
+        else if (it_n == cend()) it = dec_it(it_p);
+        else                     it = id.xorCmp(it_p->first, it_n->first) < 0 ? dec_it(it_p) : it_n++;

        if (auto n = it->second.lock())
            if ( not n->isExpired() and not n->isClient() )
@@ -104,6 +111,10 @@ NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point
    if (not node) {
        node = std::make_shared<Node>(id, addr, client);
        it.first->second = node;
+        if (cleanup_counter++ == CLEANUP_FREQ) {
+            cleanup();
+            cleanup_counter = 0;
+        }
    } else if (confirm or node->isOld(now)) {
        node->update(addr);
    }
@@ -120,6 +131,7 @@ NodeCache::NodeMap::clearBadNodes() {
            erase(it++);
        }
    }
+    cleanup_counter = 0;
 }

 void
@@ -128,6 +140,21 @@ NodeCache::NodeMap::setExpired() {
        if (auto n = wn.second.lock())
            n->setExpired();
    clear();
+    cleanup_counter = 0;
+}
+
+void
+NodeCache::NodeMap::cleanup()
+{
+    auto it = lower_bound(InfoHash::getRandom());
+    for (size_t n = 0, maxNodes = std::min(size(), CLEANUP_MAX_NODES); n != maxNodes; n++) {
+        if (it == end())
+            it = begin();
+        if (it->second.expired())
+            erase(it++);
+        else
+            ++it;
+    }
 }

 }
@@ -0,0 +1,204 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "op_cache.h"
+
+namespace dht {
+
+constexpr const std::chrono::seconds OpCache::EXPIRATION;
+
+bool
+OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
+    std::vector<Sp<Value>> newValues;
+    for (const auto& v : vals) {
+        auto viop = values.emplace(v->id, OpCacheValueStorage{v});
+        if (viop.second) {
+            newValues.emplace_back(v);
+        } else {
+            viop.first->second.refCount++;
+        }
+    }
+    return newValues.empty() ? true : callback(newValues, false);
+}
+bool
+OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
+    std::vector<Sp<Value>> expiredValues;
+    for (const auto& v : vals) {
+        auto vit = values.find(v->id);
+        if (vit != values.end()) {
+            vit->second.refCount--;
+            if (not vit->second.refCount) {
+                expiredValues.emplace_back(std::move(vit->second.data));
+                values.erase(vit);
+            }
+        }
+    }
+    return expiredValues.empty() ? true : callback(expiredValues, true);
+}
+std::vector<Sp<Value>>
+OpValueCache::get(Value::Filter& filter) const {
+    std::vector<Sp<Value>> ret;
+    if (not filter)
+        ret.reserve(values.size());
+    for (const auto& v : values)
+        if (not filter or filter(*v.second.data))
+            ret.emplace_back(v.second.data);
+    return ret;
+}
+
+Sp<Value>
+OpValueCache::get(Value::Id id) const {
+    auto v = values.find(id);
+    if (v == values.end())
+        return {};
+    return v->second.data;
+}
+
+std::vector<Sp<Value>>
+OpValueCache::getValues() const {
+    std::vector<Sp<Value>> ret;
+    ret.reserve(values.size());
+    for (const auto& v : values)
+        ret.emplace_back(v.second.data);
+    return ret;
+}
+
+void
+OpCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
+    if (not listeners.empty()) {
+        std::vector<LocalListener> list;
+        list.reserve(listeners.size());
+        for (const auto& l : listeners)
+            list.emplace_back(l.second);
+        for (auto& l : list)
+            l.get_cb(l.filter.filter(vals), false);
+    }
+}
+
+void
+OpCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
+    if (not listeners.empty()) {
+        std::vector<LocalListener> list;
+        list.reserve(listeners.size());
+        for (const auto& l : listeners)
+            list.emplace_back(l.second);
+        for (auto& l : list)
+            l.get_cb(l.filter.filter(vals), true);
+    }
+}
+
+time_point
+OpCache::getExpiration() const {
+    if (not listeners.empty())
+        return time_point::max();
+    return lastRemoved + EXPIRATION;
+}
+
+size_t
+SearchCache::listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, std::function<size_t(Sp<Query>, ValueCallback)> onListen)
+{
+    // find exact match
+    auto op = ops.find(q);
+    if (op == ops.end()) {
+        // find satisfying query
+        for (auto it = ops.begin(); it != ops.end(); it++) {
+            if (q->isSatisfiedBy(*it->first)) {
+                op = it;
+                break;
+            }
+        }
+    }
+    if (op == ops.end()) {
+        // New query
+        op = ops.emplace(q, std::unique_ptr<OpCache>(new OpCache)).first;
+        auto& cache = *op->second;
+        cache.searchToken = onListen(q, [&](const std::vector<Sp<Value>>& values, bool expired){
+            return cache.onValue(values, expired);
+        });
+    }
+    auto token = nextToken_++;
+    if (nextToken_ == 0)
+        nextToken_++;
+    op->second->addListener(token, get_cb, q, filter);
+    return token;
+}
+
+bool
+SearchCache::cancelListen(size_t gtoken, const time_point& now) {
+    for (auto& op : ops) {
+        if (op.second->removeListener(gtoken, now)) {
+            nextExpiration_ = std::min(nextExpiration_, op.second->getExpiration());
+            return true;
+        }
+    }
+    return false;
+}
+
+void
+SearchCache::cancelAll(std::function<void(size_t)> onCancel) {
+    for (auto& op : ops) {
+        auto cache = std::move(op.second);
+        cache->removeAll();
+        onCancel(cache->searchToken);
+    }
+    ops.clear();
+}
+
+time_point
+SearchCache::expire(const time_point& now, std::function<void(size_t)> onCancel) {
+    nextExpiration_ = time_point::max();
+    auto ret = nextExpiration_;
+    for (auto it = ops.begin(); it != ops.end();) {
+        auto expiration = it->second->getExpiration();
+        if (expiration < now) {
+            auto cache = std::move(it->second);
+            it = ops.erase(it);
+            onCancel(cache->searchToken);
+        } else {
+            nextExpiration_ = std::min(nextExpiration_, expiration);
+            ret = nextExpiration_;
+            ++it;
+        }
+    }
+    return ret;
+}
+
+std::vector<Sp<Value>>
+SearchCache::get(Value::Filter& filter) const {
+    if (ops.size() == 1)
+        return ops.begin()->second->get(filter);
+    std::map<Value::Id, Sp<Value>> c;
+    for (const auto& op : ops) {
+        for (const auto& v : op.second->get(filter))
+            c.emplace(v->id, v);
+    }
+    std::vector<Sp<Value>> ret;
+    ret.reserve(c.size());
+    for (auto& v : c)
+        ret.emplace_back(std::move(v.second));
+    return ret;
+}
+
+Sp<Value>
+SearchCache::get(Value::Id id) const {
+    for (const auto& op : ops)
+        if (auto v = op.second->get(id))
+            return v;
+    return {};
+}
+
+}
@@ -0,0 +1,155 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "value.h"
+#include "value_cache.h"
+#include "listener.h"
+
+namespace dht {
+
+struct OpCacheValueStorage
+{
+    Sp<Value> data {};
+    unsigned refCount {1};
+    OpCacheValueStorage(Sp<Value> val = {}) : data(val) {}
+};
+
+class OpValueCache {
+public:
+    OpValueCache(ValueCallback&& cb) : callback(std::forward<ValueCallback>(cb)) {}
+    OpValueCache(OpValueCache&& o) : values(std::move(o.values)), callback(std::move(o.callback)) {
+        o.callback = {};
+    }
+
+    static ValueCallback cacheCallback(ValueCallback&& cb) {
+        auto cache = std::make_shared<OpValueCache>(std::forward<ValueCallback>(cb));
+        return [cache](const std::vector<Sp<Value>>& vals, bool expired){
+            return cache->onValue(vals, expired);
+        };
+    }
+
+    bool onValue(const std::vector<Sp<Value>>& vals, bool expired) {
+        if (expired)
+            return onValuesExpired(vals);
+        else
+            return onValuesAdded(vals);
+    }
+
+    bool onValuesAdded(const std::vector<Sp<Value>>& vals);
+    bool onValuesExpired(const std::vector<Sp<Value>>& vals);
+
+    std::vector<Sp<Value>> get(Value::Filter& filter) const;
+    Sp<Value> get(Value::Id id) const;
+    std::vector<Sp<Value>> getValues() const;
+
+private:
+    OpValueCache(const OpValueCache&) = delete;
+    OpValueCache& operator=(const OpValueCache&) = delete;
+
+    std::map<Value::Id, OpCacheValueStorage> values {};
+    ValueCallback callback;
+};
+
+class OpCache {
+public:
+    OpCache() : cache([this](const std::vector<Sp<Value>>& vals, bool expired){
+        if (expired)
+            onValuesExpired(vals);
+        else
+            onValuesAdded(vals);
+        return true;
+    }) {}
+
+    bool onValue(const std::vector<Sp<Value>>& vals, bool expired) {
+        cache.onValue(vals, expired);
+        return not listeners.empty();
+    }
+
+    void onValuesAdded(const std::vector<Sp<Value>>& vals);
+    void onValuesExpired(const std::vector<Sp<Value>>& vals);
+
+    void addListener(size_t token, ValueCallback cb, Sp<Query> q, Value::Filter filter) {
+        listeners.emplace(token, LocalListener{q, filter, cb});
+        cb(cache.get(filter), false);
+    }
+
+    bool removeListener(size_t token, const time_point& now) {
+        lastRemoved = now;
+        return listeners.erase(token) > 0;
+    }
+
+    void removeAll() {
+        listeners.clear();
+    }
+
+    bool isDone() {
+        return listeners.empty();
+    }
+
+    std::vector<Sp<Value>> get(Value::Filter& filter) const {
+        return cache.get(filter);
+    }
+
+    Sp<Value> get(Value::Id id) const {
+        return cache.get(id);
+    }
+
+    bool isExpired(const time_point& now) const {
+        return listeners.empty() and (lastRemoved + EXPIRATION < now);
+    }
+    time_point getExpiration() const;
+
+    size_t searchToken;
+private:
+    constexpr static const std::chrono::seconds EXPIRATION {60};
+    OpCache(const OpCache&) = delete;
+    OpCache& operator=(const OpCache&) = delete;
+
+    OpValueCache cache;
+    std::map<size_t, LocalListener> listeners;
+    time_point lastRemoved {clock::now()};
+};
+
+class SearchCache {
+public:
+    SearchCache() {}
+    SearchCache(SearchCache&&) = default;
+    size_t listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, std::function<size_t(Sp<Query>, ValueCallback)> onListen);
+
+    bool cancelListen(size_t gtoken, const time_point& now);
+    void cancelAll(std::function<void(size_t)> onCancel);
+
+    time_point expire(const time_point& now, std::function<void(size_t)> onCancel);
+    time_point getExpiration() const {
+        return nextExpiration_;
+    }
+
+    std::vector<Sp<Value>> get(Value::Filter& filter) const;
+    Sp<Value> get(Value::Id id) const;
+
+private:
+    SearchCache(const SearchCache&) = delete;
+    SearchCache& operator=(const SearchCache&) = delete;
+
+    std::map<Sp<Query>, std::unique_ptr<OpCache>> ops {};
+    size_t nextToken_ {1};
+    time_point nextExpiration_ {time_point::max()};
+};
+
+}
@@ -62,6 +62,8 @@ struct ParsedMessage {
    std::vector<Sp<Node>> nodes4, nodes6;
    /* values to store or retreive request */
    std::vector<Sp<Value>> values;
+    std::vector<Value::Id> refreshed_values {};
+    std::vector<Value::Id> expired_values {};
    /* index for fields values */
    std::vector<Sp<FieldValueIndex>> fields;
    /** When part of the message header: {index -> (total size, {})}
@@ -282,6 +284,10 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
        } else {
            throw msgpack::type_error();
        }
+    } else if (auto raw_fields = findMapValue(req, "exp")) {
+        expired_values = raw_fields->as<decltype(expired_values)>();
+    } else if (auto raw_fields = findMapValue(req, "re")) {
+        refreshed_values = raw_fields->as<decltype(refreshed_values)>();
    }

    if (auto w = findMapValue(req, "w")) {
@@ -57,6 +57,15 @@ struct Request {
    bool pending() const { return state_ == State::PENDING; }
    bool over() const { return not pending(); }
    State getState() const { return state_; }
+    char getStateChar() const {
+        switch (state_) {
+            case State::PENDING:   return 'f';
+            case State::CANCELLED: return 'c';
+            case State::EXPIRED:   return 'e';
+            case State::COMPLETED: return 'a';
+            default:               return '?';
+        }
+    }

    Request(State state = State::PENDING) : state_(state) {}
    Request(MessageType type, Tid tid,
@@ -21,6 +21,8 @@
 #include "value.h"
 #include "request.h"
 #include "listener.h"
+#include "value_cache.h"
+#include "op_cache.h"

 namespace dht {

@@ -59,6 +61,17 @@ struct Dht::SearchNode {
     */
    using SyncStatus = std::map<Sp<Query>, Sp<net::Request>>;

+    struct CachedListenStatus {
+        ValueCache cache;
+        Sp<Scheduler::Job> cacheExpirationJob {};
+        Sp<net::Request> req {};
+        CachedListenStatus(ValueStateCallback&& cb) : cache(std::forward<ValueStateCallback>(cb)) {}
+        CachedListenStatus(CachedListenStatus&&) = default;
+        CachedListenStatus(const CachedListenStatus&) = delete;
+        CachedListenStatus& operator=(const CachedListenStatus&) = delete;
+    };
+    using NodeListenerStatus = std::map<Sp<Query>, CachedListenStatus>;
+
    Sp<Node> node {};                 /* the node info */

    /* queries sent for finding out values hosted by the node */
@@ -67,7 +80,7 @@ struct Dht::SearchNode {
    std::map<Sp<Query>, std::vector<Sp<Query>>> pagination_queries {};

    SyncStatus getStatus {};    /* get/sync status */
-    SyncStatus listenStatus {}; /* listen status */
+    NodeListenerStatus listenStatus {}; /* listen status */
    AnnounceStatus acked {};    /* announcement status for a given value id */

    Blob token {};                                 /* last token the node sent to us after a get request */
@@ -123,7 +136,7 @@ struct Dht::SearchNode {
     *
     * @return true if we can send get, else false.
     */
-    bool canGet(time_point now, time_point update, Sp<Query> q = {}) const {
+    bool canGet(time_point now, time_point update, const Sp<Query>& q) const {
        if (node->isExpired())
            return false;

@@ -134,9 +147,9 @@ struct Dht::SearchNode {
            if (s.second and s.second->pending())
                pending = true;
            if (s.first and q and q->isSatisfiedBy(*s.first) and s.second) {
-                if (s.second->pending() and not pending_sq_status)
+                if (s.second->pending())
                    pending_sq_status = true;
-                if (s.second->completed() and not (update > s.second->reply_time) and not completed_sq_status)
+                else if (s.second->completed() and not (update > s.second->reply_time))
                    completed_sq_status = true;
                if (completed_sq_status and pending_sq_status)
                    break;
@@ -144,7 +157,7 @@ struct Dht::SearchNode {
        }

        return (not pending and now > last_get_reply + Node::NODE_EXPIRE_TIME) or
-                not (hasStartedPagination(q) or completed_sq_status or pending_sq_status);
+                not (completed_sq_status or pending_sq_status or hasStartedPagination(q));
    }

    /**
@@ -201,6 +214,25 @@ struct Dht::SearchNode {
        getStatus.clear();
    }

+    void onValues(const Sp<Query>& q, net::RequestAnswer&& answer, const TypeStore& types, Scheduler& scheduler)
+    {
+        auto l = listenStatus.find(q);
+        if (l != listenStatus.end()) {
+            auto next = l->second.cache.onValues(answer.values,
+                                     answer.refreshed_values,
+                                     answer.expired_values, types, scheduler.time());
+            scheduler.edit(l->second.cacheExpirationJob, next);
+        }
+    }
+
+    void expireValues(const Sp<Query>& q, Scheduler& scheduler) {
+        auto l = listenStatus.find(q);
+        if (l != listenStatus.end()) {
+            auto next = l->second.cache.expireValues(scheduler.time());
+            scheduler.edit(l->second.cacheExpirationJob, next);
+        }
+    }
+
    /**
     * Tells if a request in the status map is expired.
     *
@@ -228,6 +260,12 @@ struct Dht::SearchNode {
                return r.second and r.second->pending();
            }) != status.end();
    }
+    static bool pending(const NodeListenerStatus& status) {
+        return std::find_if(status.begin(), status.end(),
+            [](const NodeListenerStatus::value_type& r){
+                return r.second.req and r.second.req->pending();
+            }) != status.end();
+    }

    bool pendingGet() const { return pending(getStatus); }

@@ -263,49 +301,48 @@ struct Dht::SearchNode {
        else
            return isListening(now, ls);
    }
-    bool isListening(time_point now, SyncStatus::const_iterator listen_status) const {
-        if (listen_status == listenStatus.end())
+    bool isListening(time_point now, NodeListenerStatus::const_iterator listen_status) const {
+        if (listen_status == listenStatus.end() or not listen_status->second.req)
            return false;
-        return listen_status->second->reply_time + LISTEN_EXPIRE_TIME > now;
+        return listen_status->second.req->reply_time + LISTEN_EXPIRE_TIME > now;
    }
    void cancelListen() {
        for (const auto& status : listenStatus)
-            node->cancelRequest(status.second);
+            node->cancelRequest(status.second.req);
        listenStatus.clear();
    }
    void cancelListen(const Sp<Query>& query) {
        auto it = listenStatus.find(query);
        if (it != listenStatus.end()) {
-            node->cancelRequest(it->second);
+            node->cancelRequest(it->second.req);
            listenStatus.erase(it);
        }
    }

    /**
-     * Assumng the node is synced, should a "put" request be sent to this node now ?
+     * Assuming the node is synced, should a "put" request be sent to this node now ?
     */
    time_point getAnnounceTime(Value::Id vid) const {
        const auto& ack = acked.find(vid);
-        const auto& gs = probe_query ? getStatus.find(probe_query) : getStatus.cend();
-        if ((ack == acked.cend() or not ack->second.first) and (gs == getStatus.cend()
-                                                          or not gs->second or not gs->second->pending()))
+        if (ack == acked.cend() or not ack->second.first) {
            return time_point::min();
-        return ((gs != getStatus.cend() and gs->second and gs->second->pending())
-                or ack == acked.cend() or not ack->second.first or ack->second.first->pending())
-                ? time_point::max()
-                : ack->second.second - REANNOUNCE_MARGIN;
+        }
+        if (ack->second.first->completed()) {
+            return ack->second.second - REANNOUNCE_MARGIN;
+        }
+        return ack->second.first->pending() ? time_point::max() : time_point::min();
    }

    /**
-     * Assumng the node is synced, should the "listen" request with Query q be
+     * Assuming the node is synced, should the "listen" request with Query q be
     * sent to this node now ?
     */
    time_point getListenTime(const Sp<Query>& q) const {
        auto listen_status = listenStatus.find(q);
-        if (listen_status == listenStatus.end() or not listen_status->second)
+        if (listen_status == listenStatus.end() or not listen_status->second.req)
            return time_point::min();
-        return listen_status->second->pending() ? time_point::max() :
-            listen_status->second->reply_time + LISTEN_EXPIRE_TIME - REANNOUNCE_MARGIN;
+        return listen_status->second.req->pending() ? time_point::max() :
+            listen_status->second.req->reply_time + LISTEN_EXPIRE_TIME - REANNOUNCE_MARGIN;
    }

    /**
@@ -340,10 +377,16 @@ struct Dht::Search {
    std::multimap<time_point, Get> callbacks {};

    /* listeners */
-    std::map<size_t, LocalListener> listeners {};
+    std::map<size_t, SearchListener> listeners {};
    size_t listener_token = 1;

+    /* Cache */
+    SearchCache cache;
+    Sp<Scheduler::Job> opExpirationJob;
+
    ~Search() {
+        if (opExpirationJob)
+            opExpirationJob->cancel();
        for (auto& get : callbacks) {
            get.second.done_cb(false, {});
            get.second.done_cb = {};
@@ -386,7 +429,8 @@ struct Dht::Search {
     *
     * @param query  The query identifying a 'get' request.
     */
-    time_point getLastGetTime(Sp<Query> query = {}) const;
+    time_point getLastGetTime(const Query&) const;
+    time_point getLastGetTime() const;

    /**
     * Is this get operation done ?
@@ -431,21 +475,51 @@ struct Dht::Search {
    bool isAnnounced(Value::Id id) const;
    bool isListening(time_point now) const;

-    void cancelListen(size_t token) {
-        Sp<Query> query;
-        const auto& ll = listeners.find(token);
-        if (ll != listeners.cend()) {
-            query = ll->second.query;
-            listeners.erase(ll);
-        }
-        for (auto& sn : nodes) {
-            if (listeners.empty())
-                sn.cancelListen();
-            else if (query)
-                sn.cancelListen(query);
+    void get(Value::Filter f, const Sp<Query>& q, const QueryCallback& qcb, const GetCallback& gcb, const DoneCallback& dcb, Scheduler& scheduler) {
+        if (gcb or qcb) {
+            const auto& now = scheduler.time();
+            callbacks.emplace(now, Get { now, f, q, qcb, gcb, dcb });
+            auto values = cache.get(f);
+            if (not values.empty())
+                gcb(values);
+            scheduler.edit(nextSearchStep, now);
        }
    }

+    size_t listen(ValueCallback cb, Value::Filter f, const Sp<Query>& q, Scheduler& scheduler) {
+        //DHT_LOG.e(id, "[search %s IPv%c] listen", id.toString().c_str(), (af == AF_INET) ? '4' : '6');
+        return cache.listen(cb, q, f, [&](const Sp<Query>& q, ValueCallback vcb){
+            done = false;
+            auto token = ++listener_token;
+            listeners.emplace(token, SearchListener{q, f, vcb});
+            scheduler.edit(nextSearchStep, scheduler.time());
+            return token;
+        });
+    }
+
+    void cancelListen(size_t token, Scheduler& scheduler) {
+        cache.cancelListen(token, scheduler.time());
+        if (not opExpirationJob)
+            opExpirationJob = scheduler.add(time_point::max(), [this,&scheduler]{
+                auto nextExpire = cache.expire(scheduler.time(), [&](size_t t){
+                    Sp<Query> query;
+                    const auto& ll = listeners.find(t);
+                    if (ll != listeners.cend()) {
+                        query = ll->second.query;
+                        listeners.erase(ll);
+                    }
+                    for (auto& sn : nodes) {
+                        if (listeners.empty())
+                            sn.cancelListen();
+                        else if (query)
+                            sn.cancelListen(query);
+                    }
+                });
+                scheduler.edit(opExpirationJob, nextExpire);
+            });
+        scheduler.edit(opExpirationJob, cache.getExpiration());
+    }
+
    /**
     * @return The number of non-good search nodes.
     */
@@ -683,11 +757,20 @@ Dht::Search::isSynced(time_point now) const
 }

 time_point
-Dht::Search::getLastGetTime(Sp<Query> q) const
+Dht::Search::getLastGetTime(const Query& q) const
 {
    time_point last = time_point::min();
    for (const auto& g : callbacks)
-        last = std::max(last, (not q or q->isSatisfiedBy(*g.second.query) ? g.second.start : time_point::min()));
+        last = std::max(last, (q.isSatisfiedBy(*g.second.query) ? g.second.start : time_point::min()));
+    return last;
+}
+
+time_point
+Dht::Search::getLastGetTime() const
+{
+    time_point last = time_point::min();
+    for (const auto& g : callbacks)
+        last = std::max(last, g.second.start);
    return last;
 }

@@ -732,7 +815,7 @@ Dht::Search::isListening(time_point now) const
    for (const auto& n : nodes) {
        if (n.isBad())
            continue;
-        SearchNode::SyncStatus::const_iterator ls {};
+        SearchNode::NodeListenerStatus::const_iterator ls {};
        for (ls = n.listenStatus.begin(); ls != n.listenStatus.end() ; ++ls) {
            if (n.isListening(now, ls))
                break;
@@ -56,7 +56,7 @@ SecureDht::SecureDht(std::unique_ptr<DhtInterface> dht, SecureDht::Config conf)
            1
        }, [this](bool ok) {
            if (ok)
-                DHT_LOG.DEBUG("SecureDht: public key announced successfully");
+                DHT_LOG.DBG("SecureDht: public key announced successfully");
        }, {}, true);
    }
 }
@@ -68,27 +68,19 @@ ValueType
 SecureDht::secureType(ValueType&& type)
 {
    type.storePolicy = [this,type](InfoHash id, Sp<Value>& v, const InfoHash& nid, const SockAddr& a) {
-        if (v->isSigned()) {
-            if (!v->owner or !v->owner->checkSignature(v->getToSign(), v->signature)) {
-                DHT_LOG.WARN("Signature verification failed");
-                return false;
-            }
-            else
-                DHT_LOG.WARN("Signature verification succeeded");
-        }
+        if (v->isSigned())
+            return v->checkSignature();
        return type.storePolicy(id, v, nid, a);
    };
    type.editPolicy = [this,type](InfoHash id, const Sp<Value>& o, Sp<Value>& n, const InfoHash& nid, const SockAddr& a) {
-        if (!o->isSigned())
+        if (not o->isSigned())
            return type.editPolicy(id, o, n, nid, a);
-        if (o->owner != n->owner) {
+        if (o->owner != n->owner or not n->isSigned()) {
            DHT_LOG.WARN("Edition forbidden: owner changed.");
            return false;
        }
-        if (!o->owner or !o->owner->checkSignature(n->getToSign(), n->signature)) {
-            DHT_LOG.WARN("Edition forbidden: signature verification failed.");
+        if (not n->checkSignature())
            return false;
-        }
        if (o->seq == n->seq) {
            // If the data is exactly the same,
            // it can be reannounced, possibly by someone else.
@@ -139,7 +131,7 @@ SecureDht::registerCertificate(const InfoHash& node, const Blob& data)
    }
    InfoHash h = crt->getPublicKey().getId();
    if (node == h) {
-        DHT_LOG.DEBUG("Registering certificate for %s", h.toString().c_str());
+        DHT_LOG.DBG("Registering certificate for %s", h.toString().c_str());
        auto it = nodesCertificates_.find(h);
        if (it == nodesCertificates_.end())
            std::tie(it, std::ignore) = nodesCertificates_.emplace(h, std::move(crt));
@@ -147,7 +139,7 @@ SecureDht::registerCertificate(const InfoHash& node, const Blob& data)
            it->second = std::move(crt);
        return it->second;
    } else {
-        DHT_LOG.DEBUG("Certificate %s for node %s does not match node id !", h.toString().c_str(), node.toString().c_str());
+        DHT_LOG.WARN("Certificate %s for node %s does not match node id !", h.toString().c_str(), node.toString().c_str());
        return nullptr;
    }
 }
@@ -164,7 +156,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function<void(const Sp<cry
 {
    Sp<crypto::Certificate> b = getCertificate(node);
    if (b && *b) {
-        DHT_LOG.DEBUG("Using certificate from cache for %s", node.toString().c_str());
+        DHT_LOG.DBG("Using certificate from cache for %s", node.toString().c_str());
        if (cb)
            cb(b);
        return;
@@ -172,7 +164,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function<void(const Sp<cry
    if (localQueryMethod_) {
        auto res = localQueryMethod_(node);
        if (not res.empty()) {
-            DHT_LOG.DEBUG("Registering certificate from local store for %s", node.toString().c_str());
+            DHT_LOG.DBG("Registering certificate from local store for %s", node.toString().c_str());
            nodesCertificates_.emplace(node, res.front());
            if (cb)
                cb(res.front());
@@ -187,7 +179,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function<void(const Sp<cry
        for (const auto& v : vals) {
            if (auto cert = registerCertificate(node, v->data)) {
                *found = true;
-                DHT_LOG.DEBUG("Found certificate for %s", node.toString().c_str());
+                DHT_LOG.DBG("Found certificate for %s", node.toString().c_str());
                if (cb)
                    cb(cert);
                return false;
@@ -205,7 +197,7 @@ SecureDht::findPublicKey(const InfoHash& node, std::function<void(const Sp<const
 {
    auto pk = getPublicKey(node);
    if (pk && *pk) {
-        DHT_LOG.DEBUG("Found public key from cache for %s", node.toString().c_str());
+        DHT_LOG.DBG("Found public key from cache for %s", node.toString().c_str());
        if (cb)
            cb(pk);
        return;
@@ -213,58 +205,82 @@ SecureDht::findPublicKey(const InfoHash& node, std::function<void(const Sp<const
    findCertificate(node, [=](const Sp<crypto::Certificate> crt) {
        if (crt && *crt) {
            auto pk = std::make_shared<crypto::PublicKey>(crt->getPublicKey());
-            nodesPubKeys_[pk->getId()] = pk;
-            if (cb) cb(pk);
-        } else {
-            if (cb) cb(nullptr);
+            if (*pk) {
+                nodesPubKeys_[pk->getId()] = pk;
+                if (cb) cb(pk);
+                return;
+            }
        }
+        if (cb) cb(nullptr);
    });
 }

+Sp<Value>
+SecureDht::checkValue(const Sp<Value>& v)
+{
+    // Decrypt encrypted values
+    if (v->isEncrypted()) {
+        if (not key_) {
+#if OPENDHT_PROXY_SERVER
+            if (forward_all_) // We are currently a proxy, send messages to clients.
+                return v;
+#endif
+            return {};
+        }
+        try {
+            if (auto decrypted_val = v->decrypt(*key_)) {
+                if (decrypted_val->owner)
+                    nodesPubKeys_[decrypted_val->owner->getId()] = decrypted_val->owner;
+                return decrypted_val;
+            }
+        } catch (const std::exception& e) {
+            DHT_LOG.WARN("Could not decrypt value %s : %s", v->toString().c_str(), e.what());
+        }
+    }
+    // Check signed values
+    else if (v->isSigned()) {
+        if (v->checkSignature()) {
+            if (v->owner)
+                nodesPubKeys_[v->owner->getId()] = v->owner;
+            return v;
+        } else
+            DHT_LOG.WARN("Signature verification failed for %s", v->toString().c_str());
+    }
+    // Forward normal values
+    else {
+        return v;
+    }
+    return {};
+}
+
+ValueCallback
+SecureDht::getCallbackFilter(ValueCallback cb, Value::Filter&& filter)
+{
+    return [=](const std::vector<Sp<Value>>& values, bool expired) {
+        std::vector<Sp<Value>> tmpvals {};
+        for (const auto& v : values) {
+            if (auto nv = checkValue(v))
+                if (not filter or filter(*nv))
+                    tmpvals.emplace_back(std::move(nv));
+        }
+        if (cb and not tmpvals.empty())
+            return cb(tmpvals, expired);
+        return true;
+    };
+}
+
+
 GetCallback
 SecureDht::getCallbackFilter(GetCallback cb, Value::Filter&& filter)
 {
    return [=](const std::vector<Sp<Value>>& values) {
        std::vector<Sp<Value>> tmpvals {};
        for (const auto& v : values) {
-            // Decrypt encrypted values
-            if (v->isEncrypted()) {
-                if (not key_) {
-#if OPENDHT_PROXY_SERVER
-                    if (forward_all_) // We are currently a proxy, send messages to clients.
-                        tmpvals.push_back(v);
-#endif
-                    continue;
-                }
-                try {
-                    Value decrypted_val (decrypt(*v));
-                    if (decrypted_val.recipient == getId()) {
-                        nodesPubKeys_[decrypted_val.owner->getId()] = decrypted_val.owner;
-                        if (not filter or filter(decrypted_val))
-                            tmpvals.push_back(std::make_shared<Value>(std::move(decrypted_val)));
-                    }
-                    // Ignore values belonging to other people
-                } catch (const std::exception& e) {
-                    DHT_LOG.WARN("Could not decrypt value %s : %s", v->toString().c_str(), e.what());
-                }
-            }
-            // Check signed values
-            else if (v->isSigned()) {
-                if (v->owner and v->owner->checkSignature(v->getToSign(), v->signature)) {
-                    nodesPubKeys_[v->owner->getId()] = v->owner;
-                    if (not filter  or filter(*v))
-                        tmpvals.push_back(v);
-                }
-                else
-                    DHT_LOG.WARN("Signature verification failed for %s", v->toString().c_str());
-            }
-            // Forward normal values
-            else {
-                if (not filter or filter(*v))
-                    tmpvals.push_back(v);
-            }
+            if (auto nv = checkValue(v))
+                if (not filter or filter(*nv))
+                    tmpvals.emplace_back(std::move(nv));
        }
-        if (cb && not tmpvals.empty())
+        if (cb and not tmpvals.empty())
            return cb(tmpvals);
        return true;
    };
@@ -276,6 +292,13 @@ SecureDht::get(const InfoHash& id, GetCallback cb, DoneCallback donecb, Value::F
    dht_->get(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)), donecb, {}, std::forward<Where>(w));
 }

+size_t
+SecureDht::listen(const InfoHash& id, ValueCallback cb, Value::Filter f, Where w)
+{
+    return dht_->listen(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)), {}, std::forward<Where>(w));
+}
+
+
 size_t
 SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where w)
 {
@@ -287,20 +310,20 @@ SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback,
 {
    if (val->id == Value::INVALID_ID) {
        crypto::random_device rdev;
+        std::uniform_int_distribution<Value::Id> rand_id;
        val->id = rand_id(rdev);
    }

    // Check if we are already announcing a value
    auto p = dht_->getPut(hash, val->id);
    if (p && val->seq <= p->seq) {
-        DHT_LOG.DEBUG("Found previous value being announced.");
        val->seq = p->seq + 1;
    }

    // Check if data already exists on the dht
    get(hash,
        [val,this] (const std::vector<Sp<Value>>& vals) {
-            DHT_LOG.DEBUG("Found online previous value being announced.");
+            DHT_LOG.DBG("Found online previous value being announced.");
            for (const auto& v : vals) {
                if (!v->isSigned())
                    DHT_LOG.ERR("Existing non-signed value seems to exists at this location.");
@@ -322,7 +345,7 @@ SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback,
 void
 SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, Sp<Value> val, DoneCallback callback, bool permanent)
 {
-    findPublicKey(to, [=](const Sp<const crypto::PublicKey> pk) {
+    findPublicKey(to, [=](const Sp<const crypto::PublicKey>& pk) {
        if(!pk || !*pk) {
            if (callback)
                callback(false, {});
@@ -154,20 +154,21 @@ struct Storage {
     *
     * @param now  The reference to now
     * @param vid  The value id
-     * @return true if a value storage was updated, false otherwise
+     * @return time of the next expiration, time_point::max() if no expiration
     */
-    bool refresh(const time_point& now, const Value::Id& vid) {
+    time_point refresh(const time_point& now, const Value::Id& vid, const TypeStore& types) {
        for (auto& vs : values)
            if (vs.data->id == vid) {
                vs.created = now;
-                return true;
+                vs.expiration = std::max(vs.expiration, now + types.getType(vs.data->type).expiration);
+                return vs.expiration;
            }
-        return false;
+        return time_point::max();
    }

    StoreDiff remove(const InfoHash& id, Value::Id);

-    StoreDiff expire(const InfoHash& id, time_point now);
+    std::pair<ssize_t, std::vector<Sp<Value>>> expire(const InfoHash& id, time_point now);

 private:
    Storage(const Storage&) = delete;
@@ -245,7 +246,7 @@ Storage::clear()
    return {-tot_size, -num_values, 0};
 }

-Storage::StoreDiff
+std::pair<ssize_t, std::vector<Sp<Value>>>
 Storage::expire(const InfoHash& id, time_point now)
 {
    // expire listeners
@@ -271,16 +272,18 @@ Storage::expire(const InfoHash& id, time_point now)
    auto r = std::partition(values.begin(), values.end(), [&](const ValueStorage& v) {
        return v.expiration > now;
    });
-    ssize_t del_num = -std::distance(r, values.end());
+    std::vector<Sp<Value>> ret;
+    ret.reserve(std::distance(r, values.end()));
    ssize_t size_diff {};
    std::for_each(r, values.end(), [&](const ValueStorage& v) {
        size_diff -= v.data->size();
        if (v.store_bucket)
            v.store_bucket->erase(id, *v.data, v.expiration);
+        ret.emplace_back(std::move(v.data));
    });
    total_size += size_diff;
    values.erase(r, values.end());
-    return {size_diff, del_num, del_listen};
+    return {size_diff, std::move(ret)};
 }

 }
@@ -29,6 +29,26 @@ namespace dht {

 static constexpr std::array<uint8_t, 12> MAPPED_IPV4_PREFIX {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}};

+std::pair<std::string, std::string>
+splitPort(const std::string& s) {
+    if (s.empty())
+        return {};
+    if (s[0] == '[') {
+        std::size_t closure = s.find_first_of(']');
+        std::size_t found = s.find_last_of(':');
+        if (closure == std::string::npos)
+            return {s, ""};
+        if (found == std::string::npos or found < closure)
+            return {s.substr(1,closure-1), ""};
+        return {s.substr(1,closure-1), s.substr(found+1)};
+    }
+    std::size_t found = s.find_last_of(':');
+    std::size_t first = s.find_first_of(':');
+    if (found == std::string::npos or found != first)
+        return {s, ""};
+    return {s.substr(0,found), s.substr(found+1)};
+}
+
 std::vector<SockAddr>
 SockAddr::resolve(const std::string& host, const std::string& service)
 {
@@ -22,9 +22,9 @@
 #include "default_types.h"
 #include "securedht.h" // print certificate ID

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
 #include "base64.h"
-#endif //OPENDHT_PROXY_SERVER
+#endif


 namespace dht {
@@ -172,43 +172,40 @@ Value::msgpack_unpack_body(const msgpack::object& o)
    }
 }

-#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_JSONCPP
 Value::Value(Json::Value& json)
 {
-   try {
-       if (json.isMember("id"))
-           id = ValueType::Id(json["id"].asInt());
-   } catch (...) { }
-   if (json.isMember("cypher")) {
-       auto cypherStr = json["cypher"].asString();
-       cypherStr = base64_decode(cypherStr);
-       cypher = std::vector<unsigned char>(cypherStr.begin(), cypherStr.end());
-   }
-   if (json.isMember("sig")) {
-       auto sigStr = json["sig"].asString();
-       sigStr = base64_decode(sigStr);
-       signature = std::vector<unsigned char>(sigStr.begin(), sigStr.end());
-   }
-   if (json.isMember("seq"))
-       seq = json["seq"].asInt();
-   if (json.isMember("owner")) {
-       auto ownerStr = json["owner"].asString();
-       auto ownerBlob = std::vector<unsigned char>(ownerStr.begin(), ownerStr.end());
-       owner = std::make_shared<const crypto::PublicKey>(ownerBlob);
-   }
-   if (json.isMember("to")) {
-       auto toStr = json["to"].asString();
-       recipient = InfoHash(toStr);
-   }
-   if (json.isMember("type"))
-       type = json["type"].asInt();
-   if (json.isMember("data")){
-       auto dataStr = json["data"].asString();
-       dataStr = base64_decode(dataStr);
-       data = std::vector<unsigned char>(dataStr.begin(), dataStr.end());
-   }
-   if (json.isMember("utype"))
-       user_type = json["utype"].asString();
+    id = Value::Id(unpackId(json, "id"));
+    if (json.isMember("cypher")) {
+        auto cypherStr = json["cypher"].asString();
+        cypherStr = base64_decode(cypherStr);
+        cypher = std::vector<unsigned char>(cypherStr.begin(), cypherStr.end());
+    }
+    if (json.isMember("sig")) {
+        auto sigStr = json["sig"].asString();
+        sigStr = base64_decode(sigStr);
+        signature = std::vector<unsigned char>(sigStr.begin(), sigStr.end());
+    }
+    if (json.isMember("seq"))
+        seq = json["seq"].asInt();
+    if (json.isMember("owner")) {
+        auto ownerStr = json["owner"].asString();
+        auto ownerBlob = std::vector<unsigned char>(ownerStr.begin(), ownerStr.end());
+        owner = std::make_shared<const crypto::PublicKey>(ownerBlob);
+    }
+    if (json.isMember("to")) {
+        auto toStr = json["to"].asString();
+        recipient = InfoHash(toStr);
+    }
+    if (json.isMember("type"))
+        type = json["type"].asInt();
+    if (json.isMember("data")){
+        auto dataStr = json["data"].asString();
+        dataStr = base64_decode(dataStr);
+        data = std::vector<unsigned char>(dataStr.begin(), dataStr.end());
+    }
+    if (json.isMember("utype"))
+        user_type = json["utype"].asString();
 }

 Json::Value
@@ -225,7 +222,7 @@ Value::toJson() const
        if (has_owner) { // isSigned
            val["seq"] = seq;
            val["owner"] = owner->toString();
-            if (recipient != InfoHash())
+            if (recipient)
                val["to"] = recipient.toString();
        }
        val["type"] = type;
@@ -235,7 +232,56 @@ Value::toJson() const
    }
    return val;
 }
-#endif //OPENDHT_PROXY_SERVER
+
+bool
+Value::checkSignature()
+{
+    if (!signatureChecked) {
+        signatureChecked = true;
+        if (isSigned()) {
+            signatureValid = owner and owner->checkSignature(getToSign(), signature);
+        }
+    }
+    return signatureValid;
+}
+
+Sp<Value>
+Value::decrypt(const crypto::PrivateKey& key)
+{
+    if (not decrypted) {
+        decrypted = true;
+        if (isEncrypted()) {
+            auto decryptedBlob = key.decrypt(cypher);
+            std::unique_ptr<Value> v {new Value(id)};
+            auto msg = msgpack::unpack((const char*)decryptedBlob.data(), decryptedBlob.size());
+            v->msgpack_unpack_body(msg.get());
+            if (v->recipient != key.getPublicKey().getId())
+                throw crypto::DecryptError("Recipient mismatch");
+            // Ignore values belonging to other people
+            if (not v->owner or not v->owner->checkSignature(v->getToSign(), v->signature))
+                throw crypto::DecryptError("Signature mismatch");
+            decryptedValue = std::move(v);
+        }
+    }
+    return decryptedValue;
+}
+
+uint64_t
+unpackId(const Json::Value& json, const std::string& key) {
+    uint64_t ret = 0;
+    try {
+        if (json.isMember(key)) {
+            const auto& t = json[key];
+            if (t.isString()) {
+                ret = std::stoull(t.asString());
+            } else {
+                ret = t.asLargestUInt();
+            }
+        }
+    } catch (...) {}
+    return ret;
+}
+#endif

 bool
 FieldValue::operator==(const FieldValue& vfd) const
@@ -0,0 +1,189 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *  Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "value.h"
+
+namespace dht {
+
+using ValueStateCallback = std::function<void(const std::vector<Sp<Value>>&, bool)>;
+using CallbackQueue = std::list<std::function<void()>>;
+
+class ValueCache {
+public:
+    ValueCache(ValueStateCallback&& cb) : callback(std::forward<ValueStateCallback>(cb)) {}
+    ValueCache(ValueCache&& o) : values(std::move(o.values)), callback(std::move(o.callback)) {
+        o.callback = {};
+    }
+
+    ~ValueCache() {
+        auto q = clear();
+        for (auto& cb: q)
+            cb();
+    }
+
+    CallbackQueue clear() {
+        std::vector<Sp<Value>> expired_values;
+        expired_values.reserve(values.size());
+        for (const auto& v : values)
+            expired_values.emplace_back(std::move(v.second.data));
+        values.clear();
+        CallbackQueue ret;
+        if (not expired_values.empty() and callback) {
+            auto cb = callback;
+            ret.emplace_back([expired_values, cb]{
+                cb(expired_values, true);
+            });
+        }
+        return ret;
+    }
+
+    time_point expireValues(const time_point& now) {
+        time_point ret = time_point::max();
+        auto cbs = expireValues(now, ret);
+        while (not cbs.empty()) {
+            cbs.front()();
+            cbs.pop_front();
+        }
+        return ret;
+    }
+
+    CallbackQueue expireValues(const time_point& now, time_point& next) {
+        std::vector<Sp<Value>> expired_values;
+        for (auto it = values.begin(); it != values.end();) {
+            if (it->second.expiration <= now) {
+                expired_values.emplace_back(std::move(it->second.data));
+                it = values.erase(it);
+            } else {
+                next = std::min(next, it->second.expiration);
+                ++it;
+            }
+        }
+        while (values.size() > MAX_VALUES) {
+            // too many values, remove oldest values
+            time_point oldest_creation = time_point::max();
+            auto oldest_value = values.end();
+            for (auto it = values.begin(); it != values.end(); ++it)
+                if (it->second.created < oldest_creation) {
+                    oldest_value = it;
+                    oldest_creation = it->second.created;
+                }
+            if (oldest_value != values.end()) {
+                expired_values.emplace_back(std::move(oldest_value->second.data));
+                values.erase(oldest_value);
+            }
+        }
+        CallbackQueue ret;
+        if (not expired_values.empty() and callback) {
+            auto cb = callback;
+            ret.emplace_back([cb, expired_values]{
+                if (cb) cb(expired_values, true);
+            });
+        }
+        return ret;
+    }
+
+    time_point onValues
+        (const std::vector<Sp<Value>>& values,
+        const std::vector<Value::Id>& refreshed_values,
+        const std::vector<Value::Id>& expired_values,
+        const TypeStore& types, const time_point& now)
+    {
+        CallbackQueue cbs;
+        time_point ret = time_point::max();
+        if (not values.empty())
+            cbs.splice(cbs.end(), addValues(values, types, now));
+        for (const auto& vid : refreshed_values)
+            refreshValue(vid, types, now);
+        for (const auto& vid : expired_values)
+            cbs.splice(cbs.end(), expireValue(vid));
+        cbs.splice(cbs.end(), expireValues(now, ret));
+        while (not cbs.empty()) {
+            cbs.front()();
+            cbs.pop_front();
+        }
+        return ret;
+    }
+
+private:
+    // prevent copy
+    ValueCache(const ValueCache&) = delete;
+    ValueCache& operator=(const ValueCache&) = delete;
+    ValueCache& operator=(ValueCache&&) = delete;
+
+    /* The maximum number of values we store in the cache. */
+    static constexpr unsigned MAX_VALUES {4096};
+
+    struct CacheValueStorage {
+        Sp<Value> data {};
+        time_point created {};
+        time_point expiration {};
+
+        CacheValueStorage() {}
+        CacheValueStorage(const Sp<Value>& v, time_point t, time_point e)
+         : data(v), created(t), expiration(e) {}
+    };
+
+    std::map<Value::Id, CacheValueStorage> values;
+    ValueStateCallback callback;
+
+    CallbackQueue addValues(const std::vector<Sp<Value>>& new_values, const TypeStore& types, const time_point& now) {
+        std::vector<Sp<Value>> nvals;
+        for (const auto& value : new_values) {
+            auto v = values.find(value->id);
+            if (v == values.end()) {
+                // new value
+                nvals.emplace_back(value);
+                values.emplace(value->id, CacheValueStorage(value, now, now + types.getType(value->type).expiration));
+            } else {
+                // refreshed value
+                v->second.created = now;
+                v->second.expiration = now + types.getType(v->second.data->type).expiration;
+            }
+        }
+        auto cb = callback;
+        CallbackQueue ret;
+        if (not nvals.empty())
+            ret.emplace_back([cb, nvals]{
+                if (cb) cb(nvals, false);
+            });
+        return ret;
+    }
+    CallbackQueue expireValue(Value::Id vid) {
+        auto v = values.find(vid);
+        if (v == values.end())
+            return {};
+        const std::vector<Sp<Value>> val {std::move(v->second.data)};
+        values.erase(v);
+        auto cb = callback;
+        CallbackQueue ret;
+        ret.emplace_back([cb, val]{
+            if (cb) cb(val, true);
+        });
+        return ret;
+    }
+    void refreshValue(Value::Id vid, const TypeStore& types, const time_point& now) {
+        auto v = values.find(vid);
+        if (v == values.end())
+            return;
+        v->second.created = now;
+        v->second.expiration = now + types.getType(v->second.data->type).expiration;
+    }
+};
+
+}
@@ -0,0 +1,9 @@
+if ENABLE_TESTS
+bin_PROGRAMS = opendht_unit_tests
+
+AM_CPPFLAGS = -I../include
+
+nobase_include_HEADERS = infohashtester.h cryptotester.h dhtrunnertester.h dhtproxytester.h
+opendht_unit_tests_SOURCES = tests_runner.cpp cryptotester.cpp infohashtester.cpp dhtrunnertester.cpp dhtproxytester.cpp
+opendht_unit_tests_LDFLAGS = -lopendht -lcppunit -L@top_builddir@/src/.libs @GnuTLS_LIBS@
+endif
@@ -0,0 +1,90 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "cryptotester.h"
+
+#include "opendht/crypto.h"
+
+namespace test {
+CPPUNIT_TEST_SUITE_REGISTRATION(CryptoTester);
+
+void
+CryptoTester::setUp() {
+
+}
+
+void
+CryptoTester::testSignatureEncryption() {
+    auto key = dht::crypto::PrivateKey::generate();
+    auto public_key = key.getPublicKey();
+
+    std::vector<uint8_t> data {5, 10};
+    std::vector<uint8_t> signature = key.sign(data);
+
+    // check signature
+    CPPUNIT_ASSERT(public_key.checkSignature(data, signature));
+
+    // encrypt data
+    std::vector<uint8_t> encrypted = public_key.encrypt(data);
+    std::vector<uint8_t> decrypted = key.decrypt(encrypted);
+    CPPUNIT_ASSERT(data == decrypted);
+}
+
+void
+CryptoTester::testCertificateRevocation()
+{
+    auto ca1 = dht::crypto::generateIdentity("ca1");
+    auto account1 = dht::crypto::generateIdentity("acc1", ca1, 4096, true);
+    auto device11 = dht::crypto::generateIdentity("dev11", account1);
+    auto device12 = dht::crypto::generateIdentity("dev12", account1);
+
+
+    dht::crypto::TrustList list;
+    list.add(*ca1.second);
+    auto v = list.verify(*account1.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), v);
+
+    list.add(*account1.second);
+    v = list.verify(*device11.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), v);
+    v = list.verify(*device12.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), v);
+
+    auto ca2 = dht::crypto::generateIdentity("ca2");
+    auto account2 = dht::crypto::generateIdentity("acc2", ca2, 4096, true);
+    auto device2 = dht::crypto::generateIdentity("dev2", account2);
+
+    v = list.verify(*device2.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), !v);
+    
+    account1.second->revoke(*account1.first, *device11.second);
+    dht::crypto::TrustList list2;
+    list2.add(*account1.second);
+
+    v = list2.verify(*device11.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), !v);
+    v = list2.verify(*device12.second);
+    CPPUNIT_ASSERT_MESSAGE(v.toString(), v);
+}
+
+void
+CryptoTester::tearDown() {
+
+}
+}  // namespace test
@@ -0,0 +1,53 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+// cppunit
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace test {
+
+class CryptoTester : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(CryptoTester);
+    CPPUNIT_TEST(testSignatureEncryption);
+    CPPUNIT_TEST(testCertificateRevocation);
+    CPPUNIT_TEST_SUITE_END();
+
+ public:
+    /**
+     * Method automatically called before each test by CppUnit
+     */
+    void setUp();
+    /**
+     * Method automatically called after each test CppUnit
+     */
+    void tearDown();
+    /**
+     * Test data signature, encryption and decryption
+     */
+    void testSignatureEncryption();
+    /**
+     * Test certificate generation, validation and revocation
+     */
+    void testCertificateRevocation();
+};
+
+}  // namespace test
@@ -0,0 +1,126 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "dhtproxytester.h"
+
+// std
+#include <iostream>
+#include <string>
+
+#include <chrono>
+#include <condition_variable>
+
+
+namespace test {
+CPPUNIT_TEST_SUITE_REGISTRATION(DhtProxyTester);
+
+void
+DhtProxyTester::setUp() {
+    nodePeer.run(42222, {}, true);
+    nodeProxy = std::make_shared<dht::DhtRunner>();
+    nodeClient = std::make_shared<dht::DhtRunner>();
+
+    nodeProxy->run(42232, {}, true);
+    nodeProxy->bootstrap(nodePeer.getBound());
+    server = std::unique_ptr<dht::DhtProxyServer>(new dht::DhtProxyServer(nodeProxy, 8080));
+
+    nodeClient->run(42242, {}, true);
+    nodeClient->bootstrap(nodePeer.getBound());
+    nodeClient->setProxyServer("127.0.0.1:8080");
+    nodeClient->enableProxy(true);
+}
+
+void
+DhtProxyTester::tearDown() {
+    nodePeer.join();
+    nodeClient->join();
+    server->stop();
+    server = nullptr;
+    nodeProxy->join();
+}
+
+void
+DhtProxyTester::testGetPut() {
+    bool done = false;
+    std::condition_variable cv;
+    std::mutex cv_m;
+
+    auto key = dht::InfoHash::get("GLaDOs");
+    dht::Value val {"Hey! It's been a long time. How have you been?"};
+    auto val_data = val.data;
+
+    nodePeer.put(key, std::move(val), [&](bool) {
+        done = true;
+        cv.notify_all();
+    });
+    std::unique_lock<std::mutex> lk(cv_m);
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+
+    auto vals = nodeClient->get(key).get();
+    CPPUNIT_ASSERT(not vals.empty());
+    CPPUNIT_ASSERT(vals.front()->data == val_data);
+}
+
+
+void
+DhtProxyTester::testListen() {
+    bool done = false;
+    std::condition_variable cv;
+    std::mutex cv_m;
+    std::unique_lock<std::mutex> lk(cv_m);
+    auto key = dht::InfoHash::get("GLaDOs");
+
+    // If a peer send a value, the listen operation from the client
+    // should retrieve this value
+    dht::Value firstVal {"Hey! It's been a long time. How have you been?"};
+    auto firstVal_data = firstVal.data;
+    nodePeer.put(key, std::move(firstVal), [&](bool) {
+        done = true;
+        cv.notify_all();
+    });
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+    done = false;
+
+    auto values = std::vector<dht::Blob>();
+    nodeClient->listen(key, [&](const std::vector<std::shared_ptr<dht::Value>>& v, bool) {
+        for (const auto& value : v)
+            values.emplace_back(value->data);
+        done = true;
+        cv.notify_all();
+        return true;
+    });
+
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+    done = false;
+    // Here values should contains 2 values
+    CPPUNIT_ASSERT_EQUAL(static_cast<int>(values.size()), 1);
+    CPPUNIT_ASSERT(values.front() == firstVal_data);
+
+    // And the listen should retrieve futures values
+    // All values
+    dht::Value secondVal {"You're a monster"};
+    auto secondVal_data = secondVal.data;
+    nodePeer.put(key, std::move(secondVal));
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+    // Here values should contains 3 values
+    CPPUNIT_ASSERT_EQUAL(static_cast<int>(values.size()), 2);
+    CPPUNIT_ASSERT(values.back() == secondVal_data);
+}
+
+}  // namespace test
@@ -0,0 +1,64 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+// cppunit
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#include <opendht/dhtrunner.h>
+#include <opendht/dht_proxy_server.h>
+
+namespace test {
+
+class DhtProxyTester : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DhtProxyTester);
+    CPPUNIT_TEST(testGetPut);
+    CPPUNIT_TEST(testListen);
+    CPPUNIT_TEST_SUITE_END();
+
+ public:
+    /**
+     * Method automatically called before each test by CppUnit
+     * Init nodes
+     */
+    void setUp();
+    /**
+     * Method automatically called after each test CppUnit
+     */
+    void tearDown();
+    /**
+     * Test get and put methods
+     */
+    void testGetPut();
+    /**
+     * Test listen
+     */
+    void testListen();
+
+ private:
+    dht::DhtRunner nodePeer {};
+
+    std::shared_ptr<dht::DhtRunner> nodeClient;
+    std::shared_ptr<dht::DhtRunner> nodeProxy;
+    std::unique_ptr<dht::DhtProxyServer> server;
+};
+
+}  // namespace test
@@ -0,0 +1,64 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "dhtrunnertester.h"
+
+// std
+#include <iostream>
+#include <string>
+
+namespace test {
+CPPUNIT_TEST_SUITE_REGISTRATION(DhtRunnerTester);
+
+void
+DhtRunnerTester::setUp() {
+    node1.run(42222, {}, true);
+    node2.run(42232, {}, true);
+    node2.bootstrap(node1.getBound());
+}
+
+void
+DhtRunnerTester::tearDown() {
+    node1.join();
+    node2.join();
+}
+
+void
+DhtRunnerTester::testConstructors() {
+    CPPUNIT_ASSERT(node1.getBoundPort() == 42222);
+    CPPUNIT_ASSERT(node2.getBoundPort() == 42232);
+}
+
+void
+DhtRunnerTester::testGetPut() {
+    auto key = dht::InfoHash::get("123");
+    dht::Value val {"hey"};
+    auto val_data = val.data;
+    node1.put(key, std::move(val));
+    auto vals = node2.get(key).get();
+    CPPUNIT_ASSERT(not vals.empty());
+    CPPUNIT_ASSERT(vals.front()->data == val_data);
+}
+
+void
+DhtRunnerTester::testListen() {
+    // TODO
+}
+
+}  // namespace test
@@ -0,0 +1,62 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+// cppunit
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#include <opendht/dhtrunner.h>
+
+namespace test {
+
+class DhtRunnerTester : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DhtRunnerTester);
+    CPPUNIT_TEST(testConstructors);
+    CPPUNIT_TEST(testGetPut);
+    CPPUNIT_TEST(testListen);
+    CPPUNIT_TEST_SUITE_END();
+
+    dht::DhtRunner node1 {};
+    dht::DhtRunner node2 {};
+ public:
+    /**
+     * Method automatically called before each test by CppUnit
+     */
+    void setUp();
+    /**
+     * Method automatically called after each test CppUnit
+     */
+    void tearDown();
+    /**
+     * Test the differents behaviors of constructors
+     */
+    void testConstructors();
+    /**
+     * Test get and put methods
+     */
+    void testGetPut();
+    /**
+     * Test listen method
+     */
+    void testListen();
+};
+
+}  // namespace test
@@ -0,0 +1,144 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "infohashtester.h"
+
+// std
+#include <iostream>
+#include <string>
+
+// opendht
+#include "opendht/infohash.h"
+
+namespace test {
+CPPUNIT_TEST_SUITE_REGISTRATION(InfoHashTester);
+
+void
+InfoHashTester::setUp() {
+
+}
+
+void
+InfoHashTester::testConstructors() {
+    // Default constructor creates a null infohash
+    auto nullHash = dht::InfoHash();
+    CPPUNIT_ASSERT(nullHash.size() == 20);
+    CPPUNIT_ASSERT(!nullHash);
+    // Build from a uint8_t. if length to short, should get a null infohash
+    uint8_t to_short[] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8};
+    auto infohash = dht::InfoHash(to_short, 8);
+    CPPUNIT_ASSERT(infohash.size() == 20);
+    CPPUNIT_ASSERT_EQUAL(infohash.toString(),
+        std::string("0000000000000000000000000000000000000000"));
+    // Build from a uint8_t. if length is enough, data should contains the uint8_t
+    uint8_t enough[] = {0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,
+                        0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa};
+    infohash = dht::InfoHash(enough, 20);
+    CPPUNIT_ASSERT(infohash.size() == 20);
+    const auto* data = infohash.data();
+    for (auto i = 0; i < 20; ++i) {
+        CPPUNIT_ASSERT_EQUAL(enough[i], data[i]);
+    }
+    // if too long, should be cutted to 20
+    uint8_t tooLong[] = {0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,
+                        0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb0};
+    infohash = dht::InfoHash(tooLong, 21);
+    CPPUNIT_ASSERT(infohash.size() == 20);
+    const auto* data2 = infohash.data();
+    for (auto i = 0; i < 20; ++i) {
+        CPPUNIT_ASSERT_EQUAL(enough[i], data2[i]);
+    }
+    // Build from string
+    auto infohashFromStr = dht::InfoHash("0102030405060708090A0102030405060708090A");
+    CPPUNIT_ASSERT(infohashFromStr.size() == 20);
+    const auto* dataStr = infohashFromStr.data();
+    for (auto i = 0; i < 20; ++i) {
+        CPPUNIT_ASSERT_EQUAL((int)dataStr[i], (int)data[i]);
+    }
+}
+
+void
+InfoHashTester::testComperators() {
+    auto nullHash = dht::InfoHash();
+    auto minHash = dht::InfoHash("0000000000000000000000000000000000111110");
+    auto maxHash = dht::InfoHash("0111110000000000000000000000000000000000");
+    // operator ==
+    CPPUNIT_ASSERT_EQUAL(minHash, minHash);
+    CPPUNIT_ASSERT_EQUAL(minHash, dht::InfoHash("0000000000000000000000000000000000111110"));
+	CPPUNIT_ASSERT(!(minHash == maxHash));
+    // operator !=
+    CPPUNIT_ASSERT(!(minHash != minHash));
+    CPPUNIT_ASSERT(!(minHash != dht::InfoHash("0000000000000000000000000000000000111110")));
+	CPPUNIT_ASSERT(minHash != maxHash);
+    // operator<
+    CPPUNIT_ASSERT(nullHash < minHash);
+    CPPUNIT_ASSERT(nullHash < maxHash);
+    CPPUNIT_ASSERT(minHash < maxHash);
+    CPPUNIT_ASSERT(!(minHash < nullHash));
+    CPPUNIT_ASSERT(!(maxHash < nullHash));
+    CPPUNIT_ASSERT(!(maxHash < minHash));
+    CPPUNIT_ASSERT(!(minHash < minHash));
+    // bool()
+    CPPUNIT_ASSERT(maxHash);
+    CPPUNIT_ASSERT(!nullHash);
+
+}
+
+void
+InfoHashTester::testLowBit() {
+    auto nullHash = dht::InfoHash();
+    auto minHash = dht::InfoHash("0000000000000000000000000000000000000010");
+    auto maxHash = dht::InfoHash("0100000000000000000000000000000000000000");
+    CPPUNIT_ASSERT_EQUAL(nullHash.lowbit(), -1);
+    CPPUNIT_ASSERT_EQUAL(minHash.lowbit(), 155);
+    CPPUNIT_ASSERT_EQUAL(maxHash.lowbit(), 7);
+}
+
+void
+InfoHashTester::testCommonBits() {
+    auto nullHash = dht::InfoHash();
+    auto minHash = dht::InfoHash("0000000000000000000000000000000000000010");
+    auto maxHash = dht::InfoHash("0100000000000000000000000000000000000000");
+    CPPUNIT_ASSERT_EQUAL(dht::InfoHash::commonBits(nullHash, nullHash), (unsigned)160);
+    CPPUNIT_ASSERT_EQUAL(dht::InfoHash::commonBits(nullHash, minHash), (unsigned)155);
+    CPPUNIT_ASSERT_EQUAL(dht::InfoHash::commonBits(nullHash, maxHash), (unsigned)7);
+    CPPUNIT_ASSERT_EQUAL(dht::InfoHash::commonBits(minHash, maxHash), (unsigned)7);
+}
+
+void
+InfoHashTester::testXorCmp() {
+    auto nullHash = dht::InfoHash();
+    auto minHash = dht::InfoHash("0000000000000000000000000000000000000010");
+    auto maxHash = dht::InfoHash("0100000000000000000000000000000000000000");
+    CPPUNIT_ASSERT_EQUAL(minHash.xorCmp(nullHash, maxHash), -1);
+    CPPUNIT_ASSERT_EQUAL(minHash.xorCmp(maxHash, nullHash), 1);
+    CPPUNIT_ASSERT_EQUAL(minHash.xorCmp(minHash, maxHash), -1);
+    CPPUNIT_ASSERT_EQUAL(minHash.xorCmp(maxHash, minHash), 1);
+    CPPUNIT_ASSERT_EQUAL(nullHash.xorCmp(minHash, maxHash), -1);
+    CPPUNIT_ASSERT_EQUAL(nullHash.xorCmp(maxHash, minHash), 1);
+    // Because hashes are circular in distance.
+    CPPUNIT_ASSERT_EQUAL(maxHash.xorCmp(nullHash, minHash), -1);
+    CPPUNIT_ASSERT_EQUAL(maxHash.xorCmp(minHash, nullHash), 1);
+}
+
+void
+InfoHashTester::tearDown() {
+
+}
+}  // namespace test
@@ -0,0 +1,69 @@
+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+// cppunit
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace test {
+
+class InfoHashTester : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(InfoHashTester);
+    CPPUNIT_TEST(testConstructors);
+    CPPUNIT_TEST(testComperators);
+    CPPUNIT_TEST(testLowBit);
+    CPPUNIT_TEST(testCommonBits);
+    CPPUNIT_TEST(testXorCmp);
+    CPPUNIT_TEST_SUITE_END();
+
+ public:
+    /**
+     * Method automatically called before each test by CppUnit
+     */
+    void setUp();
+    /**
+     * Method automatically called after each test CppUnit
+     */
+    void tearDown();
+    /**
+     * Test the differents behaviors of constructors
+     */
+    void testConstructors();
+    /**
+     * Test compare operators
+     */
+    void testComperators();
+    /**
+     * Test lowbit method
+     */
+    void testLowBit();
+    /**
+     * Test commonBits method
+     */
+    void testCommonBits();
+    /**
+     * Test xorCmp operators
+     */
+    void testXorCmp();
+
+};
+
+}  // namespace test
@@ -0,0 +1,35 @@
+/*
+ *  Copyright (C) 2017-2018 Savoir-faire Linux Inc.
+ *  Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/ui/text/TestRunner.h>
+#include <cppunit/CompilerOutputter.h>
+#include <iostream>
+
+int main(int argc, char** argv) {
+    CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
+    CppUnit::Test *suite = registry.makeTest();
+    if (suite->countTestCases() == 0) {
+        std::cout << "No test cases specified for suite" << std::endl;
+        return 1;
+    }
+    CppUnit::TextUi::TestRunner runner;
+    runner.addTest(suite);
+    auto result = runner.run() ? 0 : 1;
+    return result;
+}
@@ -1,7 +1,7 @@
 bin_PROGRAMS = dhtnode dhtchat dhtscanner
 noinst_HEADERS = tools_common.h

-AM_CPPFLAGS = -I../include
+AM_CPPFLAGS = -I../include @JsonCpp_CFLAGS@ @MsgPack_CFLAGS@

 dhtnode_SOURCES = dhtnode.cpp
 dhtnode_LDFLAGS = -lopendht -lreadline -L@top_builddir@/src/.libs @Argon2_LDFLAGS@ @GnuTLS_LIBS@
@@ -73,7 +73,7 @@ void print_help() {
 #if OPENDHT_PUSH_NOTIFICATIONS
              << "  stt [server_address] <device_key> Start the proxy client." << std::endl
              << "  rs  [token]                       Resubscribe to opendht." << std::endl
-              << "  rp  [push_notification]           Inject a push notification in Opendht." << std::endl
+              << "  rp  [token]                       Inject a push notification in Opendht." << std::endl
 #else
              << "  stt [server_address]              Start the proxy client." << std::endl
 #endif // OPENDHT_PUSH_NOTIFICATIONS
@@ -87,8 +87,10 @@ void print_help() {
              << "  cl <key> <token>      Cancel listen for <token> and <key>." << std::endl
              << "  p <key> <str>         Put string value at <key>." << std::endl
              << "  pp <key> <str>        Put string value at <key> (persistent version)." << std::endl
+              << "  cpp <key> <id>        Cancel persistent put operation for <key> and value <id>." << std::endl
              << "  s <key> <str>         Put string value at <key>, signed with our generated private key." << std::endl
-              << "  e <key> <dest> <str>  Put string value at <key>, encrypted for <dest> with its public key (if found)." << std::endl;
+              << "  e <key> <dest> <str>  Put string value at <key>, encrypted for <dest> with its public key (if found)." << std::endl
+              << "  cc                    Trigger connectivity changed signal." << std::endl;

 #ifdef OPENDHT_INDEXATION
    std::cout << std::endl << "Indexation operations on the DHT:" << std::endl
@@ -139,6 +141,12 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            std::cout << dht->getNodesStats(AF_INET).toString() << std::endl;
            std::cout << "IPv6 stats:" << std::endl;
            std::cout << dht->getNodesStats(AF_INET6).toString() << std::endl;
+#if OPENDHT_PROXY_SERVER
+            for (const auto& proxy : proxies) {
+                std::cout << "Stats for proxy on port " << proxy.first << std::endl;
+                std::cout << "  " << proxy.second->stats().toString() << std::endl;
+            }
+#endif
            continue;
        } else if (op == "lr") {
            std::cout << "IPv4 routing table:" << std::endl;
@@ -172,11 +180,8 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            iss >> idstr;
            try {
                auto addr = splitPort(idstr);
-                if (not addr.first.empty() and addr.second.empty()){
-                    std::stringstream ss;
-                    ss << DHT_DEFAULT_PORT;
-                    addr.second = ss.str();
-                }
+                if (not addr.first.empty() and addr.second.empty())
+                    addr.second = std::to_string(DHT_DEFAULT_PORT);
                dht->bootstrap(addr.first.c_str(), addr.second.c_str());
            } catch (const std::exception& e) {
                std::cerr << e.what() << std::endl;
@@ -192,6 +197,9 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
                log::disableLogging(*dht);
            dht->setLogFilter(filter);
            continue;
+        } else if (op == "cc") {
+            dht->connectivityChanged();
+            continue;
        }
 #if OPENDHT_PROXY_SERVER
        else if (op == "pst") {
@@ -230,14 +238,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
 #if OPENDHT_PUSH_NOTIFICATIONS
        else if (op == "rp") {
            iss >> value;
-            dht->pushNotificationReceived({{"token", value}});
-            continue;
-        } else if (op == "re") {
-            iss >> value;
-            try {
-                unsigned token = std::stoul(value);
-                dht->resubscribe(token);
-            } catch (...) { }
+            dht->pushNotificationReceived({{"to", "dhtnode"}, {"token", value}});
            continue;
        }
 #endif // OPENDHT_PUSH_NOTIFICATIONS
@@ -254,11 +255,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
        }
        dht::InfoHash id;

-        if (op == "cl") {
-            std::string hash, rem;
-            iss >> hash >> rem;
-            dht->cancelListen(dht::InfoHash(hash), std::stoul(rem));
-        }
+        if (false) {}
 #ifdef OPENDHT_INDEXATION
        else if (op == "il" or op == "ii") {
            // Pht syntax
@@ -268,7 +265,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
                        return i.first == index;
                    }) == indexes.end();
            if (not index.size()) {
-                std::cout << "You must enter the index name." << std::endl;
+                std::cerr << "You must enter the index name." << std::endl;
                continue;
            } else if (new_index) {
                using namespace dht::indexation;
@@ -330,14 +327,27 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
        else if (op == "l") {
            std::string rem;
            std::getline(iss, rem);
-            auto token = dht->listen(id, [](std::shared_ptr<Value> value) {
-                std::cout << "Listen: found value:" << std::endl;
-                std::cout << "\t" << *value << std::endl;
+            auto token = dht->listen(id, [](const std::vector<std::shared_ptr<Value>>& values, bool expired) {
+                std::cout << "Listen: found " << values.size() << " values" << (expired ? " expired" : "") << std::endl;
+                for (const auto& value : values)
+                    std::cout << "\t" << *value << std::endl;
                return true;
            }, {}, dht::Where {std::move(rem)});
            auto t = token.get();
            std::cout << "Listening, token: " << t << std::endl;
        }
+        if (op == "cl") {
+            std::string rem;
+            iss >> rem;
+            size_t token;
+            try {
+                token = std::stoul(rem);
+            } catch(...) {
+                std::cerr << "Syntax: cl [key] [token]" << std::endl;
+                continue;
+            }
+            dht->cancelListen(id, token);
+        }
        else if (op == "p") {
            std::string v;
            iss >> v;
@@ -487,8 +497,9 @@ main(int argc, char **argv)
            crt = dht::crypto::generateIdentity("DHT Node", ca_tmp);
        }

-        dht::DhtRunner::Config config;
+        dht::DhtRunner::Config config {};
        config.dht_config.node_config.network = params.network;
+        config.dht_config.node_config.maintain_storage = false;
        config.dht_config.id = crt;
        config.threaded = true;
        config.proxy_server = params.proxyclient;
@@ -63,11 +63,15 @@ $(function() {
                        if (!element || element.length == 0)
                            return;
                        const o = JSON.parse(element);
-                        const d = window.atob(o.data);
-                        const delay = Math.max(0, start-now);
-                        $('<li class="list-group-item"/>').append(valueGetElement(o)).appendTo(group).hide().delay(delay).slideDown(100);
-                        lastAppended = i+1;
-                        start = Math.max(start, now)+25;
+                        if (o.expired) {
+                            $('#value'+o.id).slideUp(100, (e) => $(e.target).remove());
+                        } else {
+                            const d = window.atob(o.data);
+                            const delay = Math.max(0, start-now);
+                            $('<li class="list-group-item" id="value'+o.id+'"/>').append(valueGetElement(o)).appendTo(group).hide().delay(delay).slideDown(100);
+                            lastAppended = i+1;
+                            start = Math.max(start, now)+25;
+                        }
                    }
                } else if (this.status !== 0) {
                    group.empty().append($('<li class="list-group-item list-group-item-danger"/>').text("Error loading content: " + this.statusText));
@@ -45,29 +45,6 @@
 #include <sstream>
 #include <fstream>

-/**
- * Split "[host]:port" or "host:port" to pair<"host", "port">.
- */
-std::pair<std::string, std::string>
-splitPort(const std::string& s) {
-    if (s.empty())
-        return {};
-    if (s[0] == '[') {
-        std::size_t closure = s.find_first_of(']');
-        std::size_t found = s.find_last_of(':');
-        if (closure == std::string::npos)
-            return {s, ""};
-        if (found == std::string::npos or found < closure)
-            return {s.substr(1,closure-1), ""};
-        return {s.substr(1,closure-1), s.substr(found+1)};
-    }
-    std::size_t found = s.find_last_of(':');
-    std::size_t first = s.find_first_of(':');
-    if (found == std::string::npos or found != first)
-        return {s, ""};
-    return {s.substr(0,found), s.substr(found+1)};
-}
-
 /*
 * The mapString shall have the following format:
 *
@@ -181,7 +158,7 @@ parseArgs(int argc, char **argv) {
            params.network = strtoul(optarg, nullptr, 0);
            break;
        case 'b':
-            params.bootstrap = splitPort((optarg[0] == '=') ? optarg+1 : optarg);
+            params.bootstrap = dht::splitPort((optarg[0] == '=') ? optarg+1 : optarg);
            if (not params.bootstrap.first.empty() and params.bootstrap.second.empty()) {
                params.bootstrap.second = std::to_string(DHT_DEFAULT_PORT);
            }