make move operators noexcept

storage: make non-movable
thread pool: add computation, io pools
2026-11-09 12:57:03 +00:00 · 2019-03-10 15:56:33 -04:00 · 2019-03-10 15:56:33 -04:00 · 2019-03-10 15:20:31 -04:00 · 2019-03-10 15:09:49 -04:00 · 2019-02-24 16:09:29 -05:00
40 changed files with 1242 additions and 699 deletions
@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 3.1)
 project (opendht)
 set (opendht_VERSION_MAJOR 1)
-set (opendht_VERSION_MINOR 8.0)
+set (opendht_VERSION_MINOR 8.1)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -70,6 +70,8 @@ set (CMAKE_CXX_STANDARD_REQUIRED on)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden")
 if (OPENDHT_SANITIZE)
    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong")
+else ()
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-stack-protector")
 endif ()
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT")
 if (NOT CMAKE_BUILD_TYPE)
@@ -182,40 +184,34 @@ if (OPENDHT_INDEX)
 endif()

 if (OPENDHT_PROXY_SERVER)
-  add_definitions(-DOPENDHT_PROXY_SERVER=true)
+  add_definitions(-DOPENDHT_PROXY_SERVER)
  if (OPENDHT_PROXY_SERVER_IDENTITY)
-    add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=true)
-  else ()
-    add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=false)
+    add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY)
  endif()
  list (APPEND opendht_HEADERS
    include/opendht/dht_proxy_server.h
  )
  list (APPEND opendht_SOURCES
+    src/thread_pool.h
+    src/thread_pool.cpp
    src/dht_proxy_server.cpp
  )
-else ()
-    add_definitions(-DENABLE_PROXY_SERVER=false)
 endif ()

 if (OPENDHT_PROXY_CLIENT)
-  add_definitions(-DOPENDHT_PROXY_CLIENT=true)
+  add_definitions(-DOPENDHT_PROXY_CLIENT)
  list (APPEND opendht_HEADERS
    include/opendht/dht_proxy_client.h
  )
  list (APPEND opendht_SOURCES
    src/dht_proxy_client.cpp
  )
-else ()
-  add_definitions(-DOPENDHT_PROXY_CLIENT=false)
 endif ()

 if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT)
  if (OPENDHT_PUSH_NOTIFICATIONS)
    message("Using push notification")
-    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=true)
-  else ()
-    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=false)
+    add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS)
  endif ()
  list (APPEND opendht_HEADERS
    include/opendht/proxy.h
@@ -304,7 +300,7 @@ install (EXPORT opendht DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/opendht FILE o
 install (FILES ${CMAKE_CURRENT_BINARY_DIR}/opendhtConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/opendht)

 # Unit tests
-IF(OPENDHT_TESTS)
+if (OPENDHT_TESTS)
  FIND_PACKAGE(Cppunit REQUIRED)
  # unit testing
    list (APPEND test_FILES
@@ -325,16 +321,17 @@ IF(OPENDHT_TESTS)
      tests/tests_runner.cpp
      ${test_FILES}
    )
+    target_include_directories(opendht_unit_tests SYSTEM PRIVATE ${CPPUNIT_INCLUDE_DIRS})
    if (OPENDHT_SHARED)
-      TARGET_LINK_LIBRARIES(opendht_unit_tests opendht)
+      target_link_libraries(opendht_unit_tests opendht)
    else ()
-      TARGET_LINK_LIBRARIES(opendht_unit_tests opendht-static)
+      target_link_libraries(opendht_unit_tests opendht-static)
    endif ()
-    TARGET_LINK_LIBRARIES(opendht_unit_tests
+    target_link_libraries(opendht_unit_tests
       ${CMAKE_THREAD_LIBS_INIT}
       ${CPPUNIT_LIBRARIES}
       ${GNUTLS_LIBRARIES}
    )
    enable_testing()
    add_test(TEST opendht_unit_tests)
-ENDIF()
+endif()
@@ -1,7 +1,7 @@
 dnl define macros
 m4_define([opendht_major_version], 1)
 m4_define([opendht_minor_version], 8)
-m4_define([opendht_patch_version], 0)
+m4_define([opendht_patch_version], 1)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])

@@ -177,21 +177,10 @@ AM_COND_IF([ENABLE_TOOLS], [
  ])
 ])

-AM_COND_IF(ENABLE_PROXY_SERVER,
-    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true"],
-    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false"])
-
-AM_COND_IF(ENABLE_PROXY_CLIENT,
-    [CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=true"],
-    [CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=false"])
-
-AM_COND_IF(ENABLE_PUSH_NOTIFICATIONS,
-    [CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=true"],
-    [CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=false"])
-
-AM_COND_IF(ENABLE_PROXY_SERVER_IDENTITY,
-    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=true"],
-    [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=false"])
+AM_COND_IF(ENABLE_PROXY_SERVER, [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER"], [])
+AM_COND_IF(ENABLE_PROXY_CLIENT, [CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT"], [])
+AM_COND_IF(ENABLE_PUSH_NOTIFICATIONS, [CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS"], [])
+AM_COND_IF(ENABLE_PROXY_SERVER_IDENTITY, [CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY"], [])

 AM_COND_IF([HAVE_DOXYGEN], [
  AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])
@@ -19,7 +19,7 @@
 #pragma once

 #include "opendht/dhtrunner.h"
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
 #include "opendht/dht_proxy_server.h"
 #endif
 #include "opendht/log.h"
@@ -103,6 +103,9 @@ struct OPENDHT_PUBLIC Config {

    /** Makes the DHT responsible to maintain its stored values. Consumes more ressources. */
    bool maintain_storage;
+
+    /** If set, the dht will load its state from this file on start and save its state in this file on shutdown */
+    std::string persist_path;
 };

 /**
@@ -186,7 +186,7 @@ class OPENDHT_PUBLIC RevocationList
 public:
    RevocationList();
    RevocationList(const Blob& b);
-    RevocationList(RevocationList&& o) : crl(o.crl) { o.crl = nullptr; }
+    RevocationList(RevocationList&& o) noexcept : crl(o.crl) { o.crl = nullptr; }
    ~RevocationList();

    RevocationList& operator=(RevocationList&& o) { crl = o.crl; o.crl = nullptr; return *this; }
@@ -500,7 +500,7 @@ struct OPENDHT_PUBLIC TrustList
    };

    TrustList();
-    TrustList(TrustList&& o) : trust(std::move(o.trust)) {
+    TrustList(TrustList&& o) noexcept : trust(std::move(o.trust)) {
        o.trust = nullptr;
    }
    TrustList& operator=(TrustList&& o);
@@ -163,7 +163,7 @@ public:
    /**
     * Get locally stored data for the given hash.
     */
-    std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const;
+    std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = {}) const;

    /**
     * Get locally stored data for the given key and value id.
@@ -261,11 +261,14 @@ public:
     * Get the list of good nodes for local storage saving purposes
     * The list is ordered to minimize the back-to-work delay.
     */
-    std::vector<NodeExport> exportNodes();
+    std::vector<NodeExport> exportNodes() const;

    std::vector<ValuesExport> exportValues() const;
    void importValues(const std::vector<ValuesExport>&);

+    void saveState(const std::string& path) const;
+    void loadState(const std::string& path);
+
    NodeStats getNodesStats(sa_family_t af) const;

    std::string getStorageLog() const;
@@ -324,10 +327,10 @@ private:
    static constexpr unsigned LISTEN_NODES {4};

    /* The maximum number of hashes we're willing to track. */
-    static constexpr unsigned MAX_HASHES {16384};
+    static constexpr unsigned MAX_HASHES {64 * 1024};

    /* The maximum number of searches we keep data about. */
-    static constexpr unsigned MAX_SEARCHES {16384};
+    static constexpr unsigned MAX_SEARCHES {64 * 1024};

    static constexpr std::chrono::minutes MAX_STORAGE_MAINTENANCE_EXPIRE_TIME {10};

@@ -359,13 +362,6 @@ private:
    // registred types
    TypeStore types;

-    // are we a bootstrap node ?
-    // note: Any running node can be used as a bootstrap node.
-    //       Only nodes running only as bootstrap nodes should
-    //       be put in bootstrap mode.
-    const bool is_bootstrap {false};
-    const bool maintain_storage {false};
-
    // the stuff
    RoutingTable buckets4 {};
    RoutingTable buckets6 {};
@@ -400,6 +396,15 @@ private:

    std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};

+    std::string persistPath;
+
+    // are we a bootstrap node ?
+    // note: Any running node can be used as a bootstrap node.
+    //       Only nodes running only as bootstrap nodes should
+    //       be put in bootstrap mode.
+    const bool is_bootstrap {false};
+    const bool maintain_storage {false};
+
    void rotateSecrets();

    Blob makeToken(const SockAddr&, bool old) const;
@@ -469,7 +474,7 @@ private:
    Sp<Search> search(const InfoHash& id, sa_family_t af, GetCallback = {}, QueryCallback = {}, DoneCallback = {}, Value::Filter = {}, const Sp<Query>& q = {});

    void announce(const InfoHash& id, sa_family_t af, Sp<Value> value, DoneCallback callback, time_point created=time_point::max(), bool permanent = false);
-    size_t listenTo(const InfoHash& id, sa_family_t af, ValueCallback cb, Value::Filter f = Value::AllFilter(), const Sp<Query>& q = {});
+    size_t listenTo(const InfoHash& id, sa_family_t af, ValueCallback cb, Value::Filter f = {}, const Sp<Query>& q = {});

    /**
     * Refill the search with good nodes if possible.
@@ -106,7 +106,7 @@ public:
    /**
     * Get locally stored data for the given hash.
     */
-    virtual std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const = 0;
+    virtual std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = {}) const = 0;

    /**
     * Get locally stored data for the given key and value id.
@@ -181,7 +181,7 @@ public:
     * Get the list of good nodes for local storage saving purposes
     * The list is ordered to minimize the back-to-work delay.
     */
-    virtual std::vector<NodeExport> exportNodes() = 0;
+    virtual std::vector<NodeExport> exportNodes() const = 0;

    virtual std::vector<ValuesExport> exportValues() const = 0;
    virtual void importValues(const std::vector<ValuesExport>&) = 0;
@@ -17,8 +17,6 @@
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

-#if OPENDHT_PROXY_CLIENT
-
 #pragma once

 #include <functional>
@@ -41,8 +39,6 @@ namespace Json {

 namespace dht {

-class SearchCache;
-
 class OPENDHT_PUBLIC DhtProxyClient final : public DhtInterface {
 public:

@@ -51,7 +47,7 @@ public:
    explicit DhtProxyClient(std::function<void()> loopSignal, const std::string& serverHost, const std::string& pushClientId = "");

    virtual void setPushNotificationToken(const std::string& token) {
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
        deviceKey_ = token;
 #endif
    }
@@ -239,7 +235,7 @@ public:
    void insertNode(const InfoHash&, const sockaddr*, socklen_t) { }
    void insertNode(const NodeExport&) { }
    std::pair<size_t, size_t> getStoreSize() const { return {}; }
-    std::vector<NodeExport> exportNodes() { return {}; }
+    std::vector<NodeExport> exportNodes() const { return {}; }
    std::vector<ValuesExport> exportValues() const { return {}; }
    void importValues(const std::vector<ValuesExport>&) {}
    std::string getStorageLog() const { return {}; }
@@ -276,12 +272,18 @@ private:

    void opFailed();

-    size_t doListen(const InfoHash& key, ValueCallback, Value::Filter);
    bool doCancelListen(const InfoHash& key, size_t token);

    struct ListenState;
-    void sendListen(const std::shared_ptr<restbed::Request>& request, const ValueCallback&, const Value::Filter& filter, const Sp<ListenState>& state);
-    void sendSubscribe(const std::shared_ptr<restbed::Request>& request, const Sp<proxy::ListenToken>&, const Sp<ListenState>& state);
+    enum class ListenMethod {
+        LISTEN,
+        SUBSCRIBE,
+        RESUBSCRIBE,
+    };
+    void sendListen(const std::shared_ptr<restbed::Request> &request,
+                    const ValueCallback &, const Value::Filter &filter,
+                    const Sp<ListenState> &state,
+                    ListenMethod method = ListenMethod::LISTEN);

    void doPut(const InfoHash&, Sp<Value>, DoneCallback, time_point created, bool permanent);

@@ -351,6 +353,8 @@ private:
     */
    void confirmProxy();
    Sp<Scheduler::Job> nextProxyConfirmation {};
+    Sp<Scheduler::Job> listenerRestart {};
+
    /**
     * Relaunch LISTEN requests if the client disconnect/reconnect.
     */
@@ -370,8 +374,8 @@ private:

    const std::function<void()> loopSignal_;

-#if OPENDHT_PUSH_NOTIFICATIONS
-    void fillBodyToGetToken(std::shared_ptr<restbed::Request> request, unsigned token = 0);
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
+    void fillBody(std::shared_ptr<restbed::Request> request, bool resubscribe);
    void getPushRequest(Json::Value&) const;
 #endif // OPENDHT_PUSH_NOTIFICATIONS

@@ -379,5 +383,3 @@ private:
 };

 }
-
-#endif // OPENDHT_PROXY_CLIENT
@@ -17,8 +17,6 @@
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

-#if OPENDHT_PROXY_SERVER
-
 #pragma once

 #include "callbacks.h"
@@ -45,6 +43,7 @@ namespace Json {
 namespace dht {

 class DhtRunner;
+class ThreadPool;

 /**
 * Describes the REST API
@@ -178,7 +177,7 @@ private:

    void cancelPut(const InfoHash& key, Value::Id vid);

-#if OPENDHT_PROXY_SERVER_IDENTITY
+#ifdef OPENDHT_PROXY_SERVER_IDENTITY
    /**
     * Put a value to sign by the proxy on the DHT
     * Method: SIGN "/{InfoHash: .*}"
@@ -229,7 +228,7 @@ private:
     */
    void removeClosedListeners(bool testSession = true);

-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
    /**
     * Subscribe to push notifications for an iOS or Android device.
     * Method: SUBSCRIBE "/{InfoHash: .*}"
@@ -253,7 +252,7 @@ private:
     * @param key of the device
     * @param json, the content to send
     */
-    void sendPushNotification(const std::string& key, const Json::Value& json, bool isAndroid) const;
+    void sendPushNotification(const std::string& key, Json::Value&& json, bool isAndroid) const;

    /**
     * Remove a push listener between a client and a hash
@@ -272,11 +271,13 @@ private:
    std::thread server_thread {};
    std::unique_ptr<restbed::Service> service_;
    std::shared_ptr<DhtRunner> dht_;
+    Json::StreamWriterBuilder jsonBuilder_;

    std::mutex schedulerLock_;
    std::condition_variable schedulerCv_;
    Scheduler scheduler_;
    std::thread schedulerThread_;
+    std::unique_ptr<ThreadPool> threadPool_;

    Sp<Scheduler::Job> printStatsJob_;
    mutable std::mutex statsMutex_;
@@ -305,7 +306,7 @@ private:

    mutable ServerStats stats_;

-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
    struct Listener;
    struct PushListener;
    std::mutex lockPushListeners_;
@@ -315,5 +316,3 @@ private:
 };

 }
-
-#endif //OPENDHT_PROXY_SERVER
@@ -63,20 +63,20 @@ public:
    DhtRunner();
    virtual ~DhtRunner();

-    void get(InfoHash id, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+    void get(InfoHash id, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter f = {}, Where w = {}) {
        get(id, bindGetCb(cb), donecb, f, w);
    }

-    void get(InfoHash id, GetCallbackSimple cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+    void get(InfoHash id, GetCallbackSimple cb, DoneCallbackSimple donecb={}, Value::Filter f = {}, Where w = {}) {
        get(id, bindGetCb(cb), donecb, f, w);
    }

    void get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f={}, Where w = {});

-    void get(InfoHash id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+    void get(InfoHash id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter f = {}, Where w = {}) {
        get(id, cb, bindDoneCb(donecb), f, w);
    }
-    void get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb={}, Value::Filter f = Value::AllFilter(), Where w = {});
+    void get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb={}, Value::Filter f = {}, Where w = {});

    template <class T>
    void get(InfoHash hash, std::function<bool(std::vector<T>&&)> cb, DoneCallbackSimple dcb={})
@@ -354,7 +354,8 @@ public:
                    /*.node_id = */{},
                    /*.network = */network,
                    /*.is_bootstrap = */false,
-                    /*.maintain_storage*/false
+                    /*.maintain_storage = */false,
+                    /*.persist_path = */{}
                },
                /*.id = */identity
            },
@@ -290,9 +290,9 @@ public:
     */
    Sp<Request> sendFindNode(Sp<Node> n,
                             const InfoHash& hash,
-                             want_t want,
-                             RequestCb&& on_done,
-                             RequestExpiredCb&& on_expired);
+                             want_t want = -1,
+                             RequestCb&& on_done = {},
+                             RequestExpiredCb&& on_expired = {});
    /**
     * Send a "get" request to a given node.
     *
@@ -531,6 +531,8 @@ private:
    std::set<SockAddr> blacklist {};

    Scheduler& scheduler;
+
+    bool logIncoming_ {false};
 };

 } /* namespace net  */
@@ -42,6 +42,11 @@ struct Bucket {
    Sp<Node> randomNode();

    void sendCachedPing(net::NetworkEngine& ne);
+    void connectivityChanged() {
+        time = time_point::min();
+        for (auto& node : nodes)
+            node->setTime(time_point::min());
+    }
 };

 class RoutingTable : public std::list<Bucket> {
@@ -76,7 +81,7 @@ public:
    void connectivityChanged(const time_point& now) {
        grow_time = now;
        for (auto& b : *this)
-            b.time = time_point::min();
+            b.connectivityChanged();
    }

    bool onNewNode(const Sp<Node>& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne);
@@ -168,7 +168,7 @@ public:
    void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) override {
        dht_->setStorageLimit(limit);
    }
-    std::vector<NodeExport> exportNodes() override {
+    std::vector<NodeExport> exportNodes() const override {
        return dht_->exportNodes();
    }
    std::vector<ValuesExport> exportValues() const override {
@@ -58,7 +58,7 @@ public:
    SockAddr(const SockAddr& o) {
        set(o.get(), o.getLength());
    }
-    SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) {
+    SockAddr(SockAddr&& o) noexcept : len(o.len), addr(std::move(o.addr)) {
        o.len = 0;
    }

@@ -165,8 +165,8 @@ struct OPENDHT_PUBLIC Value
            return chainOr(std::move(f1), std::move(f2));
        }
        static Filter chain(Filter&& f1, Filter&& f2) {
-            if (not f1) return f2;
-            if (not f2) return f1;
+            if (not f1) return std::move(f1);
+            if (not f2) return std::move(f2);
            return [f1,f2](const Value& v) {
                return f1(v) and f2(v);
            };
@@ -184,7 +184,7 @@ struct OPENDHT_PUBLIC Value
            return chainAll(std::vector<Filter>(l.begin(), l.end()));
        }
        static Filter chainOr(Filter&& f1, Filter&& f2) {
-            if (not f1 or not f2) return AllFilter();
+            if (not f1 or not f2) return {};
            return [f1,f2](const Value& v) {
                return f1(v) or f2(v);
            };
@@ -990,7 +990,7 @@ template <typename T,
 Value::Filter
 getFilterSet()
 {
-    return Value::AllFilter();
+    return {};
 }

 template <class T>
@@ -139,6 +139,12 @@ cdef class SockAddr(object):
        return self._addr.setPort(port)
    def setFamily(SockAddr self, cpp.sa_family_t af):
        return self._addr.setFamily(af)
+    def isLoopback(SockAddr self):
+        return self._addr.isLoopback()
+    def isPrivate(SockAddr self):
+        return self._addr.isPrivate()
+    def isUnspecified(SockAddr self):
+        return self._addr.isUnspecified()
    def __str__(self):
        return self.toString().decode()
    def __repr__(self):
@@ -77,6 +77,9 @@ cdef extern from "opendht/sockaddr.h" namespace "dht":
        void setPort(in_port_t p)
        sa_family_t getFamily() const
        void setFamily(sa_family_t f)
+        bool isLoopback() const
+        bool isPrivate() const
+        bool isUnspecified() const

 ctypedef vector[uint8_t] Blob

@@ -51,7 +51,7 @@ def ping(node, h):

 def pong(node, h):
 	print(node.getNodeId().decode(), "got ping", h, i)
-	loop.call_soon_threadsafe(ping, node, h);
+	loop.call_soon_threadsafe(ping, node, h)
 	return True

 ping_node.listen(loc_ping, lambda v: pong(pong_node, loc_pong))
@@ -56,7 +56,7 @@ nobase_include_HEADERS = \
        ../include/opendht/rng.h

 if ENABLE_PROXY_SERVER
-libopendht_la_SOURCES += dht_proxy_server.cpp
+libopendht_la_SOURCES += dht_proxy_server.cpp thread_pool.h thread_pool.cpp
 nobase_include_HEADERS += ../include/opendht/dht_proxy_server.h
 endif

@@ -29,6 +29,7 @@
 #include <algorithm>
 #include <random>
 #include <sstream>
+#include <fstream>

 namespace dht {

@@ -54,6 +55,9 @@ Dht::getStatus(sa_family_t af) const
 void
 Dht::shutdown(ShutdownCallback cb)
 {
+    if (not persistPath.empty())
+        saveState(persistPath);
+
    if (not maintain_storage) {
        if (cb) cb();
        return;
@@ -125,19 +129,16 @@ Dht::trySearchInsert(const Sp<Node>& node)
    bool inserted {false};

    // insert forward
-    auto it = closest;
-    while (it != srs.end()) {
+    for (auto it = closest; it != srs.end(); it++) {
        auto& s = *it->second;
        if (s.insertNode(node, now)) {
            inserted = true;
            scheduler.edit(s.nextSearchStep, now);
        } else if (not s.expired and not s.done)
            break;
-        ++it;
    }
    // insert backward
-    it = closest;
-    while (it != srs.begin()) {
+    for (auto it = closest; it != srs.begin();) {
        --it;
        auto& s = *it->second;
        if (s.insertNode(node, now)) {
@@ -167,8 +168,14 @@ Dht::reportedAddr(const SockAddr& addr)
 void
 Dht::onNewNode(const Sp<Node>& node, int confirm)
 {
-    if (buckets(node->getFamily()).onNewNode(node, confirm, scheduler.time(), myid, network_engine) or confirm) {
+    const auto& now = scheduler.time();
+    auto& b = buckets(node->getFamily());
+    auto wasEmpty = confirm < 2 && b.grow_time < now - std::chrono::minutes(5);
+    if (b.onNewNode(node, confirm, now, myid, network_engine) or confirm) {
        trySearchInsert(node);
+        if (wasEmpty) {
+            scheduler.edit(nextNodesConfirmation, now + std::chrono::seconds(1));
+        }
    }
 }

@@ -279,7 +286,7 @@ void Dht::paginate(std::weak_ptr<Search> ws, Sp<Query> query, SearchNode* n) {
                if (vid == Value::INVALID_ID) continue;
                auto query_for_vid = std::make_shared<Query>(Select {}, Where {}.id(vid));
                sn->pagination_queries[query].push_back(query_for_vid);
-                DHT_LOG.w(id, sn->node->id, "[search %s] [node %s] sending %s",
+                DHT_LOG.d(id, sn->node->id, "[search %s] [node %s] sending %s",
                        id.toString().c_str(), sn->node->toString().c_str(), query_for_vid->toString().c_str());
                sn->getStatus[query_for_vid] = network_engine.sendGetValues(status.node,
                        id,
@@ -298,7 +305,7 @@ void Dht::paginate(std::weak_ptr<Search> ws, Sp<Query> query, SearchNode* n) {
    /* add pagination query key for tracking ongoing requests. */
    n->pagination_queries[query].push_back(select_q);

-    DHT_LOG.w(sr->id, n->node->id, "[search %s] [node %s] sending %s",
+    DHT_LOG.d(sr->id, n->node->id, "[search %s] [node %s] sending %s",
            sr->id.toString().c_str(), n->node->toString().c_str(), select_q->toString().c_str());
    n->getStatus[select_q] = network_engine.sendGetValues(n->node,
            sr->id,
@@ -432,24 +439,20 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
            auto next_refresh_time = now + getType(a.value->type).expiration;
            /* only put the value if the node doesn't already have it */
            if (not hasValue or seq_no < a.value->seq) {
-                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)",
+                DHT_LOG.d(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)",
                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
-                sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node,
-                                                sr->id,
-                                                a.value,
-                                                a.permanent ? time_point::max() : a.created,
-                                                sn->token,
-                                                onDone,
-                                                onExpired), next_refresh_time);
+                auto created = a.permanent ? time_point::max() : a.created;
+                sn->acked[a.value->id] = {
+                    network_engine.sendAnnounceValue(sn->node, sr->id, a.value, created, sn->token, onDone, onExpired),
+                    next_refresh_time
+                };
            } else if (hasValue and a.permanent) {
                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)",
                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
-                sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node,
-                                                sr->id,
-                                                a.value->id,
-                                                sn->token,
-                                                onDone,
-                                                onExpired), next_refresh_time);
+                sn->acked[a.value->id] = {
+                    network_engine.sendRefreshValue(sn->node, sr->id, a.value->id, sn->token, onDone, onExpired),
+                    next_refresh_time
+                };
            } else {
                DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.",
                        sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
@@ -500,7 +503,7 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
        if (sendQuery) {
            if (not probe_query)
                probe_query = std::make_shared<Query>(Select {}.field(Value::Field::Id).field(Value::Field::SeqNum));
-            DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending %s",
+            DHT_LOG.d(sr->id, n.node->id, "[search %s] [node %s] sending %s",
                    sr->id.toString().c_str(), n.node->toString().c_str(), probe_query->toString().c_str());
            n.probe_query = probe_query;
            n.getStatus[probe_query] = network_engine.sendGetValues(n.node,
@@ -524,8 +527,8 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
        auto list_token = l.first;
        if (n.getListenTime(query) > scheduler.time())
            continue;
-        DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
-                sr->id.toString().c_str(), n.node->toString().c_str());
+        // DHT_LOG.d(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
+        //        sr->id.toString().c_str(), n.node->toString().c_str());

        auto r = n.listenStatus.find(query);
        if (r == n.listenStatus.end()) {
@@ -537,6 +540,13 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
                            l->second.get_cb(l->second.filter.filter(values), expired);
                        }
                    }
+                }, [ws,list_token] (ListenSyncStatus status) {
+                    if (auto sr = ws.lock()) {
+                        auto l = sr->listeners.find(list_token);
+                        if (l != sr->listeners.end()) {
+                            l->second.sync_cb(status);
+                        }
+                    }
                }
            }).first;
            auto node = n.node;
@@ -554,8 +564,10 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
            { /* on done */
                if (auto sr = ws.lock()) {
                    scheduler.edit(sr->nextSearchStep, scheduler.time());
-                    if (auto sn = sr->getNode(req.node))
+                    if (auto sn = sr->getNode(req.node)) {
                        scheduler.add(sn->getListenTime(query), std::bind(&Dht::searchStep, this, sr));
+                        sn->onListenSynced(query);
+                    }
                    onListenDone(req.node, answer, sr);
                }
            },
@@ -595,9 +607,9 @@ Dht::searchStep(Sp<Search> sr)
    if (not sr or sr->expired or sr->done) return;

    const auto& now = scheduler.time();
-    if (auto req_count = sr->currentlySolicitedNodeCount())
+    /*if (auto req_count = sr->currentlySolicitedNodeCount())
        DHT_LOG.d(sr->id, "[search %s IPv%c] step (%d requests)",
-                sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', req_count);
+                sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', req_count);*/
    sr->step_time = now;

    if (sr->refill_time + Node::NODE_EXPIRE_TIME < now and sr->nodes.size()-sr->getNumberOfBadNodes() < SEARCH_NODES)
@@ -656,18 +668,7 @@ Dht::searchStep(Sp<Search> sr)
            sr->setDone();
    }

-    if (sr->currentlySolicitedNodeCount() < MAX_REQUESTED_SEARCH_NODES) {
-        unsigned i = 0;
-        SearchNode* sent;
-        do {
-            sent = searchSendGetValues(sr);
-            if (sent and not sent->candidate)
-                i++;
-        }
-        while (sent and sr->currentlySolicitedNodeCount() < MAX_REQUESTED_SEARCH_NODES);
-        /*DHT_LOG_DBG("[search %s IPv%c] step: sent %u requests (total %u).",
-            sr->id.toString().c_str(), sr->af == AF_INET ? '4' : '6', i, sr->currentlySolicitedNodeCount());*/
-    }
+    while (sr->currentlySolicitedNodeCount() < MAX_REQUESTED_SEARCH_NODES and searchSendGetValues(sr));

    if (sr->getNumberOfConsecutiveBadNodes() >= std::min(sr->nodes.size(),
                                                             static_cast<size_t>(SEARCH_MAX_BAD_NODES)))
@@ -678,10 +679,6 @@ Dht::searchStep(Sp<Search> sr)
    }

    /* dumpSearch(*sr, std::cout); */
-
-    /* periodic searchStep scheduling. */
-    //if (not sr->done)
-    //    scheduler.edit(sr->nextSearchStep, now);
 }

 unsigned Dht::refill(Dht::Search& sr) {
@@ -843,14 +840,16 @@ Dht::listen(const InfoHash& id, ValueCallback cb, Value::Filter f, Where where)
    auto vals = std::make_shared<std::map<Value::Id, Sp<Value>>>();
    auto token = ++listener_token;

-    auto gcb = OpValueCache::cacheCallback(std::move(cb));
+    auto gcb = OpValueCache::cacheCallback(std::move(cb), [this, id, token]{
+        cancelListen(id, token);
+    });

    auto query = std::make_shared<Query>(q);
    auto filter = f.chain(q.where.getFilter());
    size_t tokenlocal = 0;
    auto st = store.find(id);
    if (st == store.end() && store.size() < MAX_HASHES)
-        st = store.emplace(id, Storage(scheduler.time() + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME)).first;
+        st = store.emplace(id, scheduler.time() + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME).first;
    if (st != store.end()) {
        if (not st->second.empty()) {
            std::vector<Sp<Value>> newvals = st->second.get(filter);
@@ -869,9 +868,12 @@ Dht::listen(const InfoHash& id, ValueCallback cb, Value::Filter f, Where where)
    }

    auto token4 = Dht::listenTo(id, AF_INET, gcb, filter, query);
-    auto token6 = Dht::listenTo(id, AF_INET6, gcb, filter, query);
+    auto token6 = token4 == 0 ? 0 : Dht::listenTo(id, AF_INET6, gcb, filter, query);
+    if (token6 == 0 && st != store.end()) {
+        st->second.local_listeners.erase(tokenlocal);
+        return 0;
+    }

-    DHT_LOG.d(id, "Added listen : %d -> %d %d %d", token, tokenlocal, token4, token6);
    listeners.emplace(token, std::make_tuple(tokenlocal, token4, token6));
    return token;
 }
@@ -916,7 +918,7 @@ struct OpStatus {

 template <typename T>
 struct GetStatus : public OpStatus {
-    std::vector<Sp<T>> values;
+    T values;
    std::vector<Sp<Node>> nodes;
 };

@@ -975,20 +977,16 @@ void doneCallbackWrapper(DoneCallback dcb, const std::vector<Sp<Node>>& nodes, G
    }
 }

-template <typename T, typename Cb>
-bool callbackWrapper(Cb get_cb,
-        DoneCallback done_cb,
-        const std::vector<Sp<T>>& values,
-        std::function<std::vector<Sp<T>>(const std::vector<Sp<T>>&)> add_values,
-        Sp<GetStatus<T>> o)
+template <typename T, typename St, typename Cb, typename Av, typename Cv>
+bool callbackWrapper(Cb get_cb, DoneCallback done_cb, const std::vector<Sp<T>>& values,
+    Av add_values, Cv cache_values, GetStatus<St>& op)
 {
-    auto& op = *o;
    if (op.status.done)
        return false;
    auto newvals = add_values(values);
    if (not newvals.empty()) {
        op.status.ok = !get_cb(newvals);
-        op.values.insert(op.values.end(), newvals.begin(), newvals.end());
+        cache_values(newvals);
    }
    doneCallbackWrapper(done_cb, {}, op);
    return !op.status.ok;
@@ -1000,23 +998,26 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
    scheduler.syncTime();

    auto q = std::make_shared<Query>(Select {}, std::move(where));
-    auto op = std::make_shared<GetStatus<Value>>();
+    auto op = std::make_shared<GetStatus<std::map<Value::Id, Sp<Value>>>>();
    auto f = filter.chain(q->where.getFilter());

-    auto add_values = [op,f](const std::vector<Sp<Value>>& values) {
-        std::vector<Sp<Value>> newvals {};
-        for (const auto& v : values) {
-            auto it = std::find_if(op->values.cbegin(), op->values.cend(), [&](const Sp<Value>& sv) {
-                return sv == v or *sv == *v;
-            });
-            if (it == op->values.cend()) {
-               if (not f or f(*v))
-                   newvals.push_back(v);
+    auto gcb = [getcb, donecb, op, f](const std::vector<Sp<Value>>& vals) {
+        auto& o = *op;
+        return callbackWrapper(getcb, donecb, vals, [&o,&f](const std::vector<Sp<Value>>& values) {
+            std::vector<Sp<Value>> newvals {};
+            for (const auto& v : values) {
+                auto it = o.values.find(v->id);
+                if (it == o.values.cend() or (it->second != v && !(*it->second == *v))) {
+                    if (not f or f(*v))
+                        newvals.push_back(v);
+                }
            }
-        }
-        return newvals;
+            return newvals;
+        }, [&o](const std::vector<Sp<Value>>& newvals) {
+            for (const auto& v : newvals)
+                o.values[v->id] = v;
+        }, o);
    };
-    auto gcb = std::bind(callbackWrapper<Value, GetCallback>, getcb, donecb, _1, add_values, op);

    /* Try to answer this search locally. */
    gcb(getLocal(id, f));
@@ -1036,36 +1037,39 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt
 void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Query&& q)
 {
    scheduler.syncTime();
-    auto op = std::make_shared<GetStatus<FieldValueIndex>>();
-
+    auto op = std::make_shared<GetStatus<std::vector<Sp<FieldValueIndex>>>>();
    auto f = q.where.getFilter();
-    auto values = getLocal(id, f);
-    auto add_fields = [=](const std::vector<Sp<FieldValueIndex>>& fields) {
-        std::vector<Sp<FieldValueIndex>> newvals {};
-        for (const auto& f : fields) {
-            auto it = std::find_if(op->values.cbegin(), op->values.cend(),
-                [&](const Sp<FieldValueIndex>& sf) {
-                    return sf == f or f->containedIn(*sf);
-                });
-            if (it == op->values.cend()) {
-                auto lesser = std::find_if(op->values.begin(), op->values.end(),
+    auto qcb = [cb, done_cb, op](const std::vector<Sp<FieldValueIndex>>& fields){
+        auto& o = *op;
+        return callbackWrapper(cb, done_cb, fields, [&](const std::vector<Sp<FieldValueIndex>>& fields) {
+            std::vector<Sp<FieldValueIndex>> newvals {};
+            for (const auto& f : fields) {
+                auto it = std::find_if(o.values.cbegin(), o.values.cend(),
                    [&](const Sp<FieldValueIndex>& sf) {
-                        return sf->containedIn(*f);
+                        return sf == f or f->containedIn(*sf);
                    });
-                if (lesser != op->values.end())
-                    op->values.erase(lesser);
-                newvals.push_back(f);
+                if (it == o.values.cend()) {
+                    auto lesser = std::find_if(o.values.begin(), o.values.end(),
+                        [&](const Sp<FieldValueIndex>& sf) {
+                            return sf->containedIn(*f);
+                        });
+                    if (lesser != o.values.end())
+                        o.values.erase(lesser);
+                    newvals.push_back(f);
+                }
            }
-        }
-        return newvals;
+            return newvals;
+        }, [&](const std::vector<Sp<FieldValueIndex>>& fields){
+            o.values.insert(o.values.end(), fields.begin(), fields.end());
+        }, o);
    };
+
+    /* Try to answer this search locally. */
+    auto values = getLocal(id, f);
    std::vector<Sp<FieldValueIndex>> local_fields(values.size());
    std::transform(values.begin(), values.end(), local_fields.begin(), [&q](const Sp<Value>& v) {
        return std::make_shared<FieldValueIndex>(*v, q.select);
    });
-    auto qcb = std::bind(callbackWrapper<FieldValueIndex, QueryCallback>, cb, done_cb, _1, add_fields, op);
-
-    /* Try to answer this search locally. */
    qcb(local_fields);

    auto sq = std::make_shared<Query>(std::move(q));
@@ -1173,6 +1177,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v, bool newVa
        if (not st.local_listeners.empty()) {
            DHT_LOG.d(id, "[store %s] %lu local listeners", id.toString().c_str(), st.local_listeners.size());
            std::vector<std::pair<ValueCallback, std::vector<Sp<Value>>>> cbs;
+            cbs.reserve(st.local_listeners.size());
            for (const auto& l : st.local_listeners) {
                std::vector<Sp<Value>> vals;
                if (not l.second.filter or l.second.filter(*v.data))
@@ -1223,7 +1228,7 @@ Dht::storageStore(const InfoHash& id, const Sp<Value>& value, time_point created
    if (st == store.end()) {
        if (store.size() >= MAX_HASHES)
            return false;
-        auto st_i = store.emplace(id, Storage(now));
+        auto st_i = store.emplace(id, now);
        st = st_i.first;
        if (maintain_storage and st_i.second)
            scheduler.add(st->second.maintenance_time, std::bind(&Dht::dataPersistence, this, id));
@@ -1269,7 +1274,7 @@ Dht::storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t socket_
    if (st == store.end()) {
        if (store.size() >= MAX_HASHES)
            return;
-        st = store.emplace(id, Storage(now)).first;
+        st = store.emplace(id, now).first;
    }
    auto node_listeners = st->second.listeners.emplace(node, std::map<size_t, Listener> {}).first;
    auto l = node_listeners->second.find(socket_id);
@@ -1388,12 +1393,6 @@ Dht::connectivityChanged(sa_family_t af)
    scheduler.edit(nextNodesConfirmation, now);
    buckets(af).connectivityChanged(now);
    network_engine.connectivityChanged(af);
-    for (auto& sp : searches(af))
-        for (auto& sn : sp.second->nodes) {
-            for (auto& ls : sn.listenStatus)
-                sn.node->cancelRequest(ls.second.req);
-            sn.listenStatus.clear();
-        }
    reported_addr.erase(std::remove_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& addr){
        return addr.second.getFamily() == af;
    }), reported_addr.end());
@@ -1727,9 +1726,7 @@ Dht::~Dht()
 Dht::Dht() : store(), network_engine(DHT_LOG, scheduler) {}

 Dht::Dht(const int& s, const int& s6, Config config)
-    : myid(config.node_id ? config.node_id : InfoHash::getRandom()),
-    is_bootstrap(config.is_bootstrap),
-    maintain_storage(config.maintain_storage), store(), store_quota(),
+    : myid(config.node_id ? config.node_id : InfoHash::getRandom()), store(), store_quota(),
    network_engine(myid, config.network, s, s6, DHT_LOG, scheduler,
            std::bind(&Dht::onError, this, _1, _2),
            std::bind(&Dht::onNewNode, this, _1, _2),
@@ -1739,7 +1736,10 @@ Dht::Dht(const int& s, const int& s6, Config config)
            std::bind(&Dht::onGetValues, this, _1, _2, _3, _4),
            std::bind(&Dht::onListen, this, _1, _2, _3, _4, _5),
            std::bind(&Dht::onAnnounce, this, _1, _2, _3, _4, _5),
-            std::bind(&Dht::onRefresh, this, _1, _2, _3, _4))
+            std::bind(&Dht::onRefresh, this, _1, _2, _3, _4)),
+    persistPath(config.persist_path),
+    is_bootstrap(config.is_bootstrap),
+    maintain_storage(config.maintain_storage)
 {
    scheduler.syncTime();
    if (s < 0 && s6 < 0)
@@ -1770,8 +1770,10 @@ Dht::Dht(const int& s, const int& s6, Config config)
    expire();

    DHT_LOG.d("DHT initialised with node ID %s", myid.toString().c_str());
-}

+    if (not persistPath.empty())
+        loadState(persistPath);
+}

 bool
 Dht::neighbourhoodMaintenance(RoutingTable& list)
@@ -1805,7 +1807,7 @@ Dht::neighbourhoodMaintenance(RoutingTable& list)
                n->toString().c_str(), id.toString().c_str());
        /* Since our node-id is the same in both DHTs, it's probably
           profitable to query both families. */
-        network_engine.sendFindNode(n, id, network_engine.want(), nullptr, nullptr);
+        network_engine.sendFindNode(n, id, network_engine.want());
    }

    return true;
@@ -2061,7 +2063,7 @@ Dht::importValues(const std::vector<ValuesExport>& import)


 std::vector<NodeExport>
-Dht::exportNodes()
+Dht::exportNodes() const
 {
    const auto& now = scheduler.time();
    std::vector<NodeExport> nodes;
@@ -2288,12 +2290,10 @@ Dht::onListen(Sp<Node> node, const InfoHash& hash, const Blob& token, size_t soc
 }

 void
-Dht::onListenDone(const Sp<Node>& node,
-        net::RequestAnswer& answer,
-        Sp<Search>& sr)
+Dht::onListenDone(const Sp<Node>& /* node */, net::RequestAnswer& /* answer */, Sp<Search>& sr)
 {
-    DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got listen confirmation",
-                sr->id.toString().c_str(), node->toString().c_str(), answer.values.size());
+    // DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got listen confirmation",
+    //            sr->id.toString().c_str(), node->toString().c_str(), answer.values.size());

    if (not sr->done) {
        const auto& now = scheduler.time();
@@ -2361,7 +2361,7 @@ Dht::onAnnounce(Sp<Node> n,
            // Allow the value to be edited by the storage policy
            const auto& type = getType(vc->type);
            if (type.storePolicy(hash, vc, node.id, node.getAddr())) {
-                DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str());
+                //DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), std::to_string(vc->id).c_str());
                storageStore(hash, vc, created, node.getAddr());
            } else {
                DHT_LOG.d(hash, node.id, "[store %s] rejecting storage of %s",
@@ -2397,6 +2397,23 @@ Dht::storageRefresh(const InfoHash& id, Value::Id vid)
    const auto& now = scheduler.time();
    auto s = store.find(id);
    if (s != store.end()) {
+        // Values like for a permanent put can be refreshed. So, inform remote listeners that the value
+        // need to be refreshed
+        auto& st = s->second;
+        if (not st.listeners.empty()) {
+            DHT_LOG.d(id, "[store %s] %lu remote listeners", id.toString().c_str(), st.listeners.size());
+            std::vector<Value::Id> ids = {vid};
+            for (const auto& node_listeners : st.listeners) {
+                for (const auto& l : node_listeners.second) {
+                    DHT_LOG.w(id, node_listeners.first->id, "[store %s] [node %s] sending refresh",
+                            id.toString().c_str(),
+                            node_listeners.first->toString().c_str());
+                    Blob ntoken = makeToken(node_listeners.first->getAddr(), false);
+                    network_engine.tellListenerRefreshed(node_listeners.first, l.first, id, ntoken, ids);
+                }
+            }
+        }
+
        auto expiration = s->second.refresh(now, vid, types);
        if (expiration != time_point::max())
            scheduler.add(expiration, std::bind(&Dht::expireStorage, this, id));
@@ -2414,4 +2431,52 @@ Dht::onAnnounceDone(const Sp<Node>& node, net::RequestAnswer& answer, Sp<Search>
    sr->checkAnnounced(answer.vid);
 }

+
+void
+Dht::saveState(const std::string& path) const
+{
+    std::ofstream file(path);
+    msgpack::pack(file, exportNodes());
+    msgpack::pack(file, exportValues());
+}
+
+void
+Dht::loadState(const std::string& path)
+{
+    DHT_LOG.d("Importing state from %s", path.c_str());
+    try {
+        // Import nodes from binary file
+        msgpack::unpacker pac;
+        {
+            // Read whole file
+            std::ifstream file(path, std::ios::binary|std::ios::ate);
+            if (!file.is_open()) {
+                return;
+            }
+            auto size = file.tellg();
+            file.seekg (0, std::ios::beg);
+            pac.reserve_buffer(size);
+            file.read (pac.buffer(), size);
+            pac.buffer_consumed(size);
+        }
+        // Import nodes
+        msgpack::object_handle oh;
+        if (pac.next(oh)) {
+            {
+                auto imported_nodes = oh.get().as<std::vector<NodeExport>>();
+                DHT_LOG.d("Importing %zu nodes", imported_nodes.size());
+                for (const auto& node : imported_nodes)
+                    insertNode(node);
+            }
+            if (pac.next(oh)) {
+                auto imported_values = oh.get().as<std::vector<ValuesExport>>();
+                DHT_LOG.d("Importing %zu values", imported_values.size());
+                importValues(imported_values);
+            }
+        }
+    } catch (const std::exception& e) {
+        DHT_LOG.w("Error importing state from %s: %s", path.c_str(), e.what());
+    }
+}
+
 }
@@ -17,8 +17,6 @@
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

-#if OPENDHT_PROXY_CLIENT
-
 #include "dht_proxy_client.h"

 #include "dhtrunner.h"
@@ -45,33 +43,29 @@ struct DhtProxyClient::ListenState {

 struct DhtProxyClient::Listener
 {
-    ValueCache cache;
-    Sp<Scheduler::Job> cacheExpirationJob {};
+    OpValueCache cache;
    ValueCallback cb;
    Value::Filter filter;
    Sp<restbed::Request> req;
    std::thread thread;
    unsigned callbackId;
    Sp<ListenState> state;
-    Sp<proxy::ListenToken> pushNotifToken;
    Sp<Scheduler::Job> refreshJob;
-    Listener(ValueCache&& c, Sp<Scheduler::Job>&& j, const Sp<restbed::Request>& r, Value::Filter&& f)
-        : cache(std::move(c)),
-          cacheExpirationJob(std::move(j)),
-          filter(std::move(f)),
-          req(r)
-    {}
+    Listener(OpValueCache&& c, const Sp<restbed::Request>& r, Value::Filter&& f)
+        : cache(std::move(c)), filter(std::move(f)),req(r) {}
 };

 struct PermanentPut {
    Sp<Value> value;
    Sp<Scheduler::Job> refreshJob;
    Sp<std::atomic_bool> ok;
+    PermanentPut(const Sp<Value>& v, Sp<Scheduler::Job>&& j, const Sp<std::atomic_bool>& o)
+        : value(v), refreshJob(std::move(j)), ok(o) {}
 };

 struct DhtProxyClient::ProxySearch {
    SearchCache ops {};
-    Sp<Scheduler::Job> opExpirationJob;
+    Sp<Scheduler::Job> opExpirationJob {};
    std::map<size_t, Listener> listeners {};
    std::map<Value::Id, PermanentPut> puts {};
 };
@@ -81,6 +75,8 @@ DhtProxyClient::DhtProxyClient() {}
 DhtProxyClient::DhtProxyClient(std::function<void()> signal, const std::string& serverHost, const std::string& pushClientId)
 : serverHost_(serverHost), pushClientId_(pushClientId), loopSignal_(signal)
 {
+    if (serverHost_.find("://") == std::string::npos)
+        serverHost_ = proxy::HTTP_PROTO + serverHost_;
    if (!serverHost_.empty())
        startProxy();
 }
@@ -98,6 +94,7 @@ DhtProxyClient::startProxy()
    if (serverHost_.empty()) return;
    DHT_LOG.w("Staring proxy client to %s", serverHost_.c_str());
    nextProxyConfirmation = scheduler.add(scheduler.time(), std::bind(&DhtProxyClient::confirmProxy, this));
+    listenerRestart = std::make_shared<Scheduler::Job>(std::bind(&DhtProxyClient::restartListeners, this));
    loopSignal_();
 }

@@ -243,7 +240,7 @@ void
 DhtProxyClient::get(const InfoHash& key, GetCallback cb, DoneCallback donecb, Value::Filter&& f, Where&& w)
 {
    DHT_LOG.d(key, "[search %s]: get", key.to_c_str());
-    restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + key.toString());
+    restbed::Uri uri(serverHost_ + "/" + key.toString());
    auto req = std::make_shared<restbed::Request>(uri);
    Value::Filter filter = w.empty() ? f : f.chain(w.getFilter());

@@ -256,45 +253,49 @@ DhtProxyClient::get(const InfoHash& key, GetCallback cb, DoneCallback donecb, Va
        // will change the connectivity status
        struct GetState{ std::atomic_bool ok {true}; std::atomic_bool stop {false}; };
        auto state = std::make_shared<GetState>();
-        restbed::Http::async(req,
-            [=](const std::shared_ptr<restbed::Request>& req,
-                const std::shared_ptr<restbed::Response>& reply) {
-            auto code = reply->get_status_code();
+        try {
+            restbed::Http::async(req,
+                [=](const std::shared_ptr<restbed::Request>& req,
+                    const std::shared_ptr<restbed::Response>& reply) {
+                auto code = reply->get_status_code();

-            if (code == 200) {
-                try {
-                    while (restbed::Http::is_open(req) and not *finished and not state->stop) {
-                        restbed::Http::fetch("\n", reply);
-                        if (*finished or state->stop)
-                            break;
-                        std::string body;
-                        reply->get_body(body);
-                        reply->set_body(""); // Reset the body for the next fetch
+                if (code == 200) {
+                    try {
+                        while (restbed::Http::is_open(req) and not *finished and not state->stop) {
+                            restbed::Http::fetch("\n", reply);
+                            if (*finished or state->stop)
+                                break;
+                            std::string body;
+                            reply->get_body(body);
+                            reply->set_body(""); // Reset the body for the next fetch

-                        std::string err;
-                        Json::Value json;
-                        Json::CharReaderBuilder rbuilder;
-                        auto* char_data = reinterpret_cast<const char*>(&body[0]);
-                        auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
-                        if (reader->parse(char_data, char_data + body.size(), &json, &err)) {
-                            auto value = std::make_shared<Value>(json);
-                            if ((not filter or filter(*value)) and cb) {
-                                std::lock_guard<std::mutex> lock(lockCallbacks);
-                                callbacks_.emplace_back([cb, value, state]() {
-                                    if (not state->stop and not cb({value}))
-                                        state->stop = true;
-                                });
-                                loopSignal_();
+                            std::string err;
+                            Json::Value json;
+                            Json::CharReaderBuilder rbuilder;
+                            auto* char_data = reinterpret_cast<const char*>(&body[0]);
+                            auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
+                            if (reader->parse(char_data, char_data + body.size(), &json, &err)) {
+                                auto value = std::make_shared<Value>(json);
+                                if ((not filter or filter(*value)) and cb) {
+                                    std::lock_guard<std::mutex> lock(lockCallbacks);
+                                    callbacks_.emplace_back([cb, value, state]() {
+                                        if (not state->stop and not cb({value}))
+                                            state->stop = true;
+                                    });
+                                    loopSignal_();
+                                }
+                            } else {
+                                state->ok = false;
                            }
-                        } else {
-                            state->ok = false;
                        }
-                    }
-                } catch (std::runtime_error& e) { }
-            } else {
-                state->ok = false;
-            }
-        }).wait();
+                    } catch (std::runtime_error& e) { }
+                } else {
+                    state->ok = false;
+                }
+            }).wait();
+        } catch(const std::exception& e) {
+            state->ok = false;
+        }
        if (donecb) {
            std::lock_guard<std::mutex> lock(lockCallbacks);
            callbacks_.emplace_back([=](){
@@ -332,24 +333,26 @@ DhtProxyClient::put(const InfoHash& key, Sp<Value> val, DoneCallback cb, time_po
    if (permanent) {
        std::lock_guard<std::mutex> lock(searchLock_);
        auto id = val->id;
-        auto search = searches_.emplace(key, ProxySearch{}).first;
+        auto& search = searches_[key];
        auto nextRefresh = scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN;
        auto ok = std::make_shared<std::atomic_bool>(false);
-        search->second.puts.erase(id);
-        search->second.puts.emplace(id, PermanentPut {val, scheduler.add(nextRefresh, [this, key, id, ok]{
-            std::lock_guard<std::mutex> lock(searchLock_);
-            auto s = searches_.find(key);
-            if (s != searches_.end()) {
-                auto p = s->second.puts.find(id);
-                if (p != s->second.puts.end()) {
-                    doPut(key, p->second.value,
-                    [ok](bool result, const std::vector<std::shared_ptr<dht::Node> >&){
-                        *ok = result;
-                    }, time_point::max(), true);
-                    scheduler.edit(p->second.refreshJob, scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+        search.puts.erase(id);
+        search.puts.emplace(std::piecewise_construct,
+            std::forward_as_tuple(id),
+            std::forward_as_tuple(val, scheduler.add(nextRefresh, [this, key, id, ok]{
+                std::lock_guard<std::mutex> lock(searchLock_);
+                auto s = searches_.find(key);
+                if (s != searches_.end()) {
+                    auto p = s->second.puts.find(id);
+                    if (p != s->second.puts.end()) {
+                        doPut(key, p->second.value,
+                        [ok](bool result, const std::vector<std::shared_ptr<dht::Node> >&){
+                            *ok = result;
+                        }, time_point::max(), true);
+                        scheduler.edit(p->second.refreshJob, scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
+                    }
                }
-            }
-        }), ok});
+            }), ok));
    }
    doPut(key, val, std::move(cb), created, permanent);
 }
@@ -358,7 +361,7 @@ void
 DhtProxyClient::doPut(const InfoHash& key, Sp<Value> val, DoneCallback cb, time_point /*created*/, bool permanent)
 {
    DHT_LOG.d(key, "[search %s] performing put of %s", key.to_c_str(), val->toString().c_str());
-    restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + key.toString());
+    restbed::Uri uri(serverHost_ + "/" + key.toString());
    auto req = std::make_shared<restbed::Request>(uri);
    req->set_method("POST");

@@ -367,7 +370,7 @@ DhtProxyClient::doPut(const InfoHash& key, Sp<Value> val, DoneCallback cb, time_
        if (deviceKey_.empty()) {
            json["permanent"] = true;
        } else {
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
            Json::Value refresh;
            getPushRequest(refresh);
            json["permanent"] = refresh;
@@ -389,32 +392,36 @@ DhtProxyClient::doPut(const InfoHash& key, Sp<Value> val, DoneCallback cb, time_
    o.finished = finished;
    o.thread = std::thread([=](){
        auto ok = std::make_shared<std::atomic_bool>(true);
-        restbed::Http::async(req,
-            [ok](const std::shared_ptr<restbed::Request>& /*req*/,
-                        const std::shared_ptr<restbed::Response>& reply) {
-            auto code = reply->get_status_code();
+        try {
+            restbed::Http::async(req,
+                [ok](const std::shared_ptr<restbed::Request>& /*req*/,
+                            const std::shared_ptr<restbed::Response>& reply) {
+                auto code = reply->get_status_code();

-            if (code == 200) {
-                restbed::Http::fetch("\n", reply);
-                std::string body;
-                reply->get_body(body);
-                reply->set_body(""); // Reset the body for the next fetch
+                if (code == 200) {
+                    restbed::Http::fetch("\n", reply);
+                    std::string body;
+                    reply->get_body(body);
+                    reply->set_body(""); // Reset the body for the next fetch

-                try {
-                    std::string err;
-                    Json::Value json;
-                    Json::CharReaderBuilder rbuilder;
-                    auto* char_data = reinterpret_cast<const char*>(&body[0]);
-                    auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
-                    if (not reader->parse(char_data, char_data + body.size(), &json, &err))
+                    try {
+                        std::string err;
+                        Json::Value json;
+                        Json::CharReaderBuilder rbuilder;
+                        auto* char_data = reinterpret_cast<const char*>(&body[0]);
+                        auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
+                        if (not reader->parse(char_data, char_data + body.size(), &json, &err))
+                            *ok = false;
+                    } catch (...) {
                        *ok = false;
-                } catch (...) {
+                    }
+                } else {
                    *ok = false;
                }
-            } else {
-                *ok = false;
-            }
-        }).wait();
+            }).wait();
+        } catch(const std::exception& e) {
+            *ok = false;
+        }
        if (cb) {
            std::lock_guard<std::mutex> lock(lockCallbacks);
            callbacks_.emplace_back([=](){
@@ -517,7 +524,14 @@ DhtProxyClient::getProxyInfos()
    }
    statusThread_ = std::thread([this, serverHost, infoState]{
        try {
-            auto hostAndService = splitPort(serverHost);
+            auto endpointStr = serverHost;
+            auto protocol = std::string(proxy::HTTP_PROTO);
+            auto protocolIdx = serverHost.find("://");
+            if (protocolIdx != std::string::npos) {
+                protocol = endpointStr.substr(0, protocolIdx + 3);
+                endpointStr = endpointStr.substr(protocolIdx + 3);
+            }
+            auto hostAndService = splitPort(endpointStr);
            auto resolved_proxies = SockAddr::resolve(hostAndService.first, hostAndService.second);
            std::vector<std::future<Sp<restbed::Response>>> reqs;
            reqs.reserve(resolved_proxies.size());
@@ -526,9 +540,9 @@ DhtProxyClient::getProxyInfos()
                if (resolved_proxy.getFamily() == AF_INET6) {
                    // HACK restbed seems to not correctly handle directly http://[ipv6]
                    // See https://github.com/Corvusoft/restbed/issues/290.
-                    server = serverHost;
+                    server = endpointStr;
                }
-                restbed::Uri uri(proxy::HTTP_PROTO + server + "/");
+                restbed::Uri uri(protocol + server + "/");
                auto req = std::make_shared<restbed::Request>(uri);
                if (infoState->cancel)
                    return;
@@ -613,7 +627,7 @@ DhtProxyClient::onProxyInfos(const Json::Value& proxyInfos, sa_family_t family)
    auto newStatus = std::max(statusIpv4_, statusIpv6_);
    if (newStatus == NodeStatus::Connected) {
        if (oldStatus == NodeStatus::Disconnected || oldStatus == NodeStatus::Connecting) {
-            restartListeners();
+            scheduler.edit(listenerRestart, scheduler.time());
        }
        scheduler.edit(nextProxyConfirmation, scheduler.time() + std::chrono::minutes(15));
    }
@@ -646,13 +660,72 @@ DhtProxyClient::getPublicAddress(sa_family_t family)
 size_t
 DhtProxyClient::listen(const InfoHash& key, ValueCallback cb, Value::Filter filter, Where where) {
    DHT_LOG.d(key, "[search %s]: listen", key.to_c_str());
-    auto it = searches_.find(key);
-    if (it == searches_.end()) {
-        it = searches_.emplace(key, ProxySearch{}).first;
-    }
+    auto& search = searches_[key];
    auto query = std::make_shared<Query>(Select{}, where);
-    auto token = it->second.ops.listen(cb, query, filter, [&](Sp<Query> /*q*/, ValueCallback vcb){
-        return doListen(key, vcb, filter);
+    auto token = search.ops.listen(cb, query, filter, [this, key, filter](Sp<Query> /*q*/, ValueCallback cb, SyncCallback /*scb*/) -> size_t {
+        scheduler.syncTime();
+        restbed::Uri uri(serverHost_ + "/" + key.toString());
+        std::lock_guard<std::mutex> lock(searchLock_);
+        auto search = searches_.find(key);
+        if (search == searches_.end()) {
+            DHT_LOG.e(key, "[search %s] listen: search not found", key.to_c_str());
+            return 0;
+        }
+        DHT_LOG.d(key, "[search %s] sending %s", key.to_c_str(), deviceKey_.empty() ? "listen" : "subscribe");
+
+        auto req = std::make_shared<restbed::Request>(uri);
+        auto token = ++listenerToken_;
+        auto l = search->second.listeners.find(token);
+        if (l == search->second.listeners.end()) {
+            auto f = filter;
+            l = search->second.listeners.emplace(std::piecewise_construct,
+                    std::forward_as_tuple(token),
+                    std::forward_as_tuple(std::move(cb), req, std::move(f))).first;
+        } else {
+            if (l->second.state)
+                l->second.state->cancel = true;
+        }
+
+        auto state = std::make_shared<ListenState>();
+        l->second.state = state;
+        l->second.cb = [this,key,token,state](const std::vector<Sp<Value>>& values, bool expired) {
+            if (state->cancel)
+                return false;
+            std::lock_guard<std::mutex> lock(searchLock_);
+            auto s = searches_.find(key);
+            if (s == searches_.end()) {
+                return false;
+            }
+            auto l = s->second.listeners.find(token);
+            if (l == s->second.listeners.end()) {
+                return false;
+            }
+            return l->second.cache.onValue(values, expired);
+        };
+
+        auto vcb = l->second.cb;
+        l->second.req = req;
+
+        if (not deviceKey_.empty()) {
+            // Relaunch push listeners even if a timeout is not received (if the proxy crash for any reason)
+            l->second.refreshJob = scheduler.add(scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN, [this, key, token, state] {
+                if (state->cancel)
+                    return;
+                std::lock_guard<std::mutex> lock(searchLock_);
+                auto s = searches_.find(key);
+                if (s != searches_.end()) {
+                    auto l = s->second.listeners.find(token);
+                    if (l != s->second.listeners.end()) {
+                        resubscribe(key, l->second);
+                    }
+                }
+            });
+        }
+        l->second.thread = std::thread([this, req, vcb, filter, state]() {
+            sendListen(req, vcb, filter, state,
+                    deviceKey_.empty() ? ListenMethod::LISTEN : ListenMethod::SUBSCRIBE);
+        });
+        return token;
    });
    return token;
 }
@@ -684,193 +757,76 @@ DhtProxyClient::cancelListen(const InfoHash& key, size_t gtoken) {
    return canceled;
 }

-
-void
-DhtProxyClient::sendListen(const std::shared_ptr<restbed::Request>& req, const ValueCallback& cb, const Value::Filter& filter, const Sp<ListenState>& state) {
+void DhtProxyClient::sendListen(const std::shared_ptr<restbed::Request> &req,
+                                const ValueCallback &cb,
+                                const Value::Filter &filter,
+                                const Sp<ListenState> &state,
+                                ListenMethod method) {
    auto settings = std::make_shared<restbed::Settings>();
-    std::chrono::milliseconds timeout(std::numeric_limits<int>::max());
-    settings->set_connection_timeout(timeout); // Avoid the client to close the socket after 5 seconds.
-    req->set_method("LISTEN");
-    restbed::Http::async(req,
-        [this, filter, cb, state](const std::shared_ptr<restbed::Request>& req,
-                                  const std::shared_ptr<restbed::Response>& reply) {
-        auto code = reply->get_status_code();
-        if (code == 200) {
-            try {
-                while (restbed::Http::is_open(req) and not state->cancel) {
-                    restbed::Http::fetch("\n", reply);
-                    if (state->cancel)
-                        break;
-                    std::string body;
-                    reply->get_body(body);
-                    reply->set_body(""); // Reset the body for the next fetch
+    if (method != ListenMethod::LISTEN) {
+        req->set_method("SUBSCRIBE");
+    } else {
+        std::chrono::milliseconds timeout(std::numeric_limits<int>::max());
+        settings->set_connection_timeout(timeout); // Avoid the client to close the socket after 5 seconds.
+        req->set_method("LISTEN");
+    }
+    try {
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
+        if (method != ListenMethod::LISTEN)
+        fillBody(req, method == ListenMethod::RESUBSCRIBE);
+    #endif
+        restbed::Http::async(req,
+              [this, filter, cb, state](const std::shared_ptr<restbed::Request>& req,
+                                        const std::shared_ptr<restbed::Response>& reply)
+        {
+            auto code = reply->get_status_code();
+            if (code == 200) {
+                try {
+                    while (restbed::Http::is_open(req) and not state->cancel) {
+                        restbed::Http::fetch("\n", reply);
+                        if (state->cancel)
+                            break;
+                        std::string body;
+                        reply->get_body(body);
+                        reply->set_body(""); // Reset the body for the next fetch

-                    Json::Value json;
-                    std::string err;
-                    Json::CharReaderBuilder rbuilder;
-                    auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
-                    if (reader->parse(body.data(), body.data() + body.size(), &json, &err)) {
-                        auto expired = json.get("expired", Json::Value(false)).asBool();
-                        auto value = std::make_shared<Value>(json);
-                        if ((not filter or filter(*value)) and cb)  {
-                            std::lock_guard<std::mutex> lock(lockCallbacks);
-                            callbacks_.emplace_back([cb, value, state, expired]() {
-                                if (not state->cancel and not cb({value}, expired))
-                                    state->cancel = true;
-                            });
-                            loopSignal_();
+                        Json::Value json;
+                        std::string err;
+                        Json::CharReaderBuilder rbuilder;
+                        auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
+                        if (reader->parse(body.data(), body.data() + body.size(), &json, &err)) {
+                            if (json.size() == 0) {
+                                // Empty value, it's the end
+                                break;
+                            }
+                            auto expired = json.get("expired", Json::Value(false)).asBool();
+                            auto value = std::make_shared<Value>(json);
+                            if ((not filter or filter(*value)) and cb) {
+                                std::lock_guard<std::mutex> lock(lockCallbacks);
+                                callbacks_.emplace_back([cb, value, state, expired]() {
+                                    if (not state->cancel and not cb({value}, expired))
+                                        state->cancel = true;
+                                });
+                                loopSignal_();
+                            }
                        }
                    }
-                }
-            } catch (const std::exception& e) {
-                if (not state->cancel) {
-                    DHT_LOG.w("Listen closed by the proxy server: %s", e.what());
-                    state->ok = false;
-                }
-            }
-        } else {
-            state->ok = false;
-        }
-    }, settings).get();
-    auto& s = *state;
-    if (not s.ok and not s.cancel)
-        opFailed();
-}
-
-void
-DhtProxyClient::sendSubscribe(const std::shared_ptr<restbed::Request>& req, const Sp<proxy::ListenToken>& token, const Sp<ListenState>& state) {
-#if OPENDHT_PUSH_NOTIFICATIONS
-    req->set_method("SUBSCRIBE");
-    fillBodyToGetToken(req);
-    restbed::Http::async(req, [this,state, token](const std::shared_ptr<restbed::Request>&,
-                                             const std::shared_ptr<restbed::Response>& reply) {
-        auto code = reply->get_status_code();
-        if (code == 200) {
-            try {
-                restbed::Http::fetch("\n", reply);
-                std::string body;
-                reply->get_body(body);
-
-                std::string err;
-                Json::Value json;
-                Json::CharReaderBuilder rbuilder;
-                auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
-                if (reader->parse(body.data(), body.data() + body.size(), &json, &err)) {
-                    if (not json.isMember("token")) {
+                } catch (const std::exception& e) {
+                    if (not state->cancel) {
+                        DHT_LOG.w("Listen closed by the proxy server: %s", e.what());
                        state->ok = false;
-                        return;
                    }
-                    *token = unpackId(json, "token");
-                }
-            } catch (const std::exception& e) {
-                if (not state->cancel) {
-                    DHT_LOG.e("sendSubscribe: error: %s", e.what());
-                    state->ok = false;
                }
+            } else {
+                state->ok = false;
            }
-        } else {
-            state->ok = false;
-        }
-    }).get();
+        }, settings).get();
+    } catch (const std::exception& e) {
+        state->ok = false;
+    }
    auto& s = *state;
    if (not s.ok and not s.cancel)
        opFailed();
-#endif
-}
-
-size_t
-DhtProxyClient::doListen(const InfoHash& key, ValueCallback cb, Value::Filter filter/*, Where where*/)
-{
-    scheduler.syncTime();
-    restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + key.toString());
-    std::lock_guard<std::mutex> lock(searchLock_);
-    auto search = searches_.find(key);
-    if (search == searches_.end()) {
-        DHT_LOG.e(key, "[search %s] listen: search not found", key.to_c_str());
-        return 0;
-    }
-    DHT_LOG.d(key, "[search %s] sending %s", key.to_c_str(), deviceKey_.empty() ? "listen" : "subscribe");
-
-    auto req = std::make_shared<restbed::Request>(uri);
-    auto token = ++listenerToken_;
-    auto l = search->second.listeners.find(token);
-    if (l == search->second.listeners.end()) {
-        auto f = filter;
-        l = search->second.listeners.emplace(token, Listener {
-            ValueCache(cb), scheduler.add(time_point::max(), [this, key, token]{
-                std::lock_guard<std::mutex> lock(searchLock_);
-                auto s = searches_.find(key);
-                if (s == searches_.end()) {
-                    return;
-                }
-                auto l = s->second.listeners.find(token);
-                if (l == s->second.listeners.end()) {
-                    return;
-                }
-                auto next = l->second.cache.expireValues(scheduler.time());
-                scheduler.edit(l->second.cacheExpirationJob, next);
-            }), req, std::move(f)
-        }).first;
-    } else {
-        if (l->second.state)
-            l->second.state->cancel = true;
-    }
-
-    auto state = std::make_shared<ListenState>();
-    l->second.state = state;
-    l->second.cb = [this,key,token,state](const std::vector<Sp<Value>>& values, bool expired) {
-        if (state->cancel)
-            return false;
-        std::lock_guard<std::mutex> lock(searchLock_);
-        auto s = searches_.find(key);
-        if (s == searches_.end()) {
-            return false;
-        }
-        auto l = s->second.listeners.find(token);
-        if (l == s->second.listeners.end()) {
-            return false;
-        }
-        const std::vector<Sp<Value>> new_values_empty;
-        std::vector<Value::Id> expired_ids;
-        if (expired) {
-            expired_ids.reserve(values.size());
-            for (const auto& v : values)
-                expired_ids.emplace_back(v->id);
-        }
-        auto next = l->second.cache.onValues(expired ? new_values_empty : values, std::vector<Value::Id>{}, expired_ids, types, scheduler.time());
-        scheduler.edit(l->second.cacheExpirationJob, next);
-        loopSignal_();
-        return true;
-    };
-
-    auto pushNotifToken = std::make_shared<proxy::ListenToken>(0);
-    auto vcb = l->second.cb;
-    l->second.pushNotifToken = pushNotifToken;
-    l->second.req = req;
-
-    if (not deviceKey_.empty()) {
-        // Relaunch push listeners even if a timeout is not received (if the proxy crash for any reason)
-        l->second.refreshJob = scheduler.add(scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN, [this, key, token, state] {
-            if (state->cancel)
-                return;
-            std::lock_guard<std::mutex> lock(searchLock_);
-            auto s = searches_.find(key);
-            if (s != searches_.end()) {
-                auto l = s->second.listeners.find(token);
-                if (l != s->second.listeners.end()) {
-                    resubscribe(key, l->second);
-                }
-            }
-        });
-        l->second.thread = std::thread([this, req,pushNotifToken, state](){
-            sendSubscribe(req,pushNotifToken,state);
-        });
-    } else {
-        l->second.thread = std::thread([this, req, vcb, filter, state]{
-            sendListen(req,vcb,filter,state);
-        });
-    }
-    return token;
 }

 bool
@@ -896,14 +852,13 @@ DhtProxyClient::doCancelListen(const InfoHash& key, size_t ltoken)
            listener.thread.join();
        }
        // UNSUBSCRIBE
-        restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + key.toString());
+        restbed::Uri uri(serverHost_ + "/" + key.toString());
        auto req = std::make_shared<restbed::Request>(uri);
        req->set_method("UNSUBSCRIBE");
        // fill request body
        Json::Value body;
        body["key"] = deviceKey_;
        body["client_id"] = pushClientId_;
-        body["token"] = std::to_string(*listener.pushNotifToken);
        Json::StreamWriterBuilder wbuilder;
        wbuilder["commentStyle"] = "None";
        wbuilder["indentation"] = "";
@@ -911,8 +866,11 @@ DhtProxyClient::doCancelListen(const InfoHash& key, size_t ltoken)
        std::replace(content.begin(), content.end(), '\n', ' ');
        req->set_body(content);
        req->set_header("Content-Length", std::to_string(content.size()));
-
-        restbed::Http::async(req, [](const std::shared_ptr<restbed::Request>&, const std::shared_ptr<restbed::Response>&){});
+        try {
+            restbed::Http::async(req, [](const std::shared_ptr<restbed::Request>&, const std::shared_ptr<restbed::Response>&){});
+        } catch (const std::exception& e) {
+            DHT_LOG.w(key, "[search %s] cancelListen: Http::async failed: %s", key.to_c_str(), e.what());
+        }
    } else {
        // Just stop the request
        if (listener.thread.joinable()) {
@@ -993,7 +951,7 @@ DhtProxyClient::restartListeners()
            state->ok = true;
            auto filter = listener.filter;
            auto cb = listener.cb;
-            restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + search.first.toString());
+            restbed::Uri uri(serverHost_ + "/" + search.first.toString());
            auto req = std::make_shared<restbed::Request>(uri);
            req->set_method("LISTEN");
            listener.req = req;
@@ -1007,7 +965,7 @@ DhtProxyClient::restartListeners()
 void
 DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string>& notification)
 {
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
    scheduler.syncTime();
    {
        // If a push notification is received, the proxy is up and running
@@ -1034,18 +992,45 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
                    resubscribe(key, list.second);
            }
        } else {
-            auto token = std::stoull(notification.at("token"));
-            for (auto& search: searches_) {
-                for (auto& list : search.second.listeners) {
-                    if (*list.second.pushNotifToken != token or list.second.state->cancel)
-                        continue;
-                    DHT_LOG.d(search.first, "[search %s] handling push notification", search.first.to_c_str());
+            auto key = InfoHash(notification.at("key"));
+            auto& search = searches_.at(key);
+            for (auto& list : search.listeners) {
+                if (list.second.state->cancel)
+                    continue;
+                DHT_LOG.d(key, "[search %s] handling push notification", key.to_c_str());
+                auto expired = notification.find("exp");
+                auto token = list.first;
+                auto state = list.second.state;
+                if (expired == notification.end()) {
                    auto cb = list.second.cb;
                    auto filter = list.second.filter;
-                    get(search.first, [cb](const std::vector<Sp<Value>>& vals) {
-                        cb(vals, false);
-                        return true;
-                    }, DoneCallbackSimple{}, std::move(filter));
+                    auto oldValues = list.second.cache.getValues();
+                    get(key, [cb](const std::vector<Sp<Value>>& vals) {
+                        return cb(vals, false);
+                    }, [cb, oldValues](bool /*ok*/) {
+                        // Decrement old values refcount to expire values not present in the new list
+                        cb(oldValues, true);
+                    }, std::move(filter));
+                } else {
+                    std::stringstream ss(expired->second);
+                    std::vector<Value::Id> ids;
+                    while(ss.good()){
+                        std::string substr;
+                        getline(ss, substr, ',');
+                        ids.emplace_back(std::stoull(substr));
+                    }
+                    std::lock_guard<std::mutex> lock(lockCallbacks);
+                    callbacks_.emplace_back([this, key, token, state, ids]() {
+                        if (state->cancel) return;
+                        std::lock_guard<std::mutex> lock(searchLock_);
+                        auto s = searches_.find(key);
+                        if (s == searches_.end()) return;
+                        auto l = s->second.listeners.find(token);
+                        if (l == s->second.listeners.end()) return;
+                        if (not state->cancel and not l->second.cache.onValuesExpired(ids))
+                            state->cancel = true;
+                    });
+                    loopSignal_();
                }
            }
        }
@@ -1058,31 +1043,33 @@ DhtProxyClient::pushNotificationReceived(const std::map<std::string, std::string
 void
 DhtProxyClient::resubscribe(const InfoHash& key, Listener& listener)
 {
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
    if (deviceKey_.empty()) return;
    scheduler.syncTime();
    DHT_LOG.d(key, "[search %s] resubscribe push listener", key.to_c_str());
    // Subscribe
-    restbed::Uri uri(proxy::HTTP_PROTO + serverHost_ + "/" + key.toString());
-    auto req = std::make_shared<restbed::Request>(uri);
-    req->set_method("SUBSCRIBE");
-
-    auto pushNotifToken = std::make_shared<proxy::ListenToken>(0);
    auto state = listener.state;
-    if (listener.thread.joinable())
+    if (listener.thread.joinable()) {
+        state->cancel = true;
+        if (listener.req)
+            restbed::Http::close(listener.req);
        listener.thread.join();
+    }
    state->cancel = false;
    state->ok = true;
+    auto req = std::make_shared<restbed::Request>(restbed::Uri {serverHost_ + "/" + key.toString()});
+    req->set_method("SUBSCRIBE");
    listener.req = req;
-    listener.pushNotifToken = pushNotifToken;
    scheduler.edit(listener.refreshJob, scheduler.time() + proxy::OP_TIMEOUT - proxy::OP_MARGIN);
-    listener.thread = std::thread([this, req, pushNotifToken, state]() {
-        sendSubscribe(req, pushNotifToken, state);
+    auto vcb = listener.cb;
+    auto filter = listener.filter;
+    listener.thread = std::thread([this, req, vcb, filter, state]() {
+        sendListen(req, vcb, filter, state, ListenMethod::RESUBSCRIBE);
    });
 #endif
 }

-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
 void
 DhtProxyClient::getPushRequest(Json::Value& body) const
 {
@@ -1097,17 +1084,18 @@ DhtProxyClient::getPushRequest(Json::Value& body) const
 }

 void
-DhtProxyClient::fillBodyToGetToken(std::shared_ptr<restbed::Request> req, unsigned token)
+DhtProxyClient::fillBody(std::shared_ptr<restbed::Request> req, bool resubscribe)
 {
    // Fill body with
    // {
    //   "key":"device_key",
-    //   "token": xxx
    // }
    Json::Value body;
    getPushRequest(body);
-    if (token > 0)
-        body["token"] = token;
+    if (resubscribe) {
+        // This is the first listen, we want to retrieve previous values.
+        body["refresh"] = true;
+    }
    Json::StreamWriterBuilder wbuilder;
    wbuilder["commentStyle"] = "None";
    wbuilder["indentation"] = "";
@@ -1119,5 +1107,3 @@ DhtProxyClient::fillBodyToGetToken(std::shared_ptr<restbed::Request> req, unsign
 #endif // OPENDHT_PUSH_NOTIFICATIONS

 } // namespace dht
-
-#endif // OPENDHT_PROXY_CLIENT
@@ -17,9 +17,9 @@
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

-#if OPENDHT_PROXY_SERVER
 #include "dht_proxy_server.h"

+#include "thread_pool.h"
 #include "default_types.h"
 #include "dhtrunner.h"

@@ -47,9 +47,11 @@ struct DhtProxyServer::SearchPuts {
 };

 constexpr const std::chrono::minutes PRINT_STATS_PERIOD {2};
+constexpr const size_t IO_THREADS_MAX {64};
+

 DhtProxyServer::DhtProxyServer(std::shared_ptr<DhtRunner> dht, in_port_t port , const std::string& pushServer)
-: dht_(dht) , pushServer_(pushServer)
+: dht_(dht), threadPool_(new ThreadPool(IO_THREADS_MAX)), pushServer_(pushServer)
 {
    if (not dht_)
        throw std::invalid_argument("A DHT instance must be provided");
@@ -58,13 +60,16 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr<DhtRunner> dht, in_port_t port ,

    std::cout << "Running DHT proxy server on port " << port << std::endl;
    if (not pushServer.empty()) {
-#if !OPENDHT_PUSH_NOTIFICATIONS
-        std::cerr << "Push server defined but built OpenDHT built without push notification support" << std::endl;
-#else
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
        std::cout << "Using push notification server: " << pushServer << std::endl;
+#else
+        std::cerr << "Push server defined but built OpenDHT built without push notification support" << std::endl;
 #endif
    }

+    jsonBuilder_["commentStyle"] = "None";
+    jsonBuilder_["indentation"] = "";
+
    server_thread = std::thread([this, port]() {
        // Create endpoints
        auto resource = std::make_shared<restbed::Resource>();
@@ -76,12 +81,12 @@ DhtProxyServer::DhtProxyServer(std::shared_ptr<DhtRunner> dht, in_port_t port ,
        resource->set_path("/{hash: .*}");
        resource->set_method_handler("GET", std::bind(&DhtProxyServer::get, this, _1));
        resource->set_method_handler("LISTEN", [this](const Sp<restbed::Session>& session) mutable { listen(session); } );
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
        resource->set_method_handler("SUBSCRIBE", [this](const Sp<restbed::Session>& session) mutable { subscribe(session); } );
        resource->set_method_handler("UNSUBSCRIBE", [this](const Sp<restbed::Session>& session) mutable { unsubscribe(session); } );
 #endif //OPENDHT_PUSH_NOTIFICATIONS
        resource->set_method_handler("POST", [this](const Sp<restbed::Session>& session) mutable { put(session); });
-#if OPENDHT_PROXY_SERVER_IDENTITY
+#ifdef OPENDHT_PROXY_SERVER_IDENTITY
        resource->set_method_handler("SIGN", std::bind(&DhtProxyServer::putSigned, this, _1));
        resource->set_method_handler("ENCRYPT", std::bind(&DhtProxyServer::putEncrypted, this, _1));
 #endif // OPENDHT_PROXY_SERVER_IDENTITY
@@ -177,6 +182,7 @@ DhtProxyServer::stop()
        schedulerThread_.join();
    if (server_thread.joinable())
        server_thread.join();
+    threadPool_->stop();
 }

 void
@@ -187,7 +193,7 @@ DhtProxyServer::updateStats() const
    auto count = requestNum_.exchange(0);
    auto dt = std::chrono::duration<double>(now - last);
    stats_.requestRate = count / dt.count();
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
    stats_.pushListenersCount = pushListeners_.size();
 #endif
    stats_.putCount = puts_.size();
@@ -219,10 +225,7 @@ DhtProxyServer::getNodeInfo(const Sp<restbed::Session>& session) const
                        result = nodeInfo_.toJson();
                    }
                    result["public_ip"] = s->get_origin(); // [ipv6:ipv4]:port or ipv4:port
-                    Json::StreamWriterBuilder wbuilder;
-                    wbuilder["commentStyle"] = "None";
-                    wbuilder["indentation"] = "";
-                    auto output = Json::writeString(wbuilder, result) + "\n";
+                    auto output = Json::writeString(jsonBuilder_, result) + "\n";
                    s->close(restbed::OK, output);
                }
                else
@@ -246,10 +249,7 @@ DhtProxyServer::getStats(const Sp<restbed::Session>& session) const
            try {
                if (dht_) {
 #ifdef OPENDHT_JSONCPP
-                    Json::StreamWriterBuilder wbuilder;
-                    wbuilder["commentStyle"] = "None";
-                    wbuilder["indentation"] = "";
-                    auto output = Json::writeString(wbuilder, stats_.toJson()) + "\n";
+                    auto output = Json::writeString(jsonBuilder_, stats_.toJson()) + "\n";
                    s->close(restbed::OK, output);
 #else
                    s->close(restbed::NotFound, "{\"err\":\"JSON not enabled on this instance\"}");
@@ -281,13 +281,10 @@ DhtProxyServer::get(const Sp<restbed::Session>& session) const
                        infoHash = InfoHash::get(hash);
                    }
                    s->yield(restbed::OK, "", [=](const Sp<restbed::Session>&) {});
-                    dht_->get(infoHash, [s](const Sp<Value>& value) {
+                    dht_->get(infoHash, [this,s](const Sp<Value>& value) {
                        if (s->is_closed()) return false;
                        // Send values as soon as we get them
-                        Json::StreamWriterBuilder wbuilder;
-                        wbuilder["commentStyle"] = "None";
-                        wbuilder["indentation"] = "";
-                        auto output = Json::writeString(wbuilder, value->toJson()) + "\n";
+                        auto output = Json::writeString(jsonBuilder_, value->toJson()) + "\n";
                        s->yield(output, [](const Sp<restbed::Session>& /*session*/){ });
                        return true;
                    }, [s](bool /*ok* */) {
@@ -335,19 +332,16 @@ DhtProxyServer::listen(const Sp<restbed::Session>& session)
                    // cache the session to avoid an incrementation of the shared_ptr's counter
                    // else, the session->close() will not close the socket.
                    auto cacheSession = std::weak_ptr<restbed::Session>(s);
-                    listener.token = dht_->listen(infoHash, [cacheSession](const std::vector<Sp<Value>>& values, bool expired) {
+                    listener.token = dht_->listen(infoHash, [this,cacheSession](const std::vector<Sp<Value>>& values, bool expired) {
                        auto s = cacheSession.lock();
                        if (!s) return false;
                        // Send values as soon as we get them
                        if (!s->is_closed()) {
-                            Json::StreamWriterBuilder wbuilder;
-                            wbuilder["commentStyle"] = "None";
-                            wbuilder["indentation"] = "";
                            for (const auto& value : values) {
                                auto val = value->toJson();
                                if (expired)
                                    val["expired"] = true;
-                                auto output = Json::writeString(wbuilder, val) + "\n";
+                                auto output = Json::writeString(jsonBuilder_, val) + "\n";
                                s->yield(output, [](const Sp<restbed::Session>&){ });
                            }
                        }
@@ -367,10 +361,9 @@ DhtProxyServer::listen(const Sp<restbed::Session>& session)
    );
 }

-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS

 struct DhtProxyServer::Listener {
-    std::shared_ptr<proxy::ListenToken> token;
    std::string clientId;
    std::future<size_t> internalToken;
    Sp<Scheduler::Job> expireJob;
@@ -409,12 +402,11 @@ DhtProxyServer::subscribe(const std::shared_ptr<restbed::Session>& session)
                    s->close(restbed::BAD_REQUEST, "{\"err\":\"No token\"}");
                    return;
                }
-                auto tokenFromReq = unpackId(root, "token");
                auto platform = root["platform"].asString();
                auto isAndroid = platform == "android";
                auto clientId = root.isMember("client_id") ? root["client_id"].asString() : std::string();

-                std::cout << "Subscribe " << infoHash << " token:" << tokenFromReq << " client:" << clientId << std::endl;
+                std::cout << "Subscribe " << infoHash << " client:" << clientId << std::endl;

                {
                    std::lock(schedulerLock_, lockListener_);
@@ -428,10 +420,34 @@ DhtProxyServer::subscribe(const std::shared_ptr<restbed::Session>& session)
                    auto listeners = pushListener->second.listeners.emplace(infoHash, std::vector<Listener>{}).first;
                    for (auto& listener: listeners->second) {
                        if (listener.clientId == clientId) {
-                            *listener.token = tokenFromReq;
                            scheduler_.edit(listener.expireJob, timeout);
                            scheduler_.edit(listener.expireNotifyJob, timeout - proxy::OP_MARGIN);
-                            s->close(restbed::OK, "{\"token\": " + std::to_string(tokenFromReq) + "}\n");
+                            s->yield(restbed::OK);
+
+                            if (!root.isMember("refresh") or !root["refresh"].asBool()) {
+                                dht_->get(
+                                    infoHash,
+                                    [this, s](const Sp<Value> &value) {
+                                        if (s->is_closed())
+                                            return false;
+                                        // Send values as soon as we get them
+                                        auto output = Json::writeString(jsonBuilder_, value->toJson()) + "\n";
+                                        s->yield(output, [](const Sp<restbed::Session>
+                                                                & /*session*/) {});
+                                        return true;
+                                    },
+                                    [s](bool /*ok* */) {
+                                        // Communication is finished
+                                        if (not s->is_closed()) {
+                                            s->close("{}\n");
+                                        }
+                                    });
+                            } else {
+                                // Communication is finished
+                                if (not s->is_closed()) {
+                                    s->close("{}\n");
+                                }
+                            }
                            schedulerCv_.notify_one();
                            return;
                        }
@@ -444,16 +460,23 @@ DhtProxyServer::subscribe(const std::shared_ptr<restbed::Session>& session)
                    pushListener->second.isAndroid = isAndroid;

                    // The listener is not found, so add it.
-                    auto token = std::make_shared<proxy::ListenToken>(tokenFromReq);
-                    listener.token = token;
                    listener.internalToken = dht_->listen(infoHash,
-                        [this, token, infoHash, pushToken, isAndroid, clientId](std::vector<std::shared_ptr<Value>> /*value*/) {
-                            // Build message content.
-                            Json::Value json;
-                            json["key"] = infoHash.toString();
-                            json["to"] = clientId;
-                            json["token"] = std::to_string(*token);
-                            sendPushNotification(pushToken, json, isAndroid);
+                        [this, infoHash, pushToken, isAndroid, clientId](const std::vector<std::shared_ptr<Value>>& values, bool expired) {
+                            threadPool_->run([this, infoHash, pushToken, isAndroid, clientId, values, expired]() {
+                                // Build message content
+                                Json::Value json;
+                                json["key"] = infoHash.toString();
+                                json["to"] = clientId;
+                                if (expired and values.size() < 3) {
+                                    std::stringstream ss;
+                                    for(size_t i = 0; i < values.size(); ++i) {
+                                        if(i != 0) ss << ",";
+                                        ss << values[i]->id;
+                                    }
+                                    json["exp"] = ss.str();
+                                }
+                                sendPushNotification(pushToken, std::move(json), isAndroid);
+                            });
                            return true;
                        }
                    );
@@ -463,18 +486,17 @@ DhtProxyServer::subscribe(const std::shared_ptr<restbed::Session>& session)
                        }
                    );
                    listener.expireNotifyJob = scheduler_.add(timeout - proxy::OP_MARGIN,
-                        [this, token, infoHash, pushToken, isAndroid, clientId] {
+                        [this, infoHash, pushToken, isAndroid, clientId] {
                            std::cout << "Listener: sending refresh " << infoHash << std::endl;
                            Json::Value json;
                            json["timeout"] = infoHash.toString();
                            json["to"] = clientId;
-                            json["token"] = std::to_string(*token);
-                            sendPushNotification(pushToken, json, isAndroid);
+                            sendPushNotification(pushToken, std::move(json), isAndroid);
                        }
                    );
                }
                schedulerCv_.notify_one();
-                s->close(restbed::OK, "{\"token\": " + std::to_string(tokenFromReq) + "}\n");
+                s->close(restbed::OK, "{}\n");
            } catch (...) {
                s->close(restbed::INTERNAL_SERVER_ERROR, "{\"err\":\"Internal server error\"}");
            }
@@ -547,8 +569,10 @@ DhtProxyServer::cancelPushListen(const std::string& pushToken, const dht::InfoHa
 }

 void
-DhtProxyServer::sendPushNotification(const std::string& token, const Json::Value& json, bool isAndroid) const
+DhtProxyServer::sendPushNotification(const std::string& token, Json::Value&& json, bool isAndroid) const
 {
+    if (pushServer_.empty())
+        return;
    restbed::Uri uri(proxy::HTTP_PROTO + pushServer_ + "/api/push");
    auto req = std::make_shared<restbed::Request>(uri);
    req->set_method("POST");
@@ -557,9 +581,9 @@ DhtProxyServer::sendPushNotification(const std::string& token, const Json::Value
    Json::Value notification(Json::objectValue);
    Json::Value tokens(Json::arrayValue);
    tokens[0] = token;
-    notification["tokens"] = tokens;
+    notification["tokens"] = std::move(tokens);
    notification["platform"] = isAndroid ? 2 : 1;
-    notification["data"] = json;
+    notification["data"] = std::move(json);
    notification["priority"] = "high";
    notification["time_to_live"] = 600;

@@ -567,7 +591,7 @@ DhtProxyServer::sendPushNotification(const std::string& token, const Json::Value
    notifications[0] = notification;

    Json::Value content;
-    content["notifications"] = notifications;
+    content["notifications"] = std::move(notifications);

    Json::StreamWriterBuilder wbuilder;
    wbuilder["commentStyle"] = "None";
@@ -658,7 +682,7 @@ DhtProxyServer::put(const std::shared_ptr<restbed::Session>& session)
                                        std::cout << "Permanent put expired: " << infoHash << " " << vid << std::endl;
                                        cancelPut(infoHash, vid);
                                    });
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
                                    if (not pushToken.empty()) {
                                        pput.expireNotifyJob = scheduler_.add(timeout - proxy::OP_MARGIN,
                                            [this, infoHash, vid, pushToken, clientId, isAndroid]
@@ -668,7 +692,7 @@ DhtProxyServer::put(const std::shared_ptr<restbed::Session>& session)
                                            json["timeout"] = infoHash.toString();
                                            json["to"] = clientId;
                                            json["vid"] = std::to_string(vid);
-                                            sendPushNotification(pushToken, json, isAndroid);
+                                            sendPushNotification(pushToken, std::move(json), isAndroid);
                                        });
                                    }
 #endif
@@ -708,7 +732,7 @@ DhtProxyServer::put(const std::shared_ptr<restbed::Session>& session)
    );
 }

-#if OPENDHT_PROXY_SERVER_IDENTITY
+#ifdef OPENDHT_PROXY_SERVER_IDENTITY
 void
 DhtProxyServer::putSigned(const std::shared_ptr<restbed::Session>& session) const
 {
@@ -814,7 +838,7 @@ void
 DhtProxyServer::handleOptionsMethod(const std::shared_ptr<restbed::Session>& session) const
 {
    requestNum_++;
-#if OPENDHT_PROXY_SERVER_IDENTITY
+#ifdef OPENDHT_PROXY_SERVER_IDENTITY
    const auto allowed = "OPTIONS, GET, POST, LISTEN, SIGN, ENCRYPT";
 #else
    const auto allowed = "OPTIONS, GET, POST, LISTEN";
@@ -884,4 +908,3 @@ DhtProxyServer::removeClosedListeners(bool testSession)
 }

 }
-#endif //OPENDHT_PROXY_SERVER
@@ -21,7 +21,7 @@
 #include "dhtrunner.h"
 #include "securedht.h"

-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
 #include "dht_proxy_client.h"
 #endif

@@ -55,7 +55,7 @@ struct DhtRunner::Listener {
 };

 DhtRunner::DhtRunner() : dht_()
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
 , dht_via_proxy_()
 #endif //OPENDHT_PROXY_CLIENT
 {
@@ -105,7 +105,7 @@ DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config
    auto dht = std::unique_ptr<DhtInterface>(new Dht(s4, s6, SecureDht::getConfig(config.dht_config)));
    dht_ = std::unique_ptr<SecureDht>(new SecureDht(std::move(dht), config.dht_config));

-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    config_ = config;
 #endif
    enableProxy(not config.proxy_server.empty());
@@ -151,7 +151,7 @@ DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config

 void
 DhtRunner::shutdown(ShutdownCallback cb) {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
        dht_via_proxy_->shutdown(cb);
 #endif
@@ -250,7 +250,7 @@ DhtRunner::setLoggers(LogMethod error, LogMethod warn, LogMethod debug) {
    std::lock_guard<std::mutex> lck(dht_mtx);
    if (dht_)
        dht_->setLoggers(error, warn, debug);
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
        dht_via_proxy_->setLoggers(error, warn, debug);
 #endif
@@ -262,7 +262,7 @@ DhtRunner::setLogFilter(const InfoHash& f) {
    activeDht()->setLogFilter(f);
    if (dht_)
        dht_->setLogFilter(f);
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
        dht_via_proxy_->setLogFilter(f);
 #endif
@@ -376,7 +376,7 @@ DhtRunner::registerCertificate(std::shared_ptr<crypto::Certificate> cert) {
 void
 DhtRunner::setLocalCertificateStore(CertificateStoreQuery&& query_method) {
    std::lock_guard<std::mutex> lck(dht_mtx);
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
        dht_via_proxy_->setLocalCertificateStore(std::forward<CertificateStoreQuery>(query_method));
 #endif
@@ -460,6 +460,7 @@ int bindSocket(const SockAddr& addr, SockAddr& bound)
        setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&set, sizeof(set));
    int rc = bind(sock, addr.get(), addr.getLength());
    if(rc < 0) {
+        rc = errno;
        close(sock);
        throw DhtException("Can't bind socket on " + addr.toString() + " " + strerror(rc));
    }
@@ -520,7 +521,7 @@ DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
 #ifndef _WIN32
    auto status = pipe(stopfds);
    if (status == -1) {
-        throw DhtException("Can't open pipe");
+        throw DhtException(std::string("Can't open pipe: ") + strerror(errno));
    }
 #else
    udpPipe(stopfds);
@@ -532,15 +533,29 @@ DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6)
    s6 = -1;

    bound4 = {};
-    if (sin4)
-        s4 = bindSocket(sin4, bound4);
+    if (sin4) {
+        try {
+            s4 = bindSocket(sin4, bound4);
+        } catch (const DhtException& e) {
+            std::cerr << "Can't bind inet socket: " << e.what() << std::endl;
+        }
+    }

 #if 1
    bound6 = {};
-    if (sin6)
-        s6 = bindSocket(sin6, bound6);
+    if (sin6) {
+        try {
+            s6 = bindSocket(sin6, bound6);
+        } catch (const DhtException& e) {
+            std::cerr << "Can't bind inet6 socket: " << e.what() << std::endl;
+        }
+    }
 #endif

+    if (s4 == -1 && s6 == -1) {
+        throw DhtException("Can't bind socket");
+    }
+
    running_network = true;
    rcv_thread = std::thread([this, stop_readfd]() {
        try {
@@ -643,7 +658,7 @@ DhtRunner::listen(InfoHash hash, ValueCallback vcb, Value::Filter f, Where w)
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
        pending_ops.emplace([=](SecureDht& dht) mutable {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
            auto tokenbGlobal = listener_token_++;
            Listener listener {};
            listener.hash = hash;
@@ -682,7 +697,7 @@ DhtRunner::cancelListen(InfoHash h, size_t token)
 {
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
        pending_ops.emplace([=](SecureDht&) {
            auto it = listeners_.find(token);
            if (it == listeners_.end()) return;
@@ -706,7 +721,7 @@ DhtRunner::cancelListen(InfoHash h, std::shared_future<size_t> ftoken)
 {
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
        pending_ops.emplace([=](SecureDht&) {
            auto it = listeners_.find(ftoken.get());
            if (it == listeners_.end()) return;
@@ -958,7 +973,7 @@ DhtRunner::findCertificate(InfoHash hash, std::function<void(const std::shared_p
 void
 DhtRunner::resetDht()
 {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    listeners_.clear();
    dht_via_proxy_.reset();
 #endif // OPENDHT_PROXY_CLIENT
@@ -968,7 +983,7 @@ DhtRunner::resetDht()
 SecureDht*
 DhtRunner::activeDht() const
 {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    return use_proxy? dht_via_proxy_.get() : dht_.get();
 #else
    return dht_.get();
@@ -978,7 +993,7 @@ DhtRunner::activeDht() const
 void
 DhtRunner::setProxyServer(const std::string& proxy, const std::string& pushNodeId)
 {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (config_.proxy_server == proxy and config_.push_node_id == pushNodeId)
        return;
    config_.proxy_server = proxy;
@@ -993,7 +1008,7 @@ DhtRunner::setProxyServer(const std::string& proxy, const std::string& pushNodeI
 void
 DhtRunner::enableProxy(bool proxify)
 {
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_) {
        dht_via_proxy_->shutdown({});
    }
@@ -1011,7 +1026,7 @@ DhtRunner::enableProxy(bool proxify)
            }, config_.proxy_server, config_.push_node_id)
        );
        dht_via_proxy_ = std::unique_ptr<SecureDht>(new SecureDht(std::move(dht_via_proxy), config_.dht_config));
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
        if (not pushToken_.empty())
            dht_via_proxy_->setPushNotificationToken(pushToken_);
 #endif
@@ -1044,8 +1059,8 @@ DhtRunner::enableProxy(bool proxify)
 void
 DhtRunner::forwardAllMessages(bool forward)
 {
-#if OPENDHT_PROXY_SERVER
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_CLIENT
    if (dht_via_proxy_)
        dht_via_proxy_->forwardAllMessages(forward);
 #endif // OPENDHT_PROXY_CLIENT
@@ -1059,7 +1074,7 @@ DhtRunner::forwardAllMessages(bool forward)
 */
 void
 DhtRunner::setPushNotificationToken(const std::string& token) {
-#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS
+#if defined(OPENDHT_PROXY_CLIENT) && defined(OPENDHT_PUSH_NOTIFICATIONS)
    pushToken_ = token;
    if (dht_via_proxy_)
        dht_via_proxy_->setPushNotificationToken(token);
@@ -1069,7 +1084,7 @@ DhtRunner::setPushNotificationToken(const std::string& token) {
 void
 DhtRunner::pushNotificationReceived(const std::map<std::string, std::string>& data)
 {
-#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS
+#if defined(OPENDHT_PROXY_CLIENT) && defined(OPENDHT_PUSH_NOTIFICATIONS)
    {
        std::lock_guard<std::mutex> lck(storage_mtx);
        pending_ops_prio.emplace([=](SecureDht&) {
@@ -48,11 +48,4 @@ struct LocalListener {
    ValueCallback get_cb;
 };

-
-struct SearchListener {
-    Sp<Query> query;
-    Value::Filter filter;
-    ValueCallback get_cb;
-};
-
 }
@@ -353,9 +353,8 @@ NetworkEngine::rateLimit(const SockAddr& addr)
        limiter_maintenance = 0;
    }

-    auto it = address_rate_limiter.emplace(addr, IpLimiter{});
    // invoke per IP, then global rate limiter
-    return it.first->second.limit(now) and rate_limiter.limit(now);
+    return address_rate_limiter[addr].limit(now) and rate_limiter.limit(now);
 }

 bool
@@ -434,7 +433,8 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
    if (msg->type == MessageType::ValueData) {
        auto pmsg_it = partial_messages.find(msg->tid);
        if (pmsg_it == partial_messages.end()) {
-            DHT_LOG.d("Can't find partial message");
+            if (logIncoming_)
+                DHT_LOG.d("Can't find partial message");
            rateLimit(from);
            return;
        }
@@ -474,15 +474,15 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr&
        process(std::move(msg), from);
    } else {
        // starting partial message session
-        PartialMessage pmsg;
-        pmsg.from = from;
-        pmsg.msg = std::move(msg);
-        pmsg.start = now;
-        pmsg.last_part = now;
-        auto wmsg = partial_messages.emplace(pmsg.msg->tid, std::move(pmsg));
-        if (wmsg.second) {
-            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
-            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, wmsg.first->first));
+        auto k = msg->tid;
+        auto& pmsg = partial_messages[k];
+        if (not pmsg.msg) {
+            pmsg.from = from;
+            pmsg.msg = std::move(msg);
+            pmsg.start = now;
+            pmsg.last_part = now;
+            scheduler.add(now + RX_MAX_PACKET_TIME, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
+            scheduler.add(now + RX_TIMEOUT, std::bind(&NetworkEngine::maintainRxBuffer, this, k));
        } else
            DHT_LOG.e("Partial message with given TID already exists");
    }
@@ -544,19 +544,21 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                req->reply_time = time_point::min();
                onError(req, DhtProtocolException {msg->error_code});
            } else {
-                DHT_LOG.w(msg->id, "[node %s %s] received unknown error message %u",
+                if (logIncoming_)
+                    DHT_LOG.w(msg->id, "[node %s %s] received unknown error message %u",
                        msg->id.toString().c_str(), from.toString().c_str(), msg->error_code);
            }
            break;
        }
        case MessageType::Reply:
            if (req) { /* request reply */
-                if (req->getType() == MessageType::AnnounceValue or req->getType() == MessageType::Listen)
-                    req->node->authSuccess();
-                req->reply_time = scheduler.time();
+                auto& r = *req;
+                if (r.getType() == MessageType::AnnounceValue or r.getType() == MessageType::Listen)
+                    r.node->authSuccess();
+                r.reply_time = scheduler.time();

                deserializeNodes(*msg, from);
-                req->setDone(std::move(*msg));
+                r.setDone(std::move(*msg));
                break;
            } else { /* request socket data */
                deserializeNodes(*msg, from);
@@ -574,7 +576,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
            switch (msg->type) {
            case MessageType::Ping:
                ++in_stats.ping;
-                DHT_LOG.d(node->id, "[node %s] sending pong", node->toString().c_str());
+                if (logIncoming_)
+                    DHT_LOG.d(node->id, "[node %s] sending pong", node->toString().c_str());
                onPing(node);
                sendPong(from, msg->tid);
                break;
@@ -595,7 +598,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                break;
            }
            case MessageType::AnnounceValue: {
-                DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'put' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_)
+                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'put' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.put;
                onAnnounce(node, msg->info_hash, msg->token, msg->values, msg->created);

@@ -608,13 +612,15 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
                break;
            }
            case MessageType::Refresh:
-                DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'refresh' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_)
+                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'refresh' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                onRefresh(node, msg->info_hash, msg->token, msg->value_id);
                /* Same note as above in MessageType::AnnounceValue applies. */
                sendValueAnnounced(from, msg->tid, msg->value_id);
                break;
            case MessageType::Listen: {
-                DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
+                if (logIncoming_)
+                    DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str());
                ++in_stats.listen;
                RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query));
                auto nnodes = bufferNodes(from.getFamily(), msg->info_hash, msg->want, answer.nodes4, answer.nodes6);
@@ -25,7 +25,7 @@ bool
 OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
    std::vector<Sp<Value>> newValues;
    for (const auto& v : vals) {
-        auto viop = values.emplace(v->id, OpCacheValueStorage{v});
+        auto viop = values.emplace(v->id, v);
        if (viop.second) {
            newValues.emplace_back(v);
        } else {
@@ -34,6 +34,7 @@ OpValueCache::onValuesAdded(const std::vector<Sp<Value>>& vals) {
    }
    return newValues.empty() ? true : callback(newValues, false);
 }
+
 bool
 OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
    std::vector<Sp<Value>> expiredValues;
@@ -49,8 +50,26 @@ OpValueCache::onValuesExpired(const std::vector<Sp<Value>>& vals) {
    }
    return expiredValues.empty() ? true : callback(expiredValues, true);
 }
+
+bool
+OpValueCache::onValuesExpired(const std::vector<Value::Id>& vids)
+{
+    std::vector<Sp<Value>> expiredValues;
+    for (const auto& vid : vids) {
+        auto vit = values.find(vid);
+        if (vit != values.end()) {
+            vit->second.refCount--;
+            if (not vit->second.refCount) {
+                expiredValues.emplace_back(std::move(vit->second.data));
+                values.erase(vit);
+            }
+        }
+    }
+    return expiredValues.empty() ? true : callback(expiredValues, true);
+}
+
 std::vector<Sp<Value>>
-OpValueCache::get(Value::Filter& filter) const {
+OpValueCache::get(const Value::Filter& filter) const {
    std::vector<Sp<Value>> ret;
    if (not filter)
        ret.reserve(values.size());
@@ -108,33 +127,57 @@ OpCache::getExpiration() const {
    return lastRemoved + EXPIRATION;
 }

-size_t
-SearchCache::listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, std::function<size_t(Sp<Query>, ValueCallback)> onListen)
+SearchCache::OpMap::iterator
+SearchCache::getOp(const Sp<Query>& q)
 {
    // find exact match
    auto op = ops.find(q);
-    if (op == ops.end()) {
-        // find satisfying query
-        for (auto it = ops.begin(); it != ops.end(); it++) {
-            if (q->isSatisfiedBy(*it->first)) {
-                op = it;
-                break;
-            }
+    if (op != ops.end())
+        return op;
+    // find satisfying query
+    for (auto it = ops.begin(); it != ops.end(); it++) {
+        if (q->isSatisfiedBy(*it->first)) {
+            return it;
        }
    }
+    return ops.end();
+}
+
+SearchCache::OpMap::const_iterator
+SearchCache::getOp(const Sp<Query>& q) const
+{
+    // find exact match
+    auto op = ops.find(q);
+    if (op != ops.cend())
+        return op;
+    // find satisfying query
+    for (auto it = ops.begin(); it != ops.end(); it++) {
+        if (q->isSatisfiedBy(*it->first)) {
+            return it;
+        }
+    }
+    return ops.cend();
+}
+
+size_t
+SearchCache::listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, OnListen onListen)
+{
+    // find exact match
+    auto op = getOp(q);
    if (op == ops.end()) {
        // New query
        op = ops.emplace(q, std::unique_ptr<OpCache>(new OpCache)).first;
        auto& cache = *op->second;
        cache.searchToken = onListen(q, [&](const std::vector<Sp<Value>>& values, bool expired){
            return cache.onValue(values, expired);
+        }, [&](ListenSyncStatus status) {
+            cache.onNodeChanged(status);
        });
    }
    auto token = nextToken_++;
    if (nextToken_ == 0)
        nextToken_++;
-    op->second->addListener(token, get_cb, q, filter);
-    return token;
+    return op->second->addListener(token, get_cb, q, filter) ? token : 0;
 }

 bool
@@ -177,8 +220,22 @@ SearchCache::expire(const time_point& now, std::function<void(size_t)> onCancel)
    return ret;
 }

+bool
+SearchCache::get(const Value::Filter& f, const Sp<Query>& q, const GetCallback& gcb, const DoneCallback& dcb) const
+{
+    auto op = getOp(q);
+    if (op != ops.end()) {
+        auto vals = op->second->get(f);
+        if ((not vals.empty() and not gcb(vals)) or op->second->isSynced()) {
+            dcb(true, {});
+            return true;
+        }
+    }
+    return false;
+}
+
 std::vector<Sp<Value>>
-SearchCache::get(Value::Filter& filter) const {
+SearchCache::get(const Value::Filter& filter) const {
    if (ops.size() == 1)
        return ops.begin()->second->get(filter);
    std::map<Value::Id, Sp<Value>> c;
@@ -27,20 +27,23 @@ struct OpCacheValueStorage
 {
    Sp<Value> data {};
    unsigned refCount {1};
-    OpCacheValueStorage(Sp<Value> val = {}) : data(val) {}
+    OpCacheValueStorage(Sp<Value> val) : data(val) {}
 };

 class OpValueCache {
 public:
-    OpValueCache(ValueCallback&& cb) : callback(std::forward<ValueCallback>(cb)) {}
-    OpValueCache(OpValueCache&& o) : values(std::move(o.values)), callback(std::move(o.callback)) {
+    OpValueCache(ValueCallback&& cb) noexcept : callback(std::forward<ValueCallback>(cb)) {}
+    OpValueCache(OpValueCache&& o) noexcept : values(std::move(o.values)), callback(std::move(o.callback)) {
        o.callback = {};
    }

-    static ValueCallback cacheCallback(ValueCallback&& cb) {
+    static ValueCallback cacheCallback(ValueCallback&& cb, std::function<void()>&& onCancel) {
        auto cache = std::make_shared<OpValueCache>(std::forward<ValueCallback>(cb));
-        return [cache](const std::vector<Sp<Value>>& vals, bool expired){
-            return cache->onValue(vals, expired);
+        return [cache, onCancel](const std::vector<Sp<Value>>& vals, bool expired){
+            auto ret = cache->onValue(vals, expired);
+            if (not ret)
+                onCancel();
+            return ret;
        };
    }

@@ -53,8 +56,20 @@ public:

    bool onValuesAdded(const std::vector<Sp<Value>>& vals);
    bool onValuesExpired(const std::vector<Sp<Value>>& vals);
+    bool onValuesExpired(const std::vector<Value::Id>& vals);

-    std::vector<Sp<Value>> get(Value::Filter& filter) const;
+    void onNodeChanged(ListenSyncStatus status) {
+        switch (status) {
+            case ListenSyncStatus::ADDED: nodes++; break;
+            case ListenSyncStatus::REMOVED: nodes--; break;
+            case ListenSyncStatus::SYNCED : syncedNodes++; break;
+            case ListenSyncStatus::UNSYNCED: syncedNodes--; break;
+        }
+    }
+
+    bool isSynced() const { return nodes > 0 and syncedNodes == nodes; }
+
+    std::vector<Sp<Value>> get(const Value::Filter& filter) const;
    Sp<Value> get(Value::Id id) const;
    std::vector<Sp<Value>> getValues() const;

@@ -62,6 +77,8 @@ private:
    OpValueCache(const OpValueCache&) = delete;
    OpValueCache& operator=(const OpValueCache&) = delete;

+    size_t nodes {0};
+    size_t syncedNodes {0};
    std::map<Value::Id, OpCacheValueStorage> values {};
    ValueCallback callback;
 };
@@ -80,13 +97,23 @@ public:
        cache.onValue(vals, expired);
        return not listeners.empty();
    }
+    void onNodeChanged(ListenSyncStatus status) {
+        cache.onNodeChanged(status);
+    }

    void onValuesAdded(const std::vector<Sp<Value>>& vals);
    void onValuesExpired(const std::vector<Sp<Value>>& vals);

-    void addListener(size_t token, ValueCallback cb, Sp<Query> q, Value::Filter filter) {
+    bool addListener(size_t token, ValueCallback cb, Sp<Query> q, Value::Filter filter) {
        listeners.emplace(token, LocalListener{q, filter, cb});
-        cb(cache.get(filter), false);
+        auto cached = cache.get(filter);
+        if (not cached.empty()) {
+            auto ret = cb(cached, false);
+            if (not ret)
+                listeners.erase(token);
+            return ret;
+        }
+        return true;
    }

    bool removeListener(size_t token, const time_point& now) {
@@ -102,7 +129,7 @@ public:
        return listeners.empty();
    }

-    std::vector<Sp<Value>> get(Value::Filter& filter) const {
+    std::vector<Sp<Value>> get(const Value::Filter& filter) const {
        return cache.get(filter);
    }

@@ -110,6 +137,10 @@ public:
        return cache.get(id);
    }

+    bool isSynced() const {
+        return cache.isSynced();
+    }
+
    bool isExpired(const time_point& now) const {
        return listeners.empty() and (lastRemoved + EXPIRATION < now);
    }
@@ -130,7 +161,9 @@ class SearchCache {
 public:
    SearchCache() {}
    SearchCache(SearchCache&&) = default;
-    size_t listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, std::function<size_t(Sp<Query>, ValueCallback)> onListen);
+
+    using OnListen = std::function<size_t(Sp<Query>, ValueCallback, SyncCallback)>;
+    size_t listen(ValueCallback get_cb, Sp<Query> q, Value::Filter filter, OnListen onListen);

    bool cancelListen(size_t gtoken, const time_point& now);
    void cancelAll(std::function<void(size_t)> onCancel);
@@ -140,14 +173,19 @@ public:
        return nextExpiration_;
    }

-    std::vector<Sp<Value>> get(Value::Filter& filter) const;
+    bool get(const Value::Filter& f, const Sp<Query>& q, const GetCallback& gcb, const DoneCallback& dcb) const;
+    std::vector<Sp<Value>> get(const Value::Filter& filter) const;
    Sp<Value> get(Value::Id id) const;

 private:
    SearchCache(const SearchCache&) = delete;
    SearchCache& operator=(const SearchCache&) = delete;

-    std::map<Sp<Query>, std::unique_ptr<OpCache>> ops {};
+    using OpMap = std::map<Sp<Query>, std::unique_ptr<OpCache>>;
+    OpMap ops {};
+    OpMap::iterator getOp(const Sp<Query>& q);
+    OpMap::const_iterator getOp(const Sp<Query>& q) const;
+
    size_t nextToken_ {1};
    time_point nextExpiration_ {time_point::max()};
 };
@@ -65,7 +65,8 @@ struct Dht::SearchNode {
        ValueCache cache;
        Sp<Scheduler::Job> cacheExpirationJob {};
        Sp<net::Request> req {};
-        CachedListenStatus(ValueStateCallback&& cb) : cache(std::forward<ValueStateCallback>(cb)) {}
+        CachedListenStatus(ValueStateCallback&& cb, SyncCallback&& scb)
+         : cache(std::forward<ValueStateCallback>(cb), std::forward<SyncCallback>(scb)) {}
        CachedListenStatus(CachedListenStatus&&) = default;
        CachedListenStatus(const CachedListenStatus&) = delete;
        CachedListenStatus& operator=(const CachedListenStatus&) = delete;
@@ -225,6 +226,13 @@ struct Dht::SearchNode {
        }
    }

+    void onListenSynced(const Sp<Query>& q, bool synced = true) {
+        auto l = listenStatus.find(q);
+        if (l != listenStatus.end()) {
+            l->second.cache.onSynced(synced);
+        }
+    }
+
    void expireValues(const Sp<Query>& q, Scheduler& scheduler) {
        auto l = listenStatus.find(q);
        if (l != listenStatus.end()) {
@@ -377,6 +385,12 @@ struct Dht::Search {
    std::multimap<time_point, Get> callbacks {};

    /* listeners */
+    struct SearchListener {
+        Sp<Query> query;
+        Value::Filter filter;
+        ValueCallback get_cb;
+        SyncCallback sync_cb;
+    };
    std::map<size_t, SearchListener> listeners {};
    size_t listener_token = 1;

@@ -477,21 +491,20 @@ struct Dht::Search {

    void get(Value::Filter f, const Sp<Query>& q, const QueryCallback& qcb, const GetCallback& gcb, const DoneCallback& dcb, Scheduler& scheduler) {
        if (gcb or qcb) {
-            const auto& now = scheduler.time();
-            callbacks.emplace(now, Get { now, f, q, qcb, gcb, dcb });
-            auto values = cache.get(f);
-            if (not values.empty())
-                gcb(values);
-            scheduler.edit(nextSearchStep, now);
+            if (not cache.get(f, q, gcb, dcb)) {
+                const auto& now = scheduler.time();
+                callbacks.emplace(now, Get { now, f, q, qcb, gcb, dcb });
+                scheduler.edit(nextSearchStep, now);
+            }
        }
    }

    size_t listen(ValueCallback cb, Value::Filter f, const Sp<Query>& q, Scheduler& scheduler) {
        //DHT_LOG.e(id, "[search %s IPv%c] listen", id.toString().c_str(), (af == AF_INET) ? '4' : '6');
-        return cache.listen(cb, q, f, [&](const Sp<Query>& q, ValueCallback vcb){
+        return cache.listen(cb, q, f, [&](const Sp<Query>& q, ValueCallback vcb, SyncCallback scb){
            done = false;
            auto token = ++listener_token;
-            listeners.emplace(token, SearchListener{q, f, vcb});
+            listeners.emplace(token, SearchListener{q, f, vcb, scb});
            scheduler.edit(nextSearchStep, scheduler.time());
            return token;
        });
@@ -235,7 +235,7 @@ SecureDht::checkValue(const Sp<Value>& v)
    // Decrypt encrypted values
    if (v->isEncrypted()) {
        if (not key_) {
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
            if (forward_all_) // We are currently a proxy, send messages to clients.
                return v;
 #endif
@@ -284,6 +284,8 @@ SecureDht::getCallbackFilter(ValueCallback cb, Value::Filter&& filter)
 {
    return [=](const std::vector<Sp<Value>>& values, bool expired) {
        std::vector<Sp<Value>> tmpvals {};
+        if (not filter)
+            tmpvals.reserve(values.size());
        for (const auto& v : values) {
            if (auto nv = checkValue(v))
                if (not filter or filter(*nv))
@@ -301,6 +303,8 @@ SecureDht::getCallbackFilter(GetCallback cb, Value::Filter&& filter)
 {
    return [=](const std::vector<Sp<Value>>& values) {
        std::vector<Sp<Value>> tmpvals {};
+        if (not filter)
+            tmpvals.reserve(values.size());
        for (const auto& v : values) {
            if (auto nv = checkValue(v))
                if (not filter or filter(*nv))
@@ -364,7 +368,8 @@ SecureDht::putSigned(const InfoHash& hash, Sp<Value> val, DoneCallback callback,
            sign(*val);
            dht_->put(hash, val, callback, time_point::max(), permanent);
        },
-        Value::IdFilter(val->id)
+        Value::IdFilter(val->id),
+        std::move(Where().id(val->id))
    );
 }

@@ -90,20 +90,7 @@ struct Storage {

    Storage() {}
    Storage(time_point t) : maintenance_time(t) {}
-
-#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 || defined(_WIN32)
-    // GCC-bug: remove me when support of GCC < 4.9.2 is abandoned
-    Storage(Storage&& o) noexcept
-        : maintenance_time(std::move(o.maintenance_time))
-        , listeners(std::move(o.listeners))
-        , local_listeners(std::move(o.local_listeners))
-        , listener_token(std::move(o.listener_token))
-        , values(std::move(o.values))
-        , total_size(std::move(o.total_size)) {}
-#else
    Storage(Storage&& o) noexcept = default;
-#endif
-
    Storage& operator=(Storage&& o) = default;

    bool empty() const {
@@ -0,0 +1,134 @@
+/*
+ *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "thread_pool.h"
+
+#include <atomic>
+#include <thread>
+
+#include <ciso646> // fix windows compiler bug
+
+namespace dht {
+
+constexpr const size_t IO_THREADS_MAX {64};
+
+struct ThreadPool::ThreadState
+{
+    std::thread thread {};
+    std::atomic_bool run {true};
+};
+
+ThreadPool&
+ThreadPool::computation()
+{
+    static ThreadPool pool;
+    return pool;
+}
+
+ThreadPool&
+ThreadPool::io()
+{
+    static ThreadPool pool(IO_THREADS_MAX);
+    return pool;
+}
+
+
+ThreadPool::ThreadPool(size_t maxThreads) : maxThreads_(maxThreads)
+{
+    threads_.reserve(maxThreads_);
+}
+
+ThreadPool::ThreadPool()
+ : ThreadPool(std::max<size_t>(std::thread::hardware_concurrency(), 4))
+{}
+
+ThreadPool::~ThreadPool()
+{
+    join();
+}
+
+void
+ThreadPool::run(std::function<void()>&& cb)
+{
+    std::unique_lock<std::mutex> l(lock_);
+    if (not running_) return;
+
+    // launch new thread if necessary
+    if (not readyThreads_ && threads_.size() < maxThreads_) {
+        threads_.emplace_back(new ThreadState());
+        auto& t = *threads_.back();
+        t.thread = std::thread([&]() {
+            while (t.run) {
+                std::function<void()> task;
+
+                // pick task from queue
+                {
+                    std::unique_lock<std::mutex> l(lock_);
+                    readyThreads_++;
+                    cv_.wait(l, [&](){
+                        return not t.run or not tasks_.empty();
+                    });
+                    readyThreads_--;
+                    if (not t.run)
+                        break;
+                    task = std::move(tasks_.front());
+                    tasks_.pop();
+                }
+
+                // run task
+                try {
+                    if (task)
+                        task();
+                } catch (const std::exception& e) {
+                    // LOG_ERR("Exception running task: %s", e.what());
+                }
+            }
+        });
+    }
+
+    // push task to queue
+    tasks_.emplace(std::move(cb));
+
+    // notify thread
+    l.unlock();
+    cv_.notify_one();
+}
+
+void
+ThreadPool::stop()
+{
+    {
+        std::lock_guard<std::mutex> l(lock_);
+        running_ = false;
+    }
+    for (auto& t : threads_)
+        t->run = false;
+    cv_.notify_all();
+}
+
+void
+ThreadPool::join()
+{
+    stop();
+    for (auto& t : threads_)
+        t->thread.join();
+    threads_.clear();
+}
+
+}
@@ -0,0 +1,69 @@
+/*
+ *  Copyright (C) 2016-2019 Savoir-faire Linux Inc.
+ *
+ *  Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <condition_variable>
+#include <vector>
+#include <queue>
+#include <future>
+#include <functional>
+
+namespace dht {
+
+class ThreadPool {
+public:
+    static ThreadPool& computation();
+    static ThreadPool& io();
+
+    ThreadPool();
+    ThreadPool(size_t maxThreads);
+    ~ThreadPool();
+
+    void run(std::function<void()>&& cb);
+
+    template<class T>
+    std::future<T> get(std::function<T()>&& cb) {
+        auto ret = std::make_shared<std::promise<T>>();
+        run(std::bind([=](std::function<T()>& mcb) mutable {
+                ret->set_value(mcb());
+            }, std::move(cb)));
+        return ret->get_future();
+    }
+    template<class T>
+    std::shared_ptr<std::future<T>> getShared(std::function<T()>&& cb) {
+        return std::make_shared<std::future<T>>(get(std::move(cb)));
+    }
+
+    void stop();
+    void join();
+
+private:
+    struct ThreadState;
+    std::queue<std::function<void()>> tasks_ {};
+    std::vector<std::unique_ptr<ThreadState>> threads_;
+    unsigned readyThreads_ {0};
+    std::mutex lock_ {};
+    std::condition_variable cv_ {};
+
+    const unsigned maxThreads_;
+    bool running_ {true};
+};
+
+}
@@ -54,12 +54,14 @@ std::ostream& operator<< (std::ostream& s, const Value& v)
            s << IpServiceAnnouncement(v.data);
        } else if (v.type == CERTIFICATE_TYPE.id) {
            s << "Certificate";
+#ifdef OPENDHT_LOG_CRT_ID
            try {
-                InfoHash h = crypto::Certificate(v.data).getPublicKey().getId();
+                auto h = crypto::Certificate(v.data).getPublicKey().getLongId();
                s << " with ID " << h;
            } catch (const std::exception& e) {
                s << " (invalid)";
            }
+#endif
        } else {
            s << "Data (type: " << v.type << " ): ";
            s << std::hex;
@@ -286,7 +288,7 @@ FieldValue::getLocalFilter() const
        case Value::Field::UserType:
            return Value::UserTypeFilter(std::string {blobValue.begin(), blobValue.end()});
        default:
-            return Value::AllFilter();
+            return {};
    }
 }

@@ -495,8 +497,7 @@ template <typename T>
 bool subset(std::vector<T> fds, std::vector<T> qfds)
 {
    for (auto& fd : fds) {
-        auto correspondance = std::find_if(qfds.begin(), qfds.end(), [&fd](T& _vfd) { return fd == _vfd; });
-        if (correspondance == qfds.end())
+        if (std::find_if(qfds.begin(), qfds.end(), [&fd](T& _vfd) { return fd == _vfd; }) == qfds.end())
            return false;
    }
    return true;
@@ -504,10 +505,9 @@ bool subset(std::vector<T> fds, std::vector<T> qfds)

 bool Select::isSatisfiedBy(const Select& os) const {
    /* empty, means all values are selected. */
-    if (fieldSelection_.empty() and not os.fieldSelection_.empty())
-        return false;
-    else
-        return subset(fieldSelection_, os.fieldSelection_);
+    return fieldSelection_.empty() ?
+        os.fieldSelection_.empty() :
+        subset(fieldSelection_, os.fieldSelection_);
 }

 bool Where::isSatisfiedBy(const Where& ow) const {
@@ -22,19 +22,32 @@
 namespace dht {

 using ValueStateCallback = std::function<void(const std::vector<Sp<Value>>&, bool)>;
+enum class ListenSyncStatus { ADDED, SYNCED, UNSYNCED, REMOVED };
+using SyncCallback = std::function<void(ListenSyncStatus)>;
 using CallbackQueue = std::list<std::function<void()>>;

 class ValueCache {
 public:
-    ValueCache(ValueStateCallback&& cb) : callback(std::forward<ValueStateCallback>(cb)) {}
-    ValueCache(ValueCache&& o) : values(std::move(o.values)), callback(std::move(o.callback)) {
+    ValueCache(ValueStateCallback&& cb, SyncCallback&& scb = {})
+        : callback(std::forward<ValueStateCallback>(cb)), syncCallback(std::move(scb))
+    {
+        if (syncCallback)
+            syncCallback(ListenSyncStatus::ADDED);
+    }
+    ValueCache(ValueCache&& o) noexcept : values(std::move(o.values)), callback(std::move(o.callback)), syncCallback(std::move(o.syncCallback)) {
        o.callback = {};
+        o.syncCallback = {};
    }

    ~ValueCache() {
        auto q = clear();
        for (auto& cb: q)
            cb();
+        if (syncCallback) {
+            if (status == ListenSyncStatus::SYNCED)
+                syncCallback(ListenSyncStatus::UNSYNCED);
+            syncCallback(ListenSyncStatus::REMOVED);
+        }
    }

    CallbackQueue clear() {
@@ -120,6 +133,15 @@ public:
        return ret;
    }

+    void onSynced(bool synced) {
+        auto newStatus = synced ? ListenSyncStatus::SYNCED : ListenSyncStatus::UNSYNCED;
+        if (status != newStatus) {
+            status = newStatus;
+            if (syncCallback)
+                syncCallback(newStatus);
+        }
+    }
+
 private:
    // prevent copy
    ValueCache(const ValueCache&) = delete;
@@ -141,6 +163,8 @@ private:

    std::map<Value::Id, CacheValueStorage> values;
    ValueStateCallback callback;
+    SyncCallback syncCallback;
+    ListenSyncStatus status {ListenSyncStatus::UNSYNCED};

    CallbackQueue addValues(const std::vector<Sp<Value>>& new_values, const TypeStore& types, const time_point& now) {
        std::vector<Sp<Value>> nvals;
@@ -77,7 +77,6 @@ DhtProxyTester::testGetPut() {
    CPPUNIT_ASSERT(vals.front()->data == val_data);
 }

-
 void
 DhtProxyTester::testListen() {
    bool done = false;
@@ -108,7 +107,7 @@ DhtProxyTester::testListen() {

    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
    done = false;
-    // Here values should contains 2 values
+    // Here values should contains 1 values
    CPPUNIT_ASSERT_EQUAL(static_cast<int>(values.size()), 1);
    CPPUNIT_ASSERT(values.front() == firstVal_data);

@@ -118,9 +117,63 @@ DhtProxyTester::testListen() {
    auto secondVal_data = secondVal.data;
    nodePeer.put(key, std::move(secondVal));
    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
-    // Here values should contains 3 values
+    // Here values should contains 2 values
    CPPUNIT_ASSERT_EQUAL(static_cast<int>(values.size()), 2);
    CPPUNIT_ASSERT(values.back() == secondVal_data);
 }

+void
+DhtProxyTester::testResubscribeGetValues() {
+    nodeClient->setPushNotificationToken("atlas");
+
+    bool done = false;
+    std::condition_variable cv;
+    std::mutex cv_m;
+    std::unique_lock<std::mutex> lk(cv_m);
+    auto key = dht::InfoHash::get("GLaDOs");
+
+    // If a peer send a value, the listen operation from the client
+    // should retrieve this value
+    dht::Value firstVal {"Hey! It's been a long time. How have you been?"};
+    auto firstVal_data = firstVal.data;
+    nodePeer.put(key, std::move(firstVal), [&](bool) {
+        done = true;
+        cv.notify_all();
+    });
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+    done = false;
+
+    // Send a first subscribe, the value is sent via a push notification
+    // So ignore values here.
+    nodeClient->listen(key, [&](const std::vector<std::shared_ptr<dht::Value>>&, bool) {
+        return true;
+    });
+    cv.wait_for(lk, std::chrono::seconds(1));
+
+    // Reboot node (to avoid cache)
+    nodeClient->join();
+    nodeClient->run(42242, {}, true);
+    nodeClient->bootstrap(nodePeer.getBound());
+    nodeClient->setProxyServer("127.0.0.1:8080");
+    nodeClient->enableProxy(true);
+    nodeClient->setPushNotificationToken("atlas");
+
+    // For the second subscribe, the proxy will return the value in the body
+    auto values = std::vector<dht::Blob>();
+    nodeClient->listen(key, [&](const std::vector<std::shared_ptr<dht::Value>>& v, bool) {
+        for (const auto& value : v)
+            values.emplace_back(value->data);
+        done = true;
+        cv.notify_all();
+        return true;
+    });
+
+    cv.wait_for(lk, std::chrono::seconds(10), [&]{ return done; });
+    done = false;
+    // Here values should still contains 1 values
+    CPPUNIT_ASSERT_EQUAL(static_cast<int>(values.size()), 1);
+    CPPUNIT_ASSERT(values.front() == firstVal_data);
+
+}
+
 }  // namespace test
@@ -32,6 +32,7 @@ class DhtProxyTester : public CppUnit::TestFixture {
    CPPUNIT_TEST_SUITE(DhtProxyTester);
    CPPUNIT_TEST(testGetPut);
    CPPUNIT_TEST(testListen);
+    CPPUNIT_TEST(testResubscribeGetValues);
    CPPUNIT_TEST_SUITE_END();

 public:
@@ -39,19 +40,25 @@ class DhtProxyTester : public CppUnit::TestFixture {
     * Method automatically called before each test by CppUnit
     * Init nodes
     */
-    void setUp();
+   void setUp();
    /**
     * Method automatically called after each test CppUnit
     */
-    void tearDown();
+   void tearDown();
    /**
     * Test get and put methods
     */
-    void testGetPut();
+   void testGetPut();
    /**
     * Test listen
     */
-    void testListen();
+   void testListen();
+
+   /**
+    * When a proxy redo a subscribe on the proxy
+    * it should retrieve existant values
+    */
+   void testResubscribeGetValues();

 private:
    dht::DhtRunner nodePeer {};
@@ -50,15 +50,56 @@ DhtRunnerTester::testGetPut() {
    auto key = dht::InfoHash::get("123");
    dht::Value val {"hey"};
    auto val_data = val.data;
-    node1.put(key, std::move(val));
-    auto vals = node2.get(key).get();
+    std::promise<bool> p;
+    node2.put(key, std::move(val), [&](bool ok){
+        p.set_value(ok);
+    });
+    CPPUNIT_ASSERT(p.get_future().get());
+    auto vals = node1.get(key).get();
    CPPUNIT_ASSERT(not vals.empty());
    CPPUNIT_ASSERT(vals.front()->data == val_data);
 }

 void
 DhtRunnerTester::testListen() {
-    // TODO
+    std::atomic_uint valueCount(0);
+    std::atomic_uint putCount(0);
+
+    auto a = dht::InfoHash::get("234");
+    auto b = dht::InfoHash::get("2345");
+    auto c = dht::InfoHash::get("23456");
+    constexpr unsigned N = 32;
+
+    auto ftokena = node1.listen(a, [&](const std::shared_ptr<dht::Value>&){
+        valueCount++;
+        return true;
+    });
+
+    auto ftokenb = node1.listen(b, [&](const std::shared_ptr<dht::Value>&){
+        valueCount++;
+        return false;
+    });
+
+    auto ftokenc = node1.listen(c, [&](const std::shared_ptr<dht::Value>&){
+        valueCount++;
+        return true;
+    });
+
+    for (unsigned i=0; i<N; i++) {
+        node2.put(a, dht::Value("v1"), [&](bool ok) { if (ok) putCount++; });
+        node2.put(b, dht::Value("v2"), [&](bool ok) { if (ok) putCount++; });
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    auto tokena = ftokena.get();
+    auto tokenb = ftokenb.get();
+    auto tokenc = ftokenc.get();
+
+    CPPUNIT_ASSERT(tokena);
+    CPPUNIT_ASSERT(tokenb);
+    CPPUNIT_ASSERT(tokenc);
+    CPPUNIT_ASSERT_EQUAL(N * 2u, putCount.load());
+    CPPUNIT_ASSERT_EQUAL(N + 1u, valueCount.load());
 }

 }  // namespace test
@@ -64,10 +64,19 @@ main(int argc, char **argv)
    try {
        dht.run(params.port, dht::crypto::generateIdentity("DHT Chat Node"), true, params.network);

+        if (params.log) {
+            if (params.syslog)
+                log::enableSyslog(dht, "dhtnode");
+            else if (not params.logfile.empty())
+                log::enableFileLogging(dht, params.logfile);
+            else
+                log::enableLogging(dht);
+        }
+
        if (not params.bootstrap.first.empty())
            dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());

-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    if (!params.proxyclient.empty()) {
        dht.setProxyServer(params.proxyclient);
        dht.enableProxy(true);
@@ -58,9 +58,9 @@ void print_help() {
              << "  ld [key]   Print basic information about currenty stored values on this node (or key)." << std::endl
              << "  lr         Print the full current routing table of this node." << std::endl;

-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
    std::cout << std::endl << "Operations with the proxy:" << std::endl
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
              << "  pst [port] <pushServer> Start the proxy interface on port." << std::endl
 #else
              << "  pst [port]              Start the proxy interface on port." << std::endl
@@ -68,9 +68,9 @@ void print_help() {
              << "  psp [port]              Stop the proxy interface on port." << std::endl;
 #endif //OPENDHT_PROXY_SERVER

-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
    std::cout << std::endl << "Operations with the proxy:" << std::endl
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
              << "  stt [server_address] <device_key> Start the proxy client." << std::endl
              << "  rs  [token]                       Resubscribe to opendht." << std::endl
              << "  rp  [token]                       Inject a push notification in Opendht." << std::endl
@@ -102,7 +102,7 @@ void print_help() {
 }

 void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
    , std::map<in_port_t, std::unique_ptr<DhtProxyServer>>& proxies
 #endif
 )
@@ -141,7 +141,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            std::cout << dht->getNodesStats(AF_INET).toString() << std::endl;
            std::cout << "IPv6 stats:" << std::endl;
            std::cout << dht->getNodesStats(AF_INET6).toString() << std::endl;
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
            for (const auto& proxy : proxies) {
                std::cout << "Stats for proxy on port " << proxy.first << std::endl;
                std::cout << "  " << proxy.second->stats().toString() << std::endl;
@@ -201,16 +201,16 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            dht->connectivityChanged();
            continue;
        }
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
        else if (op == "pst") {
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
                iss >> idstr >> pushServer;
 #else
                iss >> idstr;
 #endif // OPENDHT_PUSH_NOTIFICATIONS
            try {
                unsigned int port = std::stoi(idstr);
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
                proxies.emplace(port, std::unique_ptr<DhtProxyServer>(new DhtProxyServer(dht, port, pushServer)));
 #else
                proxies.emplace(port, std::unique_ptr<DhtProxyServer>(new DhtProxyServer(dht, port)));
@@ -227,7 +227,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            continue;
        }
 #endif //OPENDHT_PROXY_SERVER
-#if OPENDHT_PROXY_CLIENT
+#ifdef OPENDHT_PROXY_CLIENT
        else if (op == "stt") {
            dht->enableProxy(true);
            continue;
@@ -235,7 +235,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, dht_params& params
            dht->enableProxy(false);
            continue;
        }
-#if OPENDHT_PUSH_NOTIFICATIONS
+#ifdef OPENDHT_PUSH_NOTIFICATIONS
        else if (op == "rp") {
            iss >> value;
            dht->pushNotificationReceived({{"to", "dhtnode"}, {"token", value}});
@@ -500,6 +500,7 @@ main(int argc, char **argv)
        dht::DhtRunner::Config config {};
        config.dht_config.node_config.network = params.network;
        config.dht_config.node_config.maintain_storage = false;
+        config.dht_config.node_config.persist_path = params.persist_path;
        config.dht_config.id = crt;
        config.threaded = true;
        config.proxy_server = params.proxyclient;
@@ -523,11 +524,11 @@ main(int argc, char **argv)
            dht->bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
        }

-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
        std::map<in_port_t, std::unique_ptr<DhtProxyServer>> proxies;
 #endif
        if (params.proxyserver != 0) {
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
            proxies.emplace(params.proxyserver, std::unique_ptr<DhtProxyServer>(new DhtProxyServer(dht, params.proxyserver, params.pushserver)));
 #else
            std::cerr << "DHT proxy server requested but OpenDHT built without proxy server support." << std::endl;
@@ -539,7 +540,7 @@ main(int argc, char **argv)
            while (runner.wait());
        else
            cmd_loop(dht, params
-#if OPENDHT_PROXY_SERVER
+#ifdef OPENDHT_PROXY_SERVER
                , proxies
 #endif
            );
@@ -103,6 +103,7 @@ struct dht_params {
    std::string proxyclient {};
    std::string pushserver {};
    std::string devicekey {};
+    std::string persist_path {};
 };

 static const constexpr struct option long_options[] = {
@@ -114,6 +115,7 @@ static const constexpr struct option long_options[] = {
   {"verbose",    no_argument      , nullptr, 'v'},
   {"daemonize",  no_argument      , nullptr, 'd'},
   {"service",    no_argument      , nullptr, 's'},
+   {"persist",    required_argument, nullptr, 'f'},
   {"logfile",    required_argument, nullptr, 'l'},
   {"syslog",     no_argument      , nullptr, 'L'},
   {"proxyserver",required_argument, nullptr, 'S'},
@@ -127,7 +129,7 @@ dht_params
 parseArgs(int argc, char **argv) {
    dht_params params;
    int opt;
-    while ((opt = getopt_long(argc, argv, "hidsvp:n:b:l:", long_options, nullptr)) != -1) {
+    while ((opt = getopt_long(argc, argv, "hidsvp:n:b:f:l:", long_options, nullptr)) != -1) {
        switch (opt) {
        case 'p': {
                int port_arg = atoi(optarg);
@@ -154,6 +156,9 @@ parseArgs(int argc, char **argv) {
        case 'D':
            params.devicekey = optarg;
            break;
+        case 'f':
+            params.persist_path = optarg;
+            break;
        case 'n':
            params.network = strtoul(optarg, nullptr, 0);
            break;
Author	SHA1	Message	Date
Adrien Béraud	6046e0a561	make move operators noexcept	2019-03-10 15:56:33 -04:00
Adrien Béraud	244b16b978	storage: make non-movable	2019-03-10 15:56:33 -04:00
Adrien Béraud	778b389abd	thread pool: add computation, io pools	2019-03-10 15:20:31 -04:00
Adrien Béraud	92a341d8a0	dht: add configuration to persist state	2019-03-10 15:09:49 -04:00
Adrien Béraud	ecb441f31e	dht: make exportNodes const	2019-02-24 16:09:29 -05:00
Sébastien Blin	41f8b06743	Merge pull request #376 from savoirfairelinux/server_thread_pool Server thread pool	2019-02-19 19:57:13 -05:00
Adrien Béraud	1c74c2207f	proxy server: cache json StreamBuilder	2019-02-19 18:43:14 -05:00
Adrien Béraud	9319bec7b8	proxy client: close previous request on resubscribe	2019-02-19 18:43:14 -05:00
Adrien Béraud	e5f2a6eef5	proxy: send expired to sync values	2019-02-19 18:43:14 -05:00
Adrien Béraud	55f6c847b8	build: use define flags for proxy	2019-02-19 18:43:14 -05:00
Adrien Béraud	00cc4ebe28	proxy server: send push notification on thread pool	2019-02-19 18:43:14 -05:00
Adrien Béraud	aa93db4fa6	proxy server: move json to sendPushNotification	2019-02-19 18:43:14 -05:00
Adrien Béraud	683570a3da	add ThreadPool	2019-02-19 18:43:14 -05:00
Sébastien Blin	43559577f6	Merge pull request #377 from savoirfairelinux/proxy_sync_expired Proxy: sync expired state	2019-02-19 18:42:32 -05:00
Adrien Béraud	ba2e179abe	dht: optimize get	2019-02-19 01:59:59 -05:00
Adrien Béraud	a5d29c904c	op cache: optimize cache insertion	2019-02-19 01:59:59 -05:00
Adrien Béraud	989f32eb56	filter: cleanup and optimize	2019-02-19 01:59:59 -05:00
Adrien Béraud	b66fa73c56	test: include cppunit headers	2019-02-19 01:59:59 -05:00
Adrien Béraud	1ca181db1d	value: simplify expression	2019-02-19 01:59:59 -05:00
Adrien Béraud	1f3bf697c3	search: use syncStatus for get	2019-02-19 01:59:59 -05:00
Adrien Béraud	a1f8cc906c	proxy client: use SyncCallback	2019-02-19 01:59:59 -05:00
Adrien Béraud	35c6a57526	dht: add synced status	2019-02-19 01:59:59 -05:00
Adrien Béraud	39e69f813c	network engine: cleanup	2019-02-19 01:59:59 -05:00
Sébastien Blin	8045c2cbf2	dhtrunner: avoid startNetwork failure if only one binding is failing	2019-02-19 01:44:47 -05:00
Sébastien Blin	79a2f4280f	proxy: resolve url with protocol and port	2019-02-14 11:04:11 -05:00
Sébastien Blin	769ce45740	proxy_client: support prototype in serverHost	2019-01-22 07:56:27 -08:00
Sébastien Blin	75a14d3608	proxy: only get old values at first subscribe	2019-01-18 22:53:24 -08:00
Sébastien Blin	d523e9f259	tests: test retrieve values when resubscribe	2019-01-14 10:53:08 -08:00
Sébastien Blin	3a3933e6d5	dhtproxy: remove sendSubscribe	2019-01-14 10:53:08 -08:00
Sébastien Blin	70f09ded3f	dhtproxy: avoid notification after subscribe to retrieve values	2019-01-14 10:53:08 -08:00
Sébastien Blin	824a1bf5fa	proxy_server: return old values when resubscribe	2019-01-14 10:53:08 -08:00
Sébastien Blin	e5b56cc0a5	dht: link to tellListenerRefreshed When a permanent put is refreshed, the node has to tell to its remote listeners that the value is refreshed.	2019-01-09 13:34:51 -08:00
Sébastien Blin	36cf70f2a7	proxy: remove useless token for push listeners Since we authentificate listeners with the clientId and the hash and we use only one listener by hash/clientId, we doesn't need to store a token always equals to 0. Note: also avoid to get multiple hashes for one push notification.	2018-12-30 13:35:18 -05:00
Adrien Béraud	459c1344aa	update to version 1.8.1	2018-12-28 11:38:15 -05:00
Adrien Béraud	c0e376e15a	dht: reduce log verbosity	2018-12-28 01:41:10 -05:00
Adrien Béraud	71b9672b96	value: don't print certificate id in toString by default	2018-12-28 01:38:45 -05:00
Adrien Béraud	086f37a71a	op cache: remove listener if callback returned false	2018-12-28 01:38:02 -05:00
Adrien Béraud	87e76a5e4e	tests: reverse nodes	2018-12-26 13:03:36 -05:00
Adrien Béraud	7f0a5dfda3	dht: increase max number of searches and hashes	2018-12-26 10:28:55 -05:00
Adrien Béraud	6d7f8004bf	connectivity change: reset routing table instead of searches	2018-12-26 10:28:30 -05:00
Adrien Béraud	8168c39b63	dht: cleanup	2018-12-26 01:22:37 -05:00
Adrien Béraud	d0fdbe5340	dhtrunner: print proper message on bind error	2018-12-26 00:58:43 -05:00
Adrien Béraud	c7604fbb8d	tests: wait for put before performing get	2018-12-26 00:56:49 -05:00
Adrien Béraud	56773baa98	unit tests: add listen test	2018-12-24 19:09:32 -05:00
Adrien Béraud	e6cf67c575	dht: clear local listener if callback returns false	2018-12-24 18:37:06 -05:00
Adrien Béraud	00ebc5e2bf	op cache: check callback return value	2018-12-24 18:36:29 -05:00
Adrien Béraud	ccef94c11b	Revert "storage: make non-movable" This reverts commit `7ebad7d3c5`.	2018-12-24 17:16:15 -05:00
Adrien Béraud	0066be49b9	python: add more bindings for SockAddr	2018-12-24 16:55:25 -05:00
Adrien Béraud	592ed4ee90	op cache: don't call callback if no value	2018-12-24 15:50:41 -05:00
Adrien Béraud	353a6f9db6	op cache: add onCancel callback	2018-12-24 15:41:17 -05:00
Adrien Béraud	7ebad7d3c5	storage: make non-movable	2018-12-24 15:28:03 -05:00
Adrien Béraud	14f6e6e5f3	proxy client: catch exceptions in restbed::Http::async	2018-12-22 23:45:37 -05:00
Adrien Béraud	9ce5b30931	securedht: use query to check for existing value	2018-12-22 11:42:45 -05:00
Adrien Béraud	92979fa96a	dhtchat: add logging support	2018-12-21 19:55:27 -05:00
Adrien Béraud	0c1e7d2763	dht: run node confirmation if bucket was empty	2018-12-19 14:07:55 -05:00
Adrien Béraud	14a744f6cd	dht: log cleanup	2018-12-19 14:07:55 -05:00
Adrien Béraud	3b26684a87	network engine: add default values for sendFindNodes	2018-12-19 14:07:55 -05:00
Sébastien Blin	0a764ee440	Merge pull request #361 from savoirfairelinux/schedule_restart proxy client: schedule listener restart	2018-12-18 16:07:39 -05:00
Adrien Béraud	7dda5a3d5a	proxy client: schedule listener restart	2018-12-18 10:10:14 -05:00