opendht: version 1.0

* add Prefix Hash Tree (PHT) for data indexation (Nicolas Reynaud, Simon Désaulniers) * add Queries (Simon Désaulniers) * add long-term data persistance (Simon Désaulniers) * add data pagination (Simon Désaulniers) * reduced bandwith usage
pht: fix compilation with pedantic
2026-11-08 12:28:42 +00:00 · 2016-09-01 18:02:23 -04:00 · 2016-09-01 18:02:23 -04:00 · 2016-09-01 18:02:23 -04:00 · 2016-09-01 18:02:23 -04:00 · 2016-09-01 18:02:23 -04:00
54 changed files with 4302 additions and 1093 deletions
@@ -45,3 +45,13 @@ Makefile.in
 # KDevelop
 .kdev4/
 *.kdev4
+
+# Python
+*.pyc
+
+# git backup files
+*.orig
+
+# vim swap files
+*.swp
+*.swo
@@ -1,44 +1,18 @@
-sudo: false
+sudo: required

 services:
  - docker

 language: cpp

-compiler:
-  - gcc
-  - clang
-
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    packages:
-    - gcc-6
-    - g++-6
-    - libgnutls28-dev
-    - cmake
-
 before_install:
-  - if [ $TRAVIS_OS_NAME == linux ] && [ $CXX == g++ ]; then export CXX=/usr/bin/g++-6 && export CC=/usr/bin/gcc-6; fi
-  - if [ $TRAVIS_OS_NAME == linux ]; then git clone https://github.com/msgpack/msgpack-c.git; fi
-  - if [ $TRAVIS_OS_NAME == linux ]; then cd msgpack-c && git checkout cpp-1.2.0 && mkdir build && mkdir -p $HOME/usr && cd build; fi
-  - if [ $TRAVIS_OS_NAME == linux ]; then cmake -DCMAKE_INSTALL_PREFIX=$HOME/usr -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF .. && make -j && make install && cd ../.. && pwd; fi
-  - if [ $TRAVIS_OS_NAME == osx ]; then brew update && brew install llvm36 && brew install msgpack && brew install gnutls && brew link --force llvm36; fi
-
-before_script:
-  - if [ $TRAVIS_OS_NAME == linux ]; then travis_retry ./autogen.sh && ./configure --prefix=$HOME/usr; fi
-  - if [ $TRAVIS_OS_NAME == osx ]; then travis_retry ./autogen.sh && ./configure; fi
+- docker pull aberaud/opendht-deps
+- docker pull aberaud/opendht-deps-llvm

 script:
-  - make -j
+- docker build -f docker/DockerfileTravis .
+- docker build -f docker/DockerfileTravisLlvm .

 notifications:
  email:
      - adrien.beraud@savoirfairelinux.com
-
-env:
-  - LANG="en_US.UTF-8"
-
-os:
-  - osx
@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 2.8.6)
 project (opendht)
-set (opendht_VERSION_MAJOR 0)
-set (opendht_VERSION_MINOR 6.1)
+set (opendht_VERSION_MAJOR 1)
+set (opendht_VERSION_MINOR 0.0)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -21,14 +21,15 @@ option (OPENDHT_LTO "Build with LTO" OFF)
 option (OPENDHT_PYTHON "Build Python bindings" OFF)
 option (OPENDHT_TOOLS "Build DHT tools" ON)

-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++11 -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++11 -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT")
 if (OPENDHT_LTO)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -fuse-linker-plugin")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto")
    if (CMAKE_COMPILER_IS_GNUCC)
-        SET (CMAKE_AR        "gcc-ar")
-        SET (CMAKE_NM        "gcc-nm")
-        SET (CMAKE_RANLIB    "gcc-ranlib")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fuse-linker-plugin")
+        set (CMAKE_AR        "gcc-ar")
+        set (CMAKE_NM        "gcc-nm")
+        set (CMAKE_RANLIB    "gcc-ranlib")
    endif ()
 endif ()

@@ -52,6 +53,7 @@ list (APPEND opendht_SOURCES
    src/network_engine.cpp
    src/securedht.cpp
    src/dhtrunner.cpp
+	src/indexation/pht.cpp
    src/argon2/argon2.c
    src/argon2/core.c
    src/argon2/blake2/blake2b.c
@@ -77,6 +79,7 @@ list (APPEND opendht_HEADERS
    include/opendht/securedht.h
    include/opendht/log.h
    include/opendht.h
+	include/opendht/indexation/pht.h
 )

 configure_file (
@@ -132,6 +135,7 @@ endif ()
 if (OPENDHT_TOOLS)
    add_subdirectory(tools)
 endif ()
+add_subdirectory(doc)

 if (OPENDHT_PYTHON)
    add_subdirectory(python)
@@ -10,9 +10,7 @@ if USE_CYTHON
 SUBDIRS += python
 endif

-if HAVE_DOXYGEN
 SUBDIRS += doc
-endif

 ACLOCAL_AMFLAGS = -I m4

@@ -1,16 +1,19 @@
-OpenDHT
-===
+<img src="https://raw.githubusercontent.com/savoirfairelinux/opendht/master/resources/opendht_logo_512.png" width="100" align="right">
+<br>
+<h1 style="margin-top:10px">
+    <a id="user-content-opendht-" class="anchor" href="/savoirfairelinux/opendht/blob/master/README.md#opendht-" aria-hidden="true"></a>OpenDHT 
+</h1>
+
 A lightweight C++11 Distributed Hash Table implementation originally based on https://github.com/jech/dht by Juliusz Chroboczek.

 * Light and fast C++11 Kademlia DHT library.
 * Distributed shared key->value data-store
- * Clean and powerfull distributed map API with storage of arbitrary binary values of up to 56 KB.
+ * Clean and powerful distributed map API with storage of arbitrary binary values of up to 56 KB.
 * Optional public key cryptography layer providing data signature and encryption (using GnuTLS).
 * IPv4 and IPv6 support.
 * Python binding.

-Documentation
-
+## Documentation
 See the wiki: <https://github.com/savoirfairelinux/opendht/wiki>

 #### How-to build and install
@@ -22,8 +25,7 @@ Build instructions : <https://github.com/savoirfairelinux/opendht/wiki/Build-the
 g++ main.cpp -std=c++11 -lopendht -lgnutls
 ```

-Examples
-
+## Examples
 ### C++ example
 The `tools` directory includes simple example programs :
 * `dhtnode`, a command line tool, mostly used for debuging, allowing to perform operations supported by the library (get, put etc.) with text values.
@@ -85,22 +87,27 @@ for r in results:
    print(r)
 ```

-Dependencies
-
+## Dependencies
 - msgpack-c 1.2+, used for data serialization.
 - GnuTLS 3.1+, used for cryptographic operations.
 - Nettle 2.4+, a GnuTLS dependency for crypto.
 - Build tested with GCC 4.8+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, OS X).
 - Build tested with Microsoft Visual Studio 2015

-Licence
-
+## Contact
+
+IRC: join us on Freenode at [`#opendht`](https://webchat.freenode.net/?channels=%23opendht).
+
+## Licence
 Copyright (C) 2014-2016 Savoir-faire Linux Inc.

-Licenced under the GNU General Public License version 3 (see COPYING for the full licence).
+OpenDHT is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+See COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html for the full GPLv3 licence.

 This project is independent from another project called OpenDHT (Sean Rhea. Ph.D. Thesis, 2005), now extinct.

-Donations
-
+## Donations
 We gratefully accept Bitcoin donations to support OpenDHT development at: `bitcoin:3EykSd1An888efq4Bq3KaV3hJ3JQ4FPnwm`.
@@ -1,10 +1,21 @@
-AC_INIT(opendht, 0.6.1)
+dnl define macros
+m4_define([opendht_major_version], 1)
+m4_define([opendht_minor_version], 0)
+m4_define([opendht_patch_version], 0)
+m4_define([opendht_version],
+		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
+
+AC_INIT(opendht, [opendht_version])
 AC_CONFIG_AUX_DIR(ac)
 AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CANONICAL_HOST

+AC_SUBST(OPENDHT_MAJOR_VERSION, opendht_major_version)
+AC_SUBST(OPENDHT_MINOR_VERSION, opendht_minor_version)
+AC_SUBST(OPENDHT_PATCH_VERSION, opendht_patch_version)
+
 AC_ARG_ENABLE([debug], AS_HELP_STRING([--enable-debug], [Build in debug mode, adds stricter warnings, disables optimization]))
 AS_IF([test "x$enable_debug" = "xyes"],
      [CXXFLAGS="${CXXFLAGS} -g -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -O0"],
@@ -17,7 +28,7 @@ dnl Check for Doxygen
 AC_ARG_ENABLE([doc], AS_HELP_STRING([--enable-doc], [Enable documentation generation (doxygen)]))
 AS_IF([test "x$enable_doc" = "xyes"], [
       AC_CHECK_PROGS([DOXYGEN], [doxygen])
-       AS_IF([test -z "$DOXYGEN"], [AC_MSG_WARN([Doxygen not found - continuing without Doxygen support])])       
+       AS_IF([test -z "$DOXYGEN"], [AC_MSG_WARN([Doxygen not found - continuing without Doxygen support])])
 ])
 AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"])

@@ -88,7 +99,7 @@ AM_COND_IF([ENABLE_TOOLS], [
  ])
 ])

-AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])])
+AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])

 dnl Configure setup.py if we build the python module
 AM_COND_IF([USE_CYTHON], [
@@ -0,0 +1,3 @@
+if (OPENDHT_TOOLS)
+	INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dhtnode.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1)
+endif ()
@@ -1,3 +1,6 @@
+dist_man1_MANS = dhtnode.1
+
+if HAVE_DOXYGEN
 doxyfile.stamp:
 	$(DOXYGEN) Doxyfile
 	echo stamp > doxyfile.stamp
@@ -7,3 +10,4 @@ CLEANFILES = doxyfile.stamp
 all-local: doxyfile.stamp
 clean-local:
 	rm -rf $(top_srcdir)/doc/man $(top_srcdir)/doc/html
+endif
@@ -0,0 +1,87 @@
+.TH DHTNODE 1 2016-07-29
+
+.SH NAME
+.B dhtnode
+- a simple OpenDHT command line node runner.
+
+.SH SYNOPSIS
+.B dhtnode [-h]
+
+.B dhtnode [-v [-l \fIlogfile\fP]] [-i] [-d] [-n \fInetwork_id\fP] [-p \fIlocal_port\fP] [-b \fIbootstrap_host\fP[:\fIport\fP]]
+
+.SH DESCRIPTION
+
+This program runs a simple OpenDHT node in an interactive way. If you rather
+want to run the node in daemon mode, option \fB'-d'\fP is provided. When running
+in the interactive shell, you benefit from the readline capabilities for writing
+your commands such as command history. Here are the available commands in the
+interactive shell:
+
+.EE
+    h, help    Print this help message.
+    q, quit    Quit the program.
+    log        Start/stop printing DHT logs.
+
+    Node information:
+    ll         Print basic information and stats about the current node.
+    ls         Print basic information about current searches.
+    ld         Print basic information about currenty stored values on this node.
+    lr         Print the full current routing table of this node
+
+    Operations on the DHT:
+    b ip:port             Ping potential node at given IP address/port.
+    g [key]               Get values at [key].
+    l [key]               Listen for value changes at [key].
+    p [key] [str]         Put string value at [key].
+    s [key] [str]         Put string value at [key], signed with our generated
+                          private key.
+    e [key] [dest] [str]  Put string value at [key], encrypted for [dest] with
+                          its public key (if found).
+
+.SH OPTIONS
+
+.TP
+\fB-h\fP
+Prints some help.
+
+.TP
+\fB-v\fP
+Enable the verbose mode (log to stdout by default)
+
+.TP
+\fB-l\fP \fIlog_file\fP
+Write log to file instead of stdout
+
+.TP
+\fB-i\fP
+Generate cryptographic identity for the node.
+
+.TP
+\fB-d\fP
+Run the program in daemon mode (will fork in the background).
+
+.TP
+\fB-n\fP \fInetwork_id\fP
+Specify the network id. This let you connect to distinct networks and prevents
+the merge of two different networks (available since OpenDHT v0.6.1).
+
+.TP
+\fB-p\fP \fIlocal_port\fP
+Use port \fIlocal_port\fP for the program to bind to.
+
+.TP
+\fB-b\fP \fIbootstrap_host\fP[:\fIport\fP]
+The program needs to be given a node to connect to the network. You use this
+option to provide the ip address of that node.
+
+.SH AUTHORS
+.TP
+Program written by
+.IP \(bu
+.\}
+Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+.TP
+Man page written by
+.IP \(bu
+.\}
+Simon Désaulniers <sim.desaulniers@gmail.com>
@@ -0,0 +1,6 @@
+FROM aberaud/opendht-deps
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN git clone https://github.com/savoirfairelinux/opendht.git \
+	&& cd opendht && mkdir build && cd build \
+	&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On && make -j8 && make install \
+	&& cd ../.. && rm -rf opendht
@@ -0,0 +1,9 @@
+FROM ubuntu:16.04
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN apt-get update && apt-get install -y build-essential cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls-dev cython3 python3-dev python3-setuptools && apt-get clean
+RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.0.0/msgpack-2.0.0.tar.gz \
+	&& tar -xzf msgpack-2.0.0.tar.gz \
+	&& cd msgpack-2.0.0 && mkdir build && cd build \
+	&& cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
+	&& make -j8 && make install \
+	&& cd ../.. && rm -rf msgpack-2.0.0 msgpack-2.0.0.tar.gz
@@ -0,0 +1,13 @@
+FROM ubuntu:16.04
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN apt-get update \
+	&& apt-get install -y llvm llvm-dev clang make cmake git wget libncurses5-dev libreadline-dev nettle-dev libgnutls-dev cython3 python3-dev python3-setuptools \
+	&& apt-get remove -y gcc g++ && apt-get autoremove -y && apt-get clean
+ENV CC cc
+ENV CXX c++
+RUN wget https://github.com/msgpack/msgpack-c/releases/download/cpp-2.0.0/msgpack-2.0.0.tar.gz \
+	&& tar -xzf msgpack-2.0.0.tar.gz \
+	&& cd msgpack-2.0.0 && mkdir build && cd build \
+	&& cmake -DMSGPACK_CXX11=ON -DMSGPACK_BUILD_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
+	&& make -j8 && make install \
+	&& cd ../.. && rm -rf msgpack-2.0.0 msgpack-2.0.0.tar.gz
@@ -0,0 +1,6 @@
+FROM aberaud/opendht-deps-llvm
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+RUN git clone https://github.com/savoirfairelinux/opendht.git \
+	&& cd opendht && mkdir build && cd build \
+	&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On && make -j8 && make install \
+	&& cd ../.. && rm -rf opendht
@@ -0,0 +1,6 @@
+FROM aberaud/opendht-deps
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+COPY . /root/opendht
+RUN cd /root/opendht && mkdir build && cd build \
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On .. && make -j8 && make install \
+	&& cd ../.. && rm -rf opendht
@@ -0,0 +1,6 @@
+FROM aberaud/opendht-deps-llvm
+MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+COPY . /root/opendht
+RUN cd /root/opendht && mkdir build && cd build \
+	&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On .. && make -j8 && make install \
+	&& cd ../.. && rm -rf opendht
@@ -22,3 +22,4 @@
 #include "opendht/dhtrunner.h"
 #include "opendht/log.h"
 #include "opendht/default_types.h"
+#include "opendht/indexation/pht.h"
@@ -30,25 +30,47 @@ namespace dht {

 struct Node;

+/**
+ * Current status of a DHT node.
+ */
 enum class NodeStatus {
    Disconnected, // 0 nodes
    Connecting,   // 1+ nodes
    Connected     // 1+ good nodes
 };

+/**
+ * Dht configuration.
+ */
 struct Config {
+    /** DHT node ID */
    InfoHash node_id;
+
+    /** 
+     * DHT network ID. A node will only talk with other nodes having
+     * the same network ID.
+     * Network ID 0 (default) represents the main public network.
+     */
+    NetId network;
+
+    /** For testing purposes only, enables bootstrap mode */
    bool is_bootstrap;
 };

+/**
+ * SecureDht configuration.
+ */
 struct SecureDhtConfig
 {
    Config node_config;
    crypto::Identity id;
 };

+static constexpr size_t DEFAULT_STORAGE_LIMIT {1024 * 1024 * 64};
+
 using ValuesExport = std::pair<InfoHash, Blob>;

+using QueryCallback = std::function<bool(const std::vector<std::shared_ptr<FieldValueIndex>>& fields)>;
 using GetCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values)>;
 using GetCallbackSimple = std::function<bool(std::shared_ptr<Value> value)>;
 using ShutdownCallback = std::function<void()>;
@@ -57,20 +79,19 @@ using CertificateStoreQuery = std::function<std::vector<std::shared_ptr<crypto::

 typedef bool (*GetCallbackRaw)(std::shared_ptr<Value>, void *user_data);

-static constexpr size_t DEFAULT_STORAGE_LIMIT {1024 * 1024 * 64};
-
 GetCallbackSimple bindGetCb(GetCallbackRaw raw_cb, void* user_data);
 GetCallback bindGetCb(GetCallbackSimple cb);

 using DoneCallback = std::function<void(bool success, const std::vector<std::shared_ptr<Node>>& nodes)>;
 typedef void (*DoneCallbackRaw)(bool, std::vector<std::shared_ptr<Node>>*, void *user_data);
 typedef void (*ShutdownCallbackRaw)(void *user_data);
+typedef void (*DoneCallbackSimpleRaw)(bool, void *user_data);

 using DoneCallbackSimple = std::function<void(bool success)>;

 ShutdownCallback bindShutdownCb(ShutdownCallbackRaw shutdown_cb_raw, void* user_data);
 DoneCallback bindDoneCb(DoneCallbackSimple donecb);
 DoneCallback bindDoneCb(DoneCallbackRaw raw_cb, void* user_data);
-
+DoneCallbackSimple bindDoneCbSimple(DoneCallbackSimpleRaw raw_cb, void* user_data);

 }
@@ -21,6 +21,8 @@

 #include "value.h"

+MSGPACK_ADD_ENUM(dht::Value::Field)
+
 namespace dht {
 enum class ImStatus : uint8_t {
    NONE = 0,
@@ -29,7 +31,7 @@ enum class ImStatus : uint8_t {
    READ
 };
 }
-MSGPACK_ADD_ENUM(dht::ImStatus);
+MSGPACK_ADD_ENUM(dht::ImStatus)

 namespace dht {

@@ -55,7 +57,7 @@ public:

    std::string service;
    Blob data;
-    MSGPACK_DEFINE(service, data);
+    MSGPACK_DEFINE(service, data)
 };

 template <typename T>
@@ -126,7 +128,7 @@ public:
    long date {0};
    ImStatus status {ImStatus::NONE};

-    MSGPACK_DEFINE_MAP(id, msg, date, status);
+    MSGPACK_DEFINE_MAP(id, msg, date, status)
 };

 class TrustRequest : public EncryptedValue<TrustRequest>
@@ -147,7 +149,7 @@ public:

    std::string service;
    Blob payload;
-    MSGPACK_DEFINE(service, payload);
+    MSGPACK_DEFINE(service, payload)
 };

 class IceCandidates : public EncryptedValue<IceCandidates>
@@ -175,7 +177,7 @@ public:
        pk.pack_bin_body((const char*)ice_data.data(), ice_data.size());
 #else
        // hack for backward compatibility with old opendht compiled with msgpack 1.0
-        // remove when enough people have moved to new versions 
+        // remove when enough people have moved to new versions
        pk.pack_array(ice_data.size());
        for (uint8_t b : ice_data)
            pk.pack(b);
@@ -133,18 +133,32 @@ public:
     * @param cb a function called when new values are found on the network.
     *           It should return false to stop the operation.
     * @param donecb a function called when the operation is complete.
-                        cb and donecb won't be called again afterward.
+                     cb and donecb won't be called again afterward.
     * @param f a filter function used to prefilter values.
     */
-    virtual void get(const InfoHash& key, GetCallback cb, DoneCallback donecb={}, Value::Filter&& f={});
-    virtual void get(const InfoHash& key, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f={}) {
-        get(key, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& key, GetCallback cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {});
+    virtual void get(const InfoHash& key, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f={}, Where&& w = {}) {
+        get(key, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}) {
-        get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) {
+        get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}) {
-        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) {
+        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
+    }
+    /**
+     * Similar to Dht::get, but sends a Query to filter data remotely.
+     * @param key the key for which to query data for.
+     * @param cb a function called when new values are found on the network.
+     *           It should return false to stop the operation.
+     * @param donecb a function called when the operation is complete.
+                     cb and donecb won't be called again afterward.
+     * @param q a query used to filter values on the remotes before they send a
+     *          response.
+     */
+    virtual void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {});
+    virtual void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) {
+        query(key, cb, bindDoneCb(done_cb), std::forward<Query>(q));
    }

    /**
@@ -158,14 +172,10 @@ public:
    std::shared_ptr<Value> getLocalById(const InfoHash& key, Value::Id vid) const;

    /**
-     * Announce a value on all available protocols (IPv4, IPv6), and
-     * automatically re-announce when it's about to expire.
+     * Announce a value on all available protocols (IPv4, IPv6).
+     *
     * The operation will start as soon as the node is connected to the network.
     * The done callback will be called once, when the first announce succeeds, or fails.
-     *
-     * A "put" operation will never end by itself because the value will need to be
-     * reannounced on a regular basis.
-     * User can call #cancelPut(InfoHash, Value::Id) to cancel a put operation.
     */
    void put(const InfoHash& key,
            std::shared_ptr<Value>,
@@ -221,9 +231,9 @@ public:
     *
     * @return a token to cancel the listener later.
     */
-    virtual size_t listen(const InfoHash&, GetCallback, Value::Filter&&={});
-    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}) {
-        return listen(key, bindGetCb(cb), std::forward<Value::Filter>(f));
+    virtual size_t listen(const InfoHash&, GetCallback, Value::Filter&&={}, Where&& w = {});
+    virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) {
+        return listen(key, bindGetCb(cb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }

    virtual bool cancelListen(const InfoHash&, size_t token);
@@ -244,7 +254,7 @@ public:
    std::vector<ValuesExport> exportValues() const;
    void importValues(const std::vector<ValuesExport>&);

-    int getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return,
+    unsigned getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return,
            unsigned *incoming_return) const;
    std::string getStorageLog() const;
    std::string getRoutingTablesLog(sa_family_t) const;
@@ -282,8 +292,8 @@ private:
       the target 8 turn out to be dead. */
    static constexpr unsigned SEARCH_NODES {14};

-    /* Concurrent requests during a search */
-    static constexpr unsigned SEARCH_REQUESTS {4};
+    /* Concurrent search nodes requested count */
+    static constexpr unsigned MAX_REQUESTED_SEARCH_NODES {4};

    /* Number of listening nodes */
    static constexpr unsigned LISTEN_NODES {4};
@@ -317,6 +327,9 @@ private:
    struct Get {
        time_point start;
        Value::Filter filter;
+        std::shared_ptr<Query> query;
+        std::set<std::shared_ptr<Query>> pagination_queries;
+        QueryCallback query_cb;
        GetCallback get_cb;
        DoneCallback done_cb;
    };
@@ -335,6 +348,7 @@ private:
     * A single "listen" operation data
     */
    struct LocalListener {
+        std::shared_ptr<Query> query;
        Value::Filter filter;
        GetCallback get_cb;
    };
@@ -359,8 +373,9 @@ private:
    struct Listener {
        size_t rid {};
        time_point time {};
+        Query query {};

-        /*constexpr*/ Listener(size_t rid, time_point t) : rid(rid), time(t) {}
+        /*constexpr*/ Listener(size_t rid, time_point t, Query&& q) : rid(rid), time(t), query(q) {}

        void refresh(size_t tid, time_point t) {
            rid = tid;
@@ -392,7 +407,7 @@ private:
    RoutingTable buckets {};
    RoutingTable buckets6 {};

-    std::vector<Storage> store;
+    std::vector<std::unique_ptr<Storage>> store;
    size_t total_values {0};
    size_t total_store_size {0};
    size_t max_store_size {DEFAULT_STORAGE_LIMIT};
@@ -412,6 +427,8 @@ private:
    time_point mybucket_grow_time {time_point::min()}, mybucket6_grow_time {time_point::min()};

    NetworkEngine network_engine;
+    unsigned pending_pings4 {0};
+    unsigned pending_pings6 {0};

    using ReportedAddr = std::pair<unsigned, Address>;
    std::vector<ReportedAddr> reported_addr;
@@ -427,7 +444,7 @@ private:
    decltype(Dht::store)::iterator findStorage(const InfoHash& id);
    decltype(Dht::store)::const_iterator findStorage(const InfoHash& id) const;

-    void storageAddListener(const InfoHash& id, const std::shared_ptr<Node>& node, size_t tid);
+    void storageAddListener(const InfoHash& id, const std::shared_ptr<Node>& node, size_t tid, Query&& = {});
    bool storageStore(const InfoHash& id, const std::shared_ptr<Value>& value, time_point created);
    void expireStorage();
    void storageChanged(Storage& st, ValueStorage&);
@@ -472,21 +489,43 @@ private:
    // Searches

    /**
-     * Low-level method that will perform a search on the DHT for the
-     * specified infohash (id), using the specified IP version (IPv4 or IPv6).
-     * The values can be filtered by an arbitrary provided filter.
+     * Low-level method that will perform a search on the DHT for the specified
+     * infohash (id), using the specified IP version (IPv4 or IPv6).
     */
-    std::shared_ptr<Search> search(const InfoHash& id, sa_family_t af, GetCallback = nullptr, DoneCallback = nullptr, Value::Filter = Value::AllFilter());
+    std::shared_ptr<Search> search(const InfoHash& id, sa_family_t af, GetCallback = {}, QueryCallback = {}, DoneCallback = {}, Value::Filter = {}, Query q = {});
+
    void announce(const InfoHash& id, sa_family_t af, std::shared_ptr<Value> value, DoneCallback callback, time_point created=time_point::max(), bool permanent = false);
-    size_t listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter f = Value::AllFilter());
+    size_t listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter f = Value::AllFilter(), const std::shared_ptr<Query>& q = {});
+
+    /**
+     * Refill the search with good nodes if possible.
+     *
+     * @param sr  The search to refill.
+     *
+     * @return the number inserted nodes.
+     */
+    unsigned refill(Search& sr);

-    void bootstrapSearch(Search& sr);
    Search *findSearch(unsigned short tid, sa_family_t af);
    void expireSearches();

    void confirmNodes();
    void expire();

+    void searchNodeGetDone(const Request& status,
+            NetworkEngine::RequestAnswer&& answer,
+            std::weak_ptr<Search> ws,
+            std::shared_ptr<Query> query);
+    void searchNodeGetExpired(const Request& status, bool over, std::weak_ptr<Search> ws, std::shared_ptr<Query> query);
+    /**
+     * This method recovers sends individual request for values per id.
+     *
+     * @param ws     A weak pointer to the Search.
+     * @param query  The initial query passed through the API.
+     * @param n      The node to which send the requests.
+     */
+    void paginate(std::weak_ptr<Search> ws, std::shared_ptr<Query> query, SearchNode* n);
+
    /**
     * If update is true, this method will also send message to synced but non-updated search nodes.
     */
@@ -508,14 +547,27 @@ private:
    NetworkEngine::RequestAnswer onFindNode(std::shared_ptr<Node> node, InfoHash& hash, want_t want);
    void onFindNodeDone(const Request& status, NetworkEngine::RequestAnswer& a, std::shared_ptr<Search> sr);
    /* when we receive a "get values" request */
-    NetworkEngine::RequestAnswer onGetValues(std::shared_ptr<Node> node, InfoHash& hash, want_t want);
-    void onGetValuesDone(const Request& status, NetworkEngine::RequestAnswer& a, std::shared_ptr<Search> sr);
+    NetworkEngine::RequestAnswer onGetValues(std::shared_ptr<Node> node, InfoHash& hash, want_t want, const Query& q);
+    void onGetValuesDone(const Request& status,
+            NetworkEngine::RequestAnswer& a,
+            std::shared_ptr<Search>& sr,
+            const std::shared_ptr<Query>& orig_query);
    /* when we receive a listen request */
-    NetworkEngine::RequestAnswer onListen(std::shared_ptr<Node> node, InfoHash& hash, Blob& token, size_t rid);
-    void onListenDone(const Request& status, NetworkEngine::RequestAnswer& a, std::shared_ptr<Search>& sr);
+    NetworkEngine::RequestAnswer onListen(std::shared_ptr<Node> node,
+            InfoHash& hash,
+            Blob& token,
+            size_t rid,
+            Query&& query);
+    void onListenDone(const Request& status,
+            NetworkEngine::RequestAnswer& a,
+            std::shared_ptr<Search>& sr,
+            const std::shared_ptr<Query>& orig_query);
    /* when we receive an announce request */
    NetworkEngine::RequestAnswer onAnnounce(std::shared_ptr<Node> node,
-            InfoHash& hash, Blob& token, std::vector<std::shared_ptr<Value>> v, time_point created);
+            InfoHash& hash,
+            Blob& token,
+            std::vector<std::shared_ptr<Value>> v,
+            time_point created);
    void onAnnounceDone(const Request& status, NetworkEngine::RequestAnswer& a, std::shared_ptr<Search>& sr);
 };

@@ -54,20 +54,20 @@ public:
    DhtRunner();
    virtual ~DhtRunner();

-    void get(InfoHash id, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter f = Value::AllFilter()) {
-        get(id, bindGetCb(cb), donecb, f);
+    void get(InfoHash id, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+        get(id, bindGetCb(cb), donecb, f, w);
    }

-    void get(InfoHash id, GetCallbackSimple cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter()) {
-        get(id, bindGetCb(cb), donecb, f);
+    void get(InfoHash id, GetCallbackSimple cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+        get(id, bindGetCb(cb), donecb, f, w);
    }

-    void get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f={});
+    void get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f={}, Where w = {});

-    void get(InfoHash id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter()) {
-        get(id, cb, bindDoneCb(donecb), f);
+    void get(InfoHash id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter f = Value::AllFilter(), Where w = {}) {
+        get(id, cb, bindDoneCb(donecb), f, w);
    }
-    void get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb={}, Value::Filter f = Value::AllFilter());
+    void get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb={}, Value::Filter f = Value::AllFilter(), Where w = {});

    template <class T>
    void get(InfoHash hash, std::function<bool(std::vector<T>&&)> cb, DoneCallbackSimple dcb={})
@@ -96,7 +96,7 @@ public:
        getFilterSet<T>());
    }

-    std::future<std::vector<std::shared_ptr<dht::Value>>> get(InfoHash key, Value::Filter f = Value::AllFilter()) {
+    std::future<std::vector<std::shared_ptr<dht::Value>>> get(InfoHash key, Value::Filter f = Value::AllFilter(), Where w = {}) {
        auto p = std::make_shared<std::promise<std::vector<std::shared_ptr< dht::Value >>>>();
        auto values = std::make_shared<std::vector<std::shared_ptr< dht::Value >>>();
        get(key, [=](const std::vector<std::shared_ptr<dht::Value>>& vlist) {
@@ -105,7 +105,7 @@ public:
        }, [=](bool) {
            p->set_value(std::move(*values));
        },
-        f);
+        f, w);
        return p->get_future();
    }

@@ -122,10 +122,15 @@ public:
        return p->get_future();
    }

-    std::future<size_t> listen(InfoHash key, GetCallback vcb, Value::Filter f = Value::AllFilter());
-    std::future<size_t> listen(const std::string& key, GetCallback vcb, Value::Filter f = Value::AllFilter());
-    std::future<size_t> listen(InfoHash key, GetCallbackSimple cb, Value::Filter f = Value::AllFilter()) {
-        return listen(key, bindGetCb(cb), f);
+    void query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb = {}, Query q = {});
+    void query(const InfoHash& hash, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query q = {}) {
+        query(hash, cb, bindDoneCb(done_cb), q);
+    }
+
+    std::future<size_t> listen(InfoHash key, GetCallback vcb, Value::Filter f = Value::AllFilter(), Where w = {});
+    std::future<size_t> listen(const std::string& key, GetCallback vcb, Value::Filter f = Value::AllFilter(), Where w = {});
+    std::future<size_t> listen(InfoHash key, GetCallbackSimple cb, Value::Filter f = Value::AllFilter(), Where w = {}) {
+        return listen(key, bindGetCb(cb), f, w);
    }

    template <class T>
@@ -137,7 +142,7 @@ public:
        getFilterSet<T>());
    }
    template <typename T>
-    std::future<size_t> listen(InfoHash hash, std::function<bool(T&&)> cb, Value::Filter f = Value::AllFilter())
+    std::future<size_t> listen(InfoHash hash, std::function<bool(T&&)> cb, Value::Filter f = Value::AllFilter(), Where w = {})
    {
        return listen(hash, [=](const std::vector<std::shared_ptr<Value>>& vals) {
            for (const auto& v : vals) {
@@ -150,22 +155,22 @@ public:
            }
            return true;
        },
-        getFilterSet<T>(f));
+        getFilterSet<T>(f), w);
    }

    void cancelListen(InfoHash h, size_t token);
    void cancelListen(InfoHash h, std::shared_future<size_t> token);

-    void put(InfoHash hash, std::shared_ptr<Value> value, DoneCallback cb={}, bool permanent = false);
-    void put(InfoHash hash, std::shared_ptr<Value> value, DoneCallbackSimple cb, bool permanent = false) {
-        put(hash, value, bindDoneCb(cb), permanent);
+    void put(InfoHash hash, std::shared_ptr<Value> value, DoneCallback cb={}, time_point created=time_point::max(), bool permanent = false);
+    void put(InfoHash hash, std::shared_ptr<Value> value, DoneCallbackSimple cb, time_point created=time_point::max(), bool permanent = false) {
+        put(hash, value, bindDoneCb(cb), created, permanent);
    }

-    void put(InfoHash hash, Value&& value, DoneCallback cb={}, bool permanent = false);
-    void put(InfoHash hash, Value&& value, DoneCallbackSimple cb, bool permanent = false) {
-        put(hash, std::forward<Value>(value), bindDoneCb(cb), permanent);
+    void put(InfoHash hash, Value&& value, DoneCallback cb={}, time_point created=time_point::max(), bool permanent = false);
+    void put(InfoHash hash, Value&& value, DoneCallbackSimple cb, time_point created=time_point::max(), bool permanent = false) {
+        put(hash, std::forward<Value>(value), bindDoneCb(cb), created, permanent);
    }
-    void put(const std::string& key, Value&& value, DoneCallbackSimple cb={}, bool permanent = false);
+    void put(const std::string& key, Value&& value, DoneCallbackSimple cb={}, time_point created=time_point::max(), bool permanent = false);

    void cancelPut(const InfoHash& h, const Value::Id& id);

@@ -268,12 +273,13 @@ public:
     * @param threaded: If false, ::loop() must be called periodically. Otherwise a thread is launched.
     * @param cb: Optional callback to receive general state information.
     */
-    void run(in_port_t port, const crypto::Identity identity, bool threaded = false, bool is_bootstrap = false) {
+    void run(in_port_t port, const crypto::Identity identity, bool threaded = false, NetId network = 0) {
        run(port, {
            /*.dht_config = */{
                /*.node_config = */{
                    /*.node_id = */{},
-                    /*.is_bootstrap = */is_bootstrap
+                    /*.network = */network,
+                    /*.is_bootstrap = */false
                },
                /*.id = */identity
            },
@@ -0,0 +1,514 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <map>
+#include <functional>
+#include <stdexcept>
+#include <bitset>
+#include <iostream>
+#include <sstream>
+
+#include "../value.h"
+#include "../dhtrunner.h"
+#include "../infohash.h"
+
+namespace dht {
+namespace indexation {
+
+/*!
+ * @class   Prefix
+ * @brief   A blob structure which prefixes a Key in the PHT.
+ * @details
+ * Since the PHT structure is a "trie", every node in this structure have a
+ * label which is defined by the path from the root of the trie to the node. If
+ * the node in question is a leaf, *the label is a prefix of all the keys
+ * contained in the leaf*.
+ */
+struct Prefix {
+    Prefix() {}
+    Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) { }
+    Prefix(const Blob& d, const Blob& f={}) : size_(d.size()*8), flags_(f), content_(d) { }
+
+    Prefix(const Prefix& p, size_t first) :
+        size_(std::min(first, p.content_.size()*8)),
+        content_(Blob(p.content_.begin(), p.content_.begin()+size_/8))
+    {
+
+        auto rem = size_ % 8;
+        if ( not p.flags_.empty() ) {
+            flags_ = Blob(p.flags_.begin(), p.flags_.begin()+size_/8);
+            if (rem)
+                flags_.push_back(p.flags_[size_/8] & (0xFF << (8 - rem)));
+        }
+
+        if (rem)
+            content_.push_back(p.content_[size_/8] & (0xFF << (8 - rem)));
+    }
+
+    /**
+     * Get a sub prefix of the Prefix
+     *
+     * @param len lenght of the prefix to get, could be negative
+     *            if len is negativ then you will get the prefix
+     *            of size of the previous prefix minus len
+     *
+     * @return Sub-prefix of size len or if len is negative sub-prefix of size
+     *         of prefix minus len
+     *
+     * @throw out_of_range if len is larger than size of the content
+     */
+    Prefix getPrefix(ssize_t len) const {
+        if ((size_t)std::abs(len) >= content_.size() * 8)
+            throw std::out_of_range("len larger than prefix size.");
+        if (len < 0)
+            len += size_;
+
+        return Prefix(*this, len);
+    }
+
+    /**
+     * Flags are considered as active if flag is empty or if the flag
+     * at pos 'pos' is active
+ee     *
+     * @see isActiveBit in private function
+     */
+    bool isFlagActive(size_t pos) const {
+        return flags_.empty() or isActiveBit(flags_, pos);
+    }
+
+    /**
+     * @see isActiveBit in private function
+     */
+    bool isContentBitActive(size_t pos) const {
+        return isActiveBit(content_, pos);
+    }
+
+    Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
+
+    /**
+     * This methods gets the prefix of its sibling in the PHT structure.
+     *
+     * @return The prefix of this sibling.
+     */
+    Prefix getSibling() const {
+        Prefix copy = *this;
+        if ( size_ )
+            copy.swapContentBit(size_ - 1);
+
+        return copy;
+    }
+
+    InfoHash hash() const {
+        Blob copy(content_);
+        copy.push_back(size_);
+        return InfoHash::get(copy);
+    }
+
+    /**
+     * This method count total of bit in common between 2 prefix
+     *
+     * @param p1 first prefix to compared
+     * @param p2 second prefix to compared
+     * @return Lenght of the larger common prefix between both
+     */
+    static inline unsigned commonBits(const Prefix& p1, const Prefix& p2) {
+        unsigned i, j;
+        uint8_t x;
+        auto longest_prefix_size = std::min(p1.size_, p2.size_);
+
+        for (i = 0; i < longest_prefix_size; i++) {
+            if (p1.content_.data()[i] != p2.content_.data()[i]
+                or not p1.isFlagActive(i)
+                or not p2.isFlagActive(i) ) {
+
+                    break;
+            }
+        }
+
+        if (i == longest_prefix_size)
+            return 8*longest_prefix_size;
+
+        x = p1.content_.data()[i] ^ p2.content_.data()[i];
+
+        j = 0;
+        while ((x & 0x80) == 0) {
+            x <<= 1;
+            j++;
+        }
+
+        return 8 * i + j;
+    }
+
+    /**
+     * @see doc of swap private function
+     */
+    void swapContentBit(size_t bit) {
+        swapBit(content_, bit);
+    }
+
+    /**
+     * @see doc of swap private function
+     */
+    void swapFlagBit(size_t bit) {
+        swapBit(flags_, bit);
+    }
+
+    /**
+     * @see doc of addPadding private function
+     */
+    void addPaddingContent(size_t size) {
+        content_ = addPadding(content_, size);
+    }
+
+    void updateFlags() {
+        /* Fill first known bit */
+        auto csize = size_ - flags_.size() * 8;
+        while(csize >= 8) {
+            flags_.push_back(0xFF);
+            csize -= 8;
+        }
+
+        /* if needed fill remaining bit */
+        if ( csize )
+            flags_.push_back(0xFF << (8 - csize));
+
+        /* Complet vector space missing */
+        for ( auto i = flags_.size(); i < content_.size(); i++ )
+            flags_.push_back(0xFF);
+    }
+
+    std::string toString() const;
+
+    size_t size_ {0};
+
+    /* Will contain flags according to content_.
+       If flags_[i] == 0, then content_[i] is unknown
+       else if flags_[i] == 1, then content_[i] is known */
+    Blob flags_ {};
+    Blob content_ {};
+
+private:
+
+    /**
+     * Add a padding to the input blob
+     *
+     * @param toP  : Prefix where to add a padding
+     * @param size : Final size of the prefix with padding
+     *
+     * @return Copy of the input Blob but with a padding
+     */
+    Blob addPadding(Blob toP, size_t size) {
+        Blob copy = toP;
+        for ( auto i = copy.size(); i < size; i++ )
+            copy.push_back(0);
+
+        swapBit(copy, size_ + 1);
+        return copy;
+    }
+
+    /**
+     * Check if the bit a pos 'pos' is active, i.e. equal to 1
+     *
+     * @param b   : Blob to check
+     * @param pos : Position to check
+     *
+     * @return true if the bit is equal to 1, false otherwise
+     *
+     * @throw out_of_range if bit is superior to blob size * 8
+     */
+    bool isActiveBit(const Blob &b, size_t pos) const {
+        if ( pos >= content_.size() * 8 )
+            throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix");
+
+        return ((b[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
+    }
+
+    /**
+     * Swap bit at position bit [from 0 to 1 and vice-versa]
+     *
+     * @param b   : Blob to swap
+     * @param bit : Bit to swap on b
+     *
+     * @return the input prefix with the bit at pos 'bit' swapped
+     *
+     * @throw out_of_range if bit is superior to blob size * 8
+     */
+    void swapBit(Blob &b, size_t bit) {
+        if ( bit >= b.size() * 8 )
+            throw std::out_of_range("bit larger than prefix size.");
+
+        size_t offset_bit = (8 - bit) % 8;
+        b[bit / 8] ^= (1 << offset_bit);
+    }
+};
+
+using Value = std::pair<InfoHash, dht::Value::Id>;
+struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
+    static const ValueType TYPE;
+
+    virtual void unpackValue(const dht::Value& v) {
+        Serializable<IndexEntry>::unpackValue(v);
+        name = v.user_type;
+    }
+
+    virtual dht::Value packValue() const {
+        auto pack = Serializable<IndexEntry>::packValue();
+        pack.user_type = name;
+        return pack;
+    }
+
+    Blob prefix;
+    Value value;
+    std::string name;
+    MSGPACK_DEFINE_MAP(prefix, value)
+};
+
+class Pht {
+    static constexpr const char* INVALID_KEY = "Key does not match the PHT key spec.";
+
+    /* Prefixes the user_type for all dht values put on the DHT */
+    static constexpr const char* INDEX_PREFIX = "index.pht.";
+
+public:
+
+    /* This is the maximum number of entries per node. This parameter is
+     * critical and influences the traffic a lot during a lookup operation.
+     */
+    static constexpr const size_t MAX_NODE_ENTRY_COUNT {16};
+
+    /* A key for a an index entry */
+    using Key = std::map<std::string, Blob>;
+
+    /* Specifications of the keys. It defines the number, the length and the
+     * serialization order of fields. */
+    using KeySpec = std::map<std::string, size_t>;
+    using LookupCallback = std::function<void(std::vector<std::shared_ptr<Value>>& values, Prefix p)>;
+
+    typedef void (*LookupCallbackRaw)(std::vector<std::shared_ptr<Value>>* values, Prefix* p, void *user_data);
+    static LookupCallback
+    bindLookupCb(LookupCallbackRaw raw_cb, void* user_data) {
+        if (not raw_cb) return {};
+        return [=](std::vector<std::shared_ptr<Value>>& values, Prefix p) {
+            raw_cb((std::vector<std::shared_ptr<Value>>*) &values, (Prefix*) &p, user_data);
+        };
+    }
+    using LookupCallbackSimple = std::function<void(std::vector<std::shared_ptr<Value>>& values)>;
+    typedef void (*LookupCallbackSimpleRaw)(std::vector<std::shared_ptr<Value>>* values, void *user_data);
+    static LookupCallbackSimple
+    bindLookupCbSimple(LookupCallbackSimpleRaw raw_cb, void* user_data) {
+        if (not raw_cb) return {};
+        return [=](std::vector<std::shared_ptr<Value>>& values) {
+            raw_cb((std::vector<std::shared_ptr<Value>>*) &values, user_data);
+        };
+    }
+
+    Pht(std::string name, KeySpec k_spec, std::shared_ptr<DhtRunner> dht)
+        : name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht) {}
+
+    virtual ~Pht () { }
+
+    /**
+     * Lookup a key for a value.
+     */
+    void lookup(Key k, LookupCallback cb = {}, DoneCallbackSimple done_cb = {}, bool exact_match = true);
+    void lookup(Key k, LookupCallbackSimple cb = {}, DoneCallbackSimple done_cb = {}, bool exact_match = true)
+    {
+        lookup(k, [=](std::vector<std::shared_ptr<Value>>& values, Prefix) { cb(values); }, done_cb, exact_match);
+    }
+
+    /**
+     * Wrapper function which call the private one.
+     *
+     * @param k : Key to insert [i.e map of string, blob]
+     * @param v : Value to insert
+     * @param done_cb : Callbakc which going to be call when all the insert is done
+     */
+    void insert(Key k, Value v, DoneCallbackSimple done_cb = {}) {
+        Prefix p = linearize(k);
+
+        auto lo = std::make_shared<int>(0);
+        auto hi = std::make_shared<int>(p.size_);
+
+        IndexEntry entry;
+        entry.value = v;
+        entry.prefix = p.content_;
+        entry.name = name_;
+
+        Pht::insert(p, entry, lo, hi, clock::now(), true, done_cb);
+    }
+
+private:
+
+    /**
+     * Insert function which really insert onto the pht
+     *
+     * @param kp          : Prefix to insert (linearize the the key)
+     * @param entry       : Entry created from the value
+     * @param lo          : Lowest point to start in the prefix
+     * @param hi          : Highest point to end in the prefix
+     * @param time_p      : Timepoint to use for the insertion into the dht (must be < now)
+     * @param check_split : If this flag is true then the algoritm will not use the merge algorithm
+     * @param done_cb     : Callback to call when the insert is done
+     */
+
+    void insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shared_ptr<int> hi, time_point time_p,
+                bool check_split, DoneCallbackSimple done_cb = {});
+
+    class Cache {
+    public:
+        /**
+         * Insert all needed node into the tree according to a prefix
+         * @param p : Prefix that we need to insert
+         */
+        void insert(const Prefix& p);
+
+        /**
+         * Lookup into the tree to return the maximum prefix length in the cache tree
+         *
+         * @param p : Prefix that we are looking for
+         *
+         * @return  : The size of the longest prefix known in the cache between 0 and p.size_
+         */
+
+        int lookup(const Prefix& p);
+
+    private:
+        static constexpr const size_t MAX_ELEMENT {1024};
+        static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
+
+        struct Node {
+            time_point last_reply;           /* Made the assocation between leaves and leaves multimap */
+            std::shared_ptr<Node> parent;    /* Share_ptr to the parent, it allow the self destruction of tree */
+            std::weak_ptr<Node> left_child;  /* Left child, for bit equal to 1 */
+            std::weak_ptr<Node> right_child; /* Right child, for bit equal to 0 */
+        };
+
+        std::weak_ptr<Node> root_;                         /* Root of the tree */
+
+        /**
+         * This mutlimap contains all prefix insert in the tree in time order
+         * We could then delete the last one if there is too much node
+         * The tree will self destroy is branch ( thanks to share_ptr )
+         */
+        std::multimap<time_point, std::shared_ptr<Node>> leaves_;
+    };
+
+    /* Callback used for insert value by using the pht */
+    using RealInsertCallback = std::function<void(std::shared_ptr<Prefix> p, IndexEntry entry)>;
+    using LookupCallbackWrapper = std::function<void(std::vector<std::shared_ptr<IndexEntry>>& values, Prefix p)>;
+
+    /**
+     * Performs a step in the lookup operation. Each steps are performed
+     * asynchronously.
+     *
+     * @param k          : Prefix on which the lookup is performed
+     * @param lo         : lowest bound on the prefix (where to start)
+     * @param hi         : highest bound on the prefix (where to stop)
+     * @param vals       : Shared ptr to a vector of IndexEntry (going to contains all values found)
+     * @param cb         : Callback to use at the end of the lookupStep (call on the value of vals)
+     * @param done_cb    : Callback at the end of the lookupStep
+     * @param max_common_prefix_len: used in the inexacte lookup match case, indicate the longest common prefix found
+     * @param start      : If start is set then lo and hi will be ignore for the first step, if the step fail lo and hi will be used
+     * @param all_values : If all value is true then all value met during the lookupstep will be in the vector vals
+     */
+    void lookupStep(Prefix k, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
+            std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals,
+            LookupCallbackWrapper cb, DoneCallbackSimple done_cb,
+            std::shared_ptr<unsigned> max_common_prefix_len,
+            int start = -1, bool all_values = false);
+
+    /**
+     * Apply the zcurve algorithm on the list of input prefix
+     *
+     * @param all_prefix : Vector of prefix to interleave
+     *
+     * @return The output prefix where all flags and content are interleaves
+     */
+    Prefix zcurve(const std::vector<Prefix>& all_prefix) const;
+
+    /**
+     * Linearizes the key into a unidimensional key. A pht only takes
+     * unidimensional key.
+     *
+     * @param Key  The initial key.
+     *
+     * @return the prefix of the linearized key.
+     */
+    virtual Prefix linearize(Key k) const;
+
+    /**
+     * Looking where to put the data cause if there is free space on the node
+     * above then this node will became the real leave.
+     *
+     * @param p       Share_ptr on the Prefix to check
+     * @param entry   The entry to put at the prefix p
+     * @param end_cb  Callback to use at the end of counting
+     */
+    void getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb );
+
+    /**
+     * Looking where to put the data cause if there is free space on the node
+     * above then this node will became the real leave.
+     *
+     * @param p       Share_ptr on the Prefix to check
+     * @param entry   The entry to put at the prefix p
+     * @param end_cb  Callback to use at the end of counting
+     */
+    void checkPhtUpdate(Prefix p, IndexEntry entry, time_point time_p);
+
+    /**
+     * Search for the split location by comparing 'compared' to all values in vals.
+     *
+     * @param compared : Value which going to be compared
+     * @param vals     : The vector of values to compare with comapred
+     * @return position compared diverge from all others
+     */
+    size_t foundSplitLocation(Prefix compared, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals) {
+        for ( size_t i = 0; i < compared.content_.size() * 8 - 1; i++ )
+            for ( auto const& v : *vals)
+                if ( Prefix(v->prefix).isContentBitActive(i) != compared.isContentBitActive(i) )
+                    return i + 1;
+
+        return compared.content_.size() * 8 - 1;
+    }
+
+    /**
+     * Put canary from the split point until the last known canary and add the prefix at the good place
+     *
+     * @param insert : Prefix to insertm but also prefix which going to check where we need to split
+     * @param vals   : Vector of vals for the comparaison
+     * @param entry  : Entry to put on the pht
+     * @param end_cb : Callback to apply to the insert prefi (here does the insert)
+     */
+    void split(Prefix insert, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals, IndexEntry entry, RealInsertCallback end_cb);
+
+    /**
+     * Tells if the key is valid according to the key spec.
+     */
+    bool validKey(const Key& k) const {
+        return k.size() == keySpec_.size() and
+            std::equal(k.begin(), k.end(), keySpec_.begin(),
+                [&](const Key::value_type& key, const KeySpec::value_type& key_spec) {
+                    return key.first == key_spec.first and key.second.size() <= key_spec.second;
+                }
+            );
+    }
+
+    /**
+     * Updates the canary token on the node responsible for the specified
+     * Prefix.
+     */
+    void updateCanary(Prefix p);
+
+    const std::string name_;
+    const std::string canary_;
+    const KeySpec keySpec_;
+    Cache cache_;
+    std::shared_ptr<DhtRunner> dht_;
+};
+
+} /* indexation  */
+} /* dht */
+
@@ -197,6 +197,7 @@ public:
    static InfoHash getRandom();

    friend std::ostream& operator<< (std::ostream& s, const InfoHash& h);
+    friend std::istream& operator>> (std::istream& s, InfoHash& h);

    std::string toString() const;

@@ -153,6 +153,7 @@ public:
        Blob ntoken {};
        Value::Id vid {};
        std::vector<std::shared_ptr<Value>> values {};
+        std::vector<std::shared_ptr<FieldValueIndex>> fields {};
        std::vector<std::shared_ptr<Node>> nodes4 {};
        std::vector<std::shared_ptr<Node>> nodes6 {};
        RequestAnswer() {}
@@ -223,7 +224,8 @@ private:
     */
    std::function<RequestAnswer(std::shared_ptr<Node>,
            InfoHash&,
-            want_t)> onGetValues {};
+            want_t,
+            Query)> onGetValues {};
    /**
     * @brief on listen request callback.
     *
@@ -235,7 +237,8 @@ private:
    std::function<RequestAnswer(std::shared_ptr<Node>,
            InfoHash&,
            Blob&,
-            uint16_t)> onListen {};
+            uint16_t,
+            Query)> onListen {};
    /**
     * @brief on announce request callback.
     *
@@ -256,7 +259,7 @@ public:
    using RequestExpiredCb = std::function<void(const Request&, bool)>;

    NetworkEngine(Logger& log, Scheduler& scheduler) : myid(zeroes), DHT_LOG(log), scheduler(scheduler) {}
-    NetworkEngine(InfoHash& myid, int s, int s6, Logger& log, Scheduler& scheduler,
+    NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler,
            decltype(NetworkEngine::onError) onError,
            decltype(NetworkEngine::onNewNode) onNewNode,
            decltype(NetworkEngine::onReportedAddr) onReportedAddr,
@@ -266,7 +269,7 @@ public:
            decltype(NetworkEngine::onListen) onListen,
            decltype(NetworkEngine::onAnnounce) onAnnounce) :
        onError(onError), onNewNode(onNewNode), onReportedAddr(onReportedAddr), onPing(onPing), onFindNode(onFindNode),
-        onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), myid(myid),
+        onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), myid(myid), network(net),
        dht_socket(s), dht_socket6(s6), DHT_LOG(log), scheduler(scheduler)
    {
        transaction_id = std::uniform_int_distribution<decltype(transaction_id)>{1}(rd_device);
@@ -290,9 +293,9 @@ public:
     * @param nodes6  The ipv6 closest nodes.
     * @param values  The values to send.
     */
-    void tellListener(std::shared_ptr<Node> n, uint16_t rid, InfoHash hash, want_t want, Blob ntoken,
-            std::vector<std::shared_ptr<Node>> nodes, std::vector<std::shared_ptr<Node>> nodes6,
-            std::vector<std::shared_ptr<Value>> values);
+    void tellListener(std::shared_ptr<Node> n, uint16_t rid, const InfoHash& hash, want_t want, const Blob& ntoken,
+            std::vector<std::shared_ptr<Node>>&& nodes, std::vector<std::shared_ptr<Node>>&& nodes6,
+            std::vector<std::shared_ptr<Value>>&& values, const Query& q);

    bool isRunning(sa_family_t af) const;
    inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; }
@@ -314,13 +317,15 @@ public:
                RequestExpiredCb on_expired);
    std::shared_ptr<Request>
        sendGetValues(std::shared_ptr<Node> n,
-                const InfoHash& target,
+                const InfoHash& info_hash,
+                const Query& query,
                want_t want,
                RequestCb on_done,
                RequestExpiredCb on_expired);
    std::shared_ptr<Request>
        sendListen(std::shared_ptr<Node> n,
                const InfoHash& infohash,
+                const Query& query,
                const Blob& token,
                RequestCb on_done,
                RequestExpiredCb on_expired);
@@ -357,6 +362,10 @@ public:

    void blacklistNode(const std::shared_ptr<Node>& n);

+    std::vector<std::shared_ptr<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) {
+        return cache.getCachedNodes(id, sa_f, count);
+    }
+
 private:
    /***************
     *  Constants  *
@@ -374,8 +383,10 @@ private:
    /* TODO */
    static const std::string my_v;

+
    /* DHT info */
    const InfoHash& myid;
+    const NetId network {0};
    const int dht_socket {-1};
    const int dht_socket6 {-1};
    const Logger& DHT_LOG;
@@ -431,6 +442,7 @@ private:
            const Blob& nodes,
            const Blob& nodes6,
            const std::vector<std::shared_ptr<Value>>& st,
+            const Query& query,
            const Blob& token);
    Blob bufferNodes(sa_family_t af, const InfoHash& id, std::vector<std::shared_ptr<Node>>& nodes);

@@ -451,7 +463,7 @@ private:
            const std::string& message,
            bool include_id=false);

-    void deserializeNodesValues(ParsedMessage& msg);
+    void deserializeNodes(ParsedMessage& msg);

    std::queue<time_point> rate_limit_time {};
    static std::mt19937 rd_device;
@@ -28,7 +28,8 @@ namespace dht {

 struct NodeCache {
    std::shared_ptr<Node> getNode(const InfoHash& id, sa_family_t family);
-    std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirmed);
+    std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, bool confirmed);
+    std::vector<std::shared_ptr<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count);

    /**
     * Reset the connectivity state of every node,
@@ -41,7 +42,7 @@ private:
    class NodeMap : public std::map<InfoHash, std::weak_ptr<Node>> {
    public:
        std::shared_ptr<Node> getNode(const InfoHash& id);
-        std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirmed);
+        std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, bool confirmed);
        void clearBadNodes();
    };

@@ -23,6 +23,8 @@

 namespace dht {

+static constexpr unsigned TARGET_NODES {8};
+
 struct Bucket {
    Bucket() : cached() {}
    Bucket(sa_family_t af, const InfoHash& f = {}, time_point t = time_point::min())
@@ -85,18 +85,18 @@ public:
     * If the signature can't be checked, or if the data can't be decrypted, it is not returned.
     * Public, non-signed & non-encrypted data is retransmitted as-is.
     */
-    virtual void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}) override;
-    virtual void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}) override {
-        get(id, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}, Where&& w = {}) override;
+    virtual void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}, Where&& w = {}) override {
+        get(id, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}) override {
-        get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) override {
+        get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f), std::forward<Where>(w));
    }
-    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}) override {
-        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f));
+    virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) override {
+        get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
    }

-    virtual size_t listen(const InfoHash& id, GetCallback cb, Value::Filter&& = {}) override;
+    virtual size_t listen(const InfoHash& id, GetCallback cb, Value::Filter&& = {}, Where&& w = {}) override;

    /**
     * Will take ownership of the value, sign it using our private key and put it in the DHT.
@@ -126,16 +126,13 @@ public:
    Value decrypt(const Value& v);

    void findCertificate(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::Certificate>)> cb);
-    void findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::PublicKey>)> cb);
+    void findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<const crypto::PublicKey>)> cb);

    const std::shared_ptr<crypto::Certificate> registerCertificate(const InfoHash& node, const Blob& cert);
    void registerCertificate(std::shared_ptr<crypto::Certificate>& cert);

    const std::shared_ptr<crypto::Certificate> getCertificate(const InfoHash& node) const;
-    const std::shared_ptr<crypto::PublicKey> getPublicKey(const InfoHash& node) const;
-
-
-    
+    const std::shared_ptr<const crypto::PublicKey> getPublicKey(const InfoHash& node) const;

    /**
     * Allows to set a custom callback called by the library to find a locally-stored certificate.
@@ -161,7 +158,7 @@ private:

    // our certificate cache
    std::map<InfoHash, std::shared_ptr<crypto::Certificate>> nodesCertificates_ {};
-    std::map<InfoHash, std::shared_ptr<crypto::PublicKey>> nodesPubKeys_ {};
+    std::map<InfoHash, std::shared_ptr<const crypto::PublicKey>> nodesPubKeys_ {};

    std::uniform_int_distribution<Value::Id> rand_id {};
 };
@@ -40,9 +40,8 @@

 namespace dht {

-static constexpr unsigned TARGET_NODES {8};
-
 using Address = std::pair<sockaddr_storage, socklen_t>;
+using NetId = uint32_t;
 using want_t = int_fast8_t;

 std::string print_addr(const sockaddr* sa, socklen_t slen);
@@ -62,7 +61,7 @@ void erase_if(std::map<Key, Item>& map, const Condition& condition)
 class DhtException : public std::runtime_error {
    public:
        DhtException(const std::string &str = "") :
-            std::runtime_error("DhtException occured: " + str) {}
+            std::runtime_error("DhtException occurred: " + str) {}
 };


@@ -169,4 +168,6 @@ unpackMsg(Blob b) {

 msgpack::unpacked unpackMsg(Blob b);

+msgpack::object* findMapValue(msgpack::object& map, const std::string& key);
+
 } // namespace dht
@@ -35,10 +35,12 @@
 #include <functional>
 #include <memory>
 #include <chrono>
+#include <set>

 namespace dht {

 struct Value;
+struct Query;

 /**
 * A storage policy is applied once to every incoming value storage requests.
@@ -68,7 +70,7 @@ using StorePolicy = std::function<bool(InfoHash key, std::shared_ptr<Value>& val
 */
 using EditPolicy = std::function<bool(InfoHash key, const std::shared_ptr<Value>& old_val, std::shared_ptr<Value>& new_val, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>;

-static constexpr const size_t MAX_VALUE_SIZE {1024 * 14};
+static constexpr const size_t MAX_VALUE_SIZE {1024 * 16};

 struct ValueType {
    typedef uint16_t Id;
@@ -114,6 +116,14 @@ struct ValueType {
 */
 struct Value
 {
+    enum class Field {
+        None = 0,
+        Id,
+        ValueType,
+        OwnerPk,
+        UserType,
+    };
+
    typedef uint64_t Id;
    static const Id INVALID_ID {0};

@@ -135,14 +145,18 @@ struct Value
                return f1(v) and f2(v);
            };
        }
-        static Filter chain(std::initializer_list<Filter> l) {
-            const std::vector<Filter> list(l.begin(), l.end());
-            return [list](const Value& v){
-                for (const auto& f : list)
+        template <typename T>
+        static Filter chainAll(T&& set) {
+            using namespace std::placeholders;
+            return std::bind([](const Value& v, T& s) {
+                for (const auto& f : s)
                    if (f and not f(v))
                        return false;
                return true;
-            };
+            }, _1, std::move(set));
+        }
+        static Filter chain(std::initializer_list<Filter> l) {
+            return chainAll(std::move(l));
        }
        static Filter chainOr(Filter&& f1, Filter&& f2) {
            if (not f1 or not f2) return AllFilter();
@@ -162,6 +176,11 @@ struct Value
            return v.type == tid;
        };
    }
+    static Filter TypeFilter(const ValueType::Id& tid) {
+        return [tid](const Value& v) {
+            return v.type == tid;
+        };
+    }

    static Filter IdFilter(const Id id) {
        return [id](const Value& v) {
@@ -175,6 +194,23 @@ struct Value
        };
    }

+    static Filter ownerFilter(const crypto::PublicKey& pk) {
+        return ownerFilter(pk.getId());
+    }
+
+    static Filter ownerFilter(const InfoHash& pkh) {
+        return [pkh](const Value& v) {
+            return v.owner and v.owner->getId() == pkh;
+        };
+    }
+
+    static Filter userTypeFilter(const std::string& ut)
+    {
+        return [ut](const Value& v) {
+            return v.user_type == ut;
+        };
+    }
+
    class SerializableBase
    {
    public:
@@ -248,6 +284,43 @@ struct Value
        return owner and not signature.empty();
    }

+    /**
+     * Sign the value using the provided private key.
+     * Afterward, checkSignature() will return true and owner will
+     * be set to the corresponding public key.
+     */
+    void sign(const crypto::PrivateKey& key) {
+        if (isEncrypted())
+            throw DhtException("Can't sign encrypted data.");
+        owner = std::make_shared<const crypto::PublicKey>(key.getPublicKey());
+        signature = key.sign(getToSign());
+    }
+
+    /**
+     * Check that the value is signed and that the signature matches.
+     * If true, the owner field will contain the signer public key.
+     */
+    bool checkSignature() const {
+        return isSigned() and owner->checkSignature(getToSign(), signature);
+    }
+
+    std::shared_ptr<const crypto::PublicKey> getOwner() const {
+        return std::static_pointer_cast<const crypto::PublicKey>(owner);
+    }
+
+    /**
+     * Sign the value with from and returns the encrypted version for to.
+     */
+    Value encrypt(const crypto::PrivateKey& from, const crypto::PublicKey& to) {
+        if (isEncrypted())
+            throw DhtException("Data is already encrypted.");
+        setRecipient(to.getId());
+        sign(from);
+        Value nv {id};
+        nv.setCypher(to.encrypt(getToEncrypt()));
+        return nv;
+    }
+
    Value() {}

    Value (Id id) : id(id) {}
@@ -378,15 +451,48 @@ struct Value
        pk.pack(std::string("dat")); msgpack_pack_to_encrypt(pk);
    }

+    template <typename Packer>
+    void msgpack_pack_fields(const std::set<Value::Field>& fields, Packer& pk) const
+    {
+        for (const auto& field : fields)
+            switch (field) {
+                case Value::Field::Id:
+                    pk.pack(static_cast<uint64_t>(id));
+                    break;
+                case Value::Field::ValueType:
+                    pk.pack(static_cast<uint64_t>(type));
+                    break;
+                case Value::Field::OwnerPk:
+                    if (owner)
+                        owner->msgpack_pack(pk);
+                    else
+                        InfoHash().msgpack_pack(pk);
+                    break;
+                case Value::Field::UserType:
+                    pk.pack(user_type);
+                    break;
+                default:
+                    break;
+            }
+    }
+
    void msgpack_unpack(msgpack::object o);
    void msgpack_unpack_body(const msgpack::object& o);
+    Blob getPacked() const {
+        msgpack::sbuffer buffer;
+        msgpack::packer<msgpack::sbuffer> pk(&buffer);
+        pk.pack(*this);
+        return {buffer.data(), buffer.data()+buffer.size()};
+    }
+
+    void msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset);

    Id id {INVALID_ID};

    /**
     * Public key of the signer.
     */
-    std::shared_ptr<crypto::PublicKey> owner {};
+    std::shared_ptr<const crypto::PublicKey> owner {};

    /**
     * Hash of the recipient (optional).
@@ -424,6 +530,344 @@ struct Value

 using ValuesExport = std::pair<InfoHash, Blob>;

+/**
+ * @class   FieldValue
+ * @brief   Describes a value filter.
+ * @details
+ * This structure holds the value for a specified field. It's type can either be
+ * uint64_t, InfoHash or Blob.
+ */
+struct FieldValue
+{
+    FieldValue() {}
+    FieldValue(Value::Field f, uint64_t int_value) : field(f), intValue(int_value) {}
+    FieldValue(Value::Field f, InfoHash hash_value) : field(f), hashValue(hash_value) {}
+    FieldValue(Value::Field f, Blob blob_value) : field(f), blobValue(blob_value) {}
+
+    bool operator==(const FieldValue& fd) const;
+
+    // accessors
+    Value::Field getField() const { return field; }
+    uint64_t getInt() const { return intValue; }
+    InfoHash getHash() const { return hashValue; }
+    Blob getBlob() const { return blobValue; }
+
+    template <typename Packer>
+    void msgpack_pack(Packer& p) const {
+        p.pack_map(2);
+        p.pack(std::string("f")); p.pack(static_cast<uint8_t>(field));
+
+        p.pack(std::string("v"));
+        switch (field) {
+            case Value::Field::Id:
+            case Value::Field::ValueType:
+                p.pack(intValue);
+                break;
+            case Value::Field::OwnerPk:
+                p.pack(hashValue);
+                break;
+            case Value::Field::UserType:
+                p.pack_bin(blobValue.size());
+                p.pack_bin_body((const char*)blobValue.data(), blobValue.size());
+                break;
+            default:
+                throw msgpack::type_error();
+        }
+    }
+
+    void msgpack_unpack(msgpack::object msg) {
+        hashValue = {};
+        blobValue.clear();
+
+        if (auto f = findMapValue(msg, "f"))
+            field = (Value::Field)f->as<unsigned>();
+        else
+            throw msgpack::type_error();
+
+        auto v = findMapValue(msg, "v");
+        if (not v)
+            throw msgpack::type_error();
+        else
+            switch (field) {
+                case Value::Field::Id:
+                case Value::Field::ValueType:
+                    intValue = v->as<decltype(intValue)>();
+                    break;
+                case Value::Field::OwnerPk:
+                    hashValue = v->as<decltype(hashValue)>();
+                    break;
+                case Value::Field::UserType:
+                    blobValue = unpackBlob(*v);
+                    break;
+                default:
+                    throw msgpack::type_error();
+            }
+    }
+
+    Value::Filter getLocalFilter() const;
+
+private:
+    Value::Field field {Value::Field::None};
+    // three possible value types
+    uint64_t intValue {};
+    InfoHash hashValue {};
+    Blob blobValue {};
+};
+
+
+/**
+ * @struct  FieldSelectorDescription
+ * @brief   Describes a selection.
+ * @details
+ * This is meant to narrow data to a set of specified fields. This structure is
+ * used to construct a Select structure.
+ */
+struct FieldSelectorDescription
+{
+    FieldSelectorDescription() {}
+    FieldSelectorDescription(Value::Field f) : field(f) {}
+
+    Value::Field getField() const { return field; }
+
+    bool operator==(const FieldSelectorDescription& fd) const { return field == fd.field; }
+
+    template <typename Packer>
+    void msgpack_pack(Packer& p) const { p.pack(static_cast<uint8_t>(field)); }
+    void msgpack_unpack(msgpack::object msg) { field = static_cast<Value::Field>(msg.as<int>()); }
+private:
+    Value::Field field {Value::Field::None};
+};
+
+/**
+ * @class   Select
+ * @brief   Serializable Value field selection.
+ * @details
+ * This is a container for a list of FieldSelectorDescription instances. It
+ * describes a complete SELECT query for dht::Value.
+ */
+struct Select
+{
+    Select() { }
+    Select(const std::string& q_str);
+
+    bool isSatisfiedBy(const Select& os) const;
+
+    /**
+     * Selects a field of type Value::Field.
+     *
+     * @param field  the field to require.
+     *
+     * @return the resulting Select instance.
+     */
+    Select& field(Value::Field field) {
+        fieldSelection_.emplace_back(field);
+        return *this;
+    }
+
+    /**
+     * Computes the set of selected fields based on previous require* calls.
+     *
+     * @return the set of fields.
+     */
+    std::set<Value::Field> getSelection() const {
+        std::set<Value::Field> fields {};
+        for (const auto& f : fieldSelection_) {
+            fields.insert(f.getField());
+        }
+        return fields;
+    }
+
+    template <typename Packer>
+    void msgpack_pack(Packer& pk) const { pk.pack(fieldSelection_); }
+    void msgpack_unpack(const msgpack::object& o) {
+        fieldSelection_.clear();
+        fieldSelection_ = o.as<decltype(fieldSelection_)>();
+    }
+
+    friend std::ostream& operator<<(std::ostream& s, const dht::Select& q);
+private:
+    std::vector<FieldSelectorDescription> fieldSelection_ {};
+};
+
+/**
+ * @class   Where
+ * @brief   Serializable dht::Value filter.
+ * @details
+ * This is container for a list of FieldValue instances. It describes a
+ * complete WHERE query for dht::Value.
+ */
+struct Where
+{
+    Where() { }
+    Where(const std::string& q_str);
+
+    bool isSatisfiedBy(const Where& where) const;
+
+    /**
+     * Adds restriction on Value::Id based on the id argument.
+     *
+     * @param id  the id.
+     *
+     * @return the resulting Where instance.
+     */
+    Where& id(Value::Id id) {
+        filters_.emplace_back(Value::Field::Id, id);
+        return *this;
+    }
+
+    /**
+     * Adds restriction on Value::ValueType based on the type argument.
+     *
+     * @param type  the value type.
+     *
+     * @return the resulting Where instance.
+     */
+    Where& valueType(ValueType::Id type) {
+        filters_.emplace_back(Value::Field::ValueType, type);
+        return *this;
+    }
+
+    /**
+     * Adds restriction on Value::OwnerPk based on the owner_pk_hash argument.
+     *
+     * @param owner_pk_hash  the owner public key fingerprint.
+     *
+     * @return the resulting Where instance.
+     */
+    Where& owner(InfoHash owner_pk_hash) {
+        filters_.emplace_back(Value::Field::OwnerPk, owner_pk_hash);
+        return *this;
+    }
+
+    /**
+     * Adds restriction on Value::UserType based on the user_type argument.
+     *
+     * @param user_type  the user type.
+     *
+     * @return the resulting Where instance.
+     */
+    Where& userType(std::string user_type) {
+        filters_.emplace_back(Value::Field::UserType, Blob {user_type.begin(), user_type.end()});
+        return *this;
+    }
+
+    /**
+     * Computes the Value::Filter based on the list of field value set.
+     *
+     * @return the resulting Value::Filter.
+     */
+    Value::Filter getFilter() const {
+        std::vector<Value::Filter> fset(filters_.size());
+        std::transform(filters_.begin(), filters_.end(), fset.begin(), [](const FieldValue& f) {
+            return f.getLocalFilter();
+        });
+        return Value::Filter::chainAll(std::move(fset));
+    }
+
+    template <typename Packer>
+    void msgpack_pack(Packer& pk) const { pk.pack(filters_); }
+    void msgpack_unpack(const msgpack::object& o) {
+        filters_.clear();
+        filters_ = o.as<decltype(filters_)>();
+    }
+
+    friend std::ostream& operator<<(std::ostream& s, const dht::Where& q);
+
+private:
+    std::vector<FieldValue> filters_;
+};
+
+/**
+ * @class   Query
+ * @brief   Describes a query destined to another peer.
+ * @details
+ * This class describes the list of filters on field values and the field
+ * itselves to include in the peer response to a GET operation. See
+ * FieldValue.
+ */
+struct Query
+{
+    static const std::string QUERY_PARSE_ERROR;
+
+    Query(Select s = {}, Where w = {}) : select(s), where(w) { };
+
+    /**
+     * Initializes a query based on a SQL-ish formatted string. The abstract
+     * form of such a string is the following:
+     *
+     *  [SELECT <$field$> [WHERE <$field$=$value$>]]
+     *
+     *  where
+     *
+     *  - $field$ = *|id|value_type|owner_pk|user_type
+     *  - $value$ = $string$|$integer$
+     *  - $string$: a simple string WITHOUT SPACES.
+     *  - $integer$: a simple integer.
+     */
+    Query(std::string q_str) {
+        auto pos_W = q_str.find("WHERE");
+        auto pos_w = q_str.find("where");
+        auto pos = std::min(pos_W != std::string::npos ? pos_W : q_str.size(),
+                            pos_w != std::string::npos ? pos_w : q_str.size());
+        select = q_str.substr(0, pos);
+        where = q_str.substr(pos, q_str.size()-pos);
+    }
+
+    /**
+     * Tell if the query is satisfied by another query.
+     */
+    bool isSatisfiedBy(const Query& q) const;
+
+    template <typename Packer>
+    void msgpack_pack(Packer& pk) const {
+        pk.pack_map(2);
+        pk.pack(std::string("s")); pk.pack(select); /* packing field selectors */
+        pk.pack(std::string("w")); pk.pack(where);  /* packing filters */
+    }
+
+    void msgpack_unpack(const msgpack::object& o);
+
+    std::string toString() {
+        std::stringstream ss;
+        ss << *this;
+        return ss.str();
+    }
+
+    friend std::ostream& operator<<(std::ostream& s, const dht::Query& q) {
+        return s << "Query[" << q.select << " " << q.where << "]";
+    }
+
+    Select select {};
+    Where where {};
+};
+
+/*!
+ * @class   FieldValueIndex
+ * @brief   An index for field values.
+ * @details
+ * This structures is meant to manipulate a subset of fields normally contained
+ * in Value.
+ */
+struct FieldValueIndex {
+    FieldValueIndex() {}
+    FieldValueIndex(const Value& v, Select s = {});
+    /**
+     * Tells if all the fields of this are contained in the other
+     * FieldValueIndex with the same value.
+     *
+     * @param other  The other FieldValueIndex instance.
+     */
+    bool containedIn(const FieldValueIndex& other) const;
+
+    friend std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi);
+
+    void msgpack_unpack_fields(const std::set<Value::Field>& fields,
+            const msgpack::object& o,
+            unsigned offset);
+
+    std::map<Value::Field, FieldValue> index {};
+};
+
 template <typename T,
          typename std::enable_if<std::is_base_of<Value::SerializableBase, T>::value, T>::type* = nullptr>
 Value::Filter
@@ -31,6 +31,7 @@ from libcpp.map cimport map as map
 from libcpp cimport bool
 from libcpp.utility cimport pair
 from libcpp.string cimport string
+from libcpp.memory cimport shared_ptr

 from cython.parallel import parallel, prange
 from cython.operator cimport dereference as deref, preincrement as inc, predecrement as dec
@@ -40,19 +41,29 @@ cimport opendht_cpp as cpp

 import threading

+cdef inline void lookup_callback(cpp.vector[cpp.shared_ptr[cpp.IndexValue]]* values, cpp.Prefix* p, void *user_data) with gil:
+    cbs = <object>user_data
+    if 'lookup' in cbs and cbs['lookup']:
+        vals = []
+        for val in deref(values):
+            v = IndexValue()
+            v._value = val
+            vals.append(v)
+        cbs['lookup'](vals, p.toString())
+
 cdef inline void shutdown_callback(void* user_data) with gil:
    cbs = <object>user_data
    if 'shutdown' in cbs and cbs['shutdown']:
        cbs['shutdown']()
    ref.Py_DECREF(cbs)

-cdef inline bool get_callback(cpp.shared_ptr[cpp.Value] value, void *user_data) with gil:
+cdef inline bool get_callback(shared_ptr[cpp.Value] value, void *user_data) with gil:
    cb = (<object>user_data)['get']
    pv = Value()
    pv._value = value
    return cb(pv)

-cdef inline void done_callback(bool done, cpp.vector[cpp.shared_ptr[cpp.Node]]* nodes, void *user_data) with gil:
+cdef inline void done_callback(bool done, cpp.vector[shared_ptr[cpp.Node]]* nodes, void *user_data) with gil:
    node_ids = []
    for n in deref(nodes):
        h = NodeEntry()
@@ -64,6 +75,12 @@ cdef inline void done_callback(bool done, cpp.vector[cpp.shared_ptr[cpp.Node]]*
        cbs['done'](done, node_ids)
    ref.Py_DECREF(cbs)

+cdef inline void done_callback_simple(bool done, void *user_data) with gil:
+    cbs = <object>user_data
+    if 'done' in cbs and cbs['done']:
+        cbs['done'](done)
+    ref.Py_DECREF(cbs)
+
 cdef class _WithID(object):
    def __repr__(self):
        return "<%s '%s'>" % (self.__class__.__name__, str(self))
@@ -72,7 +89,7 @@ cdef class _WithID(object):

 cdef class InfoHash(_WithID):
    cdef cpp.InfoHash _infohash
-    def __init__(self, bytes str=b''):
+    def __cinit__(self, bytes str=b''):
        self._infohash = cpp.InfoHash(str)
    def __bool__(InfoHash self):
        return not (self._infohash == cpp.InfoHash())
@@ -109,7 +126,7 @@ cdef class InfoHash(_WithID):
        return h

 cdef class Node(_WithID):
-    cdef cpp.shared_ptr[cpp.Node] _node
+    cdef shared_ptr[cpp.Node] _node
    def getId(self):
        h = InfoHash()
        h._infohash = self._node.get().getId()
@@ -120,7 +137,7 @@ cdef class Node(_WithID):
        return self._node.get().isExpired()

 cdef class NodeEntry(_WithID):
-    cdef cpp.pair[cpp.InfoHash, cpp.shared_ptr[cpp.Node]] _v
+    cdef cpp.pair[cpp.InfoHash, shared_ptr[cpp.Node]] _v
    def getId(self):
        h = InfoHash()
        h._infohash = self._v.first
@@ -131,7 +148,7 @@ cdef class NodeEntry(_WithID):
        return n

 cdef class Value(object):
-    cdef cpp.shared_ptr[cpp.Value] _value
+    cdef shared_ptr[cpp.Value] _value
    def __init__(self, bytes val=b''):
        self._value.reset(new cpp.Value(val, len(val)))
    def __str__(self):
@@ -163,8 +180,8 @@ cdef class Value(object):
            return self._value.get().size()

 cdef class NodeSetIter(object):
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]]* _nodes
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]].iterator _curIter
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]]* _nodes
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]].iterator _curIter
    def __init__(self, NodeSet s):
        self._nodes = &s._nodes
        self._curIter = self._nodes.begin()
@@ -177,7 +194,7 @@ cdef class NodeSetIter(object):
        return h

 cdef class NodeSet(object):
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]] _nodes
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]] _nodes
    def size(self):
        return self._nodes.size()
    def insert(self, NodeEntry l):
@@ -199,7 +216,7 @@ cdef class NodeSet(object):
        return h
    def __str__(self):
        s = ''
-        cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]].iterator it = self._nodes.begin()
+        cdef map[cpp.InfoHash, shared_ptr[cpp.Node]].iterator it = self._nodes.begin()
        while it != self._nodes.end():
            s += deref(it).first.toString().decode() + ' ' + deref(it).second.get().getAddrStr().decode() + '\n'
            inc(it)
@@ -215,7 +232,7 @@ cdef class PublicKey(_WithID):
        return h

 cdef class Certificate(_WithID):
-    cdef cpp.shared_ptr[cpp.Certificate] _cert
+    cdef shared_ptr[cpp.Certificate] _cert
    def getId(self):
        h = InfoHash()
        h._infohash = self._cert.get().getId()
@@ -254,49 +271,49 @@ cdef class DhtConfig(object):
        self._config.dht_config.node_config.node_id = id._infohash

 cdef class DhtRunner(_WithID):
-    cdef cpp.DhtRunner* thisptr
+    cdef cpp.shared_ptr[cpp.DhtRunner] thisptr
    def __cinit__(self):
-        self.thisptr = new cpp.DhtRunner()
+        self.thisptr.reset(new cpp.DhtRunner())
    def getId(self):
        h = InfoHash()
-        h._infohash = self.thisptr.getId()
+        h._infohash = self.thisptr.get().getId()
        return h
    def getNodeId(self):
-        return self.thisptr.getNodeId().toString()
+        return self.thisptr.get().getNodeId().toString()
    def bootstrap(self, str host, str port):
-        self.thisptr.bootstrap(host.encode(), port.encode())
+        self.thisptr.get().bootstrap(host.encode(), port.encode())
    def run(self, Identity id=None, is_bootstrap=False, cpp.in_port_t port=0, str ipv4="", str ipv6="", DhtConfig config=DhtConfig()):
        if id:
            config.setIdentity(id)
        if ipv4 or ipv6:
            bind4 = ipv4.encode() if ipv4 else b''
            bind6 = ipv6.encode() if ipv6 else b''
-            self.thisptr.run(bind4, bind6, str(port).encode(), config._config)
+            self.thisptr.get().run(bind4, bind6, str(port).encode(), config._config)
        else:
-            self.thisptr.run(port, config._config)
+            self.thisptr.get().run(port, config._config)
    def join(self):
-        self.thisptr.join()
+        self.thisptr.get().join()
    def shutdown(self, shutdown_cb=None):
        cb_obj = {'shutdown':shutdown_cb}
        ref.Py_INCREF(cb_obj)
-        self.thisptr.shutdown(cpp.bindShutdownCb(shutdown_callback, <void*>cb_obj))
+        self.thisptr.get().shutdown(cpp.bindShutdownCb(shutdown_callback, <void*>cb_obj))
    def enableLogging(self):
-        cpp.enableLogging(self.thisptr[0])
+        cpp.enableLogging(self.thisptr.get()[0])
    def disableLogging(self):
-        cpp.disableLogging(self.thisptr[0])
+        cpp.disableLogging(self.thisptr.get()[0])
    def enableFileLogging(self, str path):
-        cpp.enableFileLogging(self.thisptr[0], path)
+        cpp.enableFileLogging(self.thisptr.get()[0], path)
    def isRunning(self):
-        return self.thisptr.isRunning()
+        return self.thisptr.get().isRunning()
    def getStorageLog(self):
-        return self.thisptr.getStorageLog().decode()
+        return self.thisptr.get().getStorageLog().decode()
    def getRoutingTablesLog(self, cpp.sa_family_t af):
-        return self.thisptr.getRoutingTablesLog(af).decode()
+        return self.thisptr.get().getRoutingTablesLog(af).decode()
    def getSearchesLog(self, cpp.sa_family_t af):
-        return self.thisptr.getSearchesLog(af).decode()
+        return self.thisptr.get().getSearchesLog(af).decode()
    def getNodeMessageStats(self):
        stats = []
-        cdef cpp.vector[unsigned] res = self.thisptr.getNodeMessageStats(False)
+        cdef cpp.vector[unsigned] res = self.thisptr.get().getNodeMessageStats(False)
        for n in res:
            stats.append(n)
        return stats
@@ -304,14 +321,16 @@ cdef class DhtRunner(_WithID):
    def get(self, InfoHash key, get_cb=None, done_cb=None):
        """Retreive values associated with a key on the DHT.

-        key -- the key for which to search
-        get_cb -- is set, makes the operation non-blocking. Called when a value is found on the DHT.
-        done_cb -- optional callback used when get_cb is set. Called when the operation is completed.
+        key     -- the key for which to search
+        get_cb  -- is set, makes the operation non-blocking. Called when a value
+                   is found on the DHT.
+        done_cb -- optional callback used when get_cb is set. Called when the
+                   operation is completed.
        """
        if get_cb:
            cb_obj = {'get':get_cb, 'done':done_cb}
            ref.Py_INCREF(cb_obj)
-            self.thisptr.get(key._infohash, cpp.bindGetCb(get_callback, <void*>cb_obj), cpp.bindDoneCb(done_callback, <void*>cb_obj))
+            self.thisptr.get().get(key._infohash, cpp.bindGetCb(get_callback, <void*>cb_obj), cpp.bindDoneCb(done_callback, <void*>cb_obj))
        else:
            lock = threading.Condition()
            pending = 0
@@ -334,14 +353,14 @@ cdef class DhtRunner(_WithID):
    def put(self, InfoHash key, Value val, done_cb=None):
        """Publish a new value on the DHT at key.

-        key -- the DHT key where to put the value
-        val -- the value to put on the DHT
+        key     -- the DHT key where to put the value
+        val     -- the value to put on the DHT
        done_cb -- optional callback called when the operation is completed.
        """
        if done_cb:
            cb_obj = {'done':done_cb}
            ref.Py_INCREF(cb_obj)
-            self.thisptr.put(key._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj))
+            self.thisptr.get().put(key._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj))
        else:
            lock = threading.Condition()
            pending = 0
@@ -365,9 +384,74 @@ cdef class DhtRunner(_WithID):
        t._cb['cb'] = cb_obj
        # avoid the callback being destructed if the token is destroyed
        ref.Py_INCREF(cb_obj)
-        t._t = self.thisptr.listen(t._h, cpp.bindGetCb(get_callback, <void*>cb_obj)).share()
+        t._t = self.thisptr.get().listen(t._h, cpp.bindGetCb(get_callback, <void*>cb_obj)).share()
        return t
    def cancelListen(self, ListenToken token):
-        self.thisptr.cancelListen(token._h, token._t)
+        self.thisptr.get().cancelListen(token._h, token._t)
        ref.Py_DECREF(<object>token._cb['cb'])
        # fixme: not thread safe
+
+cdef class IndexValue(object):
+    cdef cpp.shared_ptr[cpp.IndexValue] _value
+    def __init__(self, InfoHash h=None, cpp.uint64_t vid=0):
+       cdef cpp.InfoHash hh = h._infohash
+       self._value.reset(new cpp.IndexValue(hh, vid))
+    def __str__(self):
+        return "(" + self.getKey().toString().decode() +", "+ str(self.getValueId()) +")"
+    def getKey(self):
+        h = InfoHash()
+        h._infohash = self._value.get().first
+        return h
+    def getValueId(self):
+        return self._value.get().second
+
+cdef class Pht(object):
+    cdef cpp.Pht* thisptr
+    def __cinit__(self, bytes name, key_spec, DhtRunner dht):
+        cdef cpp.IndexKeySpec cpp_key_spec
+        for kk, size in key_spec.items():
+            cpp_key_spec[bytes(kk, 'utf-8')] = size
+        self.thisptr = new cpp.Pht(name, cpp_key_spec, dht.thisptr)
+    property MAX_NODE_ENTRY_COUNT:
+        def __get__(self):
+            return cpp.PHT_MAX_NODE_ENTRY_COUNT
+    def lookup(self, key, lookup_cb=None, done_cb=None):
+        """Query the Index with a specified key.
+
+        key       -- the key for to the entry in the index.
+        lookup_cb -- function called when the operation is completed. This
+                     function takes a list of IndexValue objects and a string
+                     representation of the prefix where the value was indexed in
+                     the PHT.
+        """
+        cb_obj = {'lookup':lookup_cb, 'done':done_cb} # TODO: donecallback is to be removed
+        ref.Py_INCREF(cb_obj)
+        cdef cpp.IndexKey cppk
+        for kk, v in key.items():
+            cppk[bytes(kk, 'utf-8')] = bytes(v)
+        self.thisptr.lookup(
+                cppk,
+                cpp.Pht.bindLookupCb(lookup_callback, <void*>cb_obj),
+                cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj)
+        )
+    def insert(self, key, IndexValue value, done_cb=None):
+        """Add an index entry to the Index.
+
+        key     -- the key for to the entry in the index.
+        value   -- an IndexValue object describing the indexed value.
+        done_cb -- Called when the operation is completed.
+        """
+        cb_obj = {'done':done_cb}
+        ref.Py_INCREF(cb_obj)
+        cdef cpp.IndexKey cppk
+        for kk, v in key.items():
+            cppk[bytes(kk, 'utf-8')] = bytes(v)
+        cdef cpp.IndexValue val
+        val.first = (<InfoHash>value.getKey())._infohash
+        val.second = value.getValueId()
+        self.thisptr.insert(
+                cppk,
+                val,
+                cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj)
+        )
+
@@ -20,6 +20,7 @@ from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 from libcpp.utility cimport pair
+from libcpp.map cimport map

 ctypedef uint16_t in_port_t
 ctypedef unsigned short int sa_family_t;
@@ -27,6 +28,7 @@ ctypedef unsigned short int sa_family_t;
 cdef extern from "<memory>" namespace "std" nogil:
    cdef cppclass shared_ptr[T]:
        shared_ptr() except +
+        shared_ptr(T*) except +
        T* get()
        T operator*()
        void reset(T*)
@@ -103,19 +105,21 @@ cdef extern from "opendht/callbacks.h" namespace "dht":
    ctypedef void (*ShutdownCallbackRaw)(void *user_data)
    ctypedef bool (*GetCallbackRaw)(shared_ptr[Value] values, void *user_data)
    ctypedef void (*DoneCallbackRaw)(bool done, vector[shared_ptr[Node]]* nodes, void *user_data)
+    ctypedef void (*DoneCallbackSimpleRaw)(bool done, void *user_data)

    cppclass ShutdownCallback:
        ShutdownCallback() except +
    cppclass GetCallback:
        GetCallback() except +
-        #GetCallback(GetCallbackRaw cb, void *user_data) except +
    cppclass DoneCallback:
        DoneCallback() except +
-        #DoneCallback(DoneCallbackRaw, void *user_data) except +
+    cppclass DoneCallbackSimple:
+        DoneCallbackSimple() except +

    cdef ShutdownCallback bindShutdownCb(ShutdownCallbackRaw cb, void *user_data)
    cdef GetCallback bindGetCb(GetCallbackRaw cb, void *user_data)
    cdef DoneCallback bindDoneCb(DoneCallbackRaw cb, void *user_data)
+    cdef DoneCallbackSimple bindDoneCbSimple(DoneCallbackSimpleRaw cb, void *user_data)

    cppclass Config:
        InfoHash node_id
@@ -156,3 +160,22 @@ cdef extern from "opendht/log.h" namespace "dht::log":
    void disableLogging(DhtRunner& dht)
    void enableFileLogging(DhtRunner& dht, const string& path)

+cdef extern from "opendht/indexation/pht.h" namespace "dht::indexation":
+    size_t PHT_MAX_NODE_ENTRY_COUNT "dht::indexation::Pht::MAX_NODE_ENTRY_COUNT"
+    cdef cppclass Prefix:
+        Prefix() except +
+        Prefix(vector[uint8_t]) except +
+        string toString() const
+    ctypedef pair[InfoHash, uint64_t] IndexValue "dht::indexation::Value"
+    ctypedef map[string, vector[uint8_t]] IndexKey "dht::indexation::Pht::Key"
+    ctypedef map[string, uint32_t] IndexKeySpec "dht::indexation::Pht::KeySpec"
+    ctypedef void (*LookupCallbackRaw)(vector[shared_ptr[IndexValue]]* values, Prefix* p, void* user_data);
+    cdef cppclass Pht:
+        cppclass LookupCallback:
+            LookupCallback() except +
+        Pht(string, IndexKeySpec, shared_ptr[DhtRunner]) except +
+        void lookup(IndexKey k, LookupCallback cb, DoneCallbackSimple doneCb);
+        void insert(IndexKey k, IndexValue v, DoneCallbackSimple cb)
+        @staticmethod
+        LookupCallback bindLookupCb(LookupCallbackRaw cb, void *user_data)
+
@@ -0,0 +1,45 @@
+# Benchmark
+
+The `benchmark.py` script is used for testing OpenDHT in various cases. If you
+run `benchmark.py --help`, you should find the following text:
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      --performance         Launches performance benchmark test. Available args
+                            for "-t" are: gets.
+      --data-persistence    Launches data persistence benchmark test. Available
+                            args for "-t" are: delete, replace, mult_time.
+                            Available args for "-o" are : dump_str_log,
+                            keep_alive, trigger, traffic_plot, op_plot. Use "-m"
+                            to specify the number of producers on the DHT.Use "-e"
+                            to specify the number of values to put on the DHT.
+
+These options specify the feature to be tested. Each feature has its own tests.
+You specify the test by using `-t` flag (see `benchmark.py --help` for full
+help).
+
+## Python dependencies
+
+- pyroute2 >=0.3.14
+- matplotlib
+- GeoIP (used by `scanner.py` for drawing map of the world)
+- ipaddress
+- netifaces
+- networkx
+- numpy
+
+## Usage
+
+Before running the script, you have to build and install OpenDHT and its cython
+wrapper (`cython3` has to be installed) on the system so that it can be found by
+the benchmark script.
+
+    $ cd $OPENDHT_SRC_DIR
+    $ ./autogen.sh
+    $ ./configure
+    $ make && sudo make install
+
+Then, you can use the script like so:
+
+    $ cd $OPENDHT_SRC_DIR/python/tools/
+    $ python3 benchmark.py --performance -t gets -n 2048
@@ -27,7 +27,7 @@ import random

 from dht.network import DhtNetwork
 from dht.network import DhtNetworkSubProcess
-from dht.tests import PerformanceTest, PersistenceTest
+from dht.tests import PerformanceTest, PersistenceTest, PhtTest
 from dht import virtual_network_builder
 from dht import network as dhtnetwork

@@ -66,7 +66,11 @@ class WorkBench():

    def create_virtual_net(self):
        if self.virtual_locs > 1:
-            cmd = ["python3", os.path.abspath(virtual_network_builder.__file__), "-i", self.ifname, "-n", str(self.clusters), '-l', str(self.loss), '-d', str(self.delay)]
+            cmd = ["python3", os.path.abspath(virtual_network_builder.__file__),
+                    "-i", self.ifname,
+                    "-n", str(self.clusters),
+                    '-l', str(self.loss),
+                    '-d', str(self.delay)]
            if not self.disable_ipv4:
                cmd.append('-4')
            if not self.disable_ipv6:
@@ -90,10 +94,17 @@ class WorkBench():
                    cmd.extend(['-b', self.local_bootstrap.ip4])
                if not self.disable_ipv6 and self.local_bootstrap.ip6:
                    cmd.extend(['-b6', self.local_bootstrap.ip6])
-            self.procs[i] = DhtNetworkSubProcess('node'+str(i), cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-            while DhtNetworkSubProcess.NOTIFY_TOKEN not in self.procs[i].getline():
-                # waiting for process to spawn
-                time.sleep(0.5)
+            lock = threading.Condition()
+            def dcb(success):
+                nonlocal lock
+                if not success:
+                    DhtNetwork.Log.err("Failed to initialize network...")
+                with lock:
+                    lock.notify()
+            with lock:
+                self.procs[i] = DhtNetworkSubProcess('node'+str(i), cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+                self.procs[i].sendPing(done_cb=dcb)
+                lock.wait()
        else:
            raise Exception('First create bootstrap.')

@@ -157,11 +168,16 @@ if __name__ == '__main__':
    featureArgs = parser.add_mutually_exclusive_group(required=True)
    featureArgs.add_argument('--performance', action='store_true', default=False,
            help='Launches performance benchmark test. Available args for "-t" are: gets.')
+    featureArgs.add_argument('--pht', action='store_true', default=False,
+            help='Launches PHT benchmark test. '\
+                    'Available args for "-t" are: insert. '\
+                    'Timer available by adding "timer" to "-o" args'\
+                    'Use "-m" option for fixing number of keys to create during the test.')
    featureArgs.add_argument('--data-persistence', action='store_true', default=0,
            help='Launches data persistence benchmark test. '\
                 'Available args for "-t" are: delete, replace, mult_time. '\
                 'Available args for "-o" are : dump_str_log, keep_alive, trigger, traffic_plot, op_plot. '\
-                 'Use "-m" to specify the number of producers on the DHT.'\
+                 'Use "-m" to specify the number of producers on the DHT. '\
                 'Use "-e" to specify the number of values to put on the DHT.')

    args = parser.parse_args()
@@ -206,6 +222,10 @@ if __name__ == '__main__':
            PerformanceTest(args.test, wb, test_opt).run()
        elif args.data_persistence:
            PersistenceTest(args.test, wb, test_opt).run()
+        elif args.pht:
+            if args.m:
+                test_opt.update({ 'num_keys' : args.m })
+            PhtTest(args.test, wb, test_opt).run()

    except Exception as e:
        print(e)
@@ -25,11 +25,13 @@ import time
 import threading
 import queue
 import re
+import traceback

 import ipaddress
 import netifaces
 import numpy as np
 from pyroute2.netns.process.proxy import NSPopen
+import msgpack

 from opendht import *

@@ -37,8 +39,6 @@ from opendht import *
 # useful functions
 b_space_join = lambda *l: b' '.join(map(bytes, l))

-# TODO: find where token "notifyend" gets printed... Or switch to MSGPACK for
-# serialisation of packets between both processes.
 class DhtNetworkSubProcess(NSPopen):
    """
    Handles communication with DhtNetwork sub process.
@@ -47,20 +47,21 @@ class DhtNetworkSubProcess(NSPopen):
    process' stdout until it finds 'DhtNetworkSubProcess.NOTIFY_TOKEN' token,
    therefor, waits for the sub process to spawn.
    """
+    # Packet types
+    REQUEST = 'DhtNetworkSubProcess.request'
+    ANSWER = 'DhtNetworkSubProcess.answer'
+    OUT = 'DhtNetworkSubProcess.out'
+
    # requests
-    NODE_PUT_REQ              = b"np"
-    NEW_NODE_REQ              = b"nn"
-    REMOVE_NODE_REQ           = b"rn"
-    SHUTDOWN_NODE_REQ         = b"sdn"
-    SHUTDOWN_REPLACE_NODE_REQ = b'sdrn'
-    SHUTDOWN_CLUSTER_REQ      = b"sdc"
-    DUMP_STORAGE_REQ          = b"strl"
-    MESSAGE_STATS             = b"gms"
-
-
-    # tokens
-    NOTIFY_TOKEN     = 'notify'
-    NOTIFY_END_TOKEN = 'notifyend'
+    PING_REQ                  = "p"
+    NODE_PUT_REQ              = "np"   # "np <hash> <value>"
+    NEW_NODE_REQ              = "nn"   # "nn"
+    REMOVE_NODE_REQ           = "rn"   # "rn <id0>[ <id1>[ id2[...]]]"
+    SHUTDOWN_NODE_REQ         = "sdn"  # "sdn <id0>[ <id1>[ id2[...]]]"
+    SHUTDOWN_REPLACE_NODE_REQ = "sdrn" # "sdn <id0>[ <id1>[ id2[...]]]"
+    SHUTDOWN_CLUSTER_REQ      = "sdc"  # "sdc"
+    DUMP_STORAGE_REQ          = "strl" # "strl"
+    MESSAGE_STATS             = "gms"  # "gms"

    def __init__(self, ns, cmd, quit=False, **kwargs):
        super(DhtNetworkSubProcess, self).__init__(ns, cmd, **kwargs)
@@ -70,11 +71,11 @@ class DhtNetworkSubProcess(NSPopen):
        self._quit = quit
        self._lock = threading.Condition()
        self._in_queue = queue.Queue()
-        self._out_queue = queue.Queue()
+        self._callbacks = {}
+        self._tid = 0

        # starting thread
        self._thread = threading.Thread(target=self._communicate)
-        self._thread.daemon = True
        self._thread.start()

    def __repr__(self):
@@ -92,42 +93,38 @@ class DhtNetworkSubProcess(NSPopen):
        """
        Communication thread. This reads and writes to the sub process.
        """
-        ENCODING = 'utf-8'
        sleep_time = 0.1
-        stdin_line, stdout_line = '', ''
-
-        # first read of process living. Expecting NOTIFY_TOKEN
-        while DhtNetworkSubProcess.NOTIFY_TOKEN not in stdout_line:
-            stdout_line = self.stdout.readline().decode()
-            time.sleep(sleep_time)
-
-        with self._lock:
-            self._out_queue.put(stdout_line)

        while not self._quit:
            with self._lock:
                try:
-                    stdin_line = self._in_queue.get_nowait()
+                    packet = self._in_queue.get_nowait()

                    # sending data to sub process
-                    self.stdin.write(stdin_line if isinstance(stdin_line, bytes) else
-                            bytes(str(stdin_line), encoding=ENCODING))
+                    self.stdin.write(packet)
                    self.stdin.flush()
                except queue.Empty:
-                    #waiting for next stdin req to send
-                    self._lock.wait(timeout=sleep_time)
+                    pass

-            # reading response from sub process
-            for stdout_line in iter(self.stdout.readline, b''):
-                stdout_line = stdout_line.decode().replace('\n', '')
-                if stdout_line:
-                    with self._lock:
-                        self._out_queue.put(stdout_line)
+                # reading from sub process
+                out_string = ''
+                for p in msgpack.Unpacker(self.stdout):
+                    if isinstance(p, dict):
+                        self._process_packet(p)
+                    else:
+                        # Some non-msgpack data could slip into the stream. We
+                        # have to treat those as characters.
+                        out_string += chr(p)
+                if out_string:
+                    print(out_string)
+
+                #waiting for next stdin req to send
+                self._lock.wait(timeout=sleep_time)

        with self._lock:
            self._lock.notify()

-    def stop_communicating(self):
+    def _stop_communicating(self):
        """
        Stops the I/O thread from communicating with the subprocess.
        """
@@ -142,12 +139,12 @@ class DhtNetworkSubProcess(NSPopen):
        Notifies thread and sub process to terminate. This is blocking call
        until the sub process finishes.
        """
-        self.stop_communicating()
+        self._stop_communicating()
        self.send_signal(signal.SIGINT);
        self.wait()
        self.release()

-    def send(self, msg):
+    def _send(self, msg):
        """
        Send data to sub process.
        """
@@ -155,83 +152,76 @@ class DhtNetworkSubProcess(NSPopen):
            self._in_queue.put(msg)
            self._lock.notify()

-    def getline(self):
+    def _process_packet(self, p):
        """
-        Read line from sub process.
-
-        @return:  A line on sub process' stdout.
-        @rtype :  str
+        Process msgpack packet received from
        """
-        line = ''
-        with self._lock:
-            try:
-                line = self._out_queue.get_nowait()
-            except queue.Empty:
-                pass
-        return line
+        if not b'tid' in p:
+            DhtNetwork.Log.err('Bad packet...')
+        try:
+            self._callbacks[p[b'tid']](p)
+        except KeyError:
+            DhtNetwork.Log.err('Unknown tid...')

-    def getlinesUntilNotify(self, answer_cb=None):
-        """
-        Reads the stdout queue until a proper notification is given by the sub
-        process.

-        @param answer_cb: Callback to call when an answer is given after notify.
-                          The function takes a list of strings as argument.
-        @type  answer_cb:  function
-        """
-        notified = False
-        answer = []
-        while True:
-            out = self.getline()
-            if out.split(' ')[0] == DhtNetworkSubProcess.NOTIFY_TOKEN:
-                notified = True
-            elif notified and out.split(' ')[0] == DhtNetworkSubProcess.NOTIFY_END_TOKEN:
-                if answer_cb:
-                    answer_cb(answer)
-                break
-            elif notified:
-                answer.append(out)
-            elif out:
-                yield out
-            else:
-                time.sleep(0.1)
-
-    def _sendRequest(self, request, answer_cb=None):
+    def _sendRequest(self, request, tid, done_cb):
        """
        Sends a request to the sub network and wait for output.

        @param request: The serialized request.
-        @type  request: bytes
+        @type  request: Msgpack object
        """
-        self.send(request + b'\n')
-        for line in self.getlinesUntilNotify(answer_cb=answer_cb):
-            DhtNetwork.log(line)
+        self._callbacks[tid] = done_cb
+        self._send(request)

-    def sendGetMessageStats(self):
+    def sendPing(self, done_cb=None):
+        """Sends a ping request to the DhtNetworkSubProcess.
+
+        @param done_cb: The callback to be executed when we get a response. This
+                        function takes a boolean "success" as parameter.
+        @type  done_cb: Function
+        """
+        self._tid += 1
+        def dcb(packet):
+            try:
+                done_cb(packet[b'success'])
+            except KeyError:
+                done_cb(False)
+        self._sendRequest(msgpack.packb({
+            DhtNetworkSubProcess.REQUEST : True,
+            'tid' : self._tid,
+            'req' : DhtNetworkSubProcess.PING_REQ
+        }), self._tid, dcb)
+
+    def sendGetMessageStats(self, done_cb=None):
        """
        Sends DhtNetwork sub process statistics request about nodes messages
        sent.

+        @param done_cb: A function taking as parameter the returned list of
+                        stats.
+        @type  done_cb: function
+
        @return: A list [num_nodes, ping, find, get, put, listen].
        @rtype : list
        """
-        stats = []
-        def cb(answer):
-            """
-            Callback fed to getlinesUntilNotify made to recover answer from the
-            DhtNetwork sub process.
+        self._tid += 1
+        def dcb(packet):
+            nonlocal done_cb
+            if not done_cb:
+                return
+            try:
+                stats = packet[b'stats']
+                done_cb([] if not isinstance(stats, list) else done_cb(stats))
+            except KeyError:
+                done_cb([])
+        self._sendRequest(msgpack.packb({
+            DhtNetworkSubProcess.REQUEST : True,
+            'tid' : self._tid,
+            'req' : DhtNetworkSubProcess.MESSAGE_STATS
+        }), self._tid, dcb)

-            @param answer: the list of lines answered by the sub process.
-            @type  answer: function
-            """
-            nonlocal stats
-            if answer:
-                stats = [int(v) for v in re.findall("[0-9]+", answer.pop())]
-
-        self._sendRequest(DhtNetworkSubProcess.MESSAGE_STATS, answer_cb=cb)
-        return stats
-
-    def sendNodePutRequest(self, _hash, value):
+    def sendClusterPutRequest(self, _hash, value, done_cb=None):
        """
        Sends a put operation request.

@@ -239,40 +229,89 @@ class DhtNetworkSubProcess(NSPopen):
        @type  _hash: bytes.
        @param value: the value.
        @type  value: bytes.
+        @param done_cb: A function taking as parameter a boolean "success".
+        @type  done_cb: function
        """
-        self._sendRequest(b_space_join(DhtNetworkSubProcess.NODE_PUT_REQ, _hash,
-            value))
+        self._tid += 1
+        def dcb(packet):
+            nonlocal done_cb
+            if not done_cb:
+                return
+            try:
+                done_cb(packet[b'success'])
+            except KeyError:
+                done_cb(False)
+        self._sendRequest(msgpack.packb({
+            DhtNetworkSubProcess.REQUEST : True,
+            'tid' : self._tid,
+            'req'   : DhtNetworkSubProcess.NODE_PUT_REQ,
+            'hash'  : _hash,
+            'value' : value
+        }), self._tid, dcb)

-    def sendNodesRequest(self, request, ids=b''):
+    def sendClusterRequest(self, request, ids=[], done_cb=None):
        """
        Send request to a list of nodes or the whole cluster.

        @param request: The request. Possible values are:
-            DhtNetworkSubProcess.NODE_PUT_REQ
            DhtNetworkSubProcess.REMOVE_NODE_REQ
            DhtNetworkSubProcess.SHUTDOWN_NODE_REQ
            DhtNetworkSubProcess.SHUTDOWN_REPLACE_NODE_REQ
            DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ
            DhtNetworkSubProcess.DUMP_STORAGE_REQ
-            DhtNetworkSubProcess.MESSAGE_STATS
        @type request: bytes
        @param ids: The list of ids concerned by the request.
        @type  ids: list
        """
-        self._sendRequest(b_space_join(request, b_space_join(*ids)))
+        self._tid += 1
+        def dcb(packet):
+            nonlocal done_cb
+            if not done_cb:
+                return
+            try:
+                done_cb(packet[b'success'])
+            except KeyError:
+                done_cb(False)
+        self._sendRequest(msgpack.packb({
+            DhtNetworkSubProcess.REQUEST : True,
+            'tid' : self._tid,
+            'req' : request,
+            'ids' : ids
+        }), self._tid, dcb)
+

 class DhtNetwork(object):
    nodes = []

-    @staticmethod
-    def log(*to_print):
+    class Log(object):
        BOLD   = "\033[1m"
        NORMAL = "\033[0m"
-        print('%s[DhtNetwork-%s]%s' % (BOLD, DhtNetwork.iface, NORMAL), ':' , *to_print, file=sys.stderr)
+        WHITE  = "\033[97m"
+        RED    = "\033[31m"
+        YELLOW = "\033[33m"
+
+        @staticmethod
+        def _log_with_color(*to_print, color=None):
+            color = color if color else DhtNetwork.Log.WHITE
+            print('%s%s[DhtNetwork-%s]%s%s' %
+                    (DhtNetwork.Log.BOLD, color, DhtNetwork.iface, DhtNetwork.Log.NORMAL, color),
+                    *to_print, DhtNetwork.Log.NORMAL, file=sys.stderr)
+
+        @staticmethod
+        def log(*to_print):
+            DhtNetwork.Log._log_with_color(*to_print, color=DhtNetwork.Log.WHITE)
+
+        @staticmethod
+        def warn(*to_print):
+            DhtNetwork.Log._log_with_color(*to_print, color=DhtNetwork.Log.YELLOW)
+
+        @staticmethod
+        def err(*to_print):
+            DhtNetwork.Log._log_with_color(*to_print, color=DhtNetwork.Log.RED)

    @staticmethod
    def run_node(ip4, ip6, p, bootstrap=[], is_bootstrap=False):
-        DhtNetwork.log("run_node", ip4, ip6, p, bootstrap)
+        DhtNetwork.Log.log("run_node", ip4, ip6, p, bootstrap)
        n = DhtRunner()
        n.run(ipv4=ip4 if ip4 else "", ipv6=ip6 if ip6 else "", port=p, is_bootstrap=is_bootstrap)
        for b in bootstrap:
@@ -297,7 +336,7 @@ class DhtNetwork(object):
        self.ip6 = ip6 if ip6 else ips[1]
        self.bootstrap = bootstrap
        if first_bootstrap:
-            DhtNetwork.log("Starting bootstrap node")
+            DhtNetwork.Log.log("Starting bootstrap node")
            self.nodes.append(DhtNetwork.run_node(self.ip4, self.ip6, self.port, self.bootstrap, is_bootstrap=True))
            self.bootstrap = [(self.ip4, str(self.port))]
            self.port += 1
@@ -309,6 +348,9 @@ class DhtNetwork(object):
        return self.nodes[0][1]

    def get(self, i=None):
+        if not self.nodes:
+            return None
+
        if i is None:
            l = list(self.nodes)
            random.shuffle(l)
@@ -321,14 +363,13 @@ class DhtNetwork(object):
            for n in self.nodes:
                if n[1].getNodeId() == id:
                    return n
-        else:
-            return None
+        return None

    def launch_node(self):
        n = DhtNetwork.run_node(self.ip4, self.ip6, self.port, self.bootstrap)
        self.nodes.append(n)
        if not self.bootstrap:
-            DhtNetwork.log("Using fallback bootstrap", self.ip4, self.port)
+            DhtNetwork.Log.log("Using fallback bootstrap", self.ip4, self.port)
            self.bootstrap = [(self.ip4, str(self.port))]
        self.port += 1
        return n
@@ -346,25 +387,25 @@ class DhtNetwork(object):
        lock = threading.Condition()
        def shutdown_cb():
            nonlocal lock
-            DhtNetwork.log('Done.')
+            DhtNetwork.Log.log('Done.')
            with lock:
                lock.notify()

        if not self.nodes:
            return
-        elif id is not None:
+        elif id:
            n = self.getNodeInfoById(id)
            if n:
                if shutdown:
                    with lock:
-                        DhtNetwork.log('Waiting for node to shutdown... ')
+                        DhtNetwork.Log.log('Waiting for node to shutdown... ')
                        n[1].shutdown(shutdown_cb)
                        lock.wait()
                    if last_msg_stats:
                        last_msg_stats.append(self.getMessageStats())
                n[1].join()
                self.nodes.remove(n)
-                DhtNetwork.log(id, 'deleted !')
+                DhtNetwork.Log.log(id, 'deleted !')
                return True
            else:
                return False
@@ -385,11 +426,11 @@ class DhtNetwork(object):
        if n == l:
            return
        if n > l:
-            DhtNetwork.log("Launching", n-l, "nodes", self.ip4, self.ip6)
+            DhtNetwork.Log.log("Launching", n-l, "nodes", self.ip4, self.ip6)
            for i in range(l, n):
                self.launch_node()
        else:
-            DhtNetwork.log("Ending", l-n, "nodes", self.ip4, self.ip6)
+            DhtNetwork.Log.log("Ending", l-n, "nodes", self.ip4, self.ip6)
            #random.shuffle(self.nodes)
            for i in range(n, l):
                self.end_node()
@@ -409,30 +450,39 @@ if __name__ == '__main__':
    lock = threading.Condition()
    quit = False

-    def notify_benchmark(answer=[]):
-        sys.stdout.write('%s\n' % DhtNetworkSubProcess.NOTIFY_TOKEN)
-        for line in answer:
-            sys.stdout.write(str(line)+'\n')
-        sys.stdout.write('%s\n' % DhtNetworkSubProcess.NOTIFY_END_TOKEN)
-        sys.stdout.flush()
+    def send_msgpack_packet(packet):
+        sys.stdout.buffer.write(packet)
+        sys.stdout.buffer.flush()

-    def listen_to_mother_nature(stdin, q):
+    def notify_benchmark(packet, success):
+        """Notifies the benchmark when an operation has been completed.
+
+        @param success: If the operation has been successful
+        @type  success: boolean
+        @param packet: The packet we are providing an answer for.
+        @type  packet: dict
+        """
+        send_msgpack_packet(msgpack.packb({
+            DhtNetworkSubProcess.ANSWER : True,
+            'tid'     : packet[b'tid'],
+            'success' : success
+        }))
+
+    def send_stats(packet, stats):
+        send_msgpack_packet(msgpack.packb({
+            DhtNetworkSubProcess.ANSWER : True,
+            'tid'   : packet[b'tid'],
+            'stats' : stats
+        }))
+
+    def listen_to_mother_nature(q):
        global quit
-
-        def parse_req(req):
-            split_req = req.split(' ')
-
-            op = split_req[0]
-            hashes = [this_hash.replace('\n', '').encode() for this_hash in split_req[1:]]
-
-            return (op, hashes)
-
        while not quit:
-            req = stdin.readline()
-            parsed_req = parse_req(req)
-            q.put(parsed_req)
-            with lock:
-                lock.notify()
+            for p in msgpack.Unpacker(sys.stdin.buffer.raw):
+                if isinstance(p, dict) and DhtNetworkSubProcess.REQUEST.encode() in p:
+                    with lock:
+                        q.put(p)
+                        lock.notify()

    def handler(signum, frame):
        global quit
@@ -466,48 +516,52 @@ if __name__ == '__main__':
        net.resize(args.node_num)

        q = queue.Queue()
-        t = threading.Thread(target=listen_to_mother_nature, args=(sys.stdin, q))
+        t = threading.Thread(target=listen_to_mother_nature, args=tuple([q]))
        t.daemon = True
        t.start()

-        notify_benchmark()
-
        msg_stats = []

        with lock:
            while not quit:
-                lock.wait()
                try:
-                    req,req_args = q.get_nowait()
+                    packet = q.get_nowait()
                except queue.Empty:
-                    pass
+                    lock.wait()
                else:
-                    NODE_PUT_REQ              = DhtNetworkSubProcess.NODE_PUT_REQ.decode()
-                    NEW_NODE_REQ              = DhtNetworkSubProcess.NEW_NODE_REQ.decode()
-                    REMOVE_NODE_REQ           = DhtNetworkSubProcess.REMOVE_NODE_REQ.decode()
-                    SHUTDOWN_NODE_REQ         = DhtNetworkSubProcess.SHUTDOWN_NODE_REQ.decode()
-                    SHUTDOWN_REPLACE_NODE_REQ = DhtNetworkSubProcess.SHUTDOWN_REPLACE_NODE_REQ.decode()
-                    SHUTDOWN_CLUSTER_REQ      = DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ.decode()
-                    DUMP_STORAGE_REQ          = DhtNetworkSubProcess.DUMP_STORAGE_REQ.decode()
-                    MESSAGE_STATS             = DhtNetworkSubProcess.MESSAGE_STATS.decode()
+                    NODE_PUT_REQ              = DhtNetworkSubProcess.NODE_PUT_REQ
+                    NEW_NODE_REQ              = DhtNetworkSubProcess.NEW_NODE_REQ
+                    REMOVE_NODE_REQ           = DhtNetworkSubProcess.REMOVE_NODE_REQ
+                    SHUTDOWN_NODE_REQ         = DhtNetworkSubProcess.SHUTDOWN_NODE_REQ
+                    SHUTDOWN_REPLACE_NODE_REQ = DhtNetworkSubProcess.SHUTDOWN_REPLACE_NODE_REQ
+                    SHUTDOWN_CLUSTER_REQ      = DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ
+                    DUMP_STORAGE_REQ          = DhtNetworkSubProcess.DUMP_STORAGE_REQ
+                    MESSAGE_STATS             = DhtNetworkSubProcess.MESSAGE_STATS

-                    if req in [SHUTDOWN_NODE_REQ,
-                               SHUTDOWN_REPLACE_NODE_REQ,
-                               REMOVE_NODE_REQ]:
-                        def delete_request(req, n):
+                    req = packet[b'req'].decode()
+                    success = True
+                    if req in [SHUTDOWN_NODE_REQ, SHUTDOWN_REPLACE_NODE_REQ, REMOVE_NODE_REQ]:
+                        def delete_request(req, nid):
                            global msg_stats
+                            if not nid:
+                                return
                            if req == SHUTDOWN_NODE_REQ:
-                                net.end_node(id=n, shutdown=True, last_msg_stats=msg_stats)
+                                net.end_node(id=nid, shutdown=True, last_msg_stats=msg_stats)
                            elif req == SHUTDOWN_REPLACE_NODE_REQ:
-                                net.replace_node(id=n, shutdown=True, last_msg_stats=msg_stats)
+                                net.replace_node(id=nid, shutdown=True, last_msg_stats=msg_stats)
                            elif req == REMOVE_NODE_REQ:
-                                net.end_node(id=n, last_msg_stats=msg_stats)
+                                net.end_node(id=nid, last_msg_stats=msg_stats)

-                        if len(req) > 0:
-                            for n in req_args:
-                                delete_request(req, n)
+                        nodes = packet[b'ids']
+                        if nodes:
+                            for nid in nodes:
+                                delete_request(req, nid)
                        else:
-                            delete_request(req, net.get().getNodeId())
+                            n = net.get()
+                            if n:
+                                delete_request(req, n.getNodeId())
+                            else:
+                                success = False
                    elif req == SHUTDOWN_CLUSTER_REQ:
                        for n in net.nodes:
                            net.end_node(id=n[2], shutdown=True, last_msg_stats=msg_stats)
@@ -515,20 +569,26 @@ if __name__ == '__main__':
                    elif req == NEW_NODE_REQ:
                        net.launch_node()
                    elif req == NODE_PUT_REQ:
-                        _hash, v = req_args[:2]
-                        net.get().put(InfoHash(_hash), Value(v))
-
+                        _hash = packet[b'hash']
+                        v = packet[b'value']
+                        n = net.get()
+                        if n:
+                            n.put(InfoHash(_hash), Value(v))
+                        else:
+                            success = False
                    elif req == DUMP_STORAGE_REQ:
-                        for n in [m[1] for m in net.nodes if m[1].getNodeId() in req_args]:
+                        hashes = packet[b'ids']
+                        for n in [m[1] for m in net.nodes if m[1].getNodeId() in hashes]:
                            net.log(n.getStorageLog())
-                    elif MESSAGE_STATS in req:
+                    elif req == MESSAGE_STATS:
                        stats = sum([np.array(x) for x in [net.getMessageStats()]+msg_stats])
-                        notify_benchmark(answer=[stats])
+                        send_stats(packet, [int(_) for _ in stats])
                        msg_stats.clear()
                        continue
-                    notify_benchmark()
+                    notify_benchmark(packet, success)
    except Exception as e:
-        DhtNetwork.log(e)
+        traceback.print_tb(e.__traceback__)
+        print(type(e).__name__+':', e, file=sys.stderr)
    finally:
        if net:
            net.resize(0)
@@ -3,6 +3,7 @@
 # Author(s): Adrien Béraud <adrien.beraud@savoirfairelinux.com>
 #            Simon Désaulniers <sim.desaulniers@gmail.com>

+import sys
 import os
 import threading
 import random
@@ -10,10 +11,16 @@ import string
 import time
 import subprocess
 import re
+import collections
+
+from matplotlib.ticker import FuncFormatter
+import math

 import numpy as np
 import matplotlib.pyplot as plt
-from matplotlib.ticker import FuncFormatter
+import networkx as nx
+from networkx.drawing.nx_agraph import graphviz_layout
+

 from opendht import *
 from dht.network import DhtNetwork, DhtNetworkSubProcess
@@ -27,8 +34,40 @@ bit_format = None
 Kbit_format = FuncFormatter(lambda x, pos: '%1.1f' % (x*1024**-1) + 'Kb')
 Mbit_format = FuncFormatter(lambda x, pos: '%1.1f' % (x*1024**-2) + 'Mb')

+def random_str_val(size=1024):
+    """Creates a random string value of specified size.
+
+    @param size:  Size, in bytes, of the value.
+    @type  size:  int
+
+    @return:  Random string value
+    @rtype :  str
+    """
+    return ''.join(random.choice(string.hexdigits) for _ in range(size))
+
+
 def random_hash():
-    return InfoHash(''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(40)).encode())
+    """Creates random InfoHash.
+    """
+    return InfoHash(random_str_val(size=40).encode())
+
+def timer(f, *args):
+    """
+    Start a timer which count time taken for execute function f
+
+    @param f : Function to time
+    @type  f : function
+
+    @param args : Arguments of the function f
+    @type  args : list
+
+    @rtype : timer
+    @return : Time taken by the function f
+    """
+    start = time.time()
+    f(*args)
+
+    return time.time() - start

 def reset_before_test(featureTestMethod):
    """
@@ -70,6 +109,21 @@ def display_plot(yvals, xvals=None, yformatter=None, display_time=3, **kwargs):
        plt.plot(yvals, **kwargs)
    plt.pause(display_time)

+def display_traffic_plot(ifname):
+    """Displays the traffic plot for a given interface name.
+
+    @param ifname:  Interface name.
+    @type  ifname:  string
+    """
+    ydata = []
+    xdata = []
+    # warning: infinite loop
+    interval = 2
+    for rate in iftop_traffic_data(ifname, interval=interval):
+        ydata.append(rate)
+        xdata.append((xdata[-1] if len(xdata) > 0 else 0) + interval)
+        display_plot(ydata, xvals=xdata, yformatter=Kbit_format, color='blue')
+
 def iftop_traffic_data(ifname, interval=2, rate_type='send_receive'):
    """
    Generator (yields data) function collecting traffic data from iftop
@@ -125,13 +179,13 @@ def iftop_traffic_data(ifname, interval=2, rate_type='send_receive'):
 ###########

 class FeatureTest(object):
-    done = 0
-    lock = None
-
    """
    This is a base test.
    """

+    done = 0
+    lock = None
+
    def __init__(self, test, workbench):
        """
        @param test: The test string indicating the test to run. This string is
@@ -143,6 +197,7 @@ class FeatureTest(object):
        """
        self._test = test
        self._workbench = workbench
+        self._bootstrap = self._workbench.get_bootstrap()

    def _reset(self):
        """
@@ -156,7 +211,155 @@ class FeatureTest(object):
    def run(self):
        raise NotImplementedError('This method must be implemented.')

+##################################
+#               PHT              #
+##################################

+class PhtTest(FeatureTest):
+    """TODO
+    """
+
+    indexEntries = None
+    prefix       = None
+    key          = None
+
+    def __init__(self, test, workbench, opts):
+        """
+        @param test: is one of the following:
+                     - 'insert': indexes a considerable amount of data in
+                       the PHT structure.
+                       TODO
+        @type  test: string
+
+        @param opts: Dictionnary containing options for the test. Allowed
+                     options are:
+                     - 'num_keys': this specifies the number of keys to insert
+                                   in the PHT during the test.
+        @type  opts: dict
+        """
+        super(PhtTest, self).__init__(test, workbench)
+        self._num_keys = opts['num_keys'] if 'num_keys' in opts else 32
+        self._timer = True if 'timer' in opts else False
+
+    def _reset(self):
+        super(PhtTest, self)._reset()
+        PhtTest.indexEntries = []
+
+    @staticmethod
+    def lookupCb(vals, prefix):
+        PhtTest.indexEntries = list(vals)
+        PhtTest.prefix = prefix.decode()
+        DhtNetwork.log('Index name: <todo>')
+        DhtNetwork.log('Leaf prefix:', prefix)
+        for v in vals:
+            DhtNetwork.log('[ENTRY]:', v)
+
+    @staticmethod
+    def lookupDoneCb(ok):
+        DhtNetwork.log('[LOOKUP]:', PhtTest.key, "--", "success!" if ok else "Fail...")
+        with FeatureTest.lock:
+            FeatureTest.lock.notify()
+
+    @staticmethod
+    def insertDoneCb(ok):
+        DhtNetwork.log('[INSERT]:', PhtTest.key, "--", "success!" if ok else "Fail...")
+        with FeatureTest.lock:
+            FeatureTest.lock.notify()
+
+    @staticmethod
+    def drawTrie(trie_dict):
+        """
+        Draws the trie structure of the PHT from dictionnary.
+
+        @param trie_dict: Dictionnary of index entries (prefix -> entry).
+        @type  trie_dict: dict
+        """
+        prefixes = list(trie_dict.keys())
+        if len(prefixes) == 0:
+            return
+
+        edges = list([])
+        for prefix in prefixes:
+            for i in range(-1, len(prefix)-1):
+                u = prefix[:i+1]
+                x = ("." if i == -1 else u, u+"0")
+                y = ("." if i == -1 else u, u+"1")
+                if x not in edges:
+                    edges.append(x)
+                if y not in edges:
+                    edges.append(y)
+
+        # TODO: use a binary tree position layout...
+        #   UPDATE : In a better way [change lib]
+        G = nx.Graph(sorted(edges, key=lambda x: len(x[0])))
+        plt.title("PHT: Tree")
+        pos=graphviz_layout(G,prog='dot')
+        nx.draw(G, pos, with_labels=True, node_color='white')
+        plt.show()
+
+    def run(self):
+        try:
+            if self._test == 'insert':
+                self._insertTest()
+        except Exception as e:
+            print(e)
+        finally:
+            self._bootstrap.resize(1)
+
+    ###########
+    #  Tests  #
+    ###########
+
+    @reset_before_test
+    def _insertTest(self):
+        """TODO: Docstring for _massIndexTest.
+        """
+        bootstrap = self._bootstrap
+        bootstrap.resize(2)
+        dht = bootstrap.get(1)
+
+        NUM_DIG  = max(math.log(self._num_keys, 2)/4, 5) # at least 5 digit keys.
+        keyspec = collections.OrderedDict([('foo', NUM_DIG)])
+        pht = Pht(b'foo_index', keyspec, dht)
+
+        DhtNetwork.log('PHT has',
+                       pht.MAX_NODE_ENTRY_COUNT,
+                       'node'+ ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''),
+                       'per leaf bucket.')
+        keys = [{
+            [_ for _ in keyspec.keys()][0] :
+            ''.join(random.SystemRandom().choice(string.hexdigits)
+                for _ in range(NUM_DIG)).encode()
+            } for n in range(self._num_keys)]
+        all_entries = {}
+
+        # Index all entries.
+        for key in keys:
+            PhtTest.key = key
+            with FeatureTest.lock:
+                time_taken = timer(pht.insert, key, IndexValue(random_hash()), PhtTest.insertDoneCb)
+                if self._timer:
+                    DhtNetwork.log('This insert step took : ', time_taken, 'second')
+                FeatureTest.lock.wait()
+
+        time.sleep(1)
+
+        # Recover entries now that the trie is complete.
+        for key in keys:
+            PhtTest.key = key
+            with FeatureTest.lock:
+                time_taken = timer(pht.lookup, key, PhtTest.lookupCb, PhtTest.lookupDoneCb)
+                if self._timer:
+                    DhtNetwork.log('This lookup step took : ', time_taken, 'second')
+                FeatureTest.lock.wait()
+
+            all_entries[PhtTest.prefix] = [e.__str__()
+                                           for e in PhtTest.indexEntries]
+
+        for p in all_entries.keys():
+            DhtNetwork.log('All entries under prefix', p, ':')
+            DhtNetwork.log(all_entries[p])
+        PhtTest.drawTrie(all_entries)

 ##################################
 #               DHT              #
@@ -164,7 +367,7 @@ class FeatureTest(object):

 class DhtFeatureTest(FeatureTest):
    """
-    This is base test. A method run() implementation is required.
+    This is a base dht test.
    """
    #static variables used by class callbacks
    successfullTransfer = lambda lv,fv: len(lv) == len(fv)
@@ -173,7 +376,6 @@ class DhtFeatureTest(FeatureTest):

    def __init__(self, test, workbench):
        super(DhtFeatureTest, self).__init__(test, workbench)
-        self.bootstrap = self._workbench.get_bootstrap()

    def _reset(self):
        super(DhtFeatureTest, self)._reset()
@@ -183,15 +385,15 @@ class DhtFeatureTest(FeatureTest):
    @staticmethod
    def getcb(value):
        vstr = value.__str__()[:100]
-        DhtNetwork.log('[GET]: %s' % vstr + ("..." if len(vstr) > 100 else ""))
+        DhtNetwork.Log.log('[GET]: %s' % vstr + ("..." if len(vstr) > 100 else ""))
        DhtFeatureTest.foreignValues.append(value)
        return True

    @staticmethod
    def putDoneCb(ok, nodes):
-        if not ok:
-            DhtNetwork.log("[PUT]: failed!")
        with FeatureTest.lock:
+            if not ok:
+                DhtNetwork.Log.log("[PUT]: failed!")
            FeatureTest.done -= 1
            FeatureTest.lock.notify()

@@ -199,7 +401,7 @@ class DhtFeatureTest(FeatureTest):
    def getDoneCb(ok, nodes):
        with FeatureTest.lock:
            if not ok:
-                DhtNetwork.log("[GET]: failed!")
+                DhtNetwork.Log.log("[GET]: failed!")
            else:
                for node in nodes:
                    if not node.getNode().isExpired():
@@ -210,10 +412,10 @@ class DhtFeatureTest(FeatureTest):
    def _dhtPut(self, producer, _hash, *values):
        with FeatureTest.lock:
            for val in values:
-                    vstr = val.__str__()[:100]
-                    DhtNetwork.log('[PUT]:', _hash.toString(), '->', vstr + ("..." if len(vstr) > 100 else ""))
-                    FeatureTest.done += 1
-                    producer.put(_hash, val, DhtFeatureTest.putDoneCb)
+                vstr = val.__str__()[:100]
+                DhtNetwork.Log.log('[PUT]:', _hash.toString(), '->', vstr + ("..." if len(vstr) > 100 else ""))
+                FeatureTest.done += 1
+                producer.put(_hash, val, DhtFeatureTest.putDoneCb)
            while FeatureTest.done > 0:
                FeatureTest.lock.wait()

@@ -222,7 +424,7 @@ class DhtFeatureTest(FeatureTest):
        DhtFeatureTest.foreignNodes = []
        with FeatureTest.lock:
            FeatureTest.done += 1
-            DhtNetwork.log('[GET]:', _hash.toString())
+            DhtNetwork.Log.log('[GET]:', _hash.toString())
            consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb)
            while FeatureTest.done > 0:
                FeatureTest.lock.wait()
@@ -234,7 +436,6 @@ class DhtFeatureTest(FeatureTest):
                for n in DhtFeatureTest.foreignNodes:
                    nodes.add(n)

-
 class PersistenceTest(DhtFeatureTest):
    """
    This tests persistence of data on the network.
@@ -292,8 +493,8 @@ class PersistenceTest(DhtFeatureTest):
            config.setNodeId(InfoHash(_hash_str.encode()))
            n = DhtRunner()
            n.run(config=config)
-            n.bootstrap(self.bootstrap.ip4,
-                        str(self.bootstrap.port))
+            n.bootstrap(self._bootstrap.ip4,
+                        str(self._bootstrap.port))
            DhtNetwork.log('Node','['+_hash_str+']',
                           'started around', _hash.toString().decode()
                           if n.isRunning() else
@@ -302,24 +503,24 @@ class PersistenceTest(DhtFeatureTest):
            trigger_nodes.append(n)

    def _result(self, local_values, new_nodes):
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap
        if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
-            DhtNetwork.log('[GET]: Only %s on %s values persisted.' %
+            DhtNetwork.Log.log('[GET]: Only %s on %s values persisted.' %
                    (len(DhtFeatureTest.foreignValues), len(local_values)))
        else:
-            DhtNetwork.log('[GET]: All values successfully persisted.')
+            DhtNetwork.Log.log('[GET]: All values successfully persisted.')
        if DhtFeatureTest.foreignValues:
            if new_nodes:
-                DhtNetwork.log('Values are newly found on:')
+                DhtNetwork.Log.log('Values are newly found on:')
                for node in new_nodes:
-                    DhtNetwork.log(node)
+                    DhtNetwork.Log.log(node)
                if self._dump_storage:
-                    DhtNetwork.log('Dumping all storage log from '\
+                    DhtNetwork.Log.log('Dumping all storage log from '\
                                  'hosting nodes.')
                    for proc in self._workbench.procs:
-                        proc.sendNodesRequest(DhtNetworkSubProcess.DUMP_STORAGE_REQ, DhtFeatureTest.foreignNodes)
+                        proc.sendClusterRequest(DhtNetworkSubProcess.DUMP_STORAGE_REQ, DhtFeatureTest.foreignNodes)
            else:
-                DhtNetwork.log("Values didn't reach new hosting nodes after shutdown.")
+                DhtNetwork.Log.log("Values didn't reach new hosting nodes after shutdown.")

    def run(self):
        try:
@@ -334,13 +535,14 @@ class PersistenceTest(DhtFeatureTest):
            else:
                raise NameError("This test is not defined '" + self._test + "'")
        except Exception as e:
-            print(e)
+            traceback.print_tb(e.__traceback__)
+            print(type(e).__name__+':', e, file=sys.stderr)
        finally:
            if self._traffic_plot or self._op_plot:
                plot_fname = "traffic-plot"
                print('plot saved to', plot_fname)
                plt.savefig(plot_fname)
-            self.bootstrap.resize(1)
+            self._bootstrap.resize(1)

    ###########
    #  Tests  #
@@ -353,71 +555,89 @@ class PersistenceTest(DhtFeatureTest):
        """
        trigger_nodes = []
        wb = self._workbench
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap
        # Value representing an ICE packet. Each ICE packet is around 1KB.
        VALUE_SIZE = 1024
-        NUM_VALUES = self._num_values/wb.node_num if self._num_values else 5
-        nr_values = NUM_VALUES * wb.node_num
-        nr_nodes = wb.node_num
-        nr_nodes_cv = threading.Condition()
+        num_values_per_hash = self._num_values/wb.node_num if self._num_values else 5

-        values = [b''.join(random.choice(string.hexdigits).encode() for _ in range(VALUE_SIZE)) for __ in range(NUM_VALUES)]
+        # nodes and values counters
+        total_nr_values = 0
+        nr_nodes = wb.node_num
+        op_cv = threading.Condition()
+
+        # values string in string format. Used for sending cluster request.
        hashes = [random_hash() for _ in range(wb.node_num)]

-        # initial set of values
-        i = 0
-        for h in hashes:
-           self._dhtPut(bootstrap.front(), h, *[Value(v) for v in values])
-           print("at: ", i)
-           i += 1
-
-        def normalBehavior(do, t, log=None):
-            nonlocal nr_values
+        def normalBehavior(do, t):
+            nonlocal total_nr_values, op_cv
            while True:
-                do()
-                time.sleep(random.choice(range(t)))
+                with op_cv:
+                    do()
+                time.sleep(random.uniform(0.0, float(t)))

        def putRequest():
-            nonlocal hashes, values, nr_values
-            nr_values += 1
-            DhtNetwork.log("Random value put on the DHT.", "(now "+ str(nr_values)+" values on the dht)")
-            random.choice(wb.procs).sendNodePutRequest(random.choice(hashes).toString(), random.choice(values))
-        puts = threading.Thread(target=normalBehavior, args=(putRequest, 30))
+            nonlocal hashes, VALUE_SIZE, total_nr_values
+            lock = threading.Condition()
+            def dcb(success):
+                nonlocal total_nr_values, lock
+                if success:
+                    total_nr_values += 1
+                    DhtNetwork.Log.log("INFO: "+ str(total_nr_values)+" values put on the dht since begining")
+                with lock:
+                    lock.notify()
+            with lock:
+                DhtNetwork.Log.warn("Random value put on the DHT...")
+                random.choice(wb.procs).sendClusterPutRequest(random.choice(hashes).toString(),
+                                                              random_str_val(size=VALUE_SIZE).encode(),
+                                                              done_cb=dcb)
+                lock.wait()
+
+        puts = threading.Thread(target=normalBehavior, args=(putRequest, 30.0/wb.node_num))
        puts.daemon = True
        puts.start()
+
        def newNodeRequest():
            nonlocal nr_nodes
-            with nr_nodes_cv:
+            lock = threading.Condition()
+            def dcb(success):
+                nonlocal nr_nodes, lock
                nr_nodes += 1
-                DhtNetwork.log("Node joining the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)")
-                nr_nodes_cv.notify()
-            random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.NEW_NODE_REQ)
-        connections = threading.Thread(target=normalBehavior, args=(newNodeRequest, 1*60))
+                DhtNetwork.Log.log("INFO: now "+str(nr_nodes)+" nodes on the dht")
+                with lock:
+                    lock.notify()
+            with lock:
+                DhtNetwork.Log.warn("Node joining...")
+                random.choice(wb.procs).sendClusterRequest(DhtNetworkSubProcess.NEW_NODE_REQ, done_cb=dcb)
+                lock.wait()
+
+        connections = threading.Thread(target=normalBehavior, args=(newNodeRequest, 1*50.0/wb.node_num))
        connections.daemon = True
        connections.start()
+
        def shutdownNodeRequest():
            nonlocal nr_nodes
-            with nr_nodes_cv:
-                nr_nodes -= 1
-                DhtNetwork.log("Node quitting the DHT.", "(now "+str(nr_nodes)+" nodes on the dht)")
-                nr_nodes_cv.notify()
-            random.choice(wb.procs).sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_NODE_REQ)
-        shutdowns = threading.Thread(target=normalBehavior, args=(shutdownNodeRequest, 1*60))
+            lock = threading.Condition()
+            def dcb(success):
+                nonlocal nr_nodes, lock
+                if success:
+                    nr_nodes -= 1
+                    DhtNetwork.Log.log("INFO: now "+str(nr_nodes)+" nodes on the dht")
+                else:
+                    DhtNetwork.Log.err("Oops.. No node to shutodwn.")
+
+                with lock:
+                    lock.notify()
+            with lock:
+                DhtNetwork.Log.warn("Node shutting down...")
+                random.choice(wb.procs).sendClusterRequest(DhtNetworkSubProcess.SHUTDOWN_NODE_REQ, done_cb=dcb)
+                lock.wait()
+
+        shutdowns = threading.Thread(target=normalBehavior, args=(shutdownNodeRequest, 1*60.0/wb.node_num))
        shutdowns.daemon = True
        shutdowns.start()

-        for h in hashes:
-           self._trigger_dp(trigger_nodes, h)
-
        if self._traffic_plot:
-            ydata = []
-            xdata = []
-            # warning: infinite loop
-            interval = 2
-            for rate in iftop_traffic_data("br"+wb.ifname, interval=interval):
-                ydata.append(rate)
-                xdata.append((xdata[-1] if len(xdata) > 0 else 0) + interval)
-                display_plot(ydata, xvals=xdata, yformatter=Kbit_format, color='blue')
+            display_traffic_plot('br'+wb.ifname)
        else:
            # blocks in matplotlib thread
            while True:
@@ -430,7 +650,7 @@ class PersistenceTest(DhtFeatureTest):
        It uses Dht shutdown call from the API to gracefuly finish the nodes one
        after the other.
        """
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap

        ops_count = []

@@ -447,36 +667,54 @@ class PersistenceTest(DhtFeatureTest):
        self._dhtGet(consumer, myhash)
        if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
            if DhtFeatureTest.foreignValues:
-                DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
+                DhtNetwork.Log.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                        len(local_values), ' values successfully put.')
            else:
-                DhtNetwork.log('[GET]: 0 values successfully put')
+                DhtNetwork.Log.log('[GET]: 0 values successfully put')


        if DhtFeatureTest.foreignValues and DhtFeatureTest.foreignNodes:
-            DhtNetwork.log('Values are found on :')
+            DhtNetwork.Log.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
-                DhtNetwork.log(node)
+                DhtNetwork.Log.log(node)

            for _ in range(max(1, int(self._workbench.node_num/32))):
-                DhtNetwork.log('Removing all nodes hosting target values...')
+                DhtNetwork.Log.log('Removing all nodes hosting target values...')
                cluster_ops_count = 0
                for proc in self._workbench.procs:
-                    DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
-                    proc.sendNodesRequest(
+                    DhtNetwork.Log.log('[REMOVE]: sending shutdown request to', proc)
+                    lock = threading.Condition()
+                    def dcb(success):
+                        nonlocal lock
+                        if not success:
+                            DhtNetwork.Log.err("Failed to shutdown.")
+                        with lock:
+                            lock.notify()
+
+                    with lock:
+                        proc.sendClusterRequest(
                            DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
-                            DhtFeatureTest.foreignNodes
-                    )
-                    DhtNetwork.log('sending message stats request')
-                    stats = proc.sendGetMessageStats()
-                    cluster_ops_count += sum(stats[1:])
-                    DhtNetwork.log("5 seconds wait...")
+                            DhtFeatureTest.foreignNodes,
+                            done_cb=dcb
+                        )
+                        lock.wait()
+                    DhtNetwork.Log.log('sending message stats request')
+                    def msg_dcb(stats):
+                        nonlocal cluster_ops_count, lock
+                        if stats:
+                            cluster_ops_count += sum(stats[1:])
+                        with lock:
+                            lock.notify()
+                    with lock:
+                        proc.sendGetMessageStats(done_cb=msg_dcb)
+                        lock.wait()
+                    DhtNetwork.Log.log("5 seconds wait...")
                    time.sleep(5)
                ops_count.append(cluster_ops_count/self._workbench.node_num)

                # checking if values were transfered to new nodes
                foreignNodes_before_delete = DhtFeatureTest.foreignNodes
-                DhtNetwork.log('[GET]: trying to fetch persistent values')
+                DhtNetwork.Log.log('[GET]: trying to fetch persistent values')
                self._dhtGet(consumer, myhash)
                new_nodes = set(DhtFeatureTest.foreignNodes) - set(foreignNodes_before_delete)

@@ -485,7 +723,11 @@ class PersistenceTest(DhtFeatureTest):
            if self._op_plot:
                display_plot(ops_count, color='blue')
        else:
-            DhtNetwork.log("[GET]: either couldn't fetch values or nodes hosting values...")
+            DhtNetwork.Log.log("[GET]: either couldn't fetch values or nodes hosting values...")
+
+        if traffic_plot_thread:
+            print("Traffic plot running for ever. Ctrl-c for stopping it.")
+            traffic_plot_thread.join()

    @reset_before_test
    def _replaceClusterTest(self):
@@ -494,7 +736,7 @@ class PersistenceTest(DhtFeatureTest):
        """
        clusters = 8

-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
@@ -507,16 +749,16 @@ class PersistenceTest(DhtFeatureTest):
        self._dhtGet(consumer, myhash)
        initial_nodes = DhtFeatureTest.foreignNodes

-        DhtNetwork.log('Replacing', clusters, 'random clusters successively...')
+        DhtNetwork.Log.log('Replacing', clusters, 'random clusters successively...')
        for n in range(clusters):
            i = random.randint(0, len(self._workbench.procs)-1)
            proc = self._workbench.procs[i]
-            DhtNetwork.log('Replacing', proc)
-            proc.sendNodesRequest(DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ)
+            DhtNetwork.Log.log('Replacing', proc)
+            proc.sendClusterRequest(DhtNetworkSubProcess.SHUTDOWN_CLUSTER_REQ)
            self._workbench.stop_cluster(i)
            self._workbench.start_cluster(i)

-        DhtNetwork.log('[GET]: trying to fetch persistent values')
+        DhtNetwork.Log.log('[GET]: trying to fetch persistent values')
        self._dhtGet(consumer, myhash)
        new_nodes = set(DhtFeatureTest.foreignNodes) - set(initial_nodes)

@@ -531,17 +773,17 @@ class PersistenceTest(DhtFeatureTest):
        minutes for the nodes to trigger storage maintenance.
        """
        trigger_nodes = []
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap

        N_PRODUCERS = self._num_producers if self._num_values else 16
        DP_TIMEOUT = 1

        hashes = []

-        # Generating considerable amount of values of size 1KB.
+    # Generating considerable amount of values of size 1KB.
        VALUE_SIZE = 1024
        NUM_VALUES = self._num_values if self._num_values else 50
-        values = [Value(''.join(random.choice(string.hexdigits) for _ in range(VALUE_SIZE)).encode()) for _ in range(NUM_VALUES)]
+        values = [Value(random_str_val(size=VALUE_SIZE).encode()) for _ in range(NUM_VALUES)]

        bootstrap.resize(N_PRODUCERS+2)
        consumer = bootstrap.get(N_PRODUCERS+1)
@@ -555,21 +797,21 @@ class PersistenceTest(DhtFeatureTest):
            nodes = set([])
            self._gottaGetThemAllPokeNodes(consumer, hashes, nodes=nodes)

-            DhtNetwork.log("Values are found on:")
+            DhtNetwork.Log.log("Values are found on:")
            for n in nodes:
-                DhtNetwork.log(n)
+                DhtNetwork.Log.log(n)

-            DhtNetwork.log("Creating 8 nodes around all of these hashes...")
+            DhtNetwork.Log.log("Creating 8 nodes around all of these hashes...")
            for _hash in hashes:
                self._trigger_dp(trigger_nodes, _hash, count=8)

-            DhtNetwork.log('Waiting', DP_TIMEOUT+1, 'minutes for normal storage maintenance.')
+            DhtNetwork.Log.log('Waiting', DP_TIMEOUT+1, 'minutes for normal storage maintenance.')
            time.sleep((DP_TIMEOUT+1)*60)

-            DhtNetwork.log('Deleting old nodes from previous search.')
+            DhtNetwork.Log.log('Deleting old nodes from previous search.')
            for proc in self._workbench.procs:
-                DhtNetwork.log('[REMOVE]: sending delete request to', proc)
-                proc.sendNodesRequest(
+                DhtNetwork.Log.log('[REMOVE]: sending delete request to', proc)
+                proc.sendClusterRequest(
                    DhtNetworkSubProcess.REMOVE_NODE_REQ,
                    nodes)

@@ -610,9 +852,10 @@ class PerformanceTest(DhtFeatureTest):
            else:
                raise NameError("This test is not defined '" + self._test + "'")
        except Exception as e:
-            print(e)
+            traceback.print_tb(e.__traceback__)
+            print(type(e).__name__+':', e, file=sys.stderr)
        finally:
-            self.bootstrap.resize(1)
+            self._bootstrap.resize(1)


    ###########
@@ -624,7 +867,7 @@ class PerformanceTest(DhtFeatureTest):
        """
        Tests for performance of the DHT doing multiple get() operation.
        """
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap

        plt.ion()

@@ -649,7 +892,7 @@ class PerformanceTest(DhtFeatureTest):

        def getcb(v):
            nonlocal bootstrap
-            DhtNetwork.log("found", v)
+            DhtNetwork.Log.log("found", v)
            return True

        def donecb(ok, nodes, start):
@@ -657,7 +900,7 @@ class PerformanceTest(DhtFeatureTest):
            t = time.time()-start
            with lock:
                if not ok:
-                    DhtNetwork.log("failed !")
+                    DhtNetwork.Log.log("failed !")
                times.append(t)
                done -= 1
                lock.notify()
@@ -689,7 +932,7 @@ class PerformanceTest(DhtFeatureTest):
        for n in range(10):
            self._workbench.replace_cluster()
            plt.pause(2)
-            DhtNetwork.log("Getting 50 random hashes succesively.")
+            DhtNetwork.Log.log("Getting 50 random hashes succesively.")
            for i in range(50):
                with lock:
                    for _ in range(1):
@@ -713,8 +956,7 @@ class PerformanceTest(DhtFeatureTest):
        deleting around the target hash.
        """

-
-        bootstrap = self.bootstrap
+        bootstrap = self._bootstrap

        bootstrap.resize(3)
        consumer = bootstrap.get(1)
@@ -725,28 +967,28 @@ class PerformanceTest(DhtFeatureTest):

        for _ in range(max(1, int(self._workbench.node_num/32))):
            self._dhtGet(consumer, myhash)
-            DhtNetwork.log("Waiting 15 seconds...")
+            DhtNetwork.Log.log("Waiting 15 seconds...")
            time.sleep(15)

            self._dhtPut(producer, myhash, *local_values)

            #checking if values were transfered
            self._dhtGet(consumer, myhash)
-            DhtNetwork.log('Values are found on :')
+            DhtNetwork.Log.log('Values are found on :')
            for node in DhtFeatureTest.foreignNodes:
-                DhtNetwork.log(node)
+                DhtNetwork.Log.log(node)

            if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
                if DhtFeatureTest.foreignValues:
-                    DhtNetwork.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
+                    DhtNetwork.Log.log('[GET]: Only ', len(DhtFeatureTest.foreignValues) ,' on ',
                            len(local_values), ' values successfully put.')
                else:
-                    DhtNetwork.log('[GET]: 0 values successfully put')
+                    DhtNetwork.Log.log('[GET]: 0 values successfully put')

-            DhtNetwork.log('Removing all nodes hosting target values...')
+            DhtNetwork.Log.log('Removing all nodes hosting target values...')
            for proc in self._workbench.procs:
-                DhtNetwork.log('[REMOVE]: sending shutdown request to', proc)
-                proc.sendNodesRequest(
+                DhtNetwork.Log.log('[REMOVE]: sending shutdown request to', proc)
+                proc.sendClusterRequest(
                        DhtNetworkSubProcess.SHUTDOWN_NODE_REQ,
                        DhtFeatureTest.foreignNodes
                )
@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg3492"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   xml:space="preserve"
+   width="256"
+   height="256"
+   viewBox="0 0 256 255.99999"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"><title
+     id="title3357">OpenDHT</title><metadata
+     id="metadata3498"><rdf:RDF><cc:Work
+         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title>OpenDHT</dc:title><dc:rights><cc:Agent><dc:title></dc:title></cc:Agent></dc:rights><dc:creator><cc:Agent><dc:title>Savoir-faire Linux Inc.</dc:title></cc:Agent></dc:creator><dc:relation>https://opendht.net</dc:relation></cc:Work></rdf:RDF></metadata><defs
+     id="defs3496"><clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath3534"><path
+         d="m 0,0 1152,0 0,648 L 0,648 0,0 Z"
+         id="path3536"
+         inkscape:connector-curvature="0" /></clipPath></defs><sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1855"
+     inkscape:window-height="1056"
+     id="namedview3494"
+     showgrid="false"
+     units="px"
+     fit-margin-top="5"
+     fit-margin-right="5"
+     fit-margin-bottom="5"
+     fit-margin-left="5"
+     inkscape:zoom="1.9555556"
+     inkscape:cx="63.161953"
+     inkscape:cy="141.16919"
+     inkscape:window-x="1985"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g3532" /><g
+     id="g3500"
+     inkscape:groupmode="layer"
+     inkscape:label="SFL-logo-DHT"
+     transform="matrix(1.25,0,0,-1.25,-589.00821,604.10631)"><g
+       id="g3530"><g
+         id="g3532"
+         clip-path="url(#clipPath3534)"><g
+           id="g3359"
+           transform="matrix(0.9931555,0,0,0.99058777,1.5489503,0.9813209)"><g
+             transform="translate(485.3798,383.7826)"
+             id="g3538"><path
+               inkscape:connector-curvature="0"
+               id="path3540"
+               style="fill:#47b3d1;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c 0,50.03 40.554,90.586 90.586,90.586 50.028,0 90.587,-40.556 90.587,-90.586 0,-50.03 -40.559,-90.588 -90.587,-90.588 C 40.554,-90.588 0,-50.03 0,0" /></g><g
+             transform="translate(576.0338,313.2273)"
+             id="g3542"><path
+               inkscape:connector-curvature="0"
+               id="path3544"
+               style="fill:#0091ba;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c 0,54.307 -24.009,102.977 -61.954,136.075 -17.613,-16.526 -28.632,-39.996 -28.632,-66.056 0,-49.629 39.916,-89.912 89.393,-90.555 0.396,-0.009 0.792,-0.033 1.193,-0.033 L 0,-0.082 0,0 Z" /></g><g
+             transform="translate(575.9661,292.6626)"
+             id="g3546"><path
+               inkscape:connector-curvature="0"
+               id="path3548"
+               style="fill:#007aa3;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c -1.439,49.482 -41.105,89.355 -90.504,91.12 0,-0.178 -0.015,-0.355 -0.015,-0.536 C -90.519,40.578 -49.998,0.039 0,0" /></g><g
+             transform="translate(532.4411,337.497)"
+             id="g3550"><path
+               inkscape:connector-curvature="0"
+               id="path3552"
+               style="fill:#006382;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c -10.555,0 -20.236,-3.754 -27.777,-10 16.565,-21.182 42.472,-34.815 71.442,-34.834 0.01,0.415 -0.072,0.821 -0.072,1.241 C 43.593,-19.516 24.077,0 0,0" /></g><g
+             transform="translate(576.0338,313.2273)"
+             id="g3554"><path
+               inkscape:connector-curvature="0"
+               id="path3556"
+               style="fill:#0091ba;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="M 0,0 C 0,54.307 24.014,102.977 61.954,136.075 79.567,119.549 90.586,96.079 90.586,70.019 90.586,20.39 50.675,-19.893 1.198,-20.536 0.797,-20.545 0.406,-20.569 0.005,-20.569 l 0,20.487 C 0.005,-0.053 0,-0.029 0,0" /></g><g
+             transform="translate(576.1063,292.6626)"
+             id="g3558"><path
+               inkscape:connector-curvature="0"
+               id="path3560"
+               style="fill:#007aa3;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c 1.44,49.482 41.105,89.355 90.504,91.12 0,-0.178 0.01,-0.355 0.01,-0.536 C 90.514,40.578 49.998,0.039 0,0" /></g><g
+             transform="translate(619.6266,337.497)"
+             id="g3562"><path
+               inkscape:connector-curvature="0"
+               id="path3564"
+               style="fill:#006382;fill-opacity:1;fill-rule:nonzero;stroke:none"
+               d="m 0,0 c 10.56,0 20.241,-3.754 27.782,-10 -16.57,-21.182 -42.405,-34.815 -71.375,-34.834 -0.009,0.415 0,0.821 0,1.241 C -43.593,-19.516 -24.072,0 0,0" /></g></g></g></g></g></svg>
@@ -3,7 +3,8 @@ SUBDIRS = argon2
 lib_LTLIBRARIES = libopendht.la

 AM_CPPFLAGS = -I../include/opendht
-libopendht_la_LDFLAGS = @LDFLAGS@ @GnuTLS_LIBS@ @Nettle_LIBS@
+libopendht_la_CXXFLAGS = @CXXFLAGS@
+libopendht_la_LDFLAGS = @LDFLAGS@ @GNUTLS_LIBS@ @nettle_LIBS@
 libopendht_la_LIBADD = ./argon2/libargon2.la

 libopendht_la_SOURCES = \
@@ -19,7 +20,8 @@ libopendht_la_SOURCES = \
        crypto.cpp \
        securedht.cpp \
        dhtrunner.cpp \
-        default_types.cpp
+        default_types.cpp \
+        indexation/pht.cpp

 if WIN32
 libopendht_la_SOURCES += rng.cpp
@@ -42,4 +44,5 @@ nobase_include_HEADERS = \
        ../include/opendht/dhtrunner.h \
        ../include/opendht/default_types.h \
        ../include/opendht/log.h \
-        ../include/opendht/rng.h
+        ../include/opendht/rng.h \
+        ../include/opendht/indexation/pht.h
@@ -88,18 +88,12 @@ int argon2_hash(const uint32_t t_cost, const uint32_t m_cost,
    int result;
    uint8_t *out;

-    /* Detect and reject overflowing sizes */
-    /* TODO: This should probably be fixed in the function signature */
-    if (pwdlen > ARGON2_MAX_PWD_LENGTH) {
-        return ARGON2_PWD_TOO_LONG;
-    }
-
    if (hashlen > ARGON2_MAX_OUTLEN) {
        return ARGON2_OUTPUT_TOO_LONG;
    }

-    if (saltlen > ARGON2_MAX_SALT_LENGTH) {
-        return ARGON2_SALT_TOO_LONG;
+    if (hashlen < ARGON2_MIN_OUTLEN) {
+        return ARGON2_OUTPUT_TOO_SHORT;
    }

    out = malloc(hashlen);
@@ -212,13 +206,19 @@ int argon2_verify(const char *encoded, const void *pwd, const size_t pwdlen,
    int ret;
    int decode_result;
    uint32_t encoded_len;
+    size_t encoded_len_tmp;

    if(encoded == NULL) {
        return ARGON2_DECODING_FAIL;
    }

+    encoded_len_tmp = strlen(encoded);
    /* max values, to be updated in decode_string */
-    encoded_len = strlen(encoded);
+    if (UINT32_MAX < encoded_len_tmp) {
+        return ARGON2_DECODING_FAIL;
+    }
+
+    encoded_len = (uint32_t)encoded_len_tmp;
    ctx.adlen = encoded_len;
    ctx.saltlen = encoded_len;
    ctx.outlen = encoded_len;
@@ -226,6 +226,8 @@ int argon2_verify(const char *encoded, const void *pwd, const size_t pwdlen,
    ctx.free_cbk = NULL;
    ctx.secret = NULL;
    ctx.secretlen = 0;
+    ctx.pwdlen = 0;
+    ctx.pwd = NULL;
    ctx.ad = malloc(ctx.adlen);
    ctx.salt = malloc(ctx.saltlen);
    ctx.out = malloc(ctx.outlen);
@@ -392,5 +394,5 @@ size_t argon2_encodedlen(uint32_t t_cost, uint32_t m_cost, uint32_t parallelism,
                         uint32_t saltlen, uint32_t hashlen) {
    return strlen("$argon2x$v=$m=,t=,p=$$") + numlen(t_cost) + numlen(m_cost)
        + numlen(parallelism) + b64len(saltlen) + b64len(hashlen)
-        + numlen(ARGON2_VERSION_NUMBER);
+        + numlen(ARGON2_VERSION_NUMBER) + 1;
 }
@@ -412,7 +412,18 @@ int encode_string(char *dst, size_t dst_len, argon2_context *ctx,
 }

 size_t b64len(uint32_t len) {
-    return (((size_t)len + 2) / 3) * 4;
+    size_t olen = ((size_t)len / 3) << 2;
+
+    switch (len % 3) {
+    case 2:
+        olen++;
+    /* fall through */
+    case 1:
+        olen += 2;
+        break;
+    }
+
+    return olen;
 }

 size_t numlen(uint32_t num) {
@@ -47,4 +47,12 @@ bindDoneCb(DoneCallbackRaw raw_cb, void* user_data)
    };
 }

+DoneCallbackSimple
+bindDoneCbSimple(DoneCallbackSimpleRaw raw_cb, void* user_data) {
+    if (not raw_cb) return {};
+    return [=](bool success) {
+        raw_cb(success, user_data);
+    };
+}
+
 }
@@ -273,7 +273,7 @@ PrivateKey::PrivateKey(PrivateKey&& o) noexcept : key(o.key), x509_key(o.x509_ke
 #endif
    o.key = nullptr;
    o.x509_key = nullptr;
-};
+}

 PrivateKey::~PrivateKey()
 {
@@ -311,6 +311,8 @@ PrivateKey::sign(const Blob& data) const
 {
    if (!key)
        throw CryptoException("Can't sign data: no private key set !");
+    if (std::numeric_limits<unsigned>::max() < data.size())
+        throw CryptoException("Can't sign data: too large !");
    gnutls_datum_t sig;
    const gnutls_datum_t dat {(unsigned char*)data.data(), (unsigned)data.size()};
    if (gnutls_privkey_sign_data(key, GNUTLS_DIG_SHA512, 0, &dat, &sig) != GNUTLS_E_SUCCESS)
@@ -104,7 +104,7 @@ DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner:
                    std::lock_guard<std::mutex> lck(storage_mtx);
                    if (not pending_ops_prio.empty())
                        return true;
-                    if (not pending_ops.empty() and getStatus() >= NodeStatus::Connecting)
+                    if (not pending_ops.empty() and getStatus() != NodeStatus::Connecting)
                        return true;
                }
                return false;
@@ -293,7 +293,7 @@ DhtRunner::loop_()
        ops.front()(*dht_);
        ops.pop();
    }
-    if (getStatus() >= NodeStatus::Connecting) {
+    if (getStatus() != NodeStatus::Connecting) {
        {
            std::lock_guard<std::mutex> lck(storage_mtx);
            ops = std::move(pending_ops);
@@ -429,37 +429,44 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C
 }

 void
-DhtRunner::get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f)
+DhtRunner::get(InfoHash hash, GetCallback vcb, DoneCallback dcb, Value::Filter f, Where w)
 {
    std::lock_guard<std::mutex> lck(storage_mtx);
    pending_ops.emplace([=](SecureDht& dht) mutable {
-        dht.get(hash, vcb, dcb, std::move(f));
+        dht.get(hash, vcb, dcb, std::move(f), std::move(w));
    });
    cv.notify_all();
 }

 void
-DhtRunner::get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb, Value::Filter f)
+DhtRunner::get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb, Value::Filter f, Where w)
 {
-    get(InfoHash::get(key), vcb, dcb, f);
+    get(InfoHash::get(key), vcb, dcb, f, w);
+}
+void DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Query q) {
+    std::lock_guard<std::mutex> lck(storage_mtx);
+    pending_ops.emplace([=](SecureDht& dht) mutable {
+        dht.query(hash, cb, done_cb, std::move(q));
+    });
+    cv.notify_all();
 }

 std::future<size_t>
-DhtRunner::listen(InfoHash hash, GetCallback vcb, Value::Filter f)
+DhtRunner::listen(InfoHash hash, GetCallback vcb, Value::Filter f, Where w)
 {
    std::lock_guard<std::mutex> lck(storage_mtx);
    auto ret_token = std::make_shared<std::promise<size_t>>();
    pending_ops.emplace([=](SecureDht& dht) mutable {
-        ret_token->set_value(dht.listen(hash, vcb, std::move(f)));
+        ret_token->set_value(dht.listen(hash, vcb, std::move(f), std::move(w)));
    });
    cv.notify_all();
    return ret_token->get_future();
 }

 std::future<size_t>
-DhtRunner::listen(const std::string& key, GetCallback vcb, Value::Filter f)
+DhtRunner::listen(const std::string& key, GetCallback vcb, Value::Filter f, Where w)
 {
-    return listen(InfoHash::get(key), vcb, f);
+    return listen(InfoHash::get(key), vcb, f, w);
 }

 void
@@ -484,30 +491,30 @@ DhtRunner::cancelListen(InfoHash h, std::shared_future<size_t> token)
 }

 void
-DhtRunner::put(InfoHash hash, Value&& value, DoneCallback cb, bool permanent)
+DhtRunner::put(InfoHash hash, Value&& value, DoneCallback cb, time_point created, bool permanent)
 {
    std::lock_guard<std::mutex> lck(storage_mtx);
    auto sv = std::make_shared<Value>(std::move(value));
    pending_ops.emplace([=](SecureDht& dht) {
-        dht.put(hash, sv, cb, {}, permanent);
+        dht.put(hash, sv, cb, created, permanent);
    });
    cv.notify_all();
 }

 void
-DhtRunner::put(InfoHash hash, std::shared_ptr<Value> value, DoneCallback cb, bool permanent)
+DhtRunner::put(InfoHash hash, std::shared_ptr<Value> value, DoneCallback cb, time_point created, bool permanent)
 {
    std::lock_guard<std::mutex> lck(storage_mtx);
    pending_ops.emplace([=](SecureDht& dht) {
-        dht.put(hash, value, cb, {}, permanent);
+        dht.put(hash, value, cb, created, permanent);
    });
    cv.notify_all();
 }

 void
-DhtRunner::put(const std::string& key, Value&& value, DoneCallbackSimple cb, bool permanent)
+DhtRunner::put(const std::string& key, Value&& value, DoneCallbackSimple cb, time_point created, bool permanent)
 {
-    put(InfoHash::get(key), std::forward<Value>(value), cb, permanent);
+    put(InfoHash::get(key), std::forward<Value>(value), cb, created, permanent);
 }

 void
@@ -0,0 +1,518 @@
+#include "indexation/pht.h"
+#include "rng.h"
+
+namespace dht {
+namespace indexation {
+
+/**
+ * Output the blob into string and readable way
+ *
+ * @param bl   : Blob to print
+ *
+ * @return string that represent the blob into a readable way
+ */
+static std::string blobToString(const Blob &bl) {
+    std::stringstream ss;
+    auto bn = bl.size() % 8;
+    auto n = bl.size() / 8;
+
+    for (size_t i = 0; i < bl.size(); i++)
+        ss << std::bitset<8>(bl[i]) << " ";
+    if (bn)
+        for (unsigned b=0; b < bn; b++)
+            ss << (char)((bl[n] & (1 << (7 - b))) ? '1':'0');
+
+    return ss.str();
+}
+
+std::string Prefix::toString() const {
+    std::stringstream ss;
+
+    ss << "Prefix : " << std::endl << "\tContent_ : \"";
+    ss << blobToString(content_);
+    ss << "\"" << std::endl;
+
+    ss << "\tFlags_   : \"";
+    ss << blobToString(flags_);
+    ss << "\"" << std::endl;
+
+    return ss.str();
+}
+
+void Pht::Cache::insert(const Prefix& p) {
+    size_t i = 0;
+    auto now = clock::now();
+
+    std::shared_ptr<Node> curr_node;
+
+    while ((leaves_.size() > 0
+        and leaves_.begin()->first + NODE_EXPIRE_TIME < now)
+        or  leaves_.size() > MAX_ELEMENT) {
+
+        leaves_.erase(leaves_.begin());
+    }
+
+    if (not (curr_node = root_.lock()) ) {
+        /* Root does not exist, need to create one*/
+        curr_node = std::make_shared<Node>();
+        root_ = curr_node;
+    }
+
+    curr_node->last_reply = now;
+
+    /* Iterate through all bit of the Blob */
+    for ( i = 0; i < p.size_; i++ ) {
+
+        /* According to the bit define which node is the next one */
+        auto& next = ( p.isContentBitActive(i) ) ? curr_node->right_child : curr_node->left_child;
+
+        /**
+         * If lock, node exists
+         * else create it
+         */
+        if (auto n = next.lock()) {
+            curr_node = std::move(n);
+        } else {
+            /* Create the next node if doesn't exist*/
+            auto tmp_curr_node = std::make_shared<Node>();
+            tmp_curr_node->parent = curr_node;
+            next = tmp_curr_node;
+            curr_node = std::move(tmp_curr_node);
+        }
+
+        curr_node->last_reply = now;
+    }
+
+    /* Insert the leaf (curr_node) into the multimap */
+    leaves_.emplace(std::move(now), std::move(curr_node) );
+}
+
+int Pht::Cache::lookup(const Prefix& p) {
+    int pos = -1;
+    auto now = clock::now(), last_node_time = now;
+
+    /* Before lookup remove the useless one [i.e. too old] */
+    while ( leaves_.size() > 0
+        and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
+
+        leaves_.erase(leaves_.begin());
+    }
+
+    auto next = root_;
+    std::shared_ptr<Node> curr_node;
+
+    while ( auto n = next.lock() ) {
+        ++pos;
+        /* Safe since pos is equal to 0 until here */
+        if ( (unsigned) pos >= p.content_.size() * 8) break;
+
+        curr_node = n;
+        last_node_time = curr_node->last_reply;
+        curr_node->last_reply = now;
+
+        /* Get the Prefix bit by bit, starting from left */
+        next = ( p.isContentBitActive(pos) ) ? curr_node->right_child : curr_node->left_child;
+    }
+
+    if ( pos >= 0 ) {
+        auto to_erase = leaves_.find(last_node_time);
+        if ( to_erase != leaves_.end() )
+            leaves_.erase( to_erase );
+
+        leaves_.emplace( std::move(now), std::move(curr_node) );
+    }
+
+    return pos;
+}
+
+const ValueType IndexEntry::TYPE = ValueType::USER_DATA;
+constexpr std::chrono::minutes Pht::Cache::NODE_EXPIRE_TIME;
+
+void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
+        std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals,
+        LookupCallbackWrapper cb, DoneCallbackSimple done_cb,
+        std::shared_ptr<unsigned> max_common_prefix_len,
+        int start, bool all_values)
+{
+    struct node_lookup_result {
+        bool done {false};
+        bool is_pht {false};
+    };
+
+    /* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/
+    auto mid = (start >= 0) ? (unsigned) start : (*lo + *hi)/2;
+
+    auto first_res = std::make_shared<node_lookup_result>();
+    auto second_res = std::make_shared<node_lookup_result>();
+
+    auto on_done = [=](bool ok) {
+        bool is_leaf = first_res->is_pht and not second_res->is_pht;
+        if (not ok) {
+            if (done_cb)
+                done_cb(false);
+        }
+        else if (is_leaf or *lo > *hi) {
+            // leaf node
+            Prefix to_insert = p.getPrefix(mid);
+            cache_.insert(to_insert);
+
+            if (cb) {
+                if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
+                    auto p_ = (p.getPrefix(mid)).getSibling().getFullSize();
+                    *lo = mid;
+                    *hi = p_.size_;
+                    lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
+                }
+
+                cb(*vals, to_insert);
+            }
+
+            if (done_cb)
+                done_cb(true);
+        } else if (first_res->is_pht) {
+            // internal node
+            *lo = mid+1;
+            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
+        } else {
+            // first get failed before second.
+            if (done_cb)
+                done_cb(false);
+        }
+    };
+
+    if (*lo <= *hi) {
+        auto pht_filter = [&](const dht::Value& v) {
+            return v.user_type.compare(0, name_.size(), name_) == 0;
+        };
+
+        auto on_get = [=](const std::shared_ptr<dht::Value>& value, std::shared_ptr<node_lookup_result> res) {
+            if (value->user_type == canary_) {
+                res->is_pht = true;
+            }
+            else {
+                IndexEntry entry;
+                entry.unpackValue(*value);
+
+                auto it = std::find_if(vals->cbegin(), vals->cend(), [&](const std::shared_ptr<IndexEntry>& ie) {
+                    return ie->value == entry.value;
+                });
+
+                /* If we already got the value then get the next one */
+                if (it != vals->cend())
+                    return true;
+
+                if (max_common_prefix_len) { /* inexact match case */
+                    auto common_bits = Prefix::commonBits(p, entry.prefix);
+
+                    if (vals->empty()) {
+                        vals->emplace_back(std::make_shared<IndexEntry>(entry));
+                        *max_common_prefix_len = common_bits;
+                    }
+                    else {
+                        if (common_bits == *max_common_prefix_len) /* this is the max so far */
+                            vals->emplace_back(std::make_shared<IndexEntry>(entry));
+                        else if (common_bits > *max_common_prefix_len) { /* new max found! */
+                            vals->clear();
+                            vals->emplace_back(std::make_shared<IndexEntry>(entry));
+                            *max_common_prefix_len = common_bits;
+                        }
+                    }
+                } else if (all_values or entry.prefix == p.content_) /* exact match case */
+                    vals->emplace_back(std::make_shared<IndexEntry>(entry));
+            }
+
+            return true;
+        };
+
+        dht_->get(p.getPrefix(mid).hash(),
+                std::bind(on_get, std::placeholders::_1, first_res),
+                [=](bool ok) {
+                    if (not ok) {
+                        // DHT failed
+                        first_res->done = true;
+                        if (done_cb and second_res->done)
+                            on_done(false);
+                    }
+                    else {
+                        if (not first_res->is_pht) {
+                            // Not a PHT node.
+                            *hi = mid-1;
+                            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
+                        } else {
+                            first_res->done = true;
+                            if (second_res->done or mid >= p.size_ - 1)
+                                on_done(true);
+                        }
+                    }
+                }, pht_filter);
+
+        if (mid < p.size_ - 1)
+           dht_->get(p.getPrefix(mid+1).hash(),
+                    std::bind(on_get, std::placeholders::_1, second_res),
+                    [=](bool ok) {
+                        if (not ok) {
+                            // DHT failed
+                            second_res->done = true;
+                            if (done_cb and first_res->done)
+                                on_done(false);
+                        }
+                        else {
+                            second_res->done = true;
+                            if (first_res->done)
+                                on_done(true);
+                        }
+                }, pht_filter);
+    } else {
+        on_done(true);
+    }
+}
+
+void Pht::lookup(Key k, Pht::LookupCallback cb, DoneCallbackSimple done_cb, bool exact_match) {
+    auto prefix = linearize(k);
+    auto values = std::make_shared<std::vector<std::shared_ptr<IndexEntry>>>();
+
+    auto lo = std::make_shared<int>(0);
+    auto hi = std::make_shared<int>(prefix.size_);
+    std::shared_ptr<unsigned> max_common_prefix_len = not exact_match ? std::make_shared<unsigned>(0) : nullptr;
+
+    lookupStep(prefix, lo, hi, values,
+        [=](std::vector<std::shared_ptr<IndexEntry>>& entries, Prefix p) {
+            std::vector<std::shared_ptr<Value>> vals(entries.size());
+
+            std::transform(entries.begin(), entries.end(), vals.begin(),
+                [](const std::shared_ptr<IndexEntry>& ie) {
+                    return std::make_shared<Value>(ie->value);
+            });
+
+            cb(vals, p);
+        }, done_cb, max_common_prefix_len, cache_.lookup(prefix));
+}
+
+void Pht::updateCanary(Prefix p) {
+    // TODO: change this... copy value
+    dht::Value canary_value;
+    canary_value.user_type = canary_;
+
+    dht_->put(p.hash(), std::move(canary_value),
+        [=](bool){
+            static std::bernoulli_distribution d(0.5);
+            crypto::random_device rd;
+            if (p.size_ and d(rd))
+                updateCanary(p.getPrefix(-1));
+        }
+    );
+
+    if (p.size_) {
+        dht::Value canary_second_value;
+        canary_second_value.user_type = canary_;
+        dht_->put(p.getSibling().hash(), std::move(canary_second_value));
+    }
+}
+
+void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shared_ptr<int> hi, time_point time_p,
+                 bool check_split, DoneCallbackSimple done_cb) {
+
+    if (time_p + ValueType::USER_DATA.expiration < clock::now()) return;
+
+    auto vals = std::make_shared<std::vector<std::shared_ptr<IndexEntry>>>();
+    auto final_prefix = std::make_shared<Prefix>();
+
+    lookupStep(kp, lo, hi, vals,
+        [=](std::vector<std::shared_ptr<IndexEntry>>&, Prefix p) {
+            *final_prefix = Prefix(p);
+        },
+        [=](bool ok){
+            if (not ok) {
+                if (done_cb)
+                    done_cb(false);
+            } else {
+
+                RealInsertCallback real_insert = [=]( std::shared_ptr<Prefix> p, IndexEntry entry) {
+                    updateCanary(*p);
+                    checkPhtUpdate(*p, entry, time_p);
+                    cache_.insert(*p);
+                    dht_->put(p->hash(), std::move(entry), done_cb , time_p);
+                };
+
+                if ( not check_split or final_prefix->size_ == kp.size_ ) {
+                    real_insert(final_prefix, std::move(entry));
+                } else {
+                    if ( vals->size() < MAX_NODE_ENTRY_COUNT ) {
+                        getRealPrefix(final_prefix, std::move(entry), real_insert);
+                    }
+                    else {
+                        split(*final_prefix, vals, entry, real_insert);
+                    }
+                }
+            }
+        }, nullptr, cache_.lookup(kp), true
+    );
+}
+
+Prefix Pht::zcurve(const std::vector<Prefix>& all_prefix) const {
+    Prefix p;
+
+    if ( all_prefix.size() == 1 )
+        return all_prefix[0];
+
+    /* All prefix got the same size (thanks to padding) */
+    size_t prefix_size = all_prefix[0].content_.size();
+
+    /* Loop on all uint8_t of the input prefix */
+    for ( size_t j = 0, bit = 0; j < prefix_size; j++) {
+
+        uint8_t mask = 0x80;
+        /* For each of the 8 bits of the input uint8_t */
+        for ( int i = 0; i < 8; ) {
+
+            uint8_t flags = 0;
+            uint8_t content = 0;
+
+            /* For each bit of the output uint8_t */
+            for ( int k = 0 ; k < 8; k++ ) {
+
+                auto diff = k - i;
+
+                /*get the content 'c', and the flag 'f' of the input prefix */
+                auto c = all_prefix[bit].content_[j] & mask;
+                auto f = all_prefix[bit].flags_[j] & mask;
+
+                /* Move this bit at the right position according to the diff
+                   and merge it into content and flags in the same way */
+                content |= ( diff >= 0 ) ? c >> diff : c << std::abs(diff);
+                flags   |= ( diff >= 0 ) ? f >> diff : f << std::abs(diff);
+
+                /* If we are on the last prefix of the vector get back to the first and
+                ,move the mask in order to get the n + 1nth bit */
+                if ( ++bit == all_prefix.size() ) { bit = 0; ++i; mask >>= 1; }
+            }
+
+            /* Add the next flags + content to the output prefix */
+            p.content_.push_back(content);
+            p.flags_.push_back(flags);
+            p.size_ += 8;
+        }
+    }
+
+    return p;
+}
+
+Prefix Pht::linearize(Key k) const {
+    if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
+
+    std::vector<Prefix> all_prefix;
+    all_prefix.reserve(k.size());
+
+    /* Get the max size of the keyspec and take it for size limit (for padding) */
+    auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
+        [](const std::pair<std::string, size_t>& a, const std::pair<std::string, size_t>& b) {
+            return a.second < b.second;
+        })->second + 1;
+
+    for ( auto const& it : k ) {
+        Prefix p = Blob {it.second.begin(), it.second.end()};
+        p.addPaddingContent(max);
+        p.updateFlags();
+
+        all_prefix.emplace_back(std::move(p));
+    }
+
+    return zcurve(all_prefix);
+}
+
+void Pht::getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb ) {
+
+    if ( p->size_ == 0 ) {
+        end_cb(p, std::move(entry));
+        return;
+    }
+
+    auto total = std::make_shared<unsigned int>(0); /* Will contains the total number of data on 3 nodes */
+    auto ended = std::make_shared<unsigned int>(0); /* Just indicate how many have end */
+
+    auto parent = std::make_shared<Prefix>(p->getPrefix(-1));
+    auto sibling = std::make_shared<Prefix>(p->getSibling());
+
+    auto pht_filter = [&](const dht::Value& v) {
+        return v.user_type.compare(0, name_.size(), name_) == 0;
+    };
+
+    /* Lambda will count total number of data node */
+    auto count = [=]( const std::shared_ptr<dht::Value> value ) {
+        if ( value->user_type != canary_)
+            (*total)++;
+
+        return true;
+    };
+
+    auto on_done = [=] ( bool ) {
+        (*ended)++;
+        /* Only the last one do the CallBack*/
+        if  ( *ended == 3 ) {
+            if ( *total < MAX_NODE_ENTRY_COUNT )
+                end_cb(parent, std::move(entry));
+            else
+                end_cb(p, std::move(entry));
+        }
+    };
+
+    dht_->get(parent->hash(),
+        count,
+        on_done,
+        pht_filter
+    );
+
+    dht_->get(p->hash(),
+        count,
+        on_done,
+        pht_filter
+    );
+
+    dht_->get(sibling->hash(),
+        count,
+        on_done,
+        pht_filter
+    );
+}
+
+void Pht::checkPhtUpdate(Prefix p, IndexEntry entry, time_point time_p) {
+
+    Prefix full = entry.prefix;
+    if ( p.content_.size() * 8 >= full.content_.size() * 8 ) return;
+
+    auto next_prefix = full.getPrefix( p.size_ + 1 );
+
+    dht_->listen(next_prefix.hash(),
+        [=](const std::shared_ptr<dht::Value> &value) {
+            if (value->user_type == canary_) {
+                insert(full, entry, std::make_shared<int>(0), std::make_shared<int>(full.size_), time_p, false, nullptr);
+
+                /* Cancel listen since we found where we need to update*/
+                return false;
+            }
+
+            return true;
+        },
+        [=](const dht::Value& v) {
+            /* Filter value v thats start with the same name as ours */
+            return v.user_type.compare(0, name_.size(), name_) == 0;
+        }
+    );
+}
+
+void Pht::split(Prefix insert, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals, IndexEntry entry, RealInsertCallback end_cb ) {
+    auto full = Prefix(entry.prefix);
+
+    auto loc = foundSplitLocation(full, vals);
+    auto prefix_to_insert = std::make_shared<Prefix>(full.getPrefix(loc));
+
+    for(;loc != insert.size_ - 1; loc--) {
+        updateCanary(full.getPrefix(loc));
+    }
+
+    end_cb(prefix_to_insert, entry);
+}
+
+} /* indexation  */
+
+} /* dht */
@@ -91,4 +91,17 @@ std::ostream& operator<< (std::ostream& s, const InfoHash& h)
    return s;
 }

+std::istream& operator>> (std::istream& s, InfoHash& h)
+{
+    std::array<char, HASH_LEN*2> dat;
+    s.exceptions(std::istream::eofbit | std::istream::failbit);
+    s.read(&(*dat.begin()), dat.size());
+    for (size_t i = 0; i < HASH_LEN; i++) {
+        unsigned res = 0;
+        sscanf(dat.data() + 2*i, "%02x", &res);
+        h[i] = res;
+    }
+    return s;
+}
+
 }
@@ -21,6 +21,7 @@

 #include "network_engine.h"
 #include "request.h"
+#include "default_types.h"

 #include <msgpack.hpp>

@@ -49,6 +50,7 @@ static const uint8_t v4prefix[16] = {
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0, 0, 0, 0
 };

+constexpr unsigned SEND_NODES {8};

 enum class MessageType {
    Error = 0,
@@ -62,35 +64,40 @@ enum class MessageType {

 struct ParsedMessage {
    MessageType type;
-    InfoHash id;                                /* the id of the sender */
-    InfoHash info_hash;                         /* hash for which values are requested */
-    InfoHash target;                            /* target id around which to find nodes */
-    NetworkEngine::TransId tid;                                /* transaction id */
-    Blob token;                                 /* security token */
-    Value::Id value_id;                         /* the value id */
-    time_point created { time_point::max() };   /* time when value was first created */
-    Blob nodes4_raw, nodes6_raw;                /* IPv4 nodes in response to a 'find' request */
+    InfoHash id;                                              /* the id of the sender */
+    NetId network {0};                                 /* network id */
+    InfoHash info_hash;                                       /* hash for which values are requested */
+    InfoHash target;                                          /* target id around which to find nodes */
+    NetworkEngine::TransId tid;                               /* transaction id */
+    Blob token;                                               /* security token */
+    Value::Id value_id;                                       /* the value id */
+    time_point created { time_point::max() };                 /* time when value was first created */
+    Blob nodes4_raw, nodes6_raw;                              /* IPv4 nodes in response to a 'find' request */
    std::vector<std::shared_ptr<Node>> nodes4, nodes6;
-    std::vector<std::shared_ptr<Value>> values; /* values for a 'get' request */
-    want_t want;                                /* states if ipv4 or ipv6 request */
-    uint16_t error_code;                        /* error code in case of error */
+    std::vector<std::shared_ptr<Value>> values;               /* values for a 'get' request */
+    std::vector<std::shared_ptr<FieldValueIndex>> fields; /* index for fields values */
+    Query query;                                              /* query describing a filter to apply on values. */
+    want_t want;                                              /* states if ipv4 or ipv6 request */
+    uint16_t error_code;                                      /* error code in case of error */
    std::string ua;
-    Address addr;                               /* reported address by the distant node */
+    Address addr;                                             /* reported address by the distant node */
    void msgpack_unpack(msgpack::object o);
 };

 NetworkEngine::RequestAnswer::RequestAnswer(ParsedMessage&& msg)
- : ntoken(std::move(msg.token)), values(std::move(msg.values)), nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {}
+ : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)),
+    nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {}

 void
-NetworkEngine::tellListener(std::shared_ptr<Node> node, uint16_t rid, InfoHash hash, want_t want,
-        Blob ntoken, std::vector<std::shared_ptr<Node>> nodes, std::vector<std::shared_ptr<Node>> nodes6,
-        std::vector<std::shared_ptr<Value>> values)
+NetworkEngine::tellListener(std::shared_ptr<Node> node, uint16_t rid, const InfoHash& hash, want_t want,
+        const Blob& ntoken, std::vector<std::shared_ptr<Node>>&& nodes,
+        std::vector<std::shared_ptr<Node>>&& nodes6, std::vector<std::shared_ptr<Value>>&& values,
+        const Query& query)
 {
    auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
    try {
        sendNodesValues((const sockaddr*)&node->ss, node->sslen, TransId {TransPrefix::GET_VALUES, (uint16_t)rid}, nnodes.first, nnodes.second,
-                values, ntoken);
+                values, query, ntoken);
    } catch (const std::overflow_error& e) {
        DHT_LOG.ERR("Can't send value: buffer not large enough !");
    }
@@ -175,9 +182,8 @@ NetworkEngine::sendRequest(std::shared_ptr<Request>& request)
 {
    request->start = scheduler.time();
    auto e = requests.emplace(request->tid, request);
-    if (!e.second) {
-        DHT_LOG.ERR("Request already existed !");
-    }
+    if (!e.second)
+        DHT_LOG.ERR("Request already existed (tid: %d)!", request->tid);
    request->node->requested(request);
    requestStep(request);
 }
@@ -293,6 +299,11 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
        return;
    }

+    if (msg.network != network) {
+        DHT_LOG.DEBUG("Received message from other network %u.", msg.network);
+        return;
+    }
+
    if (msg.id == myid || msg.id == zeroes) {
        DHT_LOG.DEBUG("Received message from self.");
        return;
@@ -311,7 +322,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
    if (msg.tid.length != 4) {
        DHT_LOG.ERR("Broken node truncates transaction ids (len: %d): ", msg.tid.length);
        DHT_LOG.ERR.logPrintable(buf, buflen);
-        blacklistNode(cache.getNode(msg.id, from, fromlen, now, 1));
+        blacklistNode(cache.getNode(msg.id, from, fromlen, now, true));
        return;
    }

@@ -326,7 +337,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
        auto node = req->node;
        if (node->id != msg.id) {
            bool unknown_node = node->id == zeroes;
-            node = cache.getNode(msg.id, from, fromlen, now, 2);
+            node = cache.getNode(msg.id, from, fromlen, now, true);
            if (unknown_node) {
                // received reply to a message sent when we didn't know the node ID.
                req->node = node;
@@ -373,14 +384,14 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                requests.erase(reqp);
            req->reply_time = scheduler.time();

-            deserializeNodesValues(msg);
+            deserializeNodes(msg);
            req->setDone(std::move(msg));
            break;
        default:
            break;
        }
    } else {
-        auto node = cache.getNode(msg.id, from, fromlen, now, 1);
+        auto node = cache.getNode(msg.id, from, fromlen, now, true);
        node->received(now, {});
        onNewNode(node, 1);
        try {
@@ -397,16 +408,16 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                ++in_stats.find;
                RequestAnswer answer = onFindNode(node, msg.target, msg.want);
                auto nnodes = bufferNodes(from->sa_family, msg.target, msg.want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, {}, answer.ntoken);
+                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, {}, {}, answer.ntoken);
                break;
            }
            case MessageType::GetValues: {
                DHT_LOG.DEBUG("[node %s %s] got 'get' request for %s.",
                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.info_hash.toString().c_str());
                ++in_stats.get;
-                RequestAnswer answer = onGetValues(node, msg.info_hash, msg.want);
+                RequestAnswer answer = onGetValues(node, msg.info_hash, msg.want, msg.query);
                auto nnodes = bufferNodes(from->sa_family, msg.info_hash, msg.want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, answer.values, answer.ntoken);
+                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, answer.values, msg.query, answer.ntoken);
                break;
            }
            case MessageType::AnnounceValue: {
@@ -428,7 +439,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                DHT_LOG.DEBUG("[node %s %s] got 'listen' request for %s.",
                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.info_hash.toString().c_str());
                ++in_stats.listen;
-                RequestAnswer answer = onListen(node, msg.info_hash, msg.token, msg.tid.getTid());
+                RequestAnswer answer = onListen(node, msg.info_hash, msg.token, msg.tid.getTid(), std::move(msg.query));
                sendListenConfirmation(from, fromlen, msg.tid);
                break;
            }
@@ -451,9 +462,10 @@ packToken(msgpack::packer<msgpack::sbuffer>& pk, Blob token)
 }

 void
-insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const sockaddr *sa, socklen_t)
+insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const sockaddr *sa, socklen_t sa_len)
 {
-    size_t addr_len = (sa->sa_family == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr);
+    size_t addr_len = std::min<size_t>(sa_len,
+                     (sa->sa_family == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr));
    void* addr_ptr = (sa->sa_family == AF_INET) ? (void*)&((sockaddr_in*)sa)->sin_addr
                                                : (void*)&((sockaddr_in6*)sa)->sin6_addr;
    pk.pack("sa");
@@ -485,7 +497,7 @@ NetworkEngine::sendPing(std::shared_ptr<Node> node, RequestCb on_done, RequestEx
    auto tid = TransId {TransPrefix::PING, getNewTid()};
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));

    pk.pack(std::string("a")); pk.pack_map(1);
     pk.pack(std::string("id")); pk.pack(myid);
@@ -495,6 +507,9 @@ NetworkEngine::sendPing(std::shared_ptr<Node> node, RequestCb on_done, RequestEx
                              pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("q"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    Blob b {buffer.data(), buffer.data() + buffer.size()};
    std::shared_ptr<Request> req(new Request {tid.getTid(), node, std::move(b),
@@ -519,7 +534,7 @@ void
 NetworkEngine::sendPong(const sockaddr* sa, socklen_t salen, TransId tid) {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));

    pk.pack(std::string("r")); pk.pack_map(2);
      pk.pack(std::string("id")); pk.pack(myid);
@@ -529,6 +544,9 @@ NetworkEngine::sendPong(const sockaddr* sa, socklen_t salen, TransId tid) {
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("r"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    send(buffer.data(), buffer.size(), 0, sa, salen);
 }
@@ -539,7 +557,7 @@ NetworkEngine::sendFindNode(std::shared_ptr<Node> n, const InfoHash& target, wan
    auto tid = TransId {TransPrefix::FIND_NODE, getNewTid()};
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));

    pk.pack(std::string("a")); pk.pack_map(2 + (want>0?1:0));
      pk.pack(std::string("id"));     pk.pack(myid);
@@ -556,7 +574,9 @@ NetworkEngine::sendFindNode(std::shared_ptr<Node> n, const InfoHash& target, wan
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("q"));
    pk.pack(std::string("v")); pk.pack(my_v);
-
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    Blob b {buffer.data(), buffer.data() + buffer.size()};
    std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -578,16 +598,19 @@ NetworkEngine::sendFindNode(std::shared_ptr<Node> n, const InfoHash& target, wan


 std::shared_ptr<Request>
-NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash, want_t want,
+NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash, const Query& query, want_t want,
        RequestCb on_done, RequestExpiredCb on_expired) {
    auto tid = TransId {TransPrefix::GET_VALUES, getNewTid()};
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));

-    pk.pack(std::string("a"));  pk.pack_map(2 + (want>0?1:0));
+    pk.pack(std::string("a"));  pk.pack_map(2 +
+                                (query.where.getFilter() or not query.select.getSelection().empty() ? 1:0) +
+                                (want>0?1:0));
      pk.pack(std::string("id")); pk.pack(myid);
      pk.pack(std::string("h"));  pk.pack(info_hash);
+      pk.pack(std::string("q")); pk.pack(query);
    if (want > 0) {
      pk.pack(std::string("w"));
      pk.pack_array(((want & WANT4)?1:0) + ((want & WANT6)?1:0));
@@ -600,6 +623,9 @@ NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash,
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("q"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    Blob b {buffer.data(), buffer.data() + buffer.size()};
    std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -620,7 +646,7 @@ NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash,
 }

 void
-NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {
+NetworkEngine::deserializeNodes(ParsedMessage& msg) {
    if (msg.nodes4_raw.size() % NODE4_INFO_BUF_LEN != 0 || msg.nodes6_raw.size() % NODE6_INFO_BUF_LEN != 0) {
        throw DhtProtocolException {DhtProtocolException::WRONG_NODE_INFO_BUF_LEN};
    } else {
@@ -638,7 +664,7 @@ NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {
            memcpy(&sin.sin_port, ni + ni_id.size() + 4, 2);
            if (isMartian((sockaddr*)&sin, sizeof(sin)) || isNodeBlacklisted((sockaddr*)&sin, sizeof(sin)))
                continue;
-            msg.nodes4.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin, sizeof(sin), now, 0));
+            msg.nodes4.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin, sizeof(sin), now, false));
            onNewNode(msg.nodes4.back(), 0);
        }
        for (unsigned i = 0; i < msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i++) {
@@ -653,7 +679,7 @@ NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {
            memcpy(&sin6.sin6_port, ni + HASH_LEN + 16, 2);
            if (isMartian((sockaddr*)&sin6, sizeof(sin6)) || isNodeBlacklisted((sockaddr*)&sin6, sizeof(sin6)))
                continue;
-            msg.nodes6.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin6, sizeof(sin6), now, 0));
+            msg.nodes6.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin6, sizeof(sin6), now, false));
            onNewNode(msg.nodes6.back(), 0);
        }
    }
@@ -661,10 +687,10 @@ NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {

 void
 NetworkEngine::sendNodesValues(const sockaddr* sa, socklen_t salen, TransId tid, const Blob& nodes, const Blob& nodes6,
-        const std::vector<std::shared_ptr<Value>>& st, const Blob& token) {
+        const std::vector<std::shared_ptr<Value>>& st, const Query& query, const Blob& token) {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));

    pk.pack(std::string("r"));
    pk.pack_map(2 + (not st.empty()?1:0) + (nodes.size()>0?1:0) + (nodes6.size()>0?1:0) + (not token.empty()?1:0));
@@ -683,38 +709,56 @@ NetworkEngine::sendNodesValues(const sockaddr* sa, socklen_t salen, TransId tid,
    if (not token.empty()) {
        pk.pack(std::string("token")); packToken(pk, token);
    }
-    if (not st.empty()) {
-        // We treat the storage as a circular list, and serve a randomly
-        // chosen slice.  In order to make sure we fit,
-        // we limit ourselves to 50 values.
-        std::uniform_int_distribution<> pos_dis(0, st.size()-1);
-        std::vector<Blob> subset {};
-        subset.reserve(std::min<size_t>(st.size(), 50));
-
+    if (not st.empty()) { /* pack complete values */
+        auto fields = query.select.getSelection();
        size_t total_size = 0;
-        unsigned j0 = pos_dis(rd_device);
-        unsigned j = j0;
-        unsigned k = 0;
+        if (fields.empty()) {
+            // We treat the storage as a circular list, and serve a randomly
+            // chosen slice.  In order to make sure we fit,
+            // we limit ourselves to 50 values.
+            std::uniform_int_distribution<> pos_dis(0, st.size()-1);
+            std::vector<Blob> subset {};
+            subset.reserve(std::min<size_t>(st.size(), 50));

-        do {
-            subset.emplace_back(packMsg(st[j]));
-            total_size += subset.back().size();
-            ++k;
-            j = (j + 1) % st.size();
-        } while (j != j0 && k < 50 && total_size < MAX_VALUE_SIZE);
+            unsigned j0 = pos_dis(rd_device);
+            unsigned j = j0;
+            unsigned k = 0;

-        pk.pack(std::string("values"));
-        pk.pack_array(subset.size());
-        for (const auto& b : subset)
-            buffer.write((const char*)b.data(), b.size());
-        DHT_LOG.DEBUG("sending closest nodes (%d+%d nodes.), %lu bytes of values", nodes.size(), nodes6.size(), total_size);
+            do {
+                subset.emplace_back(packMsg(st[j]));
+                total_size += subset.back().size();
+                ++k;
+                j = (j + 1) % st.size();
+            } while (j != j0 && k < 50 && total_size < MAX_VALUE_SIZE);
+
+            pk.pack(std::string("values"));
+            pk.pack_array(subset.size());
+            for (const auto& b : subset)
+                buffer.write((const char*)b.data(), b.size());
+            DHT_LOG.DEBUG("sending closest nodes (%d+%d nodes.), %lu bytes of values",
+                    nodes.size(), nodes6.size(), total_size);
+        } else { /* pack fields */
+            pk.pack(std::string("fields"));
+            pk.pack_map(2);
+            pk.pack(std::string("f")); pk.pack(fields);
+            pk.pack(std::string("v")); pk.pack_array(st.size()*fields.size());
+            for (const auto& v : st)
+                v->msgpack_pack_fields(fields, pk);
+            DHT_LOG.DEBUG("sending closest nodes (%d+%d nodes.), %u value headers containing %u fields",
+                    nodes.size(), nodes6.size(), st.size(), fields.size());
+        }
    } else
        DHT_LOG.DEBUG("sending closest nodes (%d+%d nodes.)", nodes.size(), nodes6.size());

+    DHT_LOG.DEBUG("sending closest nodes (%d+%d nodes.)", nodes.size(), nodes6.size());
+
    pk.pack(std::string("t")); pk.pack_bin(tid.size());
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("r"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    send(buffer.data(), buffer.size(), 0, sa, salen);
 }
@@ -725,7 +769,7 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vector<std::
    std::sort(nodes.begin(), nodes.end(), [&](const std::shared_ptr<Node>& a, const std::shared_ptr<Node>& b){
        return id.xorCmp(a->id, b->id) < 0;
    });
-    size_t nnode = std::min<size_t>(TARGET_NODES, nodes.size());
+    size_t nnode = std::min<size_t>(SEND_NODES, nodes.size());
    Blob bnodes;
    if (af == AF_INET) {
        bnodes.resize(NODE4_INFO_BUF_LEN * nnode);
@@ -772,16 +816,18 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, want_t want,
 }

 std::shared_ptr<Request>
-NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, const Blob& token,
+NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, const Query& query, const Blob& token,
        RequestCb on_done, RequestExpiredCb on_expired) {
    auto tid = TransId {TransPrefix::LISTEN, getNewTid()};
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));

-    pk.pack(std::string("a")); pk.pack_map(3);
+    pk.pack(std::string("a")); pk.pack_map(3 +
+                               (query.where.getFilter() or not query.select.getSelection().empty() ? 1:0));
      pk.pack(std::string("id"));    pk.pack(myid);
      pk.pack(std::string("h"));     pk.pack(infohash);
+      pk.pack(std::string("q")); pk.pack(query);
      pk.pack(std::string("token")); packToken(pk, token);

    pk.pack(std::string("q")); pk.pack(std::string("listen"));
@@ -789,7 +835,9 @@ NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, con
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("q"));
    pk.pack(std::string("v")); pk.pack(my_v);
-
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    Blob b {buffer.data(), buffer.data() + buffer.size()};
    std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -812,7 +860,7 @@ void
 NetworkEngine::sendListenConfirmation(const sockaddr* sa, socklen_t salen, TransId tid) {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));

    pk.pack(std::string("r")); pk.pack_map(2);
      pk.pack(std::string("id")); pk.pack(myid);
@@ -822,6 +870,9 @@ NetworkEngine::sendListenConfirmation(const sockaddr* sa, socklen_t salen, Trans
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("r"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    send(buffer.data(), buffer.size(), 0, sa, salen);
 }
@@ -832,7 +883,7 @@ NetworkEngine::sendAnnounceValue(std::shared_ptr<Node> n, const InfoHash& infoha
    auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, getNewTid()};
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));

    pk.pack(std::string("a")); pk.pack_map((created < scheduler.time() ? 5 : 4));
      pk.pack(std::string("id"));     pk.pack(myid);
@@ -849,6 +900,9 @@ NetworkEngine::sendAnnounceValue(std::shared_ptr<Node> n, const InfoHash& infoha
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("q"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    Blob b {buffer.data(), buffer.data() + buffer.size()};
    std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -878,7 +932,7 @@ void
 NetworkEngine::sendValueAnnounced(const sockaddr* sa, socklen_t salen, TransId tid, Value::Id vid) {
    msgpack::sbuffer buffer;
    msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));

    pk.pack(std::string("r")); pk.pack_map(3);
      pk.pack(std::string("id"));  pk.pack(myid);
@@ -889,6 +943,9 @@ NetworkEngine::sendValueAnnounced(const sockaddr* sa, socklen_t salen, TransId t
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("r"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    send(buffer.data(), buffer.size(), 0, sa, salen);
 }
@@ -917,39 +974,47 @@ NetworkEngine::sendError(const sockaddr* sa,
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
    pk.pack(std::string("y")); pk.pack(std::string("e"));
    pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }

    send(buffer.data(), buffer.size(), 0, sa, salen);
 }

-msgpack::object*
-findMapValue(msgpack::object& map, const std::string& key) {
-    if (map.type != msgpack::type::MAP) throw msgpack::type_error();
-    for (unsigned i = 0; i < map.via.map.size; i++) {
-        auto& o = map.via.map.ptr[i];
-        if(o.key.type != msgpack::type::STR)
-            continue;
-        if (o.key.as<std::string>() == key) {
-            return &o.val;
-        }
-    }
-    return nullptr;
-}
-
 void
 ParsedMessage::msgpack_unpack(msgpack::object msg)
 {
    auto y = findMapValue(msg, "y");
-    auto a = findMapValue(msg, "a");
    auto r = findMapValue(msg, "r");
    auto e = findMapValue(msg, "e");

-    std::string query;
-    if (auto q = findMapValue(msg, "q")) {
-        if (q->type != msgpack::type::STR)
+    std::string q;
+    if (auto rq = findMapValue(msg, "q")) {
+        if (rq->type != msgpack::type::STR)
            throw msgpack::type_error();
-        query = q->as<std::string>();
+        q = rq->as<std::string>();
    }

+    if (e)
+        type = MessageType::Error;
+    else if (r)
+        type = MessageType::Reply;
+    else if (y and y->as<std::string>() != "q")
+        throw msgpack::type_error();
+    else if (q == "ping")
+        type = MessageType::Ping;
+    else if (q == "find")
+        type = MessageType::FindNode;
+    else if (q == "get")
+        type = MessageType::GetValues;
+    else if (q == "listen")
+        type = MessageType::Listen;
+    else if (q == "put")
+        type = MessageType::AnnounceValue;
+    else
+        throw msgpack::type_error();
+
+    auto a = findMapValue(msg, "a");
    if (!a && !r && !e)
        throw msgpack::type_error();
    auto& req = a ? *a : (r ? *r : *e);
@@ -960,6 +1025,9 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
        error_code = e->via.array.ptr[0].as<uint16_t>();
    }

+    if (auto netid = findMapValue(msg, "n"))
+        network = netid->as<NetId>();
+
    if (auto rid = findMapValue(req, "id"))
        id = {*rid};

@@ -969,6 +1037,9 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
    if (auto rtarget = findMapValue(req, "target"))
        target = {*rtarget};

+    if (auto rquery = findMapValue(req, "q"))
+        query.msgpack_unpack(*rquery);
+
    if (auto otoken = findMapValue(req, "token"))
        token = unpackBlob(*otoken);

@@ -1015,6 +1086,23 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
            } catch (const std::exception& e) {
                //DHT_LOG.WARN("Error reading value: %s", e.what());
            }
+    } else if (auto raw_fields = findMapValue(req, "fields")) {
+        if (auto rfields = findMapValue(*raw_fields, "f")) {
+            auto fields_ = rfields->as<std::set<Value::Field>>();
+            if (auto rvalues = findMapValue(*raw_fields, "v")) {
+                if (rvalues->type != msgpack::type::ARRAY)
+                    throw msgpack::type_error();
+                for (size_t i = 0; i < rvalues->via.array.size; ++i) {
+                    try {
+                        auto v = std::make_shared<FieldValueIndex>();
+                        v->msgpack_unpack_fields(fields_, *rvalues, i*fields_.size());
+                        fields.emplace_back(std::move(v));
+                    } catch (const std::exception& e) { }
+                }
+            }
+        } else {
+            throw msgpack::type_error();
+        }
    }

    if (auto w = findMapValue(req, "w")) {
@@ -1041,24 +1129,6 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
    if (auto rv = findMapValue(msg, "v"))
        ua = rv->as<std::string>();

-    if (e)
-        type = MessageType::Error;
-    else if (r)
-        type = MessageType::Reply;
-    else if (y and y->as<std::string>() != "q")
-        throw msgpack::type_error();
-    else if (query == "ping")
-        type = MessageType::Ping;
-    else if (query == "find")
-        type = MessageType::FindNode;
-    else if (query == "get")
-        type = MessageType::GetValues;
-    else if (query == "listen")
-        type = MessageType::Listen;
-    else if (query == "put")
-        type = MessageType::AnnounceValue;
-    else
-        throw msgpack::type_error();
 }

 }
@@ -27,12 +27,44 @@ NodeCache::getNode(const InfoHash& id, sa_family_t family) {
 }

 std::shared_ptr<Node>
-NodeCache::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirm) {
+NodeCache::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, bool confirm) {
    if (id == zeroes)
        return std::make_shared<Node>(id, sa, sa_len);
    return (sa->sa_family == AF_INET ? cache_4 : cache_6).getNode(id, sa, sa_len, now, confirm);
 }

+std::vector<std::shared_ptr<Node>>
+NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) {
+    const auto& c = (sa_f == AF_INET ? cache_4 : cache_6);
+    auto it_p = c.lower_bound(id),
+         it_n = it_p;
+
+    std::vector<std::shared_ptr<Node>> nodes;
+    nodes.reserve(std::min(c.size(), count));
+    NodeMap::const_iterator it;
+
+    if (it_p != c.begin()) /* Create 2 separate iterator if we could */
+        --it_p;
+
+    while (nodes.size() < count and (it_n != c.end() or it_p != c.end())) {
+        /* If one of the iterator is at the end, then take the other one
+           If they are both in middle of somewhere comapre both and take
+           the closest to the id. */
+        if (it_p == c.end())       it = it_n++;
+        else if (it_n == c.end())  it = it_p--;
+        else                       it = id.xorCmp(it_p->first, it_n->first) < 0 ? it_p-- : it_n++;
+
+        if (it == c.begin())
+            it_p = c.end();
+
+        if (auto n = it->second.lock())
+            if ( not n->isExpired() )
+                nodes.emplace_back(std::move(n));
+    }
+
+    return nodes;
+}
+
 void
 NodeCache::clearBadNodes(sa_family_t family)
 {
@@ -57,7 +89,7 @@ NodeCache::NodeMap::getNode(const InfoHash& id)
 }

 std::shared_ptr<Node>
-NodeCache::NodeMap::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirm)
+NodeCache::NodeMap::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, bool confirm)
 {
    auto it = emplace(id, std::weak_ptr<Node>{});
    auto node = it.first->second.lock();
@@ -99,7 +99,7 @@ SecureDht::secureType(ValueType&& type)
                return false;
            }
            else
-                DHT_LOG.WARN("Signature verification succeded");
+                DHT_LOG.WARN("Signature verification succeeded");
        }
        return type.storePolicy(id, v, nid, a, al);
    };
@@ -141,7 +141,7 @@ SecureDht::getCertificate(const InfoHash& node) const
        return it->second;
 }

-const std::shared_ptr<crypto::PublicKey>
+const std::shared_ptr<const crypto::PublicKey>
 SecureDht::getPublicKey(const InfoHash& node) const
 {
    if (node == getId())
@@ -226,7 +226,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function<void(const std::s
 }

 void
-SecureDht::findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::PublicKey>)> cb)
+SecureDht::findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<const crypto::PublicKey>)> cb)
 {
    auto pk = getPublicKey(node);
    if (pk && *pk) {
@@ -291,15 +291,15 @@ SecureDht::getCallbackFilter(GetCallback cb, Value::Filter&& filter)
 }

 void
-SecureDht::get(const InfoHash& id, GetCallback cb, DoneCallback donecb, Value::Filter&& f)
+SecureDht::get(const InfoHash& id, GetCallback cb, DoneCallback donecb, Value::Filter&& f, Where&& w)
 {
-    Dht::get(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)), donecb);
+    Dht::get(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)), donecb, {}, std::forward<Where>(w));
 }

 size_t
-SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f)
+SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f, Where&& w)
 {
-    return Dht::listen(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)));
+    return Dht::listen(id, getCallbackFilter(cb, std::forward<Value::Filter>(f)), {}, std::forward<Where>(w));
 }

 void
@@ -342,7 +342,7 @@ SecureDht::putSigned(const InfoHash& hash, std::shared_ptr<Value> val, DoneCallb
 void
 SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, std::shared_ptr<Value> val, DoneCallback callback, bool permanent)
 {
-    findPublicKey(to, [=](const std::shared_ptr<crypto::PublicKey> pk) {
+    findPublicKey(to, [=](const std::shared_ptr<const crypto::PublicKey> pk) {
        if(!pk || !*pk) {
            if (callback)
                callback(false, {});
@@ -362,22 +362,13 @@ SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, std::shared_pt
 void
 SecureDht::sign(Value& v) const
 {
-    if (v.isEncrypted())
-        throw DhtException("Can't sign encrypted data.");
-    v.owner = std::make_shared<crypto::PublicKey>(key_->getPublicKey());
-    v.signature = key_->sign(v.getToSign());
+    v.sign(*key_);
 }

 Value
 SecureDht::encrypt(Value& v, const crypto::PublicKey& to) const
 {
-    if (v.isEncrypted())
-        throw DhtException("Data is already encrypted.");
-    v.setRecipient(to.getId());
-    sign(v);
-    Value nv {v.id};
-    nv.setCypher(to.encrypt(v.getToEncrypt()));
-    return nv;
+    return v.encrypt(*key_, to);
 }

 Value
@@ -85,4 +85,15 @@ unpackMsg(Blob b) {
    return msgpack::unpack((const char*)b.data(), b.size());
 }

+msgpack::object*
+findMapValue(msgpack::object& map, const std::string& key) {
+    if (map.type != msgpack::type::MAP) throw msgpack::type_error();
+    for (unsigned i = 0; i < map.via.map.size; i++) {
+        auto& o = map.via.map.ptr[i];
+        if (o.key.type == msgpack::type::STR && o.key.as<std::string>() == key)
+            return &o.val;
+    }
+    return nullptr;
+}
+
 }
@@ -25,6 +25,8 @@

 namespace dht {

+const std::string Query::QUERY_PARSE_ERROR {"Error parsing query."};
+
 std::ostream& operator<< (std::ostream& s, const Value& v)
 {
    s << "Value[id:" << std::hex << v.id << std::dec << " ";
@@ -142,8 +144,9 @@ Value::msgpack_unpack_body(const msgpack::object& o)
                seq = rseq->as<decltype(seq)>();
            else
                throw msgpack::type_error();
-            owner = std::make_shared<crypto::PublicKey>();
-            owner->msgpack_unpack(*rowner);
+            crypto::PublicKey new_owner;
+            new_owner.msgpack_unpack(*rowner);
+            owner = std::make_shared<const crypto::PublicKey>(std::move(new_owner));
            if (auto rrecipient = findMapValue(*rbody, "to")) {
                recipient = rrecipient->as<InfoHash>();
            }
@@ -156,4 +159,306 @@ Value::msgpack_unpack_body(const msgpack::object& o)
    }
 }

+bool
+FieldValue::operator==(const FieldValue& vfd) const
+{
+    if (field != vfd.field)
+        return false;
+    switch (field) {
+        case Value::Field::Id:
+        case Value::Field::ValueType:
+            return intValue == vfd.intValue;
+        case Value::Field::OwnerPk:
+            return hashValue == vfd.hashValue;
+        case Value::Field::UserType:
+            return blobValue == vfd.blobValue;
+        case Value::Field::None:
+            return true;
+        default:
+            return false;
+    }
 }
+
+Value::Filter
+FieldValue::getLocalFilter() const
+{
+    switch (field) {
+        case Value::Field::Id:
+            return Value::IdFilter(intValue);
+        case Value::Field::ValueType:
+            return Value::TypeFilter(intValue);
+        case Value::Field::OwnerPk:
+            return Value::ownerFilter(hashValue);
+        case Value::Field::UserType:
+            return Value::userTypeFilter(std::string {blobValue.begin(), blobValue.end()});
+        default:
+            return Value::AllFilter();
+    }
+}
+
+FieldValueIndex::FieldValueIndex(const Value& v, Select s)
+{
+    auto selection = s.getSelection();
+    if (not selection.empty()) {
+        std::transform(selection.begin(), selection.end(), std::inserter(index, index.end()),
+            [](const std::set<Value::Field>::value_type& f) {
+                return std::make_pair(f, FieldValue {});
+        });
+    } else {
+        index.clear();
+        for (size_t f = 1 ; f < 5 ; ++f)
+            index[static_cast<Value::Field>(f)] = {};
+    }
+    for (const auto& fvp : index) {
+        const auto& f = fvp.first;
+        switch (f) {
+            case Value::Field::Id:
+                index[f] = {f, v.id};
+                break;
+            case Value::Field::ValueType:
+                index[f] = {f, v.type};
+                break;
+            case Value::Field::OwnerPk:
+                index[f] = {f, v.owner ? v.owner->getId() : InfoHash() };
+                break;
+            case Value::Field::UserType:
+                index[f] = {f, Blob {v.user_type.begin(), v.user_type.end()}};
+                break;
+            default:
+                break;
+        }
+    }
+}
+
+bool FieldValueIndex::containedIn(const FieldValueIndex& other) const {
+    if (index.size() > other.index.size())
+        return false;
+    for (const auto& field : index) {
+        auto other_field = other.index.find(field.first);
+        if (other_field == other.index.end())
+            return false;
+    }
+    return true;
+}
+
+std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) {
+    os << "Index[";
+    for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) {
+        switch (v->first) {
+            case Value::Field::Id:
+                os << "Id:" << std::hex << v->second.getInt();
+                break;
+            case Value::Field::ValueType:
+                os << "ValueType:" << v->second.getInt();
+                break;
+            case Value::Field::OwnerPk:
+                os << "Owner:" << v->second.getHash().toString();
+                break;
+            case Value::Field::UserType: {
+                auto ut = v->second.getBlob();
+                os << "UserType:" << std::string(ut.begin(), ut.end());
+                break;
+            }
+            default:
+                break;
+        }
+        os << (std::next(v) != fvi.index.end() ? "," : "");
+    }
+    return os << "]";
+}
+
+void
+FieldValueIndex::msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset)
+{
+    index.clear();
+
+    unsigned j = 0;
+    for (const auto& field : fields) {
+        auto& field_value = o.via.array.ptr[offset+(j++)];
+        switch (field) {
+            case Value::Field::Id:
+            case Value::Field::ValueType:
+                index[field] = FieldValue(field, field_value.as<uint64_t>());
+                break;
+            case Value::Field::OwnerPk:
+                index[field] = FieldValue(field, field_value.as<InfoHash>());
+                break;
+            case Value::Field::UserType:
+                index[field] = FieldValue(field, field_value.as<Blob>());
+                break;
+            default:
+                throw msgpack::type_error();
+        }
+    }
+}
+
+void trim_str(std::string& str) {
+    auto first = std::min(str.size(), str.find_first_not_of(" "));
+    auto last = std::min(str.size(), str.find_last_not_of(" "));
+    str = str.substr(first, last - first + 1);
+}
+
+Select::Select(const std::string& q_str) {
+    std::istringstream q_iss {q_str};
+    std::string token {};
+    q_iss >> token;
+
+    if (token == "SELECT" or token == "select") {
+        q_iss >> token;
+        std::istringstream fields {token};
+
+        while (std::getline(fields, token, ',')) {
+            trim_str(token);
+            if (token == "id")
+                field(Value::Field::Id);
+            else if (token == "value_type")
+                field(Value::Field::ValueType);
+            else if (token == "owner_pk")
+                field(Value::Field::OwnerPk);
+            else if (token == "user_type")
+                field(Value::Field::UserType);
+        }
+    }
+}
+
+Where::Where(const std::string& q_str) {
+    std::istringstream q_iss {q_str};
+    std::string token {};
+    q_iss >> token;
+    if (token == "WHERE" or token == "where") {
+        std::getline(q_iss, token);
+        std::istringstream restrictions {token};
+        while (std::getline(restrictions, token, ',')) {
+            trim_str(token);
+            std::istringstream eq_ss {token};
+            std::string field_str, value_str;
+            std::getline(eq_ss, field_str, '=');
+            trim_str(field_str);
+            std::getline(eq_ss, value_str, '=');
+            trim_str(value_str);
+
+            if (not value_str.empty()) {
+                uint64_t v = 0;
+                std::string s {};
+                std::istringstream convert {value_str};
+                convert >> v;
+                if (convert.failbit and value_str.size() > 1 and value_str[0] == '\"' and value_str[value_str.size()-1] == '\"')
+                    s = value_str.substr(1, value_str.size()-2);
+                else
+                    s = value_str;
+                if (field_str == "id")
+                    id(v);
+                else if (field_str == "value_type")
+                    valueType(v);
+                else if (field_str == "owner_pk")
+                    owner(InfoHash(s));
+                else if (field_str == "user_type")
+                    userType(s);
+                else
+                    throw std::invalid_argument(Query::QUERY_PARSE_ERROR + " (WHERE) wrong token near: " + field_str);
+            }
+        }
+    }
+}
+
+void
+Query::msgpack_unpack(const msgpack::object& o)
+{
+	if (o.type != msgpack::type::MAP)
+		throw msgpack::type_error();
+
+	auto rfilters = findMapValue(o, "w"); /* unpacking filters */
+	if (rfilters)
+        where.msgpack_unpack(*rfilters);
+	else
+		throw msgpack::type_error();
+
+	auto rfield_selector = findMapValue(o, "s"); /* unpacking field selectors */
+	if (rfield_selector)
+        select.msgpack_unpack(*rfield_selector);
+	else
+		throw msgpack::type_error();
+}
+
+template <typename T>
+bool subset(std::vector<T> fds, std::vector<T> qfds)
+{
+    for (auto& fd : fds) {
+        auto correspondance = std::find_if(qfds.begin(), qfds.end(), [&fd](T& _vfd) { return fd == _vfd; });
+        if (correspondance == qfds.end())
+            return false;
+    }
+    return true;
+}
+
+bool Select::isSatisfiedBy(const Select& os) const {
+    /* empty, means all values are selected. */
+    if (fieldSelection_.empty() and not os.fieldSelection_.empty())
+        return false;
+    else
+        return subset(fieldSelection_, os.fieldSelection_);
+}
+
+bool Where::isSatisfiedBy(const Where& ow) const {
+    return subset(ow.filters_, filters_);
+}
+
+bool Query::isSatisfiedBy(const Query& q) const {
+    return where.isSatisfiedBy(q.where) and select.isSatisfiedBy(q.select);
+}
+
+std::ostream& operator<<(std::ostream& s, const dht::Select& select) {
+    s << "SELECT " << (select.fieldSelection_.empty() ? "*" : "");
+    for (auto fs = select.fieldSelection_.begin() ; fs != select.fieldSelection_.end() ; ++fs) {
+        switch (fs->getField()) {
+            case Value::Field::Id:
+                s << "id";
+                break;
+            case Value::Field::ValueType:
+                s << "value_type";
+                break;
+            case Value::Field::UserType:
+                s << "user_type";
+                break;
+            case Value::Field::OwnerPk:
+                s << "owner_public_key";
+                break;
+            default:
+                break;
+        }
+        s << (std::next(fs) != select.fieldSelection_.end() ? "," : "");
+    }
+    return s;
+}
+
+std::ostream& operator<<(std::ostream& s, const dht::Where& where) {
+    if (not where.filters_.empty()) {
+        s << "WHERE ";
+        for (auto f = where.filters_.begin() ; f != where.filters_.end() ; ++f) {
+            switch (f->getField()) {
+                case Value::Field::Id:
+                    s << "id=" << f->getInt();
+                    break;
+                case Value::Field::ValueType:
+                    s << "value_type=" << f->getInt();
+                    break;
+                case Value::Field::OwnerPk:
+                    s << "owner_pk_hash=" << f->getHash().toString();
+                    break;
+                case Value::Field::UserType: {
+                    auto b = f->getBlob();
+                    s << "user_type=" << std::string {b.begin(), b.end()};
+                    break;
+                }
+                default:
+                    break;
+            }
+            s << (std::next(f) != where.filters_.end() ? "," : "");
+        }
+    }
+    return s;
+}
+
+
+}
+
@@ -53,7 +53,7 @@ main(int argc, char **argv)
        throw std::runtime_error(std::string("Error initializing GnuTLS: ")+gnutls_strerror(rc));

    DhtRunner dht;
-    dht.run(params.port, dht::crypto::generateIdentity("DHT Chat Node"), true);
+    dht.run(params.port, dht::crypto::generateIdentity("DHT Chat Node"), true, params.network);

    if (not params.bootstrap.first.empty())
        dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
@@ -29,7 +29,7 @@ extern "C" {
 using namespace dht;

 void print_usage() {
-    std::cout << "Usage: dhtnode [-p local_port] [-b bootstrap_host:port]" << std::endl << std::endl;
+    std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl;
    std::cout << "dhtnode, a simple OpenDHT command line node runner." << std::endl;
    std::cout << "Report bugs to: http://opendht.net" << std::endl;
 }
@@ -38,19 +38,17 @@ void print_id_req() {
    std::cout << "An identity is required to perform this operation (run with -i)" << std::endl;
 }

-void print_node_info(const DhtRunner& dht, const dht_params& params) {
-    std::cout << "OpenDht node " << dht.getNodeId() << " running on port " <<  dht.getBoundPort() << std::endl;
-    if (params.is_bootstrap_node)
-        std::cout << "Running in bootstrap mode (discouraged)." << std::endl;
+void print_node_info(const std::shared_ptr<DhtRunner>& dht, const dht_params& params) {
+    std::cout << "OpenDht node " << dht->getNodeId() << " running on port " <<  dht->getBoundPort() << std::endl;
    if (params.generate_identity)
-        std::cout << "Public key ID " << dht.getId() << std::endl;
+        std::cout << "Public key ID " << dht->getId() << std::endl;
 }

 void print_help() {
    std::cout << "OpenDht command line interface (CLI)" << std::endl;
    std::cout << "Possible commands:" << std::endl
              << "  h, help    Print this help message." << std::endl
-              << "  q, quit    Quit the program." << std::endl
+              << "  x, quit    Quit the program." << std::endl
              << "  log        Start/stop printing DHT logs." << std::endl;

    std::cout << std::endl << "Node information:" << std::endl
@@ -60,16 +58,20 @@ void print_help() {
              << "  lr         Print the full current routing table of this node" << std::endl;

    std::cout << std::endl << "Operations on the DHT:" << std::endl
-              << "  b ip:port             Ping potential node at given IP address/port." << std::endl
-              << "  g [key]               Get values at [key]." << std::endl
-              << "  l [key]               Listen for value changes at [key]." << std::endl
-              << "  p [key] [str]         Put string value at [key]." << std::endl
-              << "  s [key] [str]         Put string value at [key], signed with our generated private key." << std::endl
-              << "  e [key] [dest] [str]  Put string value at [key], encrypted for [dest] with its public key (if found)." << std::endl
+              << "  b <ip:port>             Ping potential node at given IP address/port." << std::endl
+              << "  g <key>               Get values at <key>." << std::endl
+              << "  l <key>               Listen for value changes at <key>." << std::endl
+              << "  p <key> <str>         Put string value at <key>." << std::endl
+              << "  s <key> <str>         Put string value at <key>, signed with our generated private key." << std::endl
+              << "  e <key> <dest> <str>  Put string value at <key>, encrypted for <dest> with its public key (if found)." << std::endl;
+    std::cout << std::endl << "Indexation operations on the DHT:" << std::endl
+              << "  il <name> <key> [exact match]   Lookup the index named <name> with the key <key>." << std::endl
+              << "                                  Set [exact match] to 'false' for inexact match lookup." << std::endl
+              << "  ii <name> <key> <value>         Inserts the value <value> under the key <key> in the index named <name>." << std::endl
              << std::endl;
 }

-void cmd_loop(DhtRunner& dht, dht_params& params)
+void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation::Pht> indexes, dht_params& params)
 {
    print_node_info(dht, params);
    std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl << std::endl;
@@ -85,10 +87,10 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
            break;

        std::istringstream iss(line);
-        std::string op, idstr, value;
-        iss >> op >> idstr;
+        std::string op, idstr, value, index, keystr;
+        iss >> op;

-        if (op == "x" || op == "q" || op == "exit" || op == "quit") {
+        if (op == "x" || op == "exit" || op == "quit") {
            break;
        } else if (op == "h" || op == "help") {
            print_help();
@@ -97,31 +99,32 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
            print_node_info(dht, params);
            unsigned good4, dubious4, cached4, incoming4;
            unsigned good6, dubious6, cached6, incoming6;
-            dht.getNodesStats(AF_INET, &good4, &dubious4, &cached4, &incoming4);
-            dht.getNodesStats(AF_INET6, &good6, &dubious6, &cached6, &incoming6);
+            dht->getNodesStats(AF_INET, &good4, &dubious4, &cached4, &incoming4);
+            dht->getNodesStats(AF_INET6, &good6, &dubious6, &cached6, &incoming6);
            std::cout << "IPv4 nodes : " << good4 << " good, " << dubious4 << " dubious, " << incoming4 << " incoming." << std::endl;
            std::cout << "IPv6 nodes : " << good6 << " good, " << dubious6 << " dubious, " << incoming6 << " incoming." << std::endl;
            continue;
        } else if (op == "lr") {
            std::cout << "IPv4 routing table:" << std::endl;
-            std::cout << dht.getRoutingTablesLog(AF_INET) << std::endl;
+            std::cout << dht->getRoutingTablesLog(AF_INET) << std::endl;
            std::cout << "IPv6 routing table:" << std::endl;
-            std::cout << dht.getRoutingTablesLog(AF_INET6) << std::endl;
+            std::cout << dht->getRoutingTablesLog(AF_INET6) << std::endl;
            continue;
        } else if (op == "ld") {
-            std::cout << dht.getStorageLog() << std::endl;
+            std::cout << dht->getStorageLog() << std::endl;
            continue;
        } else if (op == "ls") {
            std::cout << "Searches:" << std::endl;
-            std::cout << dht.getSearchesLog() << std::endl;
+            std::cout << dht->getSearchesLog() << std::endl;
            continue;
        } else if (op == "la")  {
            std::cout << "Reported public addresses:" << std::endl;
-            auto addrs = dht.getPublicAddressStr();
+            auto addrs = dht->getPublicAddressStr();
            for (const auto& addr : addrs)
                std::cout << addr << std::endl;
            continue;
        } else if (op == "b") {
+            iss >> idstr;
            try {
                auto addr = splitPort(idstr);
                if (not addr.first.empty() and addr.second.empty()){
@@ -129,7 +132,7 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
                    ss << DHT_DEFAULT_PORT;
                    addr.second = ss.str();
                }
-                dht.bootstrap(addr.first.c_str(), addr.second.c_str());
+                dht->bootstrap(addr.first.c_str(), addr.second.c_str());
            } catch (const std::exception& e) {
                std::cerr << e.what() << std::endl;
            }
@@ -137,31 +140,59 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
        } else if (op == "log") {
            params.log = !params.log;
            if (params.log)
-                log::enableLogging(dht);
+                log::enableLogging(*dht);
            else
-                log::disableLogging(dht);
+                log::disableLogging(*dht);
            continue;
        }

        if (op.empty())
            continue;

-        dht::InfoHash id {idstr};
-        static const std::set<std::string> VALID_OPS {"g", "l", "p", "s", "e", "a"};
+        static const std::set<std::string> VALID_OPS {"g", "l", "il", "ii", "p", "s", "e", "a"};
        if (VALID_OPS.find(op) == VALID_OPS.cend()) {
            std::cout << "Unknown command: " << op << std::endl;
            std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl;
            continue;
        }
-        static constexpr dht::InfoHash INVALID_ID {};
-        if (id == INVALID_ID) {
-            std::cout << "Syntax error: invalid InfoHash." << std::endl;
-            continue;
+
+        if (op == "il" or op == "ii") {
+            // Pht syntax
+            iss >> index >> keystr;
+            auto new_index = std::find_if(indexes.begin(), indexes.end(),
+                    [&](std::pair<const std::string, indexation::Pht>& i) {
+                        return i.first == index;
+                    }) == indexes.end();
+            if (not index.size()) {
+                std::cout << "You must enter the index name." << std::endl;
+                continue;
+            } else if (new_index) {
+                using namespace dht::indexation;
+                try {
+                    auto key = createPhtKey(parseStringMap(keystr));
+                    Pht::KeySpec ks;
+                    std::transform(key.begin(), key.end(), std::inserter(ks, ks.end()), [](Pht::Key::value_type& f) {
+                        return std::make_pair(f.first, f.second.size());
+                    });
+                    indexes.emplace(index, Pht {index, std::move(ks), dht});
+                } catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
+            }
+        }
+        else {
+            // Dht syntax
+            iss >> idstr;
+            InfoHash h {idstr};
+            if (not isInfoHash(h))
+                continue;
        }

+        dht::InfoHash id {idstr};
+        // Dht
        auto start = std::chrono::high_resolution_clock::now();
        if (op == "g") {
-            dht.get(id, [start](std::shared_ptr<Value> value) {
+            std::string rem;
+            std::getline(iss, rem);
+            dht->get(id, [start](std::shared_ptr<Value> value) {
                auto now = std::chrono::high_resolution_clock::now();
                std::cout << "Get: found value (after " << print_dt(now-start) << "s)" << std::endl;
                std::cout << "\t" << *value << std::endl;
@@ -169,19 +200,36 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
            }, [start](bool ok) {
                auto end = std::chrono::high_resolution_clock::now();
                std::cout << "Get: " << (ok ? "completed" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl;
-            });
+            }, {}, dht::Where {std::move(rem)});
+        }
+        else if (op == "q") {
+            std::string rem;
+            std::getline(iss, rem);
+            dht->query(id, [start](const std::vector<std::shared_ptr<FieldValueIndex>>& field_value_indexes) {
+                auto now = std::chrono::high_resolution_clock::now();
+                for (auto& index : field_value_indexes) {
+                    std::cout << "Query: found field value index (after " << print_dt(now-start) << "s)" << std::endl;
+                    std::cout << "\t" << *index << std::endl;
+                }
+                return true;
+            }, [start](bool ok) {
+                auto end = std::chrono::high_resolution_clock::now();
+                std::cout << "Query: " << (ok ? "completed" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl;
+            }, dht::Query {std::move(rem)});
        }
        else if (op == "l") {
-            dht.listen(id, [](std::shared_ptr<Value> value) {
+            std::string rem;
+            std::getline(iss, rem);
+            dht->listen(id, [](std::shared_ptr<Value> value) {
                std::cout << "Listen: found value:" << std::endl;
                std::cout << "\t" << *value << std::endl;
                return true;
-            });
+            }, {}, dht::Where {std::move(rem)});
        }
        else if (op == "p") {
            std::string v;
            iss >> v;
-            dht.put(id, dht::Value {
+            dht->put(id, dht::Value {
                dht::ValueType::USER_DATA.id,
                std::vector<uint8_t> {v.begin(), v.end()}
            }, [start](bool ok) {
@@ -196,7 +244,7 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
            }
            std::string v;
            iss >> v;
-            dht.putSigned(id, dht::Value {
+            dht->putSigned(id, dht::Value {
                dht::ValueType::USER_DATA.id,
                std::vector<uint8_t> {v.begin(), v.end()}
            }, [start](bool ok) {
@@ -212,7 +260,7 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
            std::string tostr;
            std::string v;
            iss >> tostr >> v;
-            dht.putEncrypted(id, InfoHash(tostr), dht::Value {
+            dht->putEncrypted(id, InfoHash(tostr), dht::Value {
                dht::ValueType::USER_DATA.id,
                std::vector<uint8_t> {v.begin(), v.end()}
            }, [start](bool ok) {
@@ -223,11 +271,56 @@ void cmd_loop(DhtRunner& dht, dht_params& params)
        else if (op == "a") {
            in_port_t port;
            iss >> port;
-            dht.put(id, dht::Value {dht::IpServiceAnnouncement::TYPE.id, dht::IpServiceAnnouncement(port)}, [start](bool ok) {
+            dht->put(id, dht::Value {dht::IpServiceAnnouncement::TYPE.id, dht::IpServiceAnnouncement(port)}, [start](bool ok) {
                auto end = std::chrono::high_resolution_clock::now();
                std::cout << "Announce: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl;
            });
        }
+        else if (op == "il") {
+            std::string exact_match;
+            iss >> exact_match;
+            try {
+                auto key = createPhtKey(parseStringMap(keystr));
+                indexes.at(index).lookup(key,
+                    [=](std::vector<std::shared_ptr<indexation::Value>>& vals, indexation::Prefix p) {
+                        if (vals.empty())
+                            return;
+                        std::cout << "Pht::lookup: found entries!" << std::endl
+                                  << p.toString() << std::endl
+                                  << "   hash: " << p.hash() << std::endl;
+                        std::cout << "   entries:" << std::endl;
+                        for (auto v : vals)
+                             std::cout << "      " << v->first.toString() << "[vid: " << v->second << "]" << std::endl;
+                    },
+                    [start](bool ok) {
+                        auto end = std::chrono::high_resolution_clock::now();
+                        std::cout << "Pht::lookup: " << (ok ? "done." : "failed.")
+                                  << " took " << print_dt(end-start) << "s)" << std::endl;
+
+                    }, exact_match.size() != 0 and exact_match == "false" ? false : true
+                );
+            }
+            catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
+            catch (std::out_of_range& e) { }
+        }
+        else if (op == "ii") {
+            iss >> idstr;
+            InfoHash h {idstr};
+            if (not isInfoHash(h))
+                continue;
+
+            indexation::Value v {h, 0};
+            try {
+                auto key = createPhtKey(parseStringMap(keystr));
+                indexes.at(index).insert(key, v,
+                    [=](bool ok) {
+                        std::cout << "Pht::insert: " << (ok ? "done." : "failed.") << std::endl;
+                    }
+                );
+            }
+            catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
+            catch (std::out_of_range& e) { }
+        }
    }

    std::cout << std::endl <<  "Stopping node..." << std::endl;
@@ -239,7 +332,9 @@ main(int argc, char **argv)
    if (int rc = gnutls_global_init())  // TODO: remove with GnuTLS >= 3.3
        throw std::runtime_error(std::string("Error initializing GnuTLS: ")+gnutls_strerror(rc));

-    DhtRunner dht;
+    auto dht = std::make_shared<DhtRunner>();
+    std::map<std::string, indexation::Pht> indexes;
+
    try {
        auto params = parseArgs(argc, argv);
        if (params.help) {
@@ -256,25 +351,25 @@ main(int argc, char **argv)
            crt = dht::crypto::generateIdentity("DHT Node", ca_tmp);
        }

-        dht.run(params.port, crt, true, params.is_bootstrap_node);
+        dht->run(params.port, crt, true, params.network);

        if (params.log) {
            if (not params.logfile.empty())
-                log::enableFileLogging(dht, params.logfile);
+                log::enableFileLogging(*dht, params.logfile);
            else
-                log::enableLogging(dht);
+                log::enableLogging(*dht);
        }

        if (not params.bootstrap.first.empty()) {
            //std::cout << "Bootstrap: " << params.bootstrap.first << ":" << params.bootstrap.second << std::endl;
-            dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
+            dht->bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
        }

        if (params.daemonize) {
            while (true)
                std::this_thread::sleep_for(std::chrono::seconds(30));
        } else {
-            cmd_loop(dht, params);
+            cmd_loop(dht, indexes, params);
        }

    } catch(const std::exception&e) {
@@ -285,7 +380,7 @@ main(int argc, char **argv)
    std::mutex m;
    std::atomic_bool done {false};

-    dht.shutdown([&]()
+    dht->shutdown([&]()
    {
        std::lock_guard<std::mutex> lk(m);
        done = true;
@@ -296,7 +391,7 @@ main(int argc, char **argv)
    std::unique_lock<std::mutex> lk(m);
    cv.wait(lk, [&](){ return done.load(); });

-    dht.join();
+    dht->join();
    gnutls_global_deinit();

    return 0;
@@ -80,7 +80,7 @@ main(int argc, char **argv)
    auto crt_tmp = dht::crypto::generateIdentity("Scanner node", ca_tmp);

    DhtRunner dht;
-    dht.run(params.port, crt_tmp, true, [](dht::NodeStatus /* ipv4 */, dht::NodeStatus /* ipv6 */) {});
+    dht.run(params.port, crt_tmp, true, params.network);

    if (not params.bootstrap.first.empty())
        dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
@@ -135,6 +135,47 @@ splitPort(const std::string& s) {
    return {s.substr(0,found), s.substr(found+1)};
 }

+/*
+ * The mapString shall have the following format:
+ *
+ *      k1:v1[,k2:v2[,...]]
+ */
+std::map<std::string, std::string> parseStringMap(std::string mapString) {
+    std::istringstream keySs(mapString);
+    std::string mapStr;
+    std::map<std::string, std::string> map;
+
+    while (std::getline(keySs, mapStr, ',')) {
+        std::istringstream mapSs(mapStr);
+        std::string key, value;
+
+        while (std::getline(mapSs, key, ':')) {
+            std::getline(mapSs, value, ':');
+            map[key] = { value };
+        }
+    }
+    return map;
+}
+
+dht::indexation::Pht::Key createPhtKey(std::map<std::string, std::string> pht_key_str_map) {
+    dht::indexation::Pht::Key pht_key;
+    for (auto f : pht_key_str_map) {
+        dht::Blob prefix {f.second.begin(), f.second.end()};
+        pht_key.emplace(f.first, std::move(prefix));
+    }
+    return pht_key;
+}
+
+bool isInfoHash(dht::InfoHash& h) {
+    static constexpr dht::InfoHash INVALID_ID {};
+
+    if (h == INVALID_ID) {
+        std::cout << "Syntax error: invalid InfoHash." << std::endl;
+        return false;
+    }
+    return true;
+}
+
 static const constexpr in_port_t DHT_DEFAULT_PORT = 4222;

 struct dht_params {
@@ -142,27 +183,29 @@ struct dht_params {
    bool log {false};
    std::string logfile {};
    in_port_t port {0};
-    bool is_bootstrap_node {false};
+    dht::NetId network {0};
    bool generate_identity {false};
    bool daemonize {false};
    std::pair<std::string, std::string> bootstrap {};
 };

 static const constexpr struct option long_options[] = {
-   {"help",       no_argument,       nullptr, 'h'},
+   {"help",       no_argument      , nullptr, 'h'},
   {"port",       required_argument, nullptr, 'p'},
-   {"bootstrap",  optional_argument, nullptr, 'b'},
+   {"net",        required_argument, nullptr, 'n'},
+   {"bootstrap",  required_argument, nullptr, 'b'},
   {"identity",   no_argument      , nullptr, 'i'},
   {"verbose",    no_argument      , nullptr, 'v'},
   {"daemonize",  no_argument      , nullptr, 'd'},
-   {nullptr,      0,                 nullptr,  0}
+   {"logfile",    required_argument, nullptr, 'l'},
+   {nullptr,      0                , nullptr,  0}
 };

 dht_params
 parseArgs(int argc, char **argv) {
    dht_params params;
    int opt;
-    while ((opt = getopt_long(argc, argv, ":hidv:p:b:", long_options, nullptr)) != -1) {
+    while ((opt = getopt_long(argc, argv, "hidvp:n:b:l:", long_options, nullptr)) != -1) {
        switch (opt) {
        case 'p': {
                int port_arg = atoi(optarg);
@@ -172,24 +215,24 @@ parseArgs(int argc, char **argv) {
                    std::cout << "Invalid port: " << port_arg << std::endl;
            }
            break;
+        case 'n':
+            params.network = strtoul(optarg, nullptr, 0);
+            break;
        case 'b':
-            if (optarg) {
-                params.bootstrap = splitPort((optarg[0] == '=') ? optarg+1 : optarg);
-                if (not params.bootstrap.first.empty() and params.bootstrap.second.empty()) {
-                    std::stringstream ss;
-                    ss << DHT_DEFAULT_PORT;
-                    params.bootstrap.second = ss.str();
-                }
+            params.bootstrap = splitPort((optarg[0] == '=') ? optarg+1 : optarg);
+            if (not params.bootstrap.first.empty() and params.bootstrap.second.empty()) {
+                std::stringstream ss;
+                ss << DHT_DEFAULT_PORT;
+                params.bootstrap.second = ss.str();
            }
-            else
-                params.is_bootstrap_node = true;
            break;
        case 'h':
            params.help = true;
            break;
+        case 'l':
+            params.logfile = optarg;
+            break;
        case 'v':
-            if (optarg)
-                params.logfile = optarg;
            params.log = true;
            break;
        case 'i':
@@ -198,19 +241,6 @@ parseArgs(int argc, char **argv) {
        case 'd':
            params.daemonize = true;
            break;
-        case ':':
-            switch (optopt) {
-            case 'b':
-                params.is_bootstrap_node = true;
-                break;
-            case 'v':
-                params.log = true;
-                break;
-            default:
-                std::cout << "option requires an argument -- '" << optopt << '\'' << std::endl;
-                break;
-            }
-            break;
        default:
            break;
        }
Author	SHA1	Message	Date
Adrien Béraud	862a03713b	opendht: version 1.0 * add Prefix Hash Tree (PHT) for data indexation (Nicolas Reynaud, Simon Désaulniers) * add Queries (Simon Désaulniers) * add long-term data persistance (Simon Désaulniers) * add data pagination (Simon Désaulniers) * reduced bandwith usage	2016-09-01 18:02:23 -04:00
Adrien Béraud	89d0d2f109	pht: fix compilation with pedantic	2016-09-01 18:02:23 -04:00
Adrien Béraud	3283e34dcf	pht: rebase over queries	2016-09-01 18:02:23 -04:00
kaldoran	34c6b264df	Add filter of duplicate value	2016-09-01 18:02:23 -04:00
kaldoran	2d8fe502f2	Fix print of the lookup output	2016-09-01 18:02:23 -04:00
kaldoran	73df308250	Adding ZCurve algorithm	2016-09-01 18:02:23 -04:00
kaldoran	cdca40cb59	Rework prefix [add flags] and prepare prefix	2016-09-01 18:02:23 -04:00
kaldoran	2a3c301314	dht: Fix get and put at the same time on a same node When the put is performed just after a get, then the get will not include the value which was in the put.	2016-09-01 18:02:23 -04:00
kaldoran	6f4923da2e	dht: fix some typo	2016-09-01 18:02:23 -04:00
kaldoran	ae26a2b39a	Update of algorithm, by using new feature of api	2016-09-01 18:02:23 -04:00
kaldoran	48c40c8a27	pht: correction on the algorithm of complet node splitting	2016-09-01 18:02:23 -04:00
kaldoran	a4a4eee495	pht: introduction of complet node splitting	2016-09-01 18:02:23 -04:00
kaldoran	d185428d1d	[Python] Draw benchmark graph in a better way #104	2016-09-01 18:02:23 -04:00
kaldoran	6855aaa6f4	Correction of error on SwapBit	2016-09-01 18:02:23 -04:00
kaldoran	2aee4f418f	Correction of the size of the prefix, now size does not include the size of the padding	2016-09-01 18:02:23 -04:00
kaldoran	95aef01a59	Pht : Fix out of range in isActiveBit function	2016-09-01 18:02:23 -04:00
kaldoran	d2ccde4859	pht: bit padding when the key is too short	2016-09-01 18:02:23 -04:00
Simon Désaulniers	7ab64ea0e6	pht: validKey must check for key field value len	2016-09-01 18:02:23 -04:00
Simon Désaulniers	fe1b8b88e1	pht: provide simpler lookup callback From a user point of view, the structure Prefix doesn't make sense really. It's better to provide a simpler callback function signature for the user then.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	b6832869b1	pht: moved some defs in cpp file	2016-09-01 18:02:23 -04:00
Simon Désaulniers	6da8d39a0e	python: update cython wrapper and python benchmark API changed when the keys pec was added. This commit updates python code according to the new API.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	f8cf1316ad	pht: add key spec, handling maximum prefix length This commit adds the new KeySpec in the Pht constructor signature so that all keys used inside a same PHT structure have the same maximal length.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	736606a845	pht: a Key is rather map str:Blob not str:Prefix	2016-09-01 18:02:23 -04:00
Simon Désaulniers	cba240e15e	pht: fix invalid comparison, inexact match lookup	2016-09-01 18:02:23 -04:00
Adrien Béraud	eb90fffb5e	update for API changes	2016-09-01 18:02:23 -04:00
kaldoran	041e280bde	benchmark: timing pht insert operations	2016-09-01 18:02:23 -04:00
kaldoran	33fae1cb66	pht: add prefix cache	2016-09-01 18:02:23 -04:00
Simon Désaulniers	4e8408e225	python: lock before launching pht insert in test	2016-09-01 18:02:23 -04:00
Simon Désaulniers	e85a423f6a	Revert "[Bug] Correction of function commonBits"	2016-09-01 18:02:23 -04:00
kaldoran	532cb40665	Correction of error on commonBits	2016-09-01 18:02:23 -04:00
kaldoran	dd58b0497e	Correction of prefix, sometime some bit remain at they default value	2016-09-01 18:02:23 -04:00
Kaldoran	cc070c80ba	Fix the autotool Makefile, missing pht include Fix indentation for pull request #37 Fix include error, and use only needed file in pht Fix implementation of function in pht	2016-09-01 18:02:23 -04:00
Simon Désaulniers	e332dae576	python: insert test show entries for each prefix	2016-09-01 18:02:23 -04:00
Simon Désaulniers	727fd4e98b	fix typo in tests	2016-09-01 18:02:23 -04:00
Simon Désaulniers	5a5f7012d8	add pht insert test	2016-09-01 18:02:23 -04:00
Simon Désaulniers	5a239cddf7	pht: fix insert always to root bug This makes the pht node recover all indexed data on a DHT node (maximum MAX_NODE_ENTRY_COUNT per node) so that the pht node can be aware if it has to split its data in the leaf.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	c4928553ba	python: fix successive deref on cython object	2016-09-01 18:02:23 -04:00
Simon Désaulniers	48b0cff08e	pht: fix multiple DoneCallBack calls	2016-09-01 18:02:23 -04:00
Simon Désaulniers	c8fcffbe6b	fix possible out of range error in Prefix ctor	2016-09-01 18:02:23 -04:00
Simon Désaulniers	63c4eeb9c9	cover empty leaf case in inexact match lookup In case this is inexact match lookup and we end up on an empty leaf (diffrent than the root), the search goes on in the leaf's sibling subtree.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	d45f777099	pht tools	2016-09-01 18:02:23 -04:00
Simon Désaulniers	9c4035dd8f	add inexact match lookup This is meant to enable random lookup queries.	2016-09-01 18:02:23 -04:00
Simon Désaulniers	acc2b71c86	base pht indexation	2016-09-01 18:02:23 -04:00
Nicolas	09d849c93d	Update README.md	2016-09-01 17:46:04 -04:00
Nicolas	1be1ad88c6	Update README.md	2016-09-01 17:43:10 -04:00
Nicolas	8ecaf25339	Move logo in the right position	2016-09-01 17:41:33 -04:00
Adrien Béraud	7a6dce4727	securedht: don't complain about certificate announcement (reverted from commit `85a9dbafc6`)	2016-09-01 17:12:28 -04:00
Adrien Béraud	b357ffb0e6	add logo	2016-09-01 17:10:59 -04:00
Adrien Béraud	517bd5f22b	opendht: add logo	2016-09-01 12:49:26 -04:00
Adrien Béraud	d1d0a67d35	dht: fix public address sorting	2016-09-01 12:46:39 -04:00
Adrien Béraud	85a9dbafc6	securedht: don't complain about certificate announcement	2016-09-01 12:46:39 -04:00
Nicolas	79390f72fa	Merge pull request #121 from sim590/benchmark-msgpack-ipc [benchmark] new msgpack ipc	2016-08-31 16:25:21 -04:00
Simon Désaulniers	b7a37dd771	benchmark: msgpack with callback based IPC	2016-08-31 15:47:05 -04:00
Simon Désaulniers	b24d1ccc8b	benchmark: removing anormal behavior from normal test	2016-08-31 15:37:30 -04:00
Simon Désaulniers	4f53e10e7e	benchmark: enable traffic_plot for "delete" test	2016-08-31 15:37:30 -04:00
Simon Désaulniers	a608ad3cf6	benchmark: send new distinct values during test During totallyNormalTest, an initial set of values would be put on the dht. Then, random values from this same set would be put over time, but since values would already be there, no put were really made.	2016-08-31 15:37:29 -04:00
Simon Désaulniers	a135f81073	benchmark: create function for displaying plot	2016-08-31 15:37:15 -04:00
Nicolas	2da48557ff	Merge pull request #120 from sim590/fix-timing-loops dht: send find_node before sending 'listen'.	2016-08-30 16:14:22 -04:00
Simon Désaulniers	73686debaf	dht: before sending 'listen', send find_node Since pagination, a bug was making the listen feature unusable running in infinite loop. This also makes more sense than sending 'get' since the listen request is paired with a Query defining the values that have to be fetched.	2016-08-30 15:51:50 -04:00
Adrien Béraud	97950621cd	crypto: check max data size for sign	2016-08-25 00:42:50 -04:00
Adrien Béraud	b3401828e4	dht: set "connecting" state when waiting for a ping	2016-08-25 00:42:50 -04:00
Nicolas	0a7d4a68b3	Merge pull request #119 from sim590/find-node-timings dht: cover bad cases in SearchNode::canGet	2016-08-22 19:16:19 -04:00
Simon Désaulniers	8d7deffa6a	dht: properly find pending satisfying query In SearchNode::canGet, we don't send a 'get' request to a search node if a satisfying query for the query we want to send is already pending. This patch honors this behavior.	2016-08-22 18:45:29 -04:00
Simon Désaulniers	669c08210e	dht: erase find node request status when done This was memory leak and produced inconsistency in SearchNode::canGet timings.	2016-08-22 18:26:53 -04:00
Adrien Béraud	27a3231a55	argon2: bump source	2016-08-18 10:10:03 -04:00
Seva Ivanov	86ce695939	cython: change shared_ptr invocation to libcpp.memory	2016-08-16 10:27:34 -04:00
Adrien Béraud	2759a136df	infohash: add istream operator overload	2016-08-16 10:27:31 -04:00
Adrien Béraud	690682408b	build with -pedantic-errors, fix pedantic build errors	2016-08-16 10:26:44 -04:00
Nicolas	bfdb8a24e9	Merge pull request #113 from sim590/value-pagination-fix dht: value pagination fix	2016-08-15 19:35:33 -04:00
Simon Désaulniers	e9bf042ac5	dht: document SearchNode::canGet	2016-08-15 13:44:52 -04:00
Simon Désaulniers	5aab3078c8	dht: properly handle pagination timings	2016-08-15 12:13:16 -04:00
Simon Désaulniers	1ac5840bb1	dht: let nodes expire properly The line with `srn->getStatus[query]->setExpired()` in searchNodeGetExpired was messign with the node expiration process.	2016-08-14 22:08:52 -04:00
Simon Désaulniers	426b3d6834	dht: pagination queries should be in SearchNode Pagination queries must to be kept in seperate list per node. Otherwise, two searches could be blocking each others or generating unwanted behavior.	2016-08-11 12:27:04 -04:00
Nicolas	97b573c182	Merge pull request #110 from sim590/value-pagination dht: Value pagination using queries	2016-08-09 17:27:00 -04:00
Simon Désaulniers	5c41cbb964	dht: value pagination Instead of literally asking to a node all values for givent `h`, we first ask for all ids the node has. Then, we concurently and individually ask for each values that match each id we recovered in our first request.	2016-08-09 15:43:55 -04:00
Simon Désaulniers	e192e3beaf	better indentation and comments	2016-08-09 15:42:05 -04:00
Adrien Béraud	5dacb50d63	Merge pull request #108 from sim590/search-refill-from-cache dht: unify bootstrapSearch and refill method using node cache	2016-08-09 14:41:23 -04:00
Adrien Béraud	86d837b5a9	Merge pull request #106 from sim590/get-timings [dht] rework get timings after queries in master	2016-08-09 14:23:13 -04:00
Adrien Béraud	6454a6dc3f	Merge pull request #109 from kaldoran/fix_put-get dht: Fix, get does not include the put value if its performed just after it.	2016-08-08 18:10:05 -04:00
kaldoran	bbd1200ad7	dht: Fix get and put at the same time on a same node When the put is performed just after a get, then the get will not include the value which was in the put.	2016-08-08 17:50:56 -04:00
Adrien Béraud	aa59242843	readme: add IRC link	2016-08-07 00:13:18 -04:00
Simon Désaulniers	c04ede4909	dht: unify bootstrapSearch and refill method Since bootstrapSearch was rewritten to use the node cache, it made it clear that refill should be rewritten as such. However, then bootstrapSearch becomes irrelevant. An enhancement to refill is that it will let insertNode review its whole list of nodes and then let it insert nodes up to SEARCH_NODES nodes. The previous method would not garantee this behavior since, most of the times, it would pass less than SEARCH_NODES nodes to insertNode.	2016-08-06 21:14:17 -04:00
Adrien Béraud	a1eaf2e83c	Merge pull request #107 from sim590/minor-search-refactor dht: more accurate name for requested nodes count	2016-08-05 23:03:25 -04:00
Simon Désaulniers	a21d4d4adc	queries: fix deserialization bug Only the first two entries of the array would be accessed in a consistent way. Therefor, only the two first field value index would be returned.	2016-08-05 21:20:58 -04:00
Simon Désaulniers	f0a92d999c	dht: more accurate name for requested nodes count The number of currently requested nodes, formerly named currentGetRequests, was renamed since it was not accurately describing the behavior. The number we are counting is not the number of 'get' requests, but rather the number of nodes being requested simultaneously.	2016-08-05 21:19:46 -04:00
Simon Désaulniers	28a3498a69	dht: fixing timing in SearchNode::canGet When a 'get' request associated with a certain 'query' has been sent, we don't want to send it again, i.e. we don't care if it's pending or not as soon as it has been sent.	2016-08-05 21:17:58 -04:00
Simon Désaulniers	7eb5d892f1	search: isDone computes using Request interface There's no need to compute with time since the network engine gives us the information we need.	2016-08-05 21:17:11 -04:00
Simon Désaulniers	f6cd79952c	node_cache: fix bad condition in getCachedNodes	2016-08-05 13:07:14 -04:00
Adrien Béraud	9c1a0ae27f	freenode: fix typo	2016-08-05 10:23:08 -04:00
Simon Désaulniers	29c1a955d0	readme: add freenode irc opendht channel	2016-08-04 18:33:56 -04:00
Nicolas	b366432a8a	Merge pull request #103 from sim590/sendlisten-loop dht: fix sendListen loop bug	2016-08-04 10:25:22 -04:00
Simon Désaulniers	af8bfdc1c1	dht: fix sendListen loop bug Each time a 'listen' request is sent to the network engine, we track this request in the upper layer with a map<shared_ptr<Query>, shared_ptr<Request>>. At some point, this map has to be maintained or totally cleared. However, it was done wrong, hence the sendListen loop. Clearing a pair <Query, Request> has to be done when the API user cancels a 'listen' operation.	2016-08-03 18:49:37 -04:00
Simon Désaulniers	88040ad467	dht: display debug info according to query impact	2016-08-03 18:49:10 -04:00
Simon Désaulniers	a35e67bcdf	Merge pull request #101 from kaldoran/optimize_search dht: bootstrap from node cache instead of routing table	2016-08-03 15:31:07 -04:00
kaldoran	df3317a5b2	dht: bootstrap from cache instead of routing table	2016-08-03 10:04:02 -04:00
kaldoran	33f95c00a9	NodeCache: getNode takes bool not int Since this was old code which use int as bool I edit that for a bool [and change related code]	2016-08-02 18:57:20 -04:00
Adrien Béraud	7383505dcc	value: increase max value size to 16KiB	2016-08-01 10:47:21 -04:00
Adrien Béraud	2c9a29af0e	docker: add python, lto, msgpack 2.0	2016-08-01 01:18:37 -04:00
Adrien Béraud	5847273b79	travis: add llvm docker build	2016-07-29 21:37:43 -04:00
Adrien Béraud	98204991ec	travis with docker	2016-07-29 21:37:43 -04:00
Adrien Béraud	b8e10bfe15	const shared_ptr	2016-07-29 21:37:07 -04:00
Adrien Béraud	132fe66699	dhtnode: cleanup	2016-07-29 15:49:43 -04:00
Adrien Béraud	a28fc4dfb1	tools: remove optional arguments	2016-07-29 15:39:50 -04:00
Nicolas	7721aa5643	api: Add value creation timepoint (#97 )	2016-07-28 17:03:11 -04:00
Nicolas	68707aff17	dht: prevent outdated store of value	2016-07-28 16:58:04 -04:00
Adrien Béraud	3ecbc9adff	Merge pull request #95 from sim590/consistent-so-versionning autotools: versionning consistent with CMake	2016-07-25 17:09:56 -04:00
Simon Désaulniers	b5a7a0ca31	autotools: versionning consistent with CMake	2016-07-25 17:03:57 -04:00
Nicolas	2362db8528	Merge pull request #94 from sim590/storage-unique-ptr dht: make main store a vector<unique_ptr<Storage>>	2016-07-22 17:02:21 -04:00
Simon Désaulniers	62835bdc54	dht: make main store a vector<unique_ptr<Storage>> Since the elements inside are going to be moved constantly during the life of the node, this design is better. Also, it works around a crash when the store is resized (elements inside moved) at some point (always happenned the 3rd time) for some reason. This bug was known when compiled with LLVM >7.3.0 (>clang-703.0.31).	2016-07-22 16:57:46 -04:00
Nicolas	fe1cf209f1	Merge pull request #93 from sim590/select-consistency dht: return consistent query from local storage	2016-07-19 16:47:44 -04:00
Simon Désaulniers	2f72563b1f	dht: return consistent query from local storage When querying from local storage with SELECT f, a FieldValueIndex with empty fields was returned without considering the SELECT statement requiring only a field "f".	2016-07-19 15:52:21 -04:00
Simon Désaulniers	0d754d6ca8	doc: clarify dhtnode usage	2016-07-19 14:00:34 -04:00
Simon Désaulniers	f474f6499b	value: missing return, errors on non-gcc Gcc understands the syntax without return value, but this confuses other compilers.	2016-07-18 14:18:37 -04:00
Adrien Béraud	97baacf609	Merge pull request #79 from sim590/queries [DHT] Queries: remote values filtering	2016-07-11 20:23:15 +02:00
Simon Désaulniers	0fa8cfdb71	tools: enable queries in dhtnode	2016-07-10 19:26:13 +02:00
Simon Désaulniers	78c7cae7e9	dhtrunner: add query in dhtrunner API	2016-07-10 19:26:13 +02:00
Simon Désaulniers	a4834efd37	dht: use queries in dht	2016-07-10 19:26:13 +02:00
Simon Désaulniers	07e4435a07	value: add Query relevant structures	2016-07-10 19:26:04 +02:00
zorun	9f93771771	doc: Fix install path of man page (#90 ) Man pages go in /usr/share/man http://www.pathname.com/fhs/pub/fhs-2.3.html#USRSHAREMANMANUALPAGES	2016-07-02 14:32:43 +02:00
Simon Désaulniers	a533807fcf	doc: add missing CMakeLists.txt	2016-07-02 01:20:46 +02:00
Simon Désaulniers	49129ec1b2	tools: specify optional arg syntax -o[ARG]	2016-06-30 01:33:14 +02:00
Simon Désaulniers	803354552f	build-tools: distribute dhtnode.1 manpage	2016-06-30 00:58:04 +02:00
Simon Désaulniers	fa6c365679	doc: fix typos in dhtnode manpage	2016-06-29 20:57:30 +02:00
Adrien Béraud	6ac1ae3527	Merge pull request #84 from sim590/dhtnode-user-experience [dhtnode] user experience	2016-06-28 14:55:27 -04:00
Simon Désaulniers	29834b714e	dhtnode: manpage and add all options in help text	2016-06-28 18:51:40 +02:00
Simon Désaulniers	bba27adf4d	dhtnode: fix getopt optional arg parsing	2016-06-27 21:14:54 +02:00
Simon Désaulniers	e3a280d7a9	Merge pull request #82 from aviau/typos fixed typos	2016-06-27 19:46:17 +02:00
aviau	63f1a736f4	fixed typos	2016-06-27 19:33:52 +02:00
Simon Désaulniers	0f8720c433	Merge pull request #80 from aviau/powerful readme: fix typo s/powerfull/powerful/	2016-06-27 15:39:55 +02:00
aviau	7a6263b940	typo: powerfull ->powerful	2016-06-27 15:29:35 +02:00
Adrien Béraud	b09e3df2e2	add network id to distinguish different networks	2016-06-21 03:06:47 -04:00
Adrien Béraud	041a9e0e7f	Clarify GPLv3 "or any later version" licence	2016-06-20 11:47:21 -04:00
Simon Désaulniers	fa9730dd07	add readme help text for benchmark	2016-06-15 15:57:56 -04:00
Adrien Béraud	33f29e95f4	value: add common crypto ops	2016-06-15 13:35:44 -04:00