opendht 0.6.4

Since the elements inside are going to be moved constantly during the life of
the node, this design is better. Also, it works around a crash when the store is
resized (elements inside moved) at some point (always happenned the 3rd time)
for some reason. This bug was known when compiled with LLVM >7.3.0 (>clang-703.0.31).

.gitignore

@@ -45,13 +45,3 @@ Makefile.in
 # KDevelop
 .kdev4/
 *.kdev4
-
-# Python
-*.pyc
-
-# git backup files
-*.orig
-
-# vim swap files
-*.swp
-*.swo

CMakeLists.txt

@@ -1,9 +1,10 @@
 cmake_minimum_required (VERSION 2.8.6)
 project (opendht)
 set (opendht_VERSION_MAJOR 0)
-set (opendht_VERSION_MINOR 6.1)
+set (opendht_VERSION_MINOR 6.4)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
+set (VERSION "${opendht_VERSION}")
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 
@@ -20,7 +21,7 @@ option (OPENDHT_LTO "Build with LTO" OFF)
 option (OPENDHT_PYTHON "Build Python bindings" OFF)
 option (OPENDHT_TOOLS "Build DHT tools" ON)
 
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++11 -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++11 -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT")
 if (OPENDHT_LTO)
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto")
@@ -52,7 +53,6 @@ list (APPEND opendht_SOURCES
     src/network_engine.cpp
     src/securedht.cpp
     src/dhtrunner.cpp
-	src/indexation/pht.cpp
     src/argon2/argon2.c
     src/argon2/core.c
     src/argon2/blake2/blake2b.c
@@ -78,7 +78,6 @@ list (APPEND opendht_HEADERS
     include/opendht/securedht.h
     include/opendht/log.h
     include/opendht.h
-	include/opendht/indexation/pht.h
 )
 
 configure_file (
@@ -123,7 +122,7 @@ if (OPENDHT_SHARED)
         ${opendht_HEADERS}
     )
     set_target_properties (opendht PROPERTIES IMPORT_SUFFIX "_import.lib")
-    #set_target_properties (opendht PROPERTIES SOVERSION 1 VERSION 1.0.0)
+    set_target_properties (opendht PROPERTIES SOVERSION ${opendht_VERSION_MAJOR} VERSION ${opendht_VERSION})
     if (OPENDHT_LTO)
         target_link_libraries(opendht -flto -fuse-linker-plugin)
     endif ()
@@ -134,6 +133,7 @@ endif ()
 if (OPENDHT_TOOLS)
     add_subdirectory(tools)
 endif ()
+add_subdirectory(doc)
 
 if (OPENDHT_PYTHON)
     add_subdirectory(python)

Makefile.am

@@ -10,9 +10,7 @@ if USE_CYTHON
 SUBDIRS += python
 endif
 
-if HAVE_DOXYGEN
 SUBDIRS += doc
-endif
 
 ACLOCAL_AMFLAGS = -I m4
 

README.md

@@ -97,7 +97,11 @@ Licence
 -
 Copyright (C) 2014-2016 Savoir-faire Linux Inc.
 
-Licenced under the GNU General Public License version 3 (see COPYING for the full licence).
+OpenDHT is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+See COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html for the full GPLv3 licence.
 
 This project is independent from another project called OpenDHT (Sean Rhea. Ph.D. Thesis, 2005), now extinct.
 

configure.ac

@@ -1,10 +1,21 @@
-AC_INIT(opendht, 0.6.1)
+dnl define macros
+m4_define([opendht_major_version], 0)
+m4_define([opendht_minor_version], 6)
+m4_define([opendht_patch_version], 4)
+m4_define([opendht_version],
+		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
+
+AC_INIT(opendht, [opendht_version])
 AC_CONFIG_AUX_DIR(ac)
 AM_INIT_AUTOMAKE([foreign subdir-objects])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CANONICAL_HOST
 
+AC_SUBST(OPENDHT_MAJOR_VERSION, opendht_major_version)
+AC_SUBST(OPENDHT_MINOR_VERSION, opendht_minor_version)
+AC_SUBST(OPENDHT_PATCH_VERSION, opendht_patch_version)
+
 AC_ARG_ENABLE([debug], AS_HELP_STRING([--enable-debug], [Build in debug mode, adds stricter warnings, disables optimization]))
 AS_IF([test "x$enable_debug" = "xyes"],
       [CXXFLAGS="${CXXFLAGS} -g -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -O0"],
@@ -14,11 +25,11 @@ AC_PROG_CXX
 AM_PROG_AR
 
 dnl Check for Doxygen
-AC_ARG_ENABLE([doc], [AS_HELP_STRING([--disable-doc], [Disable documentation generation (doxygen)])])
-AS_IF([test "x$enable_doc" != "xno"], [
+AC_ARG_ENABLE([doc], AS_HELP_STRING([--enable-doc], [Enable documentation generation (doxygen)]))
+AS_IF([test "x$enable_doc" = "xyes"], [
        AC_CHECK_PROGS([DOXYGEN], [doxygen])
-       AS_IF([test -z "$DOXYGEN"],[AC_MSG_WARN([Doxygen not found - continuing without Doxygen support])])
-       ])
+       AS_IF([test -z "$DOXYGEN"], [AC_MSG_WARN([Doxygen not found - continuing without Doxygen support])])       
+])
 AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"])
 
 dnl Check for Python
@@ -75,9 +86,9 @@ LT_LANG(C++)
 AX_CXX_COMPILE_STDCXX(11,[noext],[mandatory])
 
 PKG_PROG_PKG_CONFIG()
-PKG_CHECK_MODULES([nettle], [nettle >= 2.4])
-PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.1])
-PKG_CHECK_MODULES([msgpack], [msgpack >= 1.2])
+PKG_CHECK_MODULES([Nettle], [nettle >= 2.4])
+PKG_CHECK_MODULES([GnuTLS], [gnutls >= 3.1])
+PKG_CHECK_MODULES([MsgPack], [msgpack >= 1.2])
 CXXFLAGS="${CXXFLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT"
 
 AC_ARG_ENABLE([tools], AS_HELP_STRING([--disable-tools],[Disable tools (CLI DHT node)]),,build_tools=yes)
@@ -88,7 +99,7 @@ AM_COND_IF([ENABLE_TOOLS], [
   ])
 ])
 
-AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])])
+AC_CONFIG_FILES([doc/Doxyfile doc/Makefile])
 
 dnl Configure setup.py if we build the python module
 AM_COND_IF([USE_CYTHON], [

doc/CMakeLists.txt

@@ -0,0 +1,3 @@
+if (OPENDHT_TOOLS)
+	INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dhtnode.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1)
+endif ()

doc/Makefile.am

@@ -1,3 +1,6 @@
+dist_man1_MANS = dhtnode.1
+
+if HAVE_DOXYGEN
 doxyfile.stamp:
 	$(DOXYGEN) Doxyfile
 	echo stamp > doxyfile.stamp
@@ -7,3 +10,4 @@ CLEANFILES = doxyfile.stamp
 all-local: doxyfile.stamp
 clean-local:
 	rm -rf $(top_srcdir)/doc/man $(top_srcdir)/doc/html
+endif

doc/dhtnode.1

@@ -0,0 +1,87 @@
+.TH DHTNODE 1 2016-07-29
+
+.SH NAME
+.B dhtnode
+- a simple OpenDHT command line node runner.
+
+.SH SYNOPSIS
+.B dhtnode [-h]
+
+.B dhtnode [-v [-l \fIlogfile\fP]] [-i] [-d] [-n \fInetwork_id\fP] [-p \fIlocal_port\fP] [-b \fIbootstrap_host\fP[:\fIport\fP]]
+
+.SH DESCRIPTION
+
+This program runs a simple OpenDHT node in an interactive way. If you rather
+want to run the node in daemon mode, option \fB'-d'\fP is provided. When running
+in the interactive shell, you benefit from the readline capabilities for writing
+your commands such as command history. Here are the available commands in the
+interactive shell:
+
+.EE
+    h, help    Print this help message.
+    q, quit    Quit the program.
+    log        Start/stop printing DHT logs.
+
+    Node information:
+    ll         Print basic information and stats about the current node.
+    ls         Print basic information about current searches.
+    ld         Print basic information about currenty stored values on this node.
+    lr         Print the full current routing table of this node
+
+    Operations on the DHT:
+    b ip:port             Ping potential node at given IP address/port.
+    g [key]               Get values at [key].
+    l [key]               Listen for value changes at [key].
+    p [key] [str]         Put string value at [key].
+    s [key] [str]         Put string value at [key], signed with our generated
+                          private key.
+    e [key] [dest] [str]  Put string value at [key], encrypted for [dest] with
+                          its public key (if found).
+
+.SH OPTIONS
+
+.TP
+\fB-h\fP
+Prints some help.
+
+.TP
+\fB-v\fP
+Enable the verbose mode (log to stdout by default)
+
+.TP
+\fB-l\fP \fIlog_file\fP
+Write log to file instead of stdout
+
+.TP
+\fB-i\fP
+Generate cryptographic identity for the node.
+
+.TP
+\fB-d\fP
+Run the program in daemon mode (will fork in the background).
+
+.TP
+\fB-n\fP \fInetwork_id\fP
+Specify the network id. This let you connect to distinct networks and prevents
+the merge of two different networks (available since OpenDHT v0.6.1).
+
+.TP
+\fB-p\fP \fIlocal_port\fP
+Use port \fIlocal_port\fP for the program to bind to.
+
+.TP
+\fB-b\fP \fIbootstrap_host\fP[:\fIport\fP]
+The program needs to be given a node to connect to the network. You use this
+option to provide the ip address of that node.
+
+.SH AUTHORS
+.TP
+Program written by
+.IP \(bu
+.\}
+Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+.TP
+Man page written by
+.IP \(bu
+.\}
+Simon Désaulniers <sim.desaulniers@gmail.com>

include/opendht.h

@@ -22,4 +22,3 @@
 #include "opendht/dhtrunner.h"
 #include "opendht/log.h"
 #include "opendht/default_types.h"
-#include "opendht/indexation/pht.h"

include/opendht/callbacks.h

@@ -30,23 +30,44 @@ namespace dht {
 
 struct Node;
 
+/**
+ * Current status of a DHT node.
+ */
 enum class NodeStatus {
     Disconnected, // 0 nodes
     Connecting,   // 1+ nodes
     Connected     // 1+ good nodes
 };
 
+/**
+ * Dht configuration.
+ */
 struct Config {
+    /** DHT node ID */
     InfoHash node_id;
+
+    /** 
+     * DHT network ID. A node will only talk with other nodes having
+     * the same network ID.
+     * Network ID 0 (default) represents the main public network.
+     */
+    NetId network;
+
+    /** For testing purposes only, enables bootstrap mode */
     bool is_bootstrap;
 };
 
+/**
+ * SecureDht configuration.
+ */
 struct SecureDhtConfig
 {
     Config node_config;
     crypto::Identity id;
 };
 
+static constexpr size_t DEFAULT_STORAGE_LIMIT {1024 * 1024 * 64};
+
 using ValuesExport = std::pair<InfoHash, Blob>;
 
 using GetCallback = std::function<bool(const std::vector<std::shared_ptr<Value>>& values)>;
@@ -57,21 +78,18 @@ using CertificateStoreQuery = std::function<std::vector<std::shared_ptr<crypto::
 
 typedef bool (*GetCallbackRaw)(std::shared_ptr<Value>, void *user_data);
 
-static constexpr size_t DEFAULT_STORAGE_LIMIT {1024 * 1024 * 64};
-
 GetCallbackSimple bindGetCb(GetCallbackRaw raw_cb, void* user_data);
 GetCallback bindGetCb(GetCallbackSimple cb);
 
 using DoneCallback = std::function<void(bool success, const std::vector<std::shared_ptr<Node>>& nodes)>;
 typedef void (*DoneCallbackRaw)(bool, std::vector<std::shared_ptr<Node>>*, void *user_data);
 typedef void (*ShutdownCallbackRaw)(void *user_data);
-typedef void (*DoneCallbackSimpleRaw)(bool, void *user_data);
 
 using DoneCallbackSimple = std::function<void(bool success)>;
 
 ShutdownCallback bindShutdownCb(ShutdownCallbackRaw shutdown_cb_raw, void* user_data);
 DoneCallback bindDoneCb(DoneCallbackSimple donecb);
 DoneCallback bindDoneCb(DoneCallbackRaw raw_cb, void* user_data);
-DoneCallbackSimple bindDoneCbSimple(DoneCallbackSimpleRaw raw_cb, void* user_data);
+
 
 }

include/opendht/crypto.h

@@ -331,7 +331,7 @@ Identity generateIdentity(const std::string& name = "dhtnode", Identity ca = {},
 /**
  * SHA512
  */
-Blob hash(const Blob& data);
+Blob hash(const Blob& data, size_t hash_len = 512/8);
 
 /**
  * Generates an encryption key from a text password,
@@ -340,7 +340,7 @@ Blob hash(const Blob& data);
  * that can be transmitted in clear, and will be generated if
  * not provided (32 bytes).
  */
-Blob stretchKey(const std::string& password, Blob& salt);
+Blob stretchKey(const std::string& password, Blob& salt, size_t key_length = 512/8);
 
 /**
  * AES-GCM encryption. Key must be 128, 192 or 256 bits long (16, 24 or 32 bytes).

include/opendht/default_types.h

@@ -29,7 +29,7 @@ enum class ImStatus : uint8_t {
     READ
 };
 }
-MSGPACK_ADD_ENUM(dht::ImStatus);
+MSGPACK_ADD_ENUM(dht::ImStatus)
 
 namespace dht {
 
@@ -55,7 +55,7 @@ public:
 
     std::string service;
     Blob data;
-    MSGPACK_DEFINE(service, data);
+    MSGPACK_DEFINE(service, data)
 };
 
 template <typename T>
@@ -126,7 +126,7 @@ public:
     long date {0};
     ImStatus status {ImStatus::NONE};
 
-    MSGPACK_DEFINE_MAP(id, msg, date, status);
+    MSGPACK_DEFINE_MAP(id, msg, date, status)
 };
 
 class TrustRequest : public EncryptedValue<TrustRequest>
@@ -147,7 +147,7 @@ public:
 
     std::string service;
     Blob payload;
-    MSGPACK_DEFINE(service, payload);
+    MSGPACK_DEFINE(service, payload)
 };
 
 class IceCandidates : public EncryptedValue<IceCandidates>

include/opendht/dht.h

@@ -117,14 +117,20 @@ public:
      * The node is not pinged, so this should be
      * used to bootstrap efficiently from previously known nodes.
      */
-    bool insertNode(const InfoHash& id, const sockaddr*, socklen_t);
+    bool insertNode(const InfoHash& id, const SockAddr&);
+    bool insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) {
+        return insertNode(id, SockAddr(sa, salen));
+    }
     bool insertNode(const NodeExport& n) {
-        return insertNode(n.id, reinterpret_cast<const sockaddr*>(&n.ss), n.sslen);
+        return insertNode(n.id, SockAddr(n.ss, n.sslen));
     }
 
     int pingNode(const sockaddr*, socklen_t);
 
-    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr *from, socklen_t fromlen);
+    time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr&);
+    time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) {
+        return periodic(buf, buflen, SockAddr(from, fromlen));
+    }
 
     /**
      * Get a value by searching on all available protocols (IPv4, IPv6),
@@ -244,7 +250,7 @@ public:
     std::vector<ValuesExport> exportValues() const;
     void importValues(const std::vector<ValuesExport>&);
 
-    int getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return,
+    unsigned getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return,
             unsigned *incoming_return) const;
     std::string getStorageLog() const;
     std::string getRoutingTablesLog(sa_family_t) const;
@@ -270,7 +276,7 @@ public:
         return {total_store_size, total_values};
     }
 
-    std::vector<Address> getPublicAddress(sa_family_t family = 0);
+    std::vector<SockAddr> getPublicAddress(sa_family_t family = 0);
 
 protected:
     Logger DHT_LOG;
@@ -392,7 +398,7 @@ private:
     RoutingTable buckets {};
     RoutingTable buckets6 {};
 
-    std::vector<Storage> store;
+    std::vector<std::unique_ptr<Storage>> store;
     size_t total_values {0};
     size_t total_store_size {0};
     size_t max_store_size {DEFAULT_STORAGE_LIMIT};
@@ -412,8 +418,10 @@ private:
     time_point mybucket_grow_time {time_point::min()}, mybucket6_grow_time {time_point::min()};
 
     NetworkEngine network_engine;
+    unsigned pending_pings4 {0};
+    unsigned pending_pings6 {0};
 
-    using ReportedAddr = std::pair<unsigned, Address>;
+    using ReportedAddr = std::pair<unsigned, SockAddr>;
     std::vector<ReportedAddr> reported_addr;
 
     void rotateSecrets();
@@ -421,7 +429,7 @@ private:
     Blob makeToken(const sockaddr *sa, bool old) const;
     bool tokenMatch(const Blob& token, const sockaddr *sa) const;
 
-    void reportedAddr(const sockaddr *sa, socklen_t sa_len);
+    void reportedAddr(const SockAddr&);
 
     // Storage
     decltype(Dht::store)::iterator findStorage(const InfoHash& id);
@@ -497,11 +505,11 @@ private:
 
     bool neighbourhoodMaintenance(RoutingTable&);
 
-    void processMessage(const uint8_t *buf, size_t buflen, const sockaddr *from, socklen_t fromlen);
+    void processMessage(const uint8_t *buf, size_t buflen, const SockAddr&);
 
     void onError(std::shared_ptr<Request> node, DhtProtocolException e);
     /* when our address is reported by a distant peer. */
-    void onReportedAddr(const InfoHash& id, sockaddr* sa , socklen_t salen);
+    void onReportedAddr(const InfoHash& id, const SockAddr&);
     /* when we receive a ping request */
     NetworkEngine::RequestAnswer onPing(std::shared_ptr<Node> node);
     /* when we receive a "find node" request */

include/opendht/dhtrunner.h

@@ -212,7 +212,7 @@ public:
      * Returns the currently bound address.
      * @param f: address family of the bound address to retreive.
      */
-    const Address& getBound(sa_family_t f = AF_INET) const {
+    const SockAddr& getBound(sa_family_t f = AF_INET) const {
         return (f == AF_INET) ? bound4 : bound6;
     }
 
@@ -248,7 +248,7 @@ public:
     std::string getStorageLog() const;
     std::string getRoutingTablesLog(sa_family_t af) const;
     std::string getSearchesLog(sa_family_t af = 0) const;
-    std::vector<Address> getPublicAddress(sa_family_t af = 0);
+    std::vector<SockAddr> getPublicAddress(sa_family_t af = 0);
     std::vector<std::string> getPublicAddressStr(sa_family_t af = 0);
 
     // securedht methods
@@ -268,12 +268,13 @@ public:
      * @param threaded: If false, ::loop() must be called periodically. Otherwise a thread is launched.
      * @param cb: Optional callback to receive general state information.
      */
-    void run(in_port_t port, const crypto::Identity identity, bool threaded = false, bool is_bootstrap = false) {
+    void run(in_port_t port, const crypto::Identity identity, bool threaded = false, NetId network = 0) {
         run(port, {
             /*.dht_config = */{
                 /*.node_config = */{
                     /*.node_id = */{},
-                    /*.is_bootstrap = */is_bootstrap
+                    /*.network = */network,
+                    /*.is_bootstrap = */false
                 },
                 /*.id = */identity
             },
@@ -341,7 +342,7 @@ private:
 
     std::thread rcv_thread {};
     std::mutex sock_mtx {};
-    std::vector<std::pair<Blob, std::pair<sockaddr_storage, socklen_t>>> rcv {};
+    std::vector<std::pair<Blob, SockAddr>> rcv {};
 
     std::queue<std::function<void(SecureDht&)>> pending_ops_prio {};
     std::queue<std::function<void(SecureDht&)>> pending_ops {};
@@ -353,8 +354,8 @@ private:
                status6 {NodeStatus::Disconnected};
     StatusCallback statusCb {nullptr};
 
-    Address bound4 {};
-    Address bound6 {};
+    SockAddr bound4 {};
+    SockAddr bound6 {};
 };
 
 }

include/opendht/indexation/pht.h

@@ -1,514 +0,0 @@
-#pragma once
-
-#include <string>
-#include <vector>
-#include <memory>
-#include <map>
-#include <functional>
-#include <stdexcept>
-#include <bitset>
-#include <iostream>
-#include <sstream>
-
-#include "../value.h"
-#include "../dhtrunner.h"
-#include "../infohash.h"
-
-namespace dht {
-namespace indexation {
-
-/*!
- * @class   Prefix
- * @brief   A blob structure which prefixes a Key in the PHT.
- * @details
- * Since the PHT structure is a "trie", every node in this structure have a
- * label which is defined by the path from the root of the trie to the node. If
- * the node in question is a leaf, *the label is a prefix of all the keys
- * contained in the leaf*.
- */
-struct Prefix {
-    Prefix() {}
-    Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) { }
-    Prefix(const Blob& d, const Blob& f={}) : size_(d.size()*8), flags_(f), content_(d) { }
-
-    Prefix(const Prefix& p, size_t first) :
-        size_(std::min(first, p.content_.size()*8)),
-        content_(Blob(p.content_.begin(), p.content_.begin()+size_/8))
-    {
-
-        auto rem = size_ % 8;
-        if ( not p.flags_.empty() ) {
-            flags_ = Blob(p.flags_.begin(), p.flags_.begin()+size_/8);
-            if (rem)
-                flags_.push_back(p.flags_[size_/8] & (0xFF << (8 - rem)));
-        }
-
-        if (rem)
-            content_.push_back(p.content_[size_/8] & (0xFF << (8 - rem)));
-    }
-
-    /**
-     * Get a sub prefix of the Prefix
-     *
-     * @param len lenght of the prefix to get, could be negative
-     *            if len is negativ then you will get the prefix
-     *            of size of the previous prefix minus len
-     *
-     * @return Sub-prefix of size len or if len is negative sub-prefix of size
-     *         of prefix minus len
-     *
-     * @throw out_of_range if len is larger than size of the content
-     */
-    Prefix getPrefix(ssize_t len) const {
-        if ((size_t)std::abs(len) >= content_.size() * 8)
-            throw std::out_of_range("len larger than prefix size.");
-        if (len < 0)
-            len += size_;
-
-        return Prefix(*this, len);
-    }
-
-    /**
-     * Flags are considered as active if flag is empty or if the flag
-     * at pos 'pos' is active
-ee     *
-     * @see isActiveBit in private function
-     */
-    bool isFlagActive(size_t pos) const {
-        return flags_.empty() or isActiveBit(flags_, pos);
-    }
-
-    /**
-     * @see isActiveBit in private function
-     */
-    bool isContentBitActive(size_t pos) const {
-        return isActiveBit(content_, pos);
-    }
-
-    Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
-
-    /**
-     * This methods gets the prefix of its sibling in the PHT structure.
-     *
-     * @return The prefix of this sibling.
-     */
-    Prefix getSibling() const {
-        Prefix copy = *this;
-        if ( size_ )
-            copy.swapContentBit(size_ - 1);
-
-        return copy;
-    }
-
-    InfoHash hash() const {
-        Blob copy(content_);
-        copy.push_back(size_);
-        return InfoHash::get(copy);
-    }
-
-    /**
-     * This method count total of bit in common between 2 prefix
-     *
-     * @param p1 first prefix to compared
-     * @param p2 second prefix to compared
-     * @return Lenght of the larger common prefix between both
-     */
-    static inline unsigned commonBits(const Prefix& p1, const Prefix& p2) {
-        unsigned i, j;
-        uint8_t x;
-        auto longest_prefix_size = std::min(p1.size_, p2.size_);
-
-        for (i = 0; i < longest_prefix_size; i++) {
-            if (p1.content_.data()[i] != p2.content_.data()[i]
-                or not p1.isFlagActive(i)
-                or not p2.isFlagActive(i) ) {
-
-                    break;
-            }
-        }
-
-        if (i == longest_prefix_size)
-            return 8*longest_prefix_size;
-
-        x = p1.content_.data()[i] ^ p2.content_.data()[i];
-
-        j = 0;
-        while ((x & 0x80) == 0) {
-            x <<= 1;
-            j++;
-        }
-
-        return 8 * i + j;
-    }
-
-    /**
-     * @see doc of swap private function
-     */
-    void swapContentBit(size_t bit) {
-        swapBit(content_, bit);
-    }
-
-    /**
-     * @see doc of swap private function
-     */
-    void swapFlagBit(size_t bit) {
-        swapBit(flags_, bit);
-    }
-
-    /**
-     * @see doc of addPadding private function
-     */
-    void addPaddingContent(size_t size) {
-        content_ = addPadding(content_, size);
-    }
-
-    void updateFlags() {
-        /* Fill first known bit */
-        auto csize = size_ - flags_.size() * 8;
-        while(csize >= 8) {
-            flags_.push_back(0xFF);
-            csize -= 8;
-        }
-
-        /* if needed fill remaining bit */
-        if ( csize )
-            flags_.push_back(0xFF << (8 - csize));
-
-        /* Complet vector space missing */
-        for ( auto i = flags_.size(); i < content_.size(); i++ )
-            flags_.push_back(0xFF);
-    }
-
-    std::string toString() const;
-
-    size_t size_ {0};
-
-    /* Will contain flags according to content_.
-       If flags_[i] == 0, then content_[i] is unknown
-       else if flags_[i] == 1, then content_[i] is known */
-    Blob flags_ {};
-    Blob content_ {};
-
-private:
-
-    /**
-     * Add a padding to the input blob
-     *
-     * @param toP  : Prefix where to add a padding
-     * @param size : Final size of the prefix with padding
-     *
-     * @return Copy of the input Blob but with a padding
-     */
-    Blob addPadding(Blob toP, size_t size) {
-        Blob copy = toP;
-        for ( auto i = copy.size(); i < size; i++ )
-            copy.push_back(0);
-
-        swapBit(copy, size_ + 1);
-        return copy;
-    }
-
-    /**
-     * Check if the bit a pos 'pos' is active, i.e. equal to 1
-     *
-     * @param b   : Blob to check
-     * @param pos : Position to check
-     *
-     * @return true if the bit is equal to 1, false otherwise
-     *
-     * @throw out_of_range if bit is superior to blob size * 8
-     */
-    bool isActiveBit(const Blob &b, size_t pos) const {
-        if ( pos >= content_.size() * 8 )
-            throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix");
-
-        return ((b[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
-    }
-
-    /**
-     * Swap bit at position bit [from 0 to 1 and vice-versa]
-     *
-     * @param b   : Blob to swap
-     * @param bit : Bit to swap on b
-     *
-     * @return the input prefix with the bit at pos 'bit' swapped
-     *
-     * @throw out_of_range if bit is superior to blob size * 8
-     */
-    void swapBit(Blob &b, size_t bit) {
-        if ( bit >= b.size() * 8 )
-            throw std::out_of_range("bit larger than prefix size.");
-
-        size_t offset_bit = (8 - bit) % 8;
-        b[bit / 8] ^= (1 << offset_bit);
-    }
-};
-
-using Value = std::pair<InfoHash, dht::Value::Id>;
-struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
-    static const ValueType TYPE;
-
-    virtual void unpackValue(const dht::Value& v) {
-        Serializable<IndexEntry>::unpackValue(v);
-        name = v.user_type;
-    }
-
-    virtual dht::Value packValue() const {
-        auto pack = Serializable<IndexEntry>::packValue();
-        pack.user_type = name;
-        return pack;
-    }
-
-    Blob prefix;
-    Value value;
-    std::string name;
-    MSGPACK_DEFINE_MAP(prefix, value);
-};
-
-class Pht {
-    static constexpr const char* INVALID_KEY = "Key does not match the PHT key spec.";
-
-    /* Prefixes the user_type for all dht values put on the DHT */
-    static constexpr const char* INDEX_PREFIX = "index.pht.";
-
-public:
-
-    /* This is the maximum number of entries per node. This parameter is
-     * critical and influences the traffic a lot during a lookup operation.
-     */
-    static constexpr const size_t MAX_NODE_ENTRY_COUNT {16};
-
-    /* A key for a an index entry */
-    using Key = std::map<std::string, Blob>;
-
-    /* Specifications of the keys. It defines the number, the length and the
-     * serialization order of fields. */
-    using KeySpec = std::map<std::string, size_t>;
-    using LookupCallback = std::function<void(std::vector<std::shared_ptr<Value>>& values, Prefix p)>;
-
-    typedef void (*LookupCallbackRaw)(std::vector<std::shared_ptr<Value>>* values, Prefix* p, void *user_data);
-    static LookupCallback
-    bindLookupCb(LookupCallbackRaw raw_cb, void* user_data) {
-        if (not raw_cb) return {};
-        return [=](std::vector<std::shared_ptr<Value>>& values, Prefix p) {
-            raw_cb((std::vector<std::shared_ptr<Value>>*) &values, (Prefix*) &p, user_data);
-        };
-    }
-    using LookupCallbackSimple = std::function<void(std::vector<std::shared_ptr<Value>>& values)>;
-    typedef void (*LookupCallbackSimpleRaw)(std::vector<std::shared_ptr<Value>>* values, void *user_data);
-    static LookupCallbackSimple
-    bindLookupCbSimple(LookupCallbackSimpleRaw raw_cb, void* user_data) {
-        if (not raw_cb) return {};
-        return [=](std::vector<std::shared_ptr<Value>>& values) {
-            raw_cb((std::vector<std::shared_ptr<Value>>*) &values, user_data);
-        };
-    }
-
-    Pht(std::string name, KeySpec k_spec, std::shared_ptr<DhtRunner> dht)
-        : name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht) {}
-
-    virtual ~Pht () { }
-
-    /**
-     * Lookup a key for a value.
-     */
-    void lookup(Key k, LookupCallback cb = {}, DoneCallbackSimple done_cb = {}, bool exact_match = true);
-    void lookup(Key k, LookupCallbackSimple cb = {}, DoneCallbackSimple done_cb = {}, bool exact_match = true)
-    {
-        lookup(k, [=](std::vector<std::shared_ptr<Value>>& values, Prefix) { cb(values); }, done_cb, exact_match);
-    }
-
-    /**
-     * Wrapper function which call the private one.
-     *
-     * @param k : Key to insert [i.e map of string, blob]
-     * @param v : Value to insert
-     * @param done_cb : Callbakc which going to be call when all the insert is done
-     */
-    void insert(Key k, Value v, DoneCallbackSimple done_cb = {}) {
-        Prefix p = linearize(k);
-
-        auto lo = std::make_shared<int>(0);
-        auto hi = std::make_shared<int>(p.size_);
-
-        IndexEntry entry;
-        entry.value = v;
-        entry.prefix = p.content_;
-        entry.name = name_;
-
-        Pht::insert(p, entry, lo, hi, clock::now(), true, done_cb);
-    }
-
-private:
-
-    /**
-     * Insert function which really insert onto the pht
-     *
-     * @param kp          : Prefix to insert (linearize the the key)
-     * @param entry       : Entry created from the value
-     * @param lo          : Lowest point to start in the prefix
-     * @param hi          : Highest point to end in the prefix
-     * @param time_p      : Timepoint to use for the insertion into the dht (must be < now)
-     * @param check_split : If this flag is true then the algoritm will not use the merge algorithm
-     * @param done_cb     : Callback to call when the insert is done
-     */
-
-    void insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shared_ptr<int> hi, time_point time_p,
-                bool check_split, DoneCallbackSimple done_cb = {});
-
-    class Cache {
-    public:
-        /**
-         * Insert all needed node into the tree according to a prefix
-         * @param p : Prefix that we need to insert
-         */
-        void insert(const Prefix& p);
-
-        /**
-         * Lookup into the tree to return the maximum prefix length in the cache tree
-         *
-         * @param p : Prefix that we are looking for
-         *
-         * @return  : The size of the longest prefix known in the cache between 0 and p.size_
-         */
-
-        int lookup(const Prefix& p);
-
-    private:
-        static constexpr const size_t MAX_ELEMENT {1024};
-        static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
-
-        struct Node {
-            time_point last_reply;           /* Made the assocation between leaves and leaves multimap */
-            std::shared_ptr<Node> parent;    /* Share_ptr to the parent, it allow the self destruction of tree */
-            std::weak_ptr<Node> left_child;  /* Left child, for bit equal to 1 */
-            std::weak_ptr<Node> right_child; /* Right child, for bit equal to 0 */
-        };
-
-        std::weak_ptr<Node> root_;                         /* Root of the tree */
-
-        /**
-         * This mutlimap contains all prefix insert in the tree in time order
-         * We could then delete the last one if there is too much node
-         * The tree will self destroy is branch ( thanks to share_ptr )
-         */
-        std::multimap<time_point, std::shared_ptr<Node>> leaves_;
-    };
-
-    /* Callback used for insert value by using the pht */
-    using RealInsertCallback = std::function<void(std::shared_ptr<Prefix> p, IndexEntry entry)>;
-    using LookupCallbackWrapper = std::function<void(std::vector<std::shared_ptr<IndexEntry>>& values, Prefix p)>;
-
-    /**
-     * Performs a step in the lookup operation. Each steps are performed
-     * asynchronously.
-     *
-     * @param k          : Prefix on which the lookup is performed
-     * @param lo         : lowest bound on the prefix (where to start)
-     * @param hi         : highest bound on the prefix (where to stop)
-     * @param vals       : Shared ptr to a vector of IndexEntry (going to contains all values found)
-     * @param cb         : Callback to use at the end of the lookupStep (call on the value of vals)
-     * @param done_cb    : Callback at the end of the lookupStep
-     * @param max_common_prefix_len: used in the inexacte lookup match case, indicate the longest common prefix found
-     * @param start      : If start is set then lo and hi will be ignore for the first step, if the step fail lo and hi will be used
-     * @param all_values : If all value is true then all value met during the lookupstep will be in the vector vals
-     */
-    void lookupStep(Prefix k, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
-            std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals,
-            LookupCallbackWrapper cb, DoneCallbackSimple done_cb,
-            std::shared_ptr<unsigned> max_common_prefix_len,
-            int start = -1, bool all_values = false);
-
-    /**
-     * Apply the zcurve algorithm on the list of input prefix
-     *
-     * @param all_prefix : Vector of prefix to interleave
-     *
-     * @return The output prefix where all flags and content are interleaves
-     */
-    Prefix zcurve(const std::vector<Prefix>& all_prefix) const;
-
-    /**
-     * Linearizes the key into a unidimensional key. A pht only takes
-     * unidimensional key.
-     *
-     * @param Key  The initial key.
-     *
-     * @return the prefix of the linearized key.
-     */
-    virtual Prefix linearize(Key k) const;
-
-    /**
-     * Looking where to put the data cause if there is free space on the node
-     * above then this node will became the real leave.
-     *
-     * @param p       Share_ptr on the Prefix to check
-     * @param entry   The entry to put at the prefix p
-     * @param end_cb  Callback to use at the end of counting
-     */
-    void getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb );
-
-    /**
-     * Looking where to put the data cause if there is free space on the node
-     * above then this node will became the real leave.
-     *
-     * @param p       Share_ptr on the Prefix to check
-     * @param entry   The entry to put at the prefix p
-     * @param end_cb  Callback to use at the end of counting
-     */
-    void checkPhtUpdate(Prefix p, IndexEntry entry, time_point time_p);
-
-    /**
-     * Search for the split location by comparing 'compared' to all values in vals.
-     *
-     * @param compared : Value which going to be compared
-     * @param vals     : The vector of values to compare with comapred
-     * @return position compared diverge from all others
-     */
-    size_t foundSplitLocation(Prefix compared, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals) {
-        for ( size_t i = 0; i < compared.content_.size() * 8 - 1; i++ )
-            for ( auto const& v : *vals)
-                if ( Prefix(v->prefix).isContentBitActive(i) != compared.isContentBitActive(i) )
-                    return i + 1;
-
-        return compared.content_.size() * 8 - 1;
-    }
-
-    /**
-     * Put canary from the split point until the last known canary and add the prefix at the good place
-     *
-     * @param insert : Prefix to insertm but also prefix which going to check where we need to split
-     * @param vals   : Vector of vals for the comparaison
-     * @param entry  : Entry to put on the pht
-     * @param end_cb : Callback to apply to the insert prefi (here does the insert)
-     */
-    void split(Prefix insert, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals, IndexEntry entry, RealInsertCallback end_cb);
-
-    /**
-     * Tells if the key is valid according to the key spec.
-     */
-    bool validKey(const Key& k) const {
-        return k.size() == keySpec_.size() and
-            std::equal(k.begin(), k.end(), keySpec_.begin(),
-                [&](const Key::value_type& key, const KeySpec::value_type& key_spec) {
-                    return key.first == key_spec.first and key.second.size() <= key_spec.second;
-                }
-            );
-    }
-
-    /**
-     * Updates the canary token on the node responsible for the specified
-     * Prefix.
-     */
-    void updateCanary(Prefix p);
-
-    const std::string name_;
-    const std::string canary_;
-    const KeySpec keySpec_;
-    Cache cache_;
-    std::shared_ptr<DhtRunner> dht_;
-};
-
-} /* indexation  */
-} /* dht */
-

include/opendht/infohash.h

@@ -197,6 +197,7 @@ public:
     static InfoHash getRandom();
 
     friend std::ostream& operator<< (std::ostream& s, const InfoHash& h);
+    friend std::istream& operator>> (std::istream& s, InfoHash& h);
 
     std::string toString() const;
 

include/opendht/network_engine.h

@@ -195,7 +195,7 @@ private:
      * @param saddr (type: sockaddr*) sockaddr* pointer containing address ip information.
      * @param saddr_len (type: socklen_t) lenght of the sockaddr struct.
      */
-    std::function<void(const InfoHash&, sockaddr*, socklen_t)> onReportedAddr;
+    std::function<void(const InfoHash&, const SockAddr&)> onReportedAddr;
     /**
      * @brief on ping request callback.
      *
@@ -256,7 +256,7 @@ public:
     using RequestExpiredCb = std::function<void(const Request&, bool)>;
 
     NetworkEngine(Logger& log, Scheduler& scheduler) : myid(zeroes), DHT_LOG(log), scheduler(scheduler) {}
-    NetworkEngine(InfoHash& myid, int s, int s6, Logger& log, Scheduler& scheduler,
+    NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler,
             decltype(NetworkEngine::onError) onError,
             decltype(NetworkEngine::onNewNode) onNewNode,
             decltype(NetworkEngine::onReportedAddr) onReportedAddr,
@@ -266,7 +266,7 @@ public:
             decltype(NetworkEngine::onListen) onListen,
             decltype(NetworkEngine::onAnnounce) onAnnounce) :
         onError(onError), onNewNode(onNewNode), onReportedAddr(onReportedAddr), onPing(onPing), onFindNode(onFindNode),
-        onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), myid(myid),
+        onGetValues(onGetValues), onListen(onListen), onAnnounce(onAnnounce), myid(myid), network(net),
         dht_socket(s), dht_socket6(s6), DHT_LOG(log), scheduler(scheduler)
     {
         transaction_id = std::uniform_int_distribution<decltype(transaction_id)>{1}(rd_device);
@@ -342,10 +342,10 @@ public:
      * @param fromlen  The length of the corresponding sockaddr structure.
      * @param now  The time to adjust the clock in the network engine.
      */
-    void processMessage(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen);
+    void processMessage(const uint8_t *buf, size_t buflen, const SockAddr& addr);
 
-    std::shared_ptr<Node> insertNode(const InfoHash& myid, const sockaddr* from, socklen_t fromlen) {
-        return cache.getNode(myid, from, fromlen, scheduler.time(), 0);
+    std::shared_ptr<Node> insertNode(const InfoHash& myid, const SockAddr& addr) {
+        return cache.getNode(myid, addr, scheduler.time(), 0);
     }
 
     std::vector<unsigned> getNodeMessageStats(bool in) {
@@ -374,20 +374,10 @@ private:
     /* TODO */
     static const std::string my_v;
 
-    /* DHT info */
-    const InfoHash& myid;
-    const int dht_socket {-1};
-    const int dht_socket6 {-1};
-    const Logger& DHT_LOG;
-
-    NodeCache cache {};
-    sockaddr_storage blacklist[BLACKLISTED_MAX] {};
-    unsigned next_blacklisted = 0;
-
     bool rateLimit();
 
-    static bool isMartian(const sockaddr* sa, socklen_t len);
-    bool isNodeBlacklisted(const sockaddr*, socklen_t) const;
+    static bool isMartian(const SockAddr& addr);
+    bool isNodeBlacklisted(const SockAddr& addr) const;
 
     void requestStep(std::shared_ptr<Request> req);
 
@@ -417,16 +407,15 @@ private:
 
 
     // basic wrapper for socket sendto function
-    int send(const char *buf, size_t len, int flags, const sockaddr *sa, socklen_t salen);
+    int send(const char *buf, size_t len, int flags, const SockAddr& addr);
 
     /*************
      *  Answers  *
      *************/
     /* answer to a ping  request */
-    void sendPong(const sockaddr* sa, socklen_t salen, TransId tid);
+    void sendPong(const SockAddr& addr, TransId tid);
     /* answer to findnodes/getvalues request */
-    void sendNodesValues(const sockaddr* sa,
-            socklen_t salen,
+    void sendNodesValues(const SockAddr& addr,
             TransId tid,
             const Blob& nodes,
             const Blob& nodes6,
@@ -440,12 +429,11 @@ private:
             std::vector<std::shared_ptr<Node>>& nodes,
             std::vector<std::shared_ptr<Node>>& nodes6);
     /* answer to a listen request */
-    void sendListenConfirmation(const sockaddr* sa, socklen_t salen, TransId tid);
+    void sendListenConfirmation(const SockAddr& addr, TransId tid);
     /* answer to put request */
-    void sendValueAnnounced(const sockaddr* sa, socklen_t salen, TransId, Value::Id);
+    void sendValueAnnounced(const SockAddr& addr, TransId, Value::Id);
     /* answer in case of error */
-    void sendError(const sockaddr* sa,
-            socklen_t salen,
+    void sendError(const SockAddr& addr,
             TransId tid,
             uint16_t code,
             const std::string& message,
@@ -453,6 +441,14 @@ private:
 
     void deserializeNodesValues(ParsedMessage& msg);
 
+    /* DHT info */
+    const InfoHash& myid;
+    const NetId network {0};
+    const int dht_socket {-1};
+    const int dht_socket6 {-1};
+    const Logger& DHT_LOG;
+
+    NodeCache cache {};
     std::queue<time_point> rate_limit_time {};
     static std::mt19937 rd_device;
 
@@ -460,6 +456,7 @@ private:
     uint16_t transaction_id {1};
     std::map<uint16_t, std::shared_ptr<Request>> requests {};
     MessageStats in_stats {}, out_stats {};
+    std::set<SockAddr> blacklist {};
 
     Scheduler& scheduler;
 };

include/opendht/node.h

@@ -30,40 +30,34 @@ namespace dht {
 struct Request;
 
 struct Node {
-    friend class NetworkEngine;
-
     InfoHash id;
-
-    socklen_t sslen {0};
-    sockaddr_storage ss;
+    SockAddr addr;
 
     time_point time {time_point::min()};            /* last time eared about */
     time_point reply_time {time_point::min()};      /* time of last correct reply received */
 
     Node(const InfoHash& id, const sockaddr* sa, socklen_t salen)
-        : id(id), sslen(salen), ss() {
-        std::copy_n((const uint8_t*)sa, salen, (uint8_t*)&ss);
-        if ((unsigned)salen < sizeof(ss))
-            std::fill_n((uint8_t*)&ss+salen, sizeof(ss)-salen, 0);
-    }
+        : id(id), addr(sa, salen) {}
+    Node(const InfoHash& id, const SockAddr& addr) : id(id), addr(addr) {}
+
     InfoHash getId() const {
         return id;
     }
     std::pair<const sockaddr*, socklen_t> getAddr() const {
-        return {(const sockaddr*)&ss, sslen};
+        return {(const sockaddr*)&addr.first, addr.second};
     }
     std::string getAddrStr() const {
-        return print_addr(ss, sslen);
+        return addr.toString();
     }
     bool isExpired() const { return expired_; }
     bool isGood(time_point now) const;
     bool isPendingMessage() const;
     size_t getPendingMessageCount() const;
 
-    NodeExport exportNode() const { return NodeExport {id, ss, sslen}; }
-    sa_family_t getFamily() const { return ss.ss_family; }
+    NodeExport exportNode() const { return NodeExport {id, addr.first, addr.second}; }
+    sa_family_t getFamily() const { return addr.getFamily(); }
 
-    void update(const sockaddr* sa, socklen_t salen);
+    void update(const SockAddr&);
 
     void requested(std::shared_ptr<Request>& req);
     void received(time_point now, std::shared_ptr<Request> req);
@@ -88,7 +82,6 @@ struct Node {
     static constexpr const std::chrono::seconds MAX_RESPONSE_TIME {1};
 
 private:
-
     std::list<std::weak_ptr<Request>> requests_ {};
     bool expired_ {false};
 
@@ -97,7 +90,6 @@ private:
             return w.expired();
         });
     }
-
 };
 
 }

include/opendht/node_cache.h

@@ -28,7 +28,7 @@ namespace dht {
 
 struct NodeCache {
     std::shared_ptr<Node> getNode(const InfoHash& id, sa_family_t family);
-    std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirmed);
+    std::shared_ptr<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, int confirmed);
 
     /**
      * Reset the connectivity state of every node,
@@ -41,7 +41,7 @@ private:
     class NodeMap : public std::map<InfoHash, std::weak_ptr<Node>> {
     public:
         std::shared_ptr<Node> getNode(const InfoHash& id);
-        std::shared_ptr<Node> getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirmed);
+        std::shared_ptr<Node> getNode(const InfoHash& id, const SockAddr&, time_point now, int confirmed);
         void clearBadNodes();
     };
 

include/opendht/routing_table.h

@@ -23,6 +23,8 @@
 
 namespace dht {
 
+static constexpr unsigned TARGET_NODES {8};
+
 struct Bucket {
     Bucket() : cached() {}
     Bucket(sa_family_t af, const InfoHash& f = {}, time_point t = time_point::min())

include/opendht/securedht.h

@@ -126,16 +126,13 @@ public:
     Value decrypt(const Value& v);
 
     void findCertificate(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::Certificate>)> cb);
-    void findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::PublicKey>)> cb);
+    void findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<const crypto::PublicKey>)> cb);
 
     const std::shared_ptr<crypto::Certificate> registerCertificate(const InfoHash& node, const Blob& cert);
     void registerCertificate(std::shared_ptr<crypto::Certificate>& cert);
 
     const std::shared_ptr<crypto::Certificate> getCertificate(const InfoHash& node) const;
-    const std::shared_ptr<crypto::PublicKey> getPublicKey(const InfoHash& node) const;
-
-
-    
+    const std::shared_ptr<const crypto::PublicKey> getPublicKey(const InfoHash& node) const;
 
     /**
      * Allows to set a custom callback called by the library to find a locally-stored certificate.
@@ -161,7 +158,7 @@ private:
 
     // our certificate cache
     std::map<InfoHash, std::shared_ptr<crypto::Certificate>> nodesCertificates_ {};
-    std::map<InfoHash, std::shared_ptr<crypto::PublicKey>> nodesPubKeys_ {};
+    std::map<InfoHash, std::shared_ptr<const crypto::PublicKey>> nodesPubKeys_ {};
 
     std::uniform_int_distribution<Value::Id> rand_id {};
 };

include/opendht/utils.h

@@ -40,14 +40,49 @@
 
 namespace dht {
 
-static constexpr unsigned TARGET_NODES {8};
-
-using Address = std::pair<sockaddr_storage, socklen_t>;
-using want_t = int_fast8_t;
-
 std::string print_addr(const sockaddr* sa, socklen_t slen);
 std::string print_addr(const sockaddr_storage& ss, socklen_t sslen);
-std::string printAddr(const Address& addr);
+
+struct SockAddr : public std::pair<sockaddr_storage, socklen_t> {
+public:
+    using std::pair<sockaddr_storage, socklen_t>::pair;
+
+    SockAddr() : pair<sockaddr_storage, socklen_t>::pair({},0) {}
+    SockAddr(const SockAddr& o) : pair<sockaddr_storage, socklen_t>::pair({},o.second) {
+        std::copy_n((uint8_t*)&o.first, o.second, (uint8_t*)&first);
+    }
+    SockAddr(const sockaddr* sa, socklen_t len) : pair<sockaddr_storage, socklen_t>::pair({},len) {
+        if (len > sizeof(sockaddr_storage))
+            throw std::runtime_error("Socket address length is too large");
+        std::copy_n((uint8_t*)sa, len, (uint8_t*)&first);
+    }
+
+    bool operator<(const SockAddr& o) const {
+        if (second != o.second)
+            return second < o.second;
+        return std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) < 0;
+    }
+
+    bool operator==(const SockAddr& o) const {
+        return second == o.second
+            && std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) == 0;
+    }
+    SockAddr& operator=(const SockAddr& o) {
+        std::copy_n((const uint8_t*)&o.first, o.second, (uint8_t*)&first);
+        second = o.second;
+        return *this;
+    }
+
+    std::string toString() const {
+        return print_addr(first, second);
+    }
+    sa_family_t getFamily() const { return second > sizeof(sa_family_t) ? first.ss_family : AF_UNSPEC; }
+};
+
+using NetId = uint32_t;
+using want_t = int_fast8_t;
+
+std::string printAddr(const SockAddr& addr);
 
 template <typename Key, typename Item, typename Condition>
 void erase_if(std::map<Key, Item>& map, const Condition& condition)

include/opendht/value.h

@@ -68,7 +68,7 @@ using StorePolicy = std::function<bool(InfoHash key, std::shared_ptr<Value>& val
  */
 using EditPolicy = std::function<bool(InfoHash key, const std::shared_ptr<Value>& old_val, std::shared_ptr<Value>& new_val, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>;
 
-static constexpr const size_t MAX_VALUE_SIZE {1024 * 14};
+static constexpr const size_t MAX_VALUE_SIZE {1024 * 16};
 
 struct ValueType {
     typedef uint16_t Id;
@@ -115,7 +115,7 @@ struct ValueType {
 struct Value
 {
     typedef uint64_t Id;
-    static const Id INVALID_ID {0};
+    static const constexpr Id INVALID_ID {0};
 
     class Filter : public std::function<bool(const Value&)> {
         using std::function<bool(const Value&)>::function;
@@ -135,14 +135,17 @@ struct Value
                 return f1(v) and f2(v);
             };
         }
-        static Filter chain(std::initializer_list<Filter> l) {
-            const std::vector<Filter> list(l.begin(), l.end());
-            return [list](const Value& v){
-                for (const auto& f : list)
+        static Filter chainAll(std::vector<Filter>&& set) {
+            if (set.empty()) return {};
+            return std::bind([](const Value& v, std::vector<Filter>& s) {
+                for (const auto& f : s)
                     if (f and not f(v))
                         return false;
                 return true;
-            };
+            }, std::placeholders::_1, std::move(set));
+        }
+        static Filter chain(std::initializer_list<Filter> l) {
+            return chainAll(std::vector<Filter>(l.begin(), l.end()));
         }
         static Filter chainOr(Filter&& f1, Filter&& f2) {
             if (not f1 or not f2) return AllFilter();
@@ -153,7 +156,7 @@ struct Value
     };
 
     static const Filter AllFilter() {
-        return [](const Value&){return true;};
+        return {};
     }
 
     static Filter TypeFilter(const ValueType& t) {
@@ -248,6 +251,43 @@ struct Value
         return owner and not signature.empty();
     }
 
+    /**
+     * Sign the value using the provided private key.
+     * Afterward, checkSignature() will return true and owner will
+     * be set to the corresponding public key.
+     */
+    void sign(const crypto::PrivateKey& key) {
+        if (isEncrypted())
+            throw DhtException("Can't sign encrypted data.");
+        owner = std::make_shared<const crypto::PublicKey>(key.getPublicKey());
+        signature = key.sign(getToSign());
+    }
+
+    /**
+     * Check that the value is signed and that the signature matches.
+     * If true, the owner field will contain the signer public key.
+     */
+    bool checkSignature() const {
+        return isSigned() and owner->checkSignature(getToSign(), signature);
+    }
+
+    std::shared_ptr<const crypto::PublicKey> getOwner() const {
+        return std::static_pointer_cast<const crypto::PublicKey>(owner);
+    }
+
+    /**
+     * Sign the value with from and returns the encrypted version for to.
+     */
+    Value encrypt(const crypto::PrivateKey& from, const crypto::PublicKey& to) {
+        if (isEncrypted())
+            throw DhtException("Data is already encrypted.");
+        setRecipient(to.getId());
+        sign(from);
+        Value nv {id};
+        nv.setCypher(to.encrypt(getToEncrypt()));
+        return nv;
+    }
+
     Value() {}
 
     Value (Id id) : id(id) {}
@@ -380,13 +420,19 @@ struct Value
 
     void msgpack_unpack(msgpack::object o);
     void msgpack_unpack_body(const msgpack::object& o);
+    Blob getPacked() const {
+        msgpack::sbuffer buffer;
+        msgpack::packer<msgpack::sbuffer> pk(&buffer);
+        pk.pack(*this);
+        return {buffer.data(), buffer.data()+buffer.size()};
+    }
 
     Id id {INVALID_ID};
 
     /**
      * Public key of the signer.
      */
-    std::shared_ptr<crypto::PublicKey> owner {};
+    std::shared_ptr<const crypto::PublicKey> owner {};
 
     /**
      * Hash of the recipient (optional).

opendht.pc.in

@@ -6,4 +6,6 @@ Name: OpenDHT
 Description: C++ Distributed Hash Table library
 Version: @VERSION@
 Libs: -L${libdir} -lopendht
+Libs.private: -lpthread
+Requires.private: gnutls >= 3.1
 Cflags: -I${includedir}

python/opendht.pyx

@@ -31,6 +31,7 @@ from libcpp.map cimport map as map
 from libcpp cimport bool
 from libcpp.utility cimport pair
 from libcpp.string cimport string
+from libcpp.memory cimport shared_ptr
 
 from cython.parallel import parallel, prange
 from cython.operator cimport dereference as deref, preincrement as inc, predecrement as dec
@@ -40,29 +41,19 @@ cimport opendht_cpp as cpp
 
 import threading
 
-cdef inline void lookup_callback(cpp.vector[cpp.shared_ptr[cpp.IndexValue]]* values, cpp.Prefix* p, void *user_data) with gil:
-    cbs = <object>user_data
-    if 'lookup' in cbs and cbs['lookup']:
-        vals = []
-        for val in deref(values):
-            v = IndexValue()
-            v._value = val
-            vals.append(v)
-        cbs['lookup'](vals, p.toString())
-
 cdef inline void shutdown_callback(void* user_data) with gil:
     cbs = <object>user_data
     if 'shutdown' in cbs and cbs['shutdown']:
         cbs['shutdown']()
     ref.Py_DECREF(cbs)
 
-cdef inline bool get_callback(cpp.shared_ptr[cpp.Value] value, void *user_data) with gil:
+cdef inline bool get_callback(shared_ptr[cpp.Value] value, void *user_data) with gil:
     cb = (<object>user_data)['get']
     pv = Value()
     pv._value = value
     return cb(pv)
 
-cdef inline void done_callback(bool done, cpp.vector[cpp.shared_ptr[cpp.Node]]* nodes, void *user_data) with gil:
+cdef inline void done_callback(bool done, cpp.vector[shared_ptr[cpp.Node]]* nodes, void *user_data) with gil:
     node_ids = []
     for n in deref(nodes):
         h = NodeEntry()
@@ -74,12 +65,6 @@ cdef inline void done_callback(bool done, cpp.vector[cpp.shared_ptr[cpp.Node]]*
         cbs['done'](done, node_ids)
     ref.Py_DECREF(cbs)
 
-cdef inline void done_callback_simple(bool done, void *user_data) with gil:
-    cbs = <object>user_data
-    if 'done' in cbs and cbs['done']:
-        cbs['done'](done)
-    ref.Py_DECREF(cbs)
-
 cdef class _WithID(object):
     def __repr__(self):
         return "<%s '%s'>" % (self.__class__.__name__, str(self))
@@ -88,7 +73,7 @@ cdef class _WithID(object):
 
 cdef class InfoHash(_WithID):
     cdef cpp.InfoHash _infohash
-    def __cinit__(self, bytes str=b''):
+    def __init__(self, bytes str=b''):
         self._infohash = cpp.InfoHash(str)
     def __bool__(InfoHash self):
         return not (self._infohash == cpp.InfoHash())
@@ -125,7 +110,7 @@ cdef class InfoHash(_WithID):
         return h
 
 cdef class Node(_WithID):
-    cdef cpp.shared_ptr[cpp.Node] _node
+    cdef shared_ptr[cpp.Node] _node
     def getId(self):
         h = InfoHash()
         h._infohash = self._node.get().getId()
@@ -136,7 +121,7 @@ cdef class Node(_WithID):
         return self._node.get().isExpired()
 
 cdef class NodeEntry(_WithID):
-    cdef cpp.pair[cpp.InfoHash, cpp.shared_ptr[cpp.Node]] _v
+    cdef cpp.pair[cpp.InfoHash, shared_ptr[cpp.Node]] _v
     def getId(self):
         h = InfoHash()
         h._infohash = self._v.first
@@ -147,7 +132,7 @@ cdef class NodeEntry(_WithID):
         return n
 
 cdef class Value(object):
-    cdef cpp.shared_ptr[cpp.Value] _value
+    cdef shared_ptr[cpp.Value] _value
     def __init__(self, bytes val=b''):
         self._value.reset(new cpp.Value(val, len(val)))
     def __str__(self):
@@ -179,8 +164,8 @@ cdef class Value(object):
             return self._value.get().size()
 
 cdef class NodeSetIter(object):
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]]* _nodes
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]].iterator _curIter
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]]* _nodes
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]].iterator _curIter
     def __init__(self, NodeSet s):
         self._nodes = &s._nodes
         self._curIter = self._nodes.begin()
@@ -193,7 +178,7 @@ cdef class NodeSetIter(object):
         return h
 
 cdef class NodeSet(object):
-    cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]] _nodes
+    cdef map[cpp.InfoHash, shared_ptr[cpp.Node]] _nodes
     def size(self):
         return self._nodes.size()
     def insert(self, NodeEntry l):
@@ -215,7 +200,7 @@ cdef class NodeSet(object):
         return h
     def __str__(self):
         s = ''
-        cdef map[cpp.InfoHash, cpp.shared_ptr[cpp.Node]].iterator it = self._nodes.begin()
+        cdef map[cpp.InfoHash, shared_ptr[cpp.Node]].iterator it = self._nodes.begin()
         while it != self._nodes.end():
             s += deref(it).first.toString().decode() + ' ' + deref(it).second.get().getAddrStr().decode() + '\n'
             inc(it)
@@ -231,7 +216,7 @@ cdef class PublicKey(_WithID):
         return h
 
 cdef class Certificate(_WithID):
-    cdef cpp.shared_ptr[cpp.Certificate] _cert
+    cdef shared_ptr[cpp.Certificate] _cert
     def getId(self):
         h = InfoHash()
         h._infohash = self._cert.get().getId()
@@ -270,49 +255,49 @@ cdef class DhtConfig(object):
         self._config.dht_config.node_config.node_id = id._infohash
 
 cdef class DhtRunner(_WithID):
-    cdef cpp.shared_ptr[cpp.DhtRunner] thisptr
+    cdef cpp.DhtRunner* thisptr
     def __cinit__(self):
-        self.thisptr.reset(new cpp.DhtRunner())
+        self.thisptr = new cpp.DhtRunner()
     def getId(self):
         h = InfoHash()
-        h._infohash = self.thisptr.get().getId()
+        h._infohash = self.thisptr.getId()
         return h
     def getNodeId(self):
-        return self.thisptr.get().getNodeId().toString()
+        return self.thisptr.getNodeId().toString()
     def bootstrap(self, str host, str port):
-        self.thisptr.get().bootstrap(host.encode(), port.encode())
+        self.thisptr.bootstrap(host.encode(), port.encode())
     def run(self, Identity id=None, is_bootstrap=False, cpp.in_port_t port=0, str ipv4="", str ipv6="", DhtConfig config=DhtConfig()):
         if id:
             config.setIdentity(id)
         if ipv4 or ipv6:
             bind4 = ipv4.encode() if ipv4 else b''
             bind6 = ipv6.encode() if ipv6 else b''
-            self.thisptr.get().run(bind4, bind6, str(port).encode(), config._config)
+            self.thisptr.run(bind4, bind6, str(port).encode(), config._config)
         else:
-            self.thisptr.get().run(port, config._config)
+            self.thisptr.run(port, config._config)
     def join(self):
-        self.thisptr.get().join()
+        self.thisptr.join()
     def shutdown(self, shutdown_cb=None):
         cb_obj = {'shutdown':shutdown_cb}
         ref.Py_INCREF(cb_obj)
-        self.thisptr.get().shutdown(cpp.bindShutdownCb(shutdown_callback, <void*>cb_obj))
+        self.thisptr.shutdown(cpp.bindShutdownCb(shutdown_callback, <void*>cb_obj))
     def enableLogging(self):
-        cpp.enableLogging(self.thisptr.get()[0])
+        cpp.enableLogging(self.thisptr[0])
     def disableLogging(self):
-        cpp.disableLogging(self.thisptr.get()[0])
+        cpp.disableLogging(self.thisptr[0])
     def enableFileLogging(self, str path):
-        cpp.enableFileLogging(self.thisptr.get()[0], path)
+        cpp.enableFileLogging(self.thisptr[0], path)
     def isRunning(self):
-        return self.thisptr.get().isRunning()
+        return self.thisptr.isRunning()
     def getStorageLog(self):
-        return self.thisptr.get().getStorageLog().decode()
+        return self.thisptr.getStorageLog().decode()
     def getRoutingTablesLog(self, cpp.sa_family_t af):
-        return self.thisptr.get().getRoutingTablesLog(af).decode()
+        return self.thisptr.getRoutingTablesLog(af).decode()
     def getSearchesLog(self, cpp.sa_family_t af):
-        return self.thisptr.get().getSearchesLog(af).decode()
+        return self.thisptr.getSearchesLog(af).decode()
     def getNodeMessageStats(self):
         stats = []
-        cdef cpp.vector[unsigned] res = self.thisptr.get().getNodeMessageStats(False)
+        cdef cpp.vector[unsigned] res = self.thisptr.getNodeMessageStats(False)
         for n in res:
             stats.append(n)
         return stats
@@ -320,16 +305,14 @@ cdef class DhtRunner(_WithID):
     def get(self, InfoHash key, get_cb=None, done_cb=None):
         """Retreive values associated with a key on the DHT.
 
-        key     -- the key for which to search
-        get_cb  -- is set, makes the operation non-blocking. Called when a value
-                   is found on the DHT.
-        done_cb -- optional callback used when get_cb is set. Called when the
-                   operation is completed.
+        key -- the key for which to search
+        get_cb -- is set, makes the operation non-blocking. Called when a value is found on the DHT.
+        done_cb -- optional callback used when get_cb is set. Called when the operation is completed.
         """
         if get_cb:
             cb_obj = {'get':get_cb, 'done':done_cb}
             ref.Py_INCREF(cb_obj)
-            self.thisptr.get().get(key._infohash, cpp.bindGetCb(get_callback, <void*>cb_obj), cpp.bindDoneCb(done_callback, <void*>cb_obj))
+            self.thisptr.get(key._infohash, cpp.bindGetCb(get_callback, <void*>cb_obj), cpp.bindDoneCb(done_callback, <void*>cb_obj))
         else:
             lock = threading.Condition()
             pending = 0
@@ -352,14 +335,14 @@ cdef class DhtRunner(_WithID):
     def put(self, InfoHash key, Value val, done_cb=None):
         """Publish a new value on the DHT at key.
 
-        key     -- the DHT key where to put the value
-        val     -- the value to put on the DHT
+        key -- the DHT key where to put the value
+        val -- the value to put on the DHT
         done_cb -- optional callback called when the operation is completed.
         """
         if done_cb:
             cb_obj = {'done':done_cb}
             ref.Py_INCREF(cb_obj)
-            self.thisptr.get().put(key._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj))
+            self.thisptr.put(key._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj))
         else:
             lock = threading.Condition()
             pending = 0
@@ -383,74 +366,9 @@ cdef class DhtRunner(_WithID):
         t._cb['cb'] = cb_obj
         # avoid the callback being destructed if the token is destroyed
         ref.Py_INCREF(cb_obj)
-        t._t = self.thisptr.get().listen(t._h, cpp.bindGetCb(get_callback, <void*>cb_obj)).share()
+        t._t = self.thisptr.listen(t._h, cpp.bindGetCb(get_callback, <void*>cb_obj)).share()
         return t
     def cancelListen(self, ListenToken token):
-        self.thisptr.get().cancelListen(token._h, token._t)
+        self.thisptr.cancelListen(token._h, token._t)
         ref.Py_DECREF(<object>token._cb['cb'])
         # fixme: not thread safe
-
-cdef class IndexValue(object):
-    cdef cpp.shared_ptr[cpp.IndexValue] _value
-    def __init__(self, InfoHash h=None, cpp.uint64_t vid=0):
-       cdef cpp.InfoHash hh = h._infohash
-       self._value.reset(new cpp.IndexValue(hh, vid))
-    def __str__(self):
-        return "(" + self.getKey().toString().decode() +", "+ str(self.getValueId()) +")"
-    def getKey(self):
-        h = InfoHash()
-        h._infohash = self._value.get().first
-        return h
-    def getValueId(self):
-        return self._value.get().second
-
-cdef class Pht(object):
-    cdef cpp.Pht* thisptr
-    def __cinit__(self, bytes name, key_spec, DhtRunner dht):
-        cdef cpp.IndexKeySpec cpp_key_spec
-        for kk, size in key_spec.items():
-            cpp_key_spec[bytes(kk, 'utf-8')] = size
-        self.thisptr = new cpp.Pht(name, cpp_key_spec, dht.thisptr)
-    property MAX_NODE_ENTRY_COUNT:
-        def __get__(self):
-            return cpp.PHT_MAX_NODE_ENTRY_COUNT
-    def lookup(self, key, lookup_cb=None, done_cb=None):
-        """Query the Index with a specified key.
-
-        key       -- the key for to the entry in the index.
-        lookup_cb -- function called when the operation is completed. This
-                     function takes a list of IndexValue objects and a string
-                     representation of the prefix where the value was indexed in
-                     the PHT.
-        """
-        cb_obj = {'lookup':lookup_cb, 'done':done_cb} # TODO: donecallback is to be removed
-        ref.Py_INCREF(cb_obj)
-        cdef cpp.IndexKey cppk
-        for kk, v in key.items():
-            cppk[bytes(kk, 'utf-8')] = bytes(v)
-        self.thisptr.lookup(
-                cppk,
-                cpp.Pht.bindLookupCb(lookup_callback, <void*>cb_obj),
-                cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj)
-        )
-    def insert(self, key, IndexValue value, done_cb=None):
-        """Add an index entry to the Index.
-
-        key     -- the key for to the entry in the index.
-        value   -- an IndexValue object describing the indexed value.
-        done_cb -- Called when the operation is completed.
-        """
-        cb_obj = {'done':done_cb}
-        ref.Py_INCREF(cb_obj)
-        cdef cpp.IndexKey cppk
-        for kk, v in key.items():
-            cppk[bytes(kk, 'utf-8')] = bytes(v)
-        cdef cpp.IndexValue val
-        val.first = (<InfoHash>value.getKey())._infohash
-        val.second = value.getValueId()
-        self.thisptr.insert(
-                cppk,
-                val,
-                cpp.bindDoneCbSimple(done_callback_simple, <void*>cb_obj)
-        )
-

python/opendht_cpp.pxd

@@ -20,19 +20,11 @@ from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 from libcpp.utility cimport pair
-from libcpp.map cimport map
+from libcpp.memory cimport shared_ptr
 
 ctypedef uint16_t in_port_t
 ctypedef unsigned short int sa_family_t;
 
-cdef extern from "<memory>" namespace "std" nogil:
-    cdef cppclass shared_ptr[T]:
-        shared_ptr() except +
-        shared_ptr(T*) except +
-        T* get()
-        T operator*()
-        void reset(T*)
-
 cdef extern from "<functional>" namespace "std" nogil:
     cdef cppclass hash[T]:
         hash() except +
@@ -105,21 +97,19 @@ cdef extern from "opendht/callbacks.h" namespace "dht":
     ctypedef void (*ShutdownCallbackRaw)(void *user_data)
     ctypedef bool (*GetCallbackRaw)(shared_ptr[Value] values, void *user_data)
     ctypedef void (*DoneCallbackRaw)(bool done, vector[shared_ptr[Node]]* nodes, void *user_data)
-    ctypedef void (*DoneCallbackSimpleRaw)(bool done, void *user_data)
 
     cppclass ShutdownCallback:
         ShutdownCallback() except +
     cppclass GetCallback:
         GetCallback() except +
+        #GetCallback(GetCallbackRaw cb, void *user_data) except +
     cppclass DoneCallback:
         DoneCallback() except +
-    cppclass DoneCallbackSimple:
-        DoneCallbackSimple() except +
+        #DoneCallback(DoneCallbackRaw, void *user_data) except +
 
     cdef ShutdownCallback bindShutdownCb(ShutdownCallbackRaw cb, void *user_data)
     cdef GetCallback bindGetCb(GetCallbackRaw cb, void *user_data)
     cdef DoneCallback bindDoneCb(DoneCallbackRaw cb, void *user_data)
-    cdef DoneCallbackSimple bindDoneCbSimple(DoneCallbackSimpleRaw cb, void *user_data)
 
     cppclass Config:
         InfoHash node_id
@@ -160,22 +150,3 @@ cdef extern from "opendht/log.h" namespace "dht::log":
     void disableLogging(DhtRunner& dht)
     void enableFileLogging(DhtRunner& dht, const string& path)
 
-cdef extern from "opendht/indexation/pht.h" namespace "dht::indexation":
-    size_t PHT_MAX_NODE_ENTRY_COUNT "dht::indexation::Pht::MAX_NODE_ENTRY_COUNT"
-    cdef cppclass Prefix:
-        Prefix() except +
-        Prefix(vector[uint8_t]) except +
-        string toString() const
-    ctypedef pair[InfoHash, uint64_t] IndexValue "dht::indexation::Value"
-    ctypedef map[string, vector[uint8_t]] IndexKey "dht::indexation::Pht::Key"
-    ctypedef map[string, uint32_t] IndexKeySpec "dht::indexation::Pht::KeySpec"
-    ctypedef void (*LookupCallbackRaw)(vector[shared_ptr[IndexValue]]* values, Prefix* p, void* user_data);
-    cdef cppclass Pht:
-        cppclass LookupCallback:
-            LookupCallback() except +
-        Pht(string, IndexKeySpec, shared_ptr[DhtRunner]) except +
-        void lookup(IndexKey k, LookupCallback cb, DoneCallbackSimple doneCb);
-        void insert(IndexKey k, IndexValue v, DoneCallbackSimple cb)
-        @staticmethod
-        LookupCallback bindLookupCb(LookupCallbackRaw cb, void *user_data)
-

python/tools/README.md

@@ -0,0 +1,45 @@
+# Benchmark
+
+The `benchmark.py` script is used for testing OpenDHT in various cases. If you
+run `benchmark.py --help`, you should find the following text:
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      --performance         Launches performance benchmark test. Available args
+                            for "-t" are: gets.
+      --data-persistence    Launches data persistence benchmark test. Available
+                            args for "-t" are: delete, replace, mult_time.
+                            Available args for "-o" are : dump_str_log,
+                            keep_alive, trigger, traffic_plot, op_plot. Use "-m"
+                            to specify the number of producers on the DHT.Use "-e"
+                            to specify the number of values to put on the DHT.
+
+These options specify the feature to be tested. Each feature has its own tests.
+You specify the test by using `-t` flag (see `benchmark.py --help` for full
+help).
+
+## Python dependencies
+
+- pyroute2 >=0.3.14
+- matplotlib
+- GeoIP (used by `scanner.py` for drawing map of the world)
+- ipaddress
+- netifaces
+- networkx
+- numpy
+
+## Usage
+
+Before running the script, you have to build and install OpenDHT and its cython
+wrapper (`cython3` has to be installed) on the system so that it can be found by
+the benchmark script.
+
+    $ cd $OPENDHT_SRC_DIR
+    $ ./autogen.sh
+    $ ./configure
+    $ make && sudo make install
+
+Then, you can use the script like so:
+
+    $ cd $OPENDHT_SRC_DIR/python/tools/
+    $ python3 benchmark.py --performance -t gets -n 2048

python/tools/benchmark.py

@@ -27,7 +27,7 @@ import random
 
 from dht.network import DhtNetwork
 from dht.network import DhtNetworkSubProcess
-from dht.tests import PerformanceTest, PersistenceTest, PhtTest
+from dht.tests import PerformanceTest, PersistenceTest
 from dht import virtual_network_builder
 from dht import network as dhtnetwork
 
@@ -157,11 +157,6 @@ if __name__ == '__main__':
     featureArgs = parser.add_mutually_exclusive_group(required=True)
     featureArgs.add_argument('--performance', action='store_true', default=False,
             help='Launches performance benchmark test. Available args for "-t" are: gets.')
-    featureArgs.add_argument('--pht', action='store_true', default=False,
-            help='Launches PHT benchmark test. '\
-                    'Available args for "-t" are: insert. '\
-                    'Timer available by adding "timer" to "-o" args'\
-                    'Use "-m" option for fixing number of keys to create during the test.')
     featureArgs.add_argument('--data-persistence', action='store_true', default=0,
             help='Launches data persistence benchmark test. '\
                  'Available args for "-t" are: delete, replace, mult_time. '\
@@ -211,10 +206,6 @@ if __name__ == '__main__':
             PerformanceTest(args.test, wb, test_opt).run()
         elif args.data_persistence:
             PersistenceTest(args.test, wb, test_opt).run()
-        elif args.pht:
-            if args.m:
-                test_opt.update({ 'num_keys' : args.m })
-            PhtTest(args.test, wb, test_opt).run()
 
     except Exception as e:
         print(e)

python/tools/dht/tests.py

@@ -10,16 +10,10 @@ import string
 import time
 import subprocess
 import re
-import collections
-
-from matplotlib.ticker import FuncFormatter
-import math
 
 import numpy as np
 import matplotlib.pyplot as plt
-import networkx as nx
-from networkx.drawing.nx_agraph import graphviz_layout
-
+from matplotlib.ticker import FuncFormatter
 
 from opendht import *
 from dht.network import DhtNetwork, DhtNetworkSubProcess
@@ -36,24 +30,6 @@ Mbit_format = FuncFormatter(lambda x, pos: '%1.1f' % (x*1024**-2) + 'Mb')
 def random_hash():
     return InfoHash(''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(40)).encode())
 
-def timer(f, *args):
-    """
-    Start a timer which count time taken for execute function f
-
-    @param f : Function to time
-    @type  f : function
-
-    @param args : Arguments of the function f
-    @type  args : list
-
-    @rtype : timer
-    @return : Time taken by the function f
-    """
-    start = time.time()
-    f(*args)
-
-    return time.time() - start
-
 def reset_before_test(featureTestMethod):
     """
     This is a decorator for all test methods needing reset().
@@ -149,13 +125,13 @@ def iftop_traffic_data(ifname, interval=2, rate_type='send_receive'):
 ###########
 
 class FeatureTest(object):
+    done = 0
+    lock = None
+
     """
     This is a base test.
     """
 
-    done = 0
-    lock = None
-
     def __init__(self, test, workbench):
         """
         @param test: The test string indicating the test to run. This string is
@@ -167,7 +143,6 @@ class FeatureTest(object):
         """
         self._test = test
         self._workbench = workbench
-        self._bootstrap = self._workbench.get_bootstrap()
 
     def _reset(self):
         """
@@ -181,155 +156,7 @@ class FeatureTest(object):
     def run(self):
         raise NotImplementedError('This method must be implemented.')
 
-##################################
-#               PHT              #
-##################################
 
-class PhtTest(FeatureTest):
-    """TODO
-    """
-
-    indexEntries = None
-    prefix       = None
-    key          = None
-
-    def __init__(self, test, workbench, opts):
-        """
-        @param test: is one of the following:
-                     - 'insert': indexes a considerable amount of data in
-                       the PHT structure.
-                       TODO
-        @type  test: string
-
-        @param opts: Dictionnary containing options for the test. Allowed
-                     options are:
-                     - 'num_keys': this specifies the number of keys to insert
-                                   in the PHT during the test.
-        @type  opts: dict
-        """
-        super(PhtTest, self).__init__(test, workbench)
-        self._num_keys = opts['num_keys'] if 'num_keys' in opts else 32
-        self._timer = True if 'timer' in opts else False
-
-    def _reset(self):
-        super(PhtTest, self)._reset()
-        PhtTest.indexEntries = []
-
-    @staticmethod
-    def lookupCb(vals, prefix):
-        PhtTest.indexEntries = list(vals)
-        PhtTest.prefix = prefix.decode()
-        DhtNetwork.log('Index name: <todo>')
-        DhtNetwork.log('Leaf prefix:', prefix)
-        for v in vals:
-            DhtNetwork.log('[ENTRY]:', v)
-
-    @staticmethod
-    def lookupDoneCb(ok):
-        DhtNetwork.log('[LOOKUP]:', PhtTest.key, "--", "success!" if ok else "Fail...")
-        with FeatureTest.lock:
-            FeatureTest.lock.notify()
-
-    @staticmethod
-    def insertDoneCb(ok):
-        DhtNetwork.log('[INSERT]:', PhtTest.key, "--", "success!" if ok else "Fail...")
-        with FeatureTest.lock:
-            FeatureTest.lock.notify()
-
-    @staticmethod
-    def drawTrie(trie_dict):
-        """
-        Draws the trie structure of the PHT from dictionnary.
-
-        @param trie_dict: Dictionnary of index entries (prefix -> entry).
-        @type  trie_dict: dict
-        """
-        prefixes = list(trie_dict.keys())
-        if len(prefixes) == 0:
-            return
-
-        edges = list([])
-        for prefix in prefixes:
-            for i in range(-1, len(prefix)-1):
-                u = prefix[:i+1]
-                x = ("." if i == -1 else u, u+"0")
-                y = ("." if i == -1 else u, u+"1")
-                if x not in edges:
-                    edges.append(x)
-                if y not in edges:
-                    edges.append(y)
-
-        # TODO: use a binary tree position layout...
-        #   UPDATE : In a better way [change lib]
-        G = nx.Graph(sorted(edges, key=lambda x: len(x[0])))
-        plt.title("PHT: Tree")
-        pos=graphviz_layout(G,prog='dot')
-        nx.draw(G, pos, with_labels=True, node_color='white')
-        plt.show()
-
-    def run(self):
-        try:
-            if self._test == 'insert':
-                self._insertTest()
-        except Exception as e:
-            print(e)
-        finally:
-            self._bootstrap.resize(1)
-
-    ###########
-    #  Tests  #
-    ###########
-
-    @reset_before_test
-    def _insertTest(self):
-        """TODO: Docstring for _massIndexTest.
-        """
-        bootstrap = self._bootstrap
-        bootstrap.resize(2)
-        dht = bootstrap.get(1)
-
-        NUM_DIG  = max(math.log(self._num_keys, 2)/4, 5) # at least 5 digit keys.
-        keyspec = collections.OrderedDict([('foo', NUM_DIG)])
-        pht = Pht(b'foo_index', keyspec, dht)
-
-        DhtNetwork.log('PHT has',
-                       pht.MAX_NODE_ENTRY_COUNT,
-                       'node'+ ('s' if pht.MAX_NODE_ENTRY_COUNT > 1 else ''),
-                       'per leaf bucket.')
-        keys = [{
-            [_ for _ in keyspec.keys()][0] :
-            ''.join(random.SystemRandom().choice(string.hexdigits)
-                for _ in range(NUM_DIG)).encode()
-            } for n in range(self._num_keys)]
-        all_entries = {}
-
-        # Index all entries.
-        for key in keys:
-            PhtTest.key = key
-            with FeatureTest.lock:
-                time_taken = timer(pht.insert, key, IndexValue(random_hash()), PhtTest.insertDoneCb)
-                if self._timer:
-                    DhtNetwork.log('This insert step took : ', time_taken, 'second')
-                FeatureTest.lock.wait()
-
-        time.sleep(1)
-
-        # Recover entries now that the trie is complete.
-        for key in keys:
-            PhtTest.key = key
-            with FeatureTest.lock:
-                time_taken = timer(pht.lookup, key, PhtTest.lookupCb, PhtTest.lookupDoneCb)
-                if self._timer:
-                    DhtNetwork.log('This lookup step took : ', time_taken, 'second')
-                FeatureTest.lock.wait()
-
-            all_entries[PhtTest.prefix] = [e.__str__()
-                                           for e in PhtTest.indexEntries]
-
-        for p in all_entries.keys():
-            DhtNetwork.log('All entries under prefix', p, ':')
-            DhtNetwork.log(all_entries[p])
-        PhtTest.drawTrie(all_entries)
 
 ##################################
 #               DHT              #
@@ -337,7 +164,7 @@ class PhtTest(FeatureTest):
 
 class DhtFeatureTest(FeatureTest):
     """
-    This is a base dht test.
+    This is base test. A method run() implementation is required.
     """
     #static variables used by class callbacks
     successfullTransfer = lambda lv,fv: len(lv) == len(fv)
@@ -346,6 +173,7 @@ class DhtFeatureTest(FeatureTest):
 
     def __init__(self, test, workbench):
         super(DhtFeatureTest, self).__init__(test, workbench)
+        self.bootstrap = self._workbench.get_bootstrap()
 
     def _reset(self):
         super(DhtFeatureTest, self)._reset()
@@ -380,23 +208,23 @@ class DhtFeatureTest(FeatureTest):
             FeatureTest.lock.notify()
 
     def _dhtPut(self, producer, _hash, *values):
-        with FeatureTest.lock:
-            for val in values:
-                    vstr = val.__str__()[:100]
-                    DhtNetwork.log('[PUT]:', _hash.toString(), '->', vstr + ("..." if len(vstr) > 100 else ""))
-                    FeatureTest.done += 1
+        for val in values:
+            with FeatureTest.lock:
+                vstr = val.__str__()[:100]
+                DhtNetwork.log('[PUT]:', _hash.toString(), '->', vstr + ("..." if len(vstr) > 100 else ""))
+                FeatureTest.done += 1
+                while FeatureTest.done > 0:
                     producer.put(_hash, val, DhtFeatureTest.putDoneCb)
-            while FeatureTest.done > 0:
-                FeatureTest.lock.wait()
+                    FeatureTest.lock.wait()
 
     def _dhtGet(self, consumer, _hash):
         DhtFeatureTest.foreignValues = []
         DhtFeatureTest.foreignNodes = []
         with FeatureTest.lock:
             FeatureTest.done += 1
-            DhtNetwork.log('[GET]:', _hash.toString())
-            consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb)
             while FeatureTest.done > 0:
+                DhtNetwork.log('[GET]:', _hash.toString())
+                consumer.get(_hash, DhtFeatureTest.getcb, DhtFeatureTest.getDoneCb)
                 FeatureTest.lock.wait()
 
     def _gottaGetThemAllPokeNodes(self, consumer, hashes, nodes=None):
@@ -406,6 +234,7 @@ class DhtFeatureTest(FeatureTest):
                 for n in DhtFeatureTest.foreignNodes:
                     nodes.add(n)
 
+
 class PersistenceTest(DhtFeatureTest):
     """
     This tests persistence of data on the network.
@@ -463,8 +292,8 @@ class PersistenceTest(DhtFeatureTest):
             config.setNodeId(InfoHash(_hash_str.encode()))
             n = DhtRunner()
             n.run(config=config)
-            n.bootstrap(self._bootstrap.ip4,
-                        str(self._bootstrap.port))
+            n.bootstrap(self.bootstrap.ip4,
+                        str(self.bootstrap.port))
             DhtNetwork.log('Node','['+_hash_str+']',
                            'started around', _hash.toString().decode()
                            if n.isRunning() else
@@ -473,7 +302,7 @@ class PersistenceTest(DhtFeatureTest):
             trigger_nodes.append(n)
 
     def _result(self, local_values, new_nodes):
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
         if not DhtFeatureTest.successfullTransfer(local_values, DhtFeatureTest.foreignValues):
             DhtNetwork.log('[GET]: Only %s on %s values persisted.' %
                     (len(DhtFeatureTest.foreignValues), len(local_values)))
@@ -511,7 +340,7 @@ class PersistenceTest(DhtFeatureTest):
                 plot_fname = "traffic-plot"
                 print('plot saved to', plot_fname)
                 plt.savefig(plot_fname)
-            self._bootstrap.resize(1)
+            self.bootstrap.resize(1)
 
     ###########
     #  Tests  #
@@ -524,10 +353,10 @@ class PersistenceTest(DhtFeatureTest):
         """
         trigger_nodes = []
         wb = self._workbench
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
         # Value representing an ICE packet. Each ICE packet is around 1KB.
         VALUE_SIZE = 1024
-        NUM_VALUES = self._num_values/wb.node_num if self._num_values else 5
+        NUM_VALUES = self._num_values/wb.node_num if self._num_values else 50
         nr_values = NUM_VALUES * wb.node_num
         nr_nodes = wb.node_num
         nr_nodes_cv = threading.Condition()
@@ -536,11 +365,8 @@ class PersistenceTest(DhtFeatureTest):
         hashes = [random_hash() for _ in range(wb.node_num)]
 
         # initial set of values
-        i = 0
         for h in hashes:
            self._dhtPut(bootstrap.front(), h, *[Value(v) for v in values])
-           print("at: ", i)
-           i += 1
 
         def normalBehavior(do, t, log=None):
             nonlocal nr_values
@@ -601,7 +427,7 @@ class PersistenceTest(DhtFeatureTest):
         It uses Dht shutdown call from the API to gracefuly finish the nodes one
         after the other.
         """
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
 
         ops_count = []
 
@@ -665,7 +491,7 @@ class PersistenceTest(DhtFeatureTest):
         """
         clusters = 8
 
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
 
         bootstrap.resize(3)
         consumer = bootstrap.get(1)
@@ -702,14 +528,14 @@ class PersistenceTest(DhtFeatureTest):
         minutes for the nodes to trigger storage maintenance.
         """
         trigger_nodes = []
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
 
         N_PRODUCERS = self._num_producers if self._num_values else 16
         DP_TIMEOUT = 1
 
         hashes = []
 
-    # Generating considerable amount of values of size 1KB.
+        # Generating considerable amount of values of size 1KB.
         VALUE_SIZE = 1024
         NUM_VALUES = self._num_values if self._num_values else 50
         values = [Value(''.join(random.choice(string.hexdigits) for _ in range(VALUE_SIZE)).encode()) for _ in range(NUM_VALUES)]
@@ -783,7 +609,7 @@ class PerformanceTest(DhtFeatureTest):
         except Exception as e:
             print(e)
         finally:
-            self._bootstrap.resize(1)
+            self.bootstrap.resize(1)
 
 
     ###########
@@ -795,7 +621,7 @@ class PerformanceTest(DhtFeatureTest):
         """
         Tests for performance of the DHT doing multiple get() operation.
         """
-        bootstrap = self._bootstrap
+        bootstrap = self.bootstrap
 
         plt.ion()
 
@@ -884,7 +710,8 @@ class PerformanceTest(DhtFeatureTest):
         deleting around the target hash.
         """
 
-        bootstrap = self._bootstrap
+
+        bootstrap = self.bootstrap
 
         bootstrap.resize(3)
         consumer = bootstrap.get(1)

src/Makefile.am

@@ -3,8 +3,7 @@ SUBDIRS = argon2
 lib_LTLIBRARIES = libopendht.la
 
 AM_CPPFLAGS = -I../include/opendht
-libopendht_la_CXXFLAGS = @CXXFLAGS@
-libopendht_la_LDFLAGS = @LDFLAGS@ @GNUTLS_LIBS@ @nettle_LIBS@
+libopendht_la_LDFLAGS = @LDFLAGS@ @GnuTLS_LIBS@ @Nettle_LIBS@ -version-number @OPENDHT_MAJOR_VERSION@:@OPENDHT_MINOR_VERSION@:@OPENDHT_PATCH_VERSION@
 libopendht_la_LIBADD = ./argon2/libargon2.la
 
 libopendht_la_SOURCES = \
@@ -20,8 +19,7 @@ libopendht_la_SOURCES = \
         crypto.cpp \
         securedht.cpp \
         dhtrunner.cpp \
-        default_types.cpp \
-        indexation/pht.cpp
+        default_types.cpp
 
 if WIN32
 libopendht_la_SOURCES += rng.cpp
@@ -44,5 +42,4 @@ nobase_include_HEADERS = \
         ../include/opendht/dhtrunner.h \
         ../include/opendht/default_types.h \
         ../include/opendht/log.h \
-        ../include/opendht/rng.h \
-        ../include/opendht/indexation/pht.h
+        ../include/opendht/rng.h

src/argon2/argon2.c

@@ -88,18 +88,12 @@ int argon2_hash(const uint32_t t_cost, const uint32_t m_cost,
     int result;
     uint8_t *out;
 
-    /* Detect and reject overflowing sizes */
-    /* TODO: This should probably be fixed in the function signature */
-    if (pwdlen > ARGON2_MAX_PWD_LENGTH) {
-        return ARGON2_PWD_TOO_LONG;
-    }
-
     if (hashlen > ARGON2_MAX_OUTLEN) {
         return ARGON2_OUTPUT_TOO_LONG;
     }
 
-    if (saltlen > ARGON2_MAX_SALT_LENGTH) {
-        return ARGON2_SALT_TOO_LONG;
+    if (hashlen < ARGON2_MIN_OUTLEN) {
+        return ARGON2_OUTPUT_TOO_SHORT;
     }
 
     out = malloc(hashlen);
@@ -212,13 +206,19 @@ int argon2_verify(const char *encoded, const void *pwd, const size_t pwdlen,
     int ret;
     int decode_result;
     uint32_t encoded_len;
+    size_t encoded_len_tmp;
 
     if(encoded == NULL) {
         return ARGON2_DECODING_FAIL;
     }
 
+    encoded_len_tmp = strlen(encoded);
     /* max values, to be updated in decode_string */
-    encoded_len = strlen(encoded);
+    if (UINT32_MAX < encoded_len_tmp) {
+        return ARGON2_DECODING_FAIL;
+    }
+
+    encoded_len = (uint32_t)encoded_len_tmp;
     ctx.adlen = encoded_len;
     ctx.saltlen = encoded_len;
     ctx.outlen = encoded_len;
@@ -226,6 +226,8 @@ int argon2_verify(const char *encoded, const void *pwd, const size_t pwdlen,
     ctx.free_cbk = NULL;
     ctx.secret = NULL;
     ctx.secretlen = 0;
+    ctx.pwdlen = 0;
+    ctx.pwd = NULL;
     ctx.ad = malloc(ctx.adlen);
     ctx.salt = malloc(ctx.saltlen);
     ctx.out = malloc(ctx.outlen);
@@ -392,5 +394,5 @@ size_t argon2_encodedlen(uint32_t t_cost, uint32_t m_cost, uint32_t parallelism,
                          uint32_t saltlen, uint32_t hashlen) {
     return strlen("$argon2x$v=$m=,t=,p=$$") + numlen(t_cost) + numlen(m_cost)
         + numlen(parallelism) + b64len(saltlen) + b64len(hashlen)
-        + numlen(ARGON2_VERSION_NUMBER);
+        + numlen(ARGON2_VERSION_NUMBER) + 1;
 }

src/argon2/encoding.c

@@ -412,7 +412,18 @@ int encode_string(char *dst, size_t dst_len, argon2_context *ctx,
 }
 
 size_t b64len(uint32_t len) {
-    return (((size_t)len + 2) / 3) * 4;
+    size_t olen = ((size_t)len / 3) << 2;
+
+    switch (len % 3) {
+    case 2:
+        olen++;
+    /* fall through */
+    case 1:
+        olen += 2;
+        break;
+    }
+
+    return olen;
 }
 
 size_t numlen(uint32_t num) {

src/callbacks.cpp

@@ -47,12 +47,4 @@ bindDoneCb(DoneCallbackRaw raw_cb, void* user_data)
     };
 }
 
-DoneCallbackSimple
-bindDoneCbSimple(DoneCallbackSimpleRaw raw_cb, void* user_data) {
-    if (not raw_cb) return {};
-    return [=](bool success) {
-        raw_cb(success, user_data);
-    };
-}
-
 }

src/crypto.cpp

@@ -83,6 +83,18 @@ namespace crypto {
 static constexpr std::array<size_t, 3> AES_LENGTHS {{128/8, 192/8, 256/8}};
 static constexpr size_t PASSWORD_SALT_LENGTH {16};
 
+constexpr gnutls_digest_algorithm_t gnutlsHashAlgo(size_t min_res) {
+    return (min_res > 256/8) ? GNUTLS_DIG_SHA512 : (
+           (min_res > 128/8) ? GNUTLS_DIG_SHA256 : (
+                               GNUTLS_DIG_SHA1));
+}
+
+constexpr size_t gnutlsHashSize(int algo) {
+    return (algo == GNUTLS_DIG_SHA512) ? 512/8 : (
+           (algo == GNUTLS_DIG_SHA256) ? 256/8 : (
+           (algo == GNUTLS_DIG_SHA1)   ? 160/8 : 0 ));
+}
+
 size_t aesKeySize(size_t max)
 {
     size_t aes_key_len = 0;
@@ -181,7 +193,7 @@ Blob aesDecrypt(const Blob& data, const std::string& password)
     return aesDecrypt(encrypted, key);
 }
 
-Blob stretchKey(const std::string& password, Blob& salt)
+Blob stretchKey(const std::string& password, Blob& salt, size_t key_length)
 {
     if (salt.empty()) {
         salt.resize(PASSWORD_SALT_LENGTH);
@@ -193,18 +205,19 @@ Blob stretchKey(const std::string& password, Blob& salt)
     auto ret = argon2i_hash_raw(16, 64*1024, 1, password.data(), password.size(), salt.data(), salt.size(), res.data(), res.size());
     if (ret != ARGON2_OK)
         throw CryptoException("Can't compute argon2i !");
-    return hash(res);
+    return hash(res, key_length);
 }
 
-Blob hash(const Blob& data)
+Blob hash(const Blob& data, size_t hash_len)
 {
+    auto algo = gnutlsHashAlgo(hash_len);    
+    size_t res_size = gnutlsHashSize(algo);
     Blob res;
-    size_t res_size = 64;
     res.resize(res_size);
     const gnutls_datum_t gdat {(uint8_t*)data.data(), (unsigned)data.size()};
-    if (gnutls_fingerprint(GNUTLS_DIG_SHA512, &gdat, res.data(), &res_size))
+    if (gnutls_fingerprint(algo, &gdat, res.data(), &res_size))
         throw CryptoException("Can't compute hash !");
-    res.resize(res_size);
+    res.resize(std::min(hash_len, res_size));
     return res;
 }
 
@@ -273,7 +286,7 @@ PrivateKey::PrivateKey(PrivateKey&& o) noexcept : key(o.key), x509_key(o.x509_ke
 #endif
     o.key = nullptr;
     o.x509_key = nullptr;
-};
+}
 
 PrivateKey::~PrivateKey()
 {
@@ -311,6 +324,8 @@ PrivateKey::sign(const Blob& data) const
 {
     if (!key)
         throw CryptoException("Can't sign data: no private key set !");
+    if (std::numeric_limits<unsigned>::max() < data.size())
+        throw CryptoException("Can't sign data: too large !");
     gnutls_datum_t sig;
     const gnutls_datum_t dat {(unsigned char*)data.data(), (unsigned)data.size()};
     if (gnutls_privkey_sign_data(key, GNUTLS_DIG_SHA512, 0, &dat, &sig) != GNUTLS_E_SUCCESS)

src/dht.cpp

@@ -184,6 +184,11 @@ struct Dht::SearchNode {
         return not node->isExpired() and
                not token.empty() and last_get_reply >= now - Node::NODE_EXPIRE_TIME;
     }
+
+    /**
+     * Could a "get" request be sent to this node now ?
+     * update: time of the last "get" op for the search.
+     */
     bool canGet(time_point now, time_point update) const {
         return not node->isExpired() and
                (now > last_get_reply + Node::NODE_EXPIRE_TIME or update > last_get_reply)
@@ -204,6 +209,9 @@ struct Dht::SearchNode {
         return listenStatus->reply_time + LISTEN_EXPIRE_TIME > now;
     }
 
+    /**
+     * Assumng the node is synced, should a "put" request be sent to this node now ?
+     */
     time_point getAnnounceTime(AnnounceStatusMap::const_iterator ack, const ValueType& type) const {
         if (ack == acked.end() or not ack->second)
             return time_point::min();
@@ -214,12 +222,19 @@ struct Dht::SearchNode {
         return getAnnounceTime(acked.find(vid), type);
     }
 
+    /**
+     * Assumng the node is synced, should a "listen" request be sent to this node now ?
+     */
     time_point getListenTime() const {
         if (not listenStatus)
             return time_point::min();
 
         return listenStatus->pending() ? time_point::max() : listenStatus->reply_time + LISTEN_EXPIRE_TIME - REANNOUNCE_MARGIN;
     }
+
+    /**
+     * Is this node expired or candidate
+     */
     bool isBad() const {
         return !node || node->isExpired() || candidate;
     }
@@ -290,6 +305,10 @@ struct Dht::Search {
      */
     bool isSynced(time_point now) const;
 
+    /**
+     * Get the time of the last "get" operation performed on this search,
+     * or time_point::min() if no such operation have been performed.
+     */
     time_point getLastGetTime() const;
 
     /**
@@ -308,17 +327,23 @@ struct Dht::Search {
     unsigned getNumberOfBadNodes() const;
 
     /**
-     * ret = 0 : no announce required.
-     * ret > 0 : (re-)announce required at time ret.
+     * Returns the time of the next "announce" event for this search,
+     * or time_point::max() if no such event is planned.
+     * Only makes sense when the search is synced.
      */
     time_point getAnnounceTime(const std::map<ValueType::Id, ValueType>& types, time_point now) const;
 
     /**
-     * ret = 0 : no listen required.
-     * ret > 0 : (re-)announce required at time ret.
+     * Returns the time of the next "listen" event for this search,
+     * or time_point::max() if no such event is planned.
+     * Only makes sense when the search is synced.
      */
     time_point getListenTime(time_point now) const;
 
+    /**
+     * Returns the time of the next event for this search,
+     * or time_point::max() if no such event is planned.
+     */
     time_point getNextStepTime(const std::map<ValueType::Id, ValueType>& types, time_point now) const;
 
     bool removeExpiredNode(time_point now);
@@ -332,7 +357,7 @@ struct Dht::Search {
         callbacks.clear();
         listeners.clear();
         nodes.clear();
-        nextSearchStep = {};
+        nextSearchStep.reset();
     }
 };
 
@@ -348,12 +373,13 @@ NodeStatus
 Dht::getStatus(sa_family_t af) const
 {
     unsigned good = 0, dubious = 0, cached = 0, incoming = 0;
-    int tot = getNodesStats(af, &good, &dubious, &cached, &incoming);
-    if (tot < 1)
-        return NodeStatus::Disconnected;
-    else if (good < 1)
+    unsigned tot = getNodesStats(af, &good, &dubious, &cached, &incoming);
+    auto& ping = af == AF_INET ? pending_pings4 : pending_pings6;
+    if (good)
+        return NodeStatus::Connected;
+    if (ping or tot)
         return NodeStatus::Connecting;
-    return NodeStatus::Connected;
+    return NodeStatus::Disconnected;
 }
 
 void
@@ -371,7 +397,7 @@ Dht::shutdown(ShutdownCallback cb) {
     };
 
     for (const auto& str : store) {
-        *remaining += maintainStorage(str.id, true, str_donecb);
+        *remaining += maintainStorage(str->id, true, str_donecb);
     }
     DHT_LOG.WARN("Shuting down node: %u ops remaining.", *remaining);
     if (!*remaining && cb) { cb(); }
@@ -417,13 +443,13 @@ Dht::sendCachedPing(Bucket& b)
     return 0;
 }
 
-std::vector<Address>
+std::vector<SockAddr>
 Dht::getPublicAddress(sa_family_t family)
 {
     std::sort(reported_addr.begin(), reported_addr.end(), [](const ReportedAddr& a, const ReportedAddr& b) {
-        return a.first < b.first;
+        return a.first > b.first;
     });
-    std::vector<Address> ret;
+    std::vector<SockAddr> ret;
     for (const auto& addr : reported_addr)
         if (!family || family == addr.second.first.ss_family)
             ret.emplace_back(addr.second);
@@ -440,25 +466,24 @@ Dht::trySearchInsert(const std::shared_ptr<Node>& node)
     auto family = node->getFamily();
     auto& srs = family == AF_INET ? searches4 : searches6;
     for (auto& srp : srs) {
-        auto& s = srp.second;
-        if (s->insertNode(node, now)) {
+        auto& s = *srp.second;
+        if (s.insertNode(node, now)) {
             inserted = true;
-            scheduler.edit(s->nextSearchStep, s->getNextStepTime(types, now));
+            scheduler.edit(s.nextSearchStep, s.getNextStepTime(types, now));
         }
     }
     return inserted;
 }
 
 void
-Dht::reportedAddr(const sockaddr *sa, socklen_t sa_len)
+Dht::reportedAddr(const SockAddr& addr)
 {
-    auto it = std::find_if(reported_addr.begin(), reported_addr.end(), [=](const ReportedAddr& addr){
-        return (addr.second.second == sa_len) &&
-            std::equal((uint8_t*)&addr.second.first, (uint8_t*)&addr.second.first + addr.second.second, (uint8_t*)sa);
+    auto it = std::find_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& a){
+        return a.second == addr;
     });
     if (it == reported_addr.end()) {
         if (reported_addr.size() < 32)
-            reported_addr.emplace_back(1, std::make_pair(*((sockaddr_storage*)sa), sa_len));
+            reported_addr.emplace_back(1, addr);
     } else
         it->first++;
 }
@@ -679,10 +704,11 @@ Dht::expireSearches()
 {
     auto t = scheduler.time() - SEARCH_EXPIRE_TIME;
     auto expired = [&](std::pair<const InfoHash, std::shared_ptr<Search>>& srp) {
-        auto& sr = srp.second;
-        auto b = sr->callbacks.empty() && sr->announce.empty() && sr->listeners.empty() && sr->step_time < t;
+        auto& sr = *srp.second;
+        auto b = sr.callbacks.empty() && sr.announce.empty() && sr.listeners.empty() && sr.step_time < t;
         if (b) {
-            DHT_LOG.DEBUG("Removing search %s ", srp.first.toString().c_str());
+            DHT_LOG.DEBUG("Removing search %s", srp.first.toString().c_str());
+            sr.clear();
             return b;
         } else { return false; }
     };
@@ -841,7 +867,7 @@ Dht::searchStep(std::shared_ptr<Search> sr)
                     continue;
                 }
             }
-
+            if (in) storageStore(sr->id, a.value, a.created);
             unsigned i = 0;
             for (auto& n : sr->nodes) {
                 if (not n.isSynced(now))
@@ -1243,9 +1269,6 @@ Dht::announce(const InfoHash& id, sa_family_t af, std::shared_ptr<Value> value,
             callback(false, {});
         return;
     }
-
-    storageStore(id, value, created);
-
     auto& srs = af == AF_INET ? searches4 : searches6;
     auto srp = srs.find(id);
     auto sr = srp == srs.end() ? search(id, af, nullptr, nullptr) : srp->second;
@@ -1347,12 +1370,12 @@ Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f)
     auto st = findStorage(id);
     size_t tokenlocal = 0;
     if (st == store.end() && store.size() < MAX_HASHES) {
-        store.emplace_back(id, scheduler.time());
+        store.emplace_back(new Storage(id, scheduler.time()));
         st = std::prev(store.end());
     }
     if (st != store.end()) {
-        if (not st->empty()) {
-            std::vector<std::shared_ptr<Value>> newvals = st->get(f);
+        if (not (*st)->empty()) {
+            std::vector<std::shared_ptr<Value>> newvals = (*st)->get(f);
             if (not newvals.empty()) {
                 if (!cb(newvals))
                     return 0;
@@ -1363,8 +1386,8 @@ Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f)
                 }
             }
         }
-        tokenlocal = ++st->listener_token;
-        st->local_listeners.emplace(tokenlocal, LocalListener{f, gcb});
+        tokenlocal = ++(*st)->listener_token;
+        (*st)->local_listeners.emplace(tokenlocal, LocalListener{f, gcb});
     }
 
     auto token4 = Dht::listenTo(id, AF_INET, gcb, f);
@@ -1389,7 +1412,7 @@ Dht::cancelListen(const InfoHash& id, size_t token)
     auto st = findStorage(id);
     auto tokenlocal = std::get<0>(it->second);
     if (st != store.end() && tokenlocal)
-        st->local_listeners.erase(tokenlocal);
+        (*st)->local_listeners.erase(tokenlocal);
 
     auto searches_cancel_listen = [&](std::map<InfoHash, std::shared_ptr<Search>> srs) {
         for (auto& sp : srs) {
@@ -1522,7 +1545,7 @@ Dht::getLocal(const InfoHash& id, Value::Filter f) const
 {
     auto s = findStorage(id);
     if (s == store.end()) return {};
-    return s->get(f);
+    return (*s)->get(f);
 }
 
 std::shared_ptr<Value>
@@ -1530,7 +1553,7 @@ Dht::getLocalById(const InfoHash& id, Value::Id vid) const
 {
     auto s = findStorage(id);
     if (s != store.end())
-        return s->getById(vid);
+        return (*s)->getById(vid);
     return {};
 }
 
@@ -1603,15 +1626,15 @@ Dht::cancelPut(const InfoHash& id, const Value::Id& vid)
 decltype(Dht::store)::iterator
 Dht::findStorage(const InfoHash& id)
 {
-    return std::find_if(store.begin(), store.end(), [&](const Storage& st) {
-        return st.id == id;
+    return std::find_if(store.begin(), store.end(), [&](const std::unique_ptr<Storage>& st) {
+        return st->id == id;
     });
 }
 decltype(Dht::store)::const_iterator
 Dht::findStorage(const InfoHash& id) const
 {
-    return std::find_if(store.cbegin(), store.cend(), [&](const Storage& st) {
-        return st.id == id;
+    return std::find_if(store.cbegin(), store.cend(), [&](const std::unique_ptr<Storage>& st) {
+        return st->id == id;
     });
 }
 
@@ -1637,7 +1660,7 @@ Dht::storageChanged(Storage& st, ValueStorage& v)
         DHT_LOG.DEBUG("Storage changed. Sending update to %s.", l.first->toString().c_str());
         std::vector<std::shared_ptr<Value>> vals;
         vals.push_back(v.data);
-        Blob ntoken = makeToken((const sockaddr*)&l.first->ss, false);
+        Blob ntoken = makeToken((const sockaddr*)&l.first->addr.first, false);
         network_engine.tellListener(l.first, l.second.rid, st.id, 0, ntoken, {}, {}, vals);
     }
 }
@@ -1647,19 +1670,25 @@ Dht::storageStore(const InfoHash& id, const std::shared_ptr<Value>& value, time_
 {
     const auto& now = scheduler.time();
     created = std::min(created, now);
+
+    if (created + getType(value->type).expiration < now) {
+        DHT_LOG.WARN("[store %s] won't store already expired value", id.toString().c_str());
+        return false;
+    }
+
     auto st = findStorage(id);
     if (st == store.end()) {
         if (store.size() >= MAX_HASHES)
             return false;
-        store.emplace_back(id, now);
+        store.emplace_back(new Storage(id, now));
         st = std::prev(store.end());
     }
 
-    auto store = st->store(value, created, max_store_size - total_store_size);
+    auto store = (*st)->store(value, created, max_store_size - total_store_size);
     if (std::get<0>(store)) {
         total_store_size += std::get<1>(store);
         total_values += std::get<2>(store);
-        storageChanged(*st, *std::get<0>(store));
+        storageChanged(*(*st), *std::get<0>(store));
     }
     return std::get<0>(store);
 }
@@ -1708,21 +1737,21 @@ Dht::storageAddListener(const InfoHash& id, const std::shared_ptr<Node>& node, s
     if (st == store.end()) {
         if (store.size() >= MAX_HASHES)
             return;
-        store.emplace_back(id, now);
+        store.emplace_back(new Storage(id, now));
         st = std::prev(store.end());
     }
-    auto l = st->listeners.find(node);
-    if (l == st->listeners.end()) {
-        const auto& stvalues = st->getValues();
+    auto l = (*st)->listeners.find(node);
+    if (l == (*st)->listeners.end()) {
+        const auto& stvalues = (*st)->getValues();
         if (not stvalues.empty()) {
             std::vector<std::shared_ptr<Value>> values(stvalues.size());
             std::transform(stvalues.begin(), stvalues.end(), values.begin(), [=](const ValueStorage& vs) { return vs.data; });
 
-            network_engine.tellListener(node, rid, id, WANT4 | WANT6, makeToken((sockaddr*)&node->ss, false),
+            network_engine.tellListener(node, rid, id, WANT4 | WANT6, makeToken((sockaddr*)&node->addr.first, false),
                     buckets.findClosestNodes(id, now, TARGET_NODES), buckets6.findClosestNodes(id, now, TARGET_NODES),
                     values);
         }
-        st->listeners.emplace(node, Listener {rid, now});
+        (*st)->listeners.emplace(node, Listener {rid, now});
     }
     else
         l->second.refresh(rid, now);
@@ -1734,21 +1763,21 @@ Dht::expireStorage()
     const auto& now = scheduler.time();
     auto i = store.begin();
     while (i != store.end()) {
-        for (auto l = i->listeners.cbegin(); l != i->listeners.cend();){
+        for (auto l = (*i)->listeners.cbegin(); l != (*i)->listeners.cend();){
             bool expired = l->second.time + Node::NODE_EXPIRE_TIME < now;
             if (expired) {
                 DHT_LOG.DEBUG("Discarding expired listener %s", l->first->id.toString().c_str());
-                i->listeners.erase(l++);
+                (*i)->listeners.erase(l++);
             } else
                 ++l;
         }
 
-        auto stats = i->expire(types, now);
+        auto stats = (*i)->expire(types, now);
         total_store_size += stats.first;
         total_values += stats.second;
 
-        if (i->empty() && i->listeners.empty() && i->local_listeners.empty()) {
-            DHT_LOG.DEBUG("Discarding expired value %s", i->id.toString().c_str());
+        if ((*i)->empty() && (*i)->listeners.empty() && (*i)->local_listeners.empty()) {
+            DHT_LOG.DEBUG("Discarding expired value %s", (*i)->id.toString().c_str());
             i = store.erase(i);
         }
         else
@@ -1861,7 +1890,7 @@ Dht::tokenMatch(const Blob& token, const sockaddr *sa) const
     return false;
 }
 
-int
+unsigned
 Dht::getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return, unsigned *incoming_return) const
 {
     const auto& now = scheduler.time();
@@ -1991,8 +2020,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const
                 out << "] ";
             }
         }
-        out << print_addr(n.node->ss, n.node->sslen);
-        out << std::endl;
+        out << n.node->addr.toString() << std::endl;
     }
 }
 
@@ -2029,10 +2057,10 @@ Dht::getStorageLog() const
     using namespace std::chrono;
     std::stringstream out;
     for (const auto& st : store) {
-        out << "Storage " << st.id << " " << st.listeners.size() << " list., " << st.valueCount() << " values (" << st.totalSize() << " bytes)" << std::endl;
-        if (not st.local_listeners.empty())
-            out << "   " << st.local_listeners.size() << " local listeners" << std::endl;
-        for (const auto& l : st.listeners) {
+        out << "Storage " << (*st).id << " " << (*st).listeners.size() << " list., " << (*st).valueCount() << " values (" << (*st).totalSize() << " bytes)" << std::endl;
+        if (not (*st).local_listeners.empty())
+            out << "   " << (*st).local_listeners.size() << " local listeners" << std::endl;
+        for (const auto& l : (*st).listeners) {
             out << "   " << "Listener " << l.first->toString();
             auto since = duration_cast<seconds>(now - l.second.time);
             auto expires = duration_cast<seconds>(l.second.time + Node::NODE_EXPIRE_TIME - now);
@@ -2079,10 +2107,10 @@ Dht::Dht() : store(), network_engine(DHT_LOG, scheduler) {}
 
 Dht::Dht(int s, int s6, Config config)
  : myid(config.node_id), is_bootstrap(config.is_bootstrap), store(),
-    network_engine(myid, s, s6, DHT_LOG, scheduler,
+    network_engine(myid, config.network, s, s6, DHT_LOG, scheduler,
             std::bind(&Dht::onError, this, _1, _2),
             std::bind(&Dht::onNewNode, this, _1, _2),
-            std::bind(&Dht::onReportedAddr, this, _1, _2, _3),
+            std::bind(&Dht::onReportedAddr, this, _1, _2),
             std::bind(&Dht::onPing, this, _1),
             std::bind(&Dht::onFindNode, this, _1, _2, _3),
             std::bind(&Dht::onGetValues, this, _1, _2, _3),
@@ -2215,11 +2243,11 @@ Dht::dataPersistence() {
     const auto& now = scheduler.time();
     auto storage_maintenance_time = time_point::max();
     for (auto &str : store) {
-        if (now > str.maintenance_time) {
-            maintainStorage(str.id);
-            str.maintenance_time = now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME;
+        if (now > str->maintenance_time) {
+            maintainStorage(str->id);
+            str->maintenance_time = now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME;
         }
-        storage_maintenance_time = std::min(storage_maintenance_time, str.maintenance_time);
+        storage_maintenance_time = std::min(storage_maintenance_time, str->maintenance_time);
     }
     scheduler.add(storage_maintenance_time, std::bind(&Dht::dataPersistence, this));
 }
@@ -2236,7 +2264,7 @@ Dht::maintainStorage(InfoHash id, bool force, DoneCallback donecb) {
     auto nodes = buckets.findClosestNodes(id, now);
     if (!nodes.empty()) {
         if (force || id.xorCmp(nodes.back()->id, myid) < 0) {
-            for (auto &value : local_storage->getValues()) {
+            for (auto &value : (*local_storage)->getValues()) {
                 const auto& vt = getType(value.data->type);
                 if (force || value.time + vt.expiration > now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME) {
                     // gotta put that value there
@@ -2252,7 +2280,7 @@ Dht::maintainStorage(InfoHash id, bool force, DoneCallback donecb) {
     auto nodes6 = buckets6.findClosestNodes(id, now);
     if (!nodes6.empty()) {
         if (force || id.xorCmp(nodes6.back()->id, myid) < 0) {
-            for (auto &value : local_storage->getValues()) {
+            for (auto &value : (*local_storage)->getValues()) {
                 const auto& vt = getType(value.data->type);
                 if (force || value.time + vt.expiration > now + MAX_STORAGE_MAINTENANCE_EXPIRE_TIME) {
                     // gotta put that value there
@@ -2267,22 +2295,22 @@ Dht::maintainStorage(InfoHash id, bool force, DoneCallback donecb) {
 
     if (not want4 and not want6) {
         DHT_LOG.DEBUG("Discarding storage values %s", id.toString().c_str());
-        local_storage->clear();
+        (*local_storage)->clear();
     }
 
     return announce_per_af;
 }
 
 void
-Dht::processMessage(const uint8_t *buf, size_t buflen, const sockaddr *from, socklen_t fromlen)
+Dht::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& from)
 {
     if (buflen == 0)
         return;
 
     try {
-        network_engine.processMessage(buf, buflen, from, fromlen);
+        network_engine.processMessage(buf, buflen, from);
     } catch (const std::exception& e) {
-        DHT_LOG.ERR("Can't parse message from %s: %s", print_addr(from, fromlen).c_str(), e.what());
+        DHT_LOG.ERR("Can't parse message from %s: %s", from.toString().c_str(), e.what());
         //auto code = e.getCode();
         //if (code == DhtProtocolException::INVALID_TID_SIZE or code == DhtProtocolException::WRONG_NODE_INFO_BUF_LEN) {
             /* This is really annoying, as it means that we will
@@ -2295,10 +2323,10 @@ Dht::processMessage(const uint8_t *buf, size_t buflen, const sockaddr *from, soc
 }
 
 time_point
-Dht::periodic(const uint8_t *buf, size_t buflen, const sockaddr *from, socklen_t fromlen)
+Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from)
 {
     scheduler.syncTime();
-    processMessage(buf, buflen, from, fromlen);
+    processMessage(buf, buflen, from);
     return scheduler.run();
 }
 
@@ -2322,11 +2350,11 @@ Dht::confirmNodes()
     bool soon = false;
     const auto& now = scheduler.time();
 
-    if (searches4.empty() and getStatus(AF_INET) != NodeStatus::Disconnected) {
+    if (searches4.empty() and getStatus(AF_INET) == NodeStatus::Connected) {
         DHT_LOG.DEBUG("[confirm nodes] initial IPv4 'get' for my id (%s).", myid.toString().c_str());
         search(myid, AF_INET);
     }
-    if (searches6.empty() and getStatus(AF_INET6) != NodeStatus::Disconnected) {
+    if (searches6.empty() and getStatus(AF_INET6) == NodeStatus::Connected) {
         DHT_LOG.DEBUG("[confirm nodes] initial IPv6 'get' for my id (%s).", myid.toString().c_str());
         search(myid, AF_INET6);
     }
@@ -2360,12 +2388,12 @@ Dht::exportValues() const
     e.reserve(store.size());
     for (const auto& h : store) {
         ValuesExport ve;
-        ve.first = h.id;
+        ve.first = h->id;
 
         msgpack::sbuffer buffer;
         msgpack::packer<msgpack::sbuffer> pk(&buffer);
-        pk.pack_array(h.getValues().size());
-        for (const auto& v : h.getValues()) {
+        pk.pack_array(h->getValues().size());
+        for (const auto& v : h->getValues()) {
             pk.pack_array(2);
             pk.pack(v.time.time_since_epoch().count());
             v.data->msgpack_pack(pk);
@@ -2449,21 +2477,28 @@ Dht::exportNodes()
 }
 
 bool
-Dht::insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen)
+Dht::insertNode(const InfoHash& id, const SockAddr& addr)
 {
-    if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6)
+    if (addr.getFamily() != AF_INET && addr.getFamily() != AF_INET6)
         return false;
     scheduler.syncTime();
-    auto n = network_engine.insertNode(id, sa, salen);
+    auto n = network_engine.insertNode(id, addr);
     return !!n;
 }
 
 int
-Dht::pingNode(const sockaddr *sa, socklen_t salen)
+Dht::pingNode(const sockaddr* sa, socklen_t salen)
 {
     scheduler.syncTime();
     DHT_LOG.DEBUG("Sending ping to %s", print_addr(sa, salen).c_str());
-    network_engine.sendPing(sa, salen, nullptr, nullptr);
+    auto& count = sa->sa_family == AF_INET ? pending_pings4 : pending_pings6;
+    count++;
+    network_engine.sendPing(sa, salen, [&](const Request&, NetworkEngine::RequestAnswer&&){
+        count--;
+    }, [&](const Request&, bool last){
+        if (last)
+            count--;
+    });
     return -1;
 }
 
@@ -2488,12 +2523,12 @@ Dht::onError(std::shared_ptr<Request> req, DhtProtocolException e) {
 }
 
 void
-Dht::onReportedAddr(const InfoHash& id, sockaddr* addr , socklen_t addr_length)
+Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr)
 {
-    const auto& b = (addr->sa_family == AF_INET ? buckets : buckets6).findBucket(id);
+    const auto& b = (addr.getFamily() == AF_INET ? buckets : buckets6).findBucket(id);
     b->time = scheduler.time();
-    if (addr and addr_length)
-        reportedAddr(addr, addr_length);
+    if (addr.second)
+        reportedAddr(addr);
 }
 
 NetworkEngine::RequestAnswer
@@ -2507,7 +2542,7 @@ Dht::onFindNode(std::shared_ptr<Node> node, InfoHash& target, want_t want)
 {
     const auto& now = scheduler.time();
     NetworkEngine::RequestAnswer answer;
-    answer.ntoken = makeToken((sockaddr*)&node->ss, false);
+    answer.ntoken = makeToken((sockaddr*)&node->addr.first, false);
     if (want & WANT4)
         answer.nodes4 = buckets.findClosestNodes(target, now, TARGET_NODES);
     if (want & WANT6)
@@ -2525,11 +2560,11 @@ Dht::onGetValues(std::shared_ptr<Node> node, InfoHash& hash, want_t)
     const auto& now = scheduler.time();
     NetworkEngine::RequestAnswer answer {};
     auto st = findStorage(hash);
-    answer.ntoken = makeToken((sockaddr*)&node->ss, false);
+    answer.ntoken = makeToken((sockaddr*)&node->addr.first, false);
     answer.nodes4 = buckets.findClosestNodes(hash, now, TARGET_NODES);
     answer.nodes6 = buckets6.findClosestNodes(hash, now, TARGET_NODES);
-    if (st != store.end() && not st->empty()) {
-        auto values = st->getValues();
+    if (st != store.end() && not (*st)->empty()) {
+        auto values = (*st)->getValues();
         answer.values.resize(values.size());
         std::transform(values.begin(), values.end(), answer.values.begin(), [](const ValueStorage& vs) {
             return vs.data;
@@ -2606,7 +2641,7 @@ Dht::onListen(std::shared_ptr<Node> node, InfoHash& hash, Blob& token, size_t ri
             DhtProtocolException::LISTEN_NO_INFOHASH
         };
     }
-    if (!tokenMatch(token, (sockaddr*)&node->ss)) {
+    if (!tokenMatch(token, (sockaddr*)&node->addr.first)) {
         DHT_LOG.WARN("[node %s] incorrect token %s for 'listen'.", node->toString().c_str(), hash.toString().c_str());
         throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::LISTEN_WRONG_TOKEN};
     }
@@ -2644,7 +2679,7 @@ Dht::onAnnounce(std::shared_ptr<Node> node, InfoHash& hash, Blob& token, std::ve
             DhtProtocolException::PUT_NO_INFOHASH
         };
     }
-    if (!tokenMatch(token, (sockaddr*)&node->ss)) {
+    if (!tokenMatch(token, (sockaddr*)&node->addr.first)) {
         DHT_LOG.WARN("[node %s] incorrect token %s for 'put'.", node->toString().c_str(), hash.toString().c_str());
         throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::PUT_WRONG_TOKEN};
     }
@@ -2674,7 +2709,7 @@ Dht::onAnnounce(std::shared_ptr<Node> node, InfoHash& hash, Blob& token, std::ve
                 DHT_LOG.WARN("[value %s %lu] nothing to do.", hash.toString().c_str(), lv->id);
             } else {
                 const auto& type = getType(lv->type);
-                if (type.editPolicy(hash, lv, vc, node->id, (sockaddr*)&node->ss, node->sslen)) {
+                if (type.editPolicy(hash, lv, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) {
                     DHT_LOG.DEBUG("[value %s %lu] editing %s.",
                             hash.toString().c_str(), lv->id, vc->toString().c_str());
                     storageStore(hash, vc, created);
@@ -2686,7 +2721,7 @@ Dht::onAnnounce(std::shared_ptr<Node> node, InfoHash& hash, Blob& token, std::ve
         } else {
             // Allow the value to be edited by the storage policy
             const auto& type = getType(vc->type);
-            if (type.storePolicy(hash, vc, node->id, (sockaddr*)&node->ss, node->sslen)) {
+            if (type.storePolicy(hash, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) {
                 DHT_LOG.DEBUG("[value %s %lu] storing %s.", hash.toString().c_str(), vc->id, vc->toString().c_str());
                 storageStore(hash, vc, created);
             } else {

src/dhtrunner.cpp

@@ -104,7 +104,7 @@ DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner:
                     std::lock_guard<std::mutex> lck(storage_mtx);
                     if (not pending_ops_prio.empty())
                         return true;
-                    if (not pending_ops.empty() and getStatus() >= NodeStatus::Connecting)
+                    if (not pending_ops.empty() and getStatus() != NodeStatus::Connecting)
                         return true;
                 }
                 return false;
@@ -252,7 +252,7 @@ DhtRunner::getSearchesLog(sa_family_t af) const
     std::lock_guard<std::mutex> lck(dht_mtx);
     return dht_->getSearchesLog(af);
 }
-std::vector<Address>
+std::vector<SockAddr>
 DhtRunner::getPublicAddress(sa_family_t af)
 {
     std::lock_guard<std::mutex> lck(dht_mtx);
@@ -293,7 +293,7 @@ DhtRunner::loop_()
         ops.front()(*dht_);
         ops.pop();
     }
-    if (getStatus() >= NodeStatus::Connecting) {
+    if (getStatus() != NodeStatus::Connecting) {
         {
             std::lock_guard<std::mutex> lck(storage_mtx);
             ops = std::move(pending_ops);
@@ -315,7 +315,7 @@ DhtRunner::loop_()
         for (const auto& pck : received) {
             auto& buf = pck.first;
             auto& from = pck.second;
-            wakeup = dht_->periodic(buf.data(), buf.size()-1, (sockaddr*)&from.first, from.second);
+            wakeup = dht_->periodic(buf.data(), buf.size()-1, from);
         }
         received.clear();
     } else {
@@ -400,19 +400,19 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C
 
                 if(rc > 0) {
                     std::array<uint8_t, 1024 * 64> buf;
-                    sockaddr_storage from;
-                    socklen_t fromlen {sizeof(from)};
+                    SockAddr from;
+                    from.second = sizeof(from.first);
 
                     if(s4 >= 0 && FD_ISSET(s4, &readfds))
-                        rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from, &fromlen);
+                        rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second);
                     else if(s6 >= 0 && FD_ISSET(s6, &readfds))
-                        rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from, &fromlen);
+                        rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second);
                     else
                         break;
                     if (rc > 0) {
                         {
                             std::lock_guard<std::mutex> lck(sock_mtx);
-                            rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, std::make_pair(from, fromlen));
+                            rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, from);
                         }
                         cv.notify_all();
                     }

src/indexation/pht.cpp

@@ -1,518 +0,0 @@
-#include "indexation/pht.h"
-#include "rng.h"
-
-namespace dht {
-namespace indexation {
-
-/**
- * Output the blob into string and readable way
- *
- * @param bl   : Blob to print
- *
- * @return string that represent the blob into a readable way
- */
-static std::string blobToString(const Blob &bl) {
-    std::stringstream ss;
-    auto bn = bl.size() % 8;
-    auto n = bl.size() / 8;
-
-    for (size_t i = 0; i < bl.size(); i++)
-        ss << std::bitset<8>(bl[i]) << " ";
-    if (bn)
-        for (unsigned b=0; b < bn; b++)
-            ss << (char)((bl[n] & (1 << (7 - b))) ? '1':'0');
-
-    return ss.str();
-}
-
-std::string Prefix::toString() const {
-    std::stringstream ss;
-
-    ss << "Prefix : " << std::endl << "\tContent_ : \"";
-    ss << blobToString(content_);
-    ss << "\"" << std::endl;
-
-    ss << "\tFlags_   : \"";
-    ss << blobToString(flags_);
-    ss << "\"" << std::endl;
-
-    return ss.str();
-}
-
-void Pht::Cache::insert(const Prefix& p) {
-    size_t i = 0;
-    auto now = clock::now();
-
-    std::shared_ptr<Node> curr_node;
-
-    while ((leaves_.size() > 0
-        and leaves_.begin()->first + NODE_EXPIRE_TIME < now)
-        or  leaves_.size() > MAX_ELEMENT) {
-
-        leaves_.erase(leaves_.begin());
-    }
-
-    if (not (curr_node = root_.lock()) ) {
-        /* Root does not exist, need to create one*/
-        curr_node = std::make_shared<Node>();
-        root_ = curr_node;
-    }
-
-    curr_node->last_reply = now;
-
-    /* Iterate through all bit of the Blob */
-    for ( i = 0; i < p.size_; i++ ) {
-
-        /* According to the bit define which node is the next one */
-        auto& next = ( p.isContentBitActive(i) ) ? curr_node->right_child : curr_node->left_child;
-
-        /**
-         * If lock, node exists
-         * else create it
-         */
-        if (auto n = next.lock()) {
-            curr_node = std::move(n);
-        } else {
-            /* Create the next node if doesn't exist*/
-            auto tmp_curr_node = std::make_shared<Node>();
-            tmp_curr_node->parent = curr_node;
-            next = tmp_curr_node;
-            curr_node = std::move(tmp_curr_node);
-        }
-
-        curr_node->last_reply = now;
-    }
-
-    /* Insert the leaf (curr_node) into the multimap */
-    leaves_.emplace(std::move(now), std::move(curr_node) );
-}
-
-int Pht::Cache::lookup(const Prefix& p) {
-    int pos = -1;
-    auto now = clock::now(), last_node_time = now;
-
-    /* Before lookup remove the useless one [i.e. too old] */
-    while ( leaves_.size() > 0
-        and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
-
-        leaves_.erase(leaves_.begin());
-    }
-
-    auto next = root_;
-    std::shared_ptr<Node> curr_node;
-
-    while ( auto n = next.lock() ) {
-        ++pos;
-        /* Safe since pos is equal to 0 until here */
-        if ( (unsigned) pos >= p.content_.size() * 8) break;
-
-        curr_node = n;
-        last_node_time = curr_node->last_reply;
-        curr_node->last_reply = now;
-
-        /* Get the Prefix bit by bit, starting from left */
-        next = ( p.isContentBitActive(pos) ) ? curr_node->right_child : curr_node->left_child;
-    }
-
-    if ( pos >= 0 ) {
-        auto to_erase = leaves_.find(last_node_time);
-        if ( to_erase != leaves_.end() )
-            leaves_.erase( to_erase );
-
-        leaves_.emplace( std::move(now), std::move(curr_node) );
-    }
-
-    return pos;
-}
-
-const ValueType IndexEntry::TYPE = ValueType::USER_DATA;
-constexpr std::chrono::minutes Pht::Cache::NODE_EXPIRE_TIME;
-
-void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
-        std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals,
-        LookupCallbackWrapper cb, DoneCallbackSimple done_cb,
-        std::shared_ptr<unsigned> max_common_prefix_len,
-        int start, bool all_values)
-{
-    struct node_lookup_result {
-        bool done {false};
-        bool is_pht {false};
-    };
-
-    /* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/
-    auto mid = (start >= 0) ? (unsigned) start : (*lo + *hi)/2;
-
-    auto first_res = std::make_shared<node_lookup_result>();
-    auto second_res = std::make_shared<node_lookup_result>();
-
-    auto on_done = [=](bool ok) {
-        bool is_leaf = first_res->is_pht and not second_res->is_pht;
-        if (not ok) {
-            if (done_cb)
-                done_cb(false);
-        }
-        else if (is_leaf or *lo > *hi) {
-            // leaf node
-            Prefix to_insert = p.getPrefix(mid);
-            cache_.insert(to_insert);
-
-            if (cb) {
-                if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
-                    auto p_ = (p.getPrefix(mid)).getSibling().getFullSize();
-                    *lo = mid;
-                    *hi = p_.size_;
-                    lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
-                }
-
-                cb(*vals, to_insert);
-            }
-
-            if (done_cb)
-                done_cb(true);
-        } else if (first_res->is_pht) {
-            // internal node
-            *lo = mid+1;
-            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
-        } else {
-            // first get failed before second.
-            if (done_cb)
-                done_cb(false);
-        }
-    };
-
-    if (*lo <= *hi) {
-        auto pht_filter = [&](const dht::Value& v) {
-            return v.user_type.compare(0, name_.size(), name_) == 0;
-        };
-
-        auto on_get = [=](const std::shared_ptr<dht::Value>& value, std::shared_ptr<node_lookup_result> res) {
-            if (value->user_type == canary_) {
-                res->is_pht = true;
-            }
-            else {
-                IndexEntry entry;
-                entry.unpackValue(*value);
-
-                auto it = std::find_if(vals->cbegin(), vals->cend(), [&](const std::shared_ptr<IndexEntry>& ie) {
-                    return ie->value == entry.value;
-                });
-
-                /* If we already got the value then get the next one */
-                if (it != vals->cend())
-                    return true;
-
-                if (max_common_prefix_len) { /* inexact match case */
-                    auto common_bits = Prefix::commonBits(p, entry.prefix);
-
-                    if (vals->empty()) {
-                        vals->emplace_back(std::make_shared<IndexEntry>(entry));
-                        *max_common_prefix_len = common_bits;
-                    }
-                    else {
-                        if (common_bits == *max_common_prefix_len) /* this is the max so far */
-                            vals->emplace_back(std::make_shared<IndexEntry>(entry));
-                        else if (common_bits > *max_common_prefix_len) { /* new max found! */
-                            vals->clear();
-                            vals->emplace_back(std::make_shared<IndexEntry>(entry));
-                            *max_common_prefix_len = common_bits;
-                        }
-                    }
-                } else if (all_values or entry.prefix == p.content_) /* exact match case */
-                    vals->emplace_back(std::make_shared<IndexEntry>(entry));
-            }
-
-            return true;
-        };
-
-        dht_->get(p.getPrefix(mid).hash(),
-                std::bind(on_get, std::placeholders::_1, first_res),
-                [=](bool ok) {
-                    if (not ok) {
-                        // DHT failed
-                        first_res->done = true;
-                        if (done_cb and second_res->done)
-                            on_done(false);
-                    }
-                    else {
-                        if (not first_res->is_pht) {
-                            // Not a PHT node.
-                            *hi = mid-1;
-                            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
-                        } else {
-                            first_res->done = true;
-                            if (second_res->done or mid >= p.size_ - 1)
-                                on_done(true);
-                        }
-                    }
-                }, pht_filter);
-
-        if (mid < p.size_ - 1)
-           dht_->get(p.getPrefix(mid+1).hash(),
-                    std::bind(on_get, std::placeholders::_1, second_res),
-                    [=](bool ok) {
-                        if (not ok) {
-                            // DHT failed
-                            second_res->done = true;
-                            if (done_cb and first_res->done)
-                                on_done(false);
-                        }
-                        else {
-                            second_res->done = true;
-                            if (first_res->done)
-                                on_done(true);
-                        }
-                }, pht_filter);
-    } else {
-        on_done(true);
-    }
-}
-
-void Pht::lookup(Key k, Pht::LookupCallback cb, DoneCallbackSimple done_cb, bool exact_match) {
-    auto prefix = linearize(k);
-    auto values = std::make_shared<std::vector<std::shared_ptr<IndexEntry>>>();
-
-    auto lo = std::make_shared<int>(0);
-    auto hi = std::make_shared<int>(prefix.size_);
-    std::shared_ptr<unsigned> max_common_prefix_len = not exact_match ? std::make_shared<unsigned>(0) : nullptr;
-
-    lookupStep(prefix, lo, hi, values,
-        [=](std::vector<std::shared_ptr<IndexEntry>>& entries, Prefix p) {
-            std::vector<std::shared_ptr<Value>> vals(entries.size());
-
-            std::transform(entries.begin(), entries.end(), vals.begin(),
-                [](const std::shared_ptr<IndexEntry>& ie) {
-                    return std::make_shared<Value>(ie->value);
-            });
-
-            cb(vals, p);
-        }, done_cb, max_common_prefix_len, cache_.lookup(prefix));
-}
-
-void Pht::updateCanary(Prefix p) {
-    // TODO: change this... copy value
-    dht::Value canary_value;
-    canary_value.user_type = canary_;
-
-    dht_->put(p.hash(), std::move(canary_value),
-        [=](bool){
-            static std::bernoulli_distribution d(0.5);
-            crypto::random_device rd;
-            if (p.size_ and d(rd))
-                updateCanary(p.getPrefix(-1));
-        }
-    );
-
-    if (p.size_) {
-        dht::Value canary_second_value;
-        canary_second_value.user_type = canary_;
-        dht_->put(p.getSibling().hash(), std::move(canary_second_value));
-    }
-}
-
-void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shared_ptr<int> hi, time_point time_p,
-                 bool check_split, DoneCallbackSimple done_cb) {
-
-    if (time_p + ValueType::USER_DATA.expiration < clock::now()) return;
-
-    auto vals = std::make_shared<std::vector<std::shared_ptr<IndexEntry>>>();
-    auto final_prefix = std::make_shared<Prefix>();
-
-    lookupStep(kp, lo, hi, vals,
-        [=](std::vector<std::shared_ptr<IndexEntry>>&, Prefix p) {
-            *final_prefix = Prefix(p);
-        },
-        [=](bool ok){
-            if (not ok) {
-                if (done_cb)
-                    done_cb(false);
-            } else {
-
-                RealInsertCallback real_insert = [=]( std::shared_ptr<Prefix> p, IndexEntry entry) {
-                    updateCanary(*p);
-                    checkPhtUpdate(*p, entry, time_p);
-                    cache_.insert(*p);
-                    dht_->put(p->hash(), std::move(entry), done_cb , time_p);
-                };
-
-                if ( not check_split or final_prefix->size_ == kp.size_ ) {
-                    real_insert(final_prefix, std::move(entry));
-                } else {
-                    if ( vals->size() < MAX_NODE_ENTRY_COUNT ) {
-                        getRealPrefix(final_prefix, std::move(entry), real_insert);
-                    }
-                    else {
-                        split(*final_prefix, vals, entry, real_insert);
-                    }
-                }
-            }
-        }, nullptr, cache_.lookup(kp), true
-    );
-}
-
-Prefix Pht::zcurve(const std::vector<Prefix>& all_prefix) const {
-    Prefix p;
-
-    if ( all_prefix.size() == 1 )
-        return all_prefix[0];
-
-    /* All prefix got the same size (thanks to padding) */
-    size_t prefix_size = all_prefix[0].content_.size();
-
-    /* Loop on all uint8_t of the input prefix */
-    for ( size_t j = 0, bit = 0; j < prefix_size; j++) {
-
-        uint8_t mask = 0x80;
-        /* For each of the 8 bits of the input uint8_t */
-        for ( int i = 0; i < 8; ) {
-
-            uint8_t flags = 0;
-            uint8_t content = 0;
-
-            /* For each bit of the output uint8_t */
-            for ( int k = 0 ; k < 8; k++ ) {
-
-                auto diff = k - i;
-
-                /*get the content 'c', and the flag 'f' of the input prefix */
-                auto c = all_prefix[bit].content_[j] & mask;
-                auto f = all_prefix[bit].flags_[j] & mask;
-
-                /* Move this bit at the right position according to the diff
-                   and merge it into content and flags in the same way */
-                content |= ( diff >= 0 ) ? c >> diff : c << std::abs(diff);
-                flags   |= ( diff >= 0 ) ? f >> diff : f << std::abs(diff);
-
-                /* If we are on the last prefix of the vector get back to the first and
-                ,move the mask in order to get the n + 1nth bit */
-                if ( ++bit == all_prefix.size() ) { bit = 0; ++i; mask >>= 1; }
-            }
-
-            /* Add the next flags + content to the output prefix */
-            p.content_.push_back(content);
-            p.flags_.push_back(flags);
-            p.size_ += 8;
-        }
-    }
-
-    return p;
-}
-
-Prefix Pht::linearize(Key k) const {
-    if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
-
-    std::vector<Prefix> all_prefix;
-    all_prefix.reserve(k.size());
-
-    /* Get the max size of the keyspec and take it for size limit (for padding) */
-    auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
-        [](const std::pair<std::string, size_t>& a, const std::pair<std::string, size_t>& b) {
-            return a.second < b.second;
-        })->second + 1;
-
-    for ( auto const& it : k ) {
-        Prefix p = Blob {it.second.begin(), it.second.end()};
-        p.addPaddingContent(max);
-        p.updateFlags();
-
-        all_prefix.emplace_back(std::move(p));
-    }
-
-    return zcurve(all_prefix);
-}
-
-void Pht::getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb ) {
-
-    if ( p->size_ == 0 ) {
-        end_cb(p, std::move(entry));
-        return;
-    }
-
-    auto total = std::make_shared<unsigned int>(0); /* Will contains the total number of data on 3 nodes */
-    auto ended = std::make_shared<unsigned int>(0); /* Just indicate how many have end */
-
-    auto parent = std::make_shared<Prefix>(p->getPrefix(-1));
-    auto sibling = std::make_shared<Prefix>(p->getSibling());
-
-    auto pht_filter = [&](const dht::Value& v) {
-        return v.user_type.compare(0, name_.size(), name_) == 0;
-    };
-
-    /* Lambda will count total number of data node */
-    auto count = [=]( const std::shared_ptr<dht::Value> value ) {
-        if ( value->user_type != canary_)
-            (*total)++;
-
-        return true;
-    };
-
-    auto on_done = [=] ( bool ) {
-        (*ended)++;
-        /* Only the last one do the CallBack*/
-        if  ( *ended == 3 ) {
-            if ( *total < MAX_NODE_ENTRY_COUNT )
-                end_cb(parent, std::move(entry));
-            else
-                end_cb(p, std::move(entry));
-        }
-    };
-
-    dht_->get(parent->hash(),
-        count,
-        on_done,
-        pht_filter
-    );
-
-    dht_->get(p->hash(),
-        count,
-        on_done,
-        pht_filter
-    );
-
-    dht_->get(sibling->hash(),
-        count,
-        on_done,
-        pht_filter
-    );
-}
-
-void Pht::checkPhtUpdate(Prefix p, IndexEntry entry, time_point time_p) {
-
-    Prefix full = entry.prefix;
-    if ( p.content_.size() * 8 >= full.content_.size() * 8 ) return;
-
-    auto next_prefix = full.getPrefix( p.size_ + 1 );
-
-    dht_->listen(next_prefix.hash(),
-        [=](const std::shared_ptr<dht::Value> &value) {
-            if (value->user_type == canary_) {
-                insert(full, entry, std::make_shared<int>(0), std::make_shared<int>(full.size_), time_p, false, nullptr);
-
-                /* Cancel listen since we found where we need to update*/
-                return false;
-            }
-
-            return true;
-        },
-        [=](const dht::Value& v) {
-            /* Filter value v thats start with the same name as ours */
-            return v.user_type.compare(0, name_.size(), name_) == 0;
-        }
-    );
-}
-
-void Pht::split(Prefix insert, std::shared_ptr<std::vector<std::shared_ptr<IndexEntry>>> vals, IndexEntry entry, RealInsertCallback end_cb ) {
-    auto full = Prefix(entry.prefix);
-
-    auto loc = foundSplitLocation(full, vals);
-    auto prefix_to_insert = std::make_shared<Prefix>(full.getPrefix(loc));
-
-    for(;loc != insert.size_ - 1; loc--) {
-        updateCanary(full.getPrefix(loc));
-    }
-
-    end_cb(prefix_to_insert, entry);
-}
-
-} /* indexation  */
-
-} /* dht */

src/infohash.cpp

@@ -91,4 +91,17 @@ std::ostream& operator<< (std::ostream& s, const InfoHash& h)
     return s;
 }
 
+std::istream& operator>> (std::istream& s, InfoHash& h)
+{
+    std::array<char, HASH_LEN*2> dat;
+    s.exceptions(std::istream::eofbit | std::istream::failbit);
+    s.read(&(*dat.begin()), dat.size());
+    for (size_t i = 0; i < HASH_LEN; i++) {
+        unsigned res = 0;
+        sscanf(dat.data() + 2*i, "%02x", &res);
+        h[i] = res;
+    }
+    return s;
+}
+
 }

src/network_engine.cpp

@@ -49,6 +49,7 @@ static const uint8_t v4prefix[16] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0, 0, 0, 0
 };
 
+constexpr unsigned SEND_NODES {8};
 
 enum class MessageType {
     Error = 0,
@@ -63,6 +64,7 @@ enum class MessageType {
 struct ParsedMessage {
     MessageType type;
     InfoHash id;                                /* the id of the sender */
+    NetId network {0};                          /* network id */
     InfoHash info_hash;                         /* hash for which values are requested */
     InfoHash target;                            /* target id around which to find nodes */
     NetworkEngine::TransId tid;                                /* transaction id */
@@ -75,7 +77,7 @@ struct ParsedMessage {
     want_t want;                                /* states if ipv4 or ipv6 request */
     uint16_t error_code;                        /* error code in case of error */
     std::string ua;
-    Address addr;                               /* reported address by the distant node */
+    SockAddr addr;                               /* reported address by the distant node */
     void msgpack_unpack(msgpack::object o);
 };
 
@@ -89,7 +91,7 @@ NetworkEngine::tellListener(std::shared_ptr<Node> node, uint16_t rid, InfoHash h
 {
     auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6);
     try {
-        sendNodesValues((const sockaddr*)&node->ss, node->sslen, TransId {TransPrefix::GET_VALUES, (uint16_t)rid}, nnodes.first, nnodes.second,
+        sendNodesValues(node->addr, TransId {TransPrefix::GET_VALUES, (uint16_t)rid}, nnodes.first, nnodes.second,
                 values, ntoken);
     } catch (const std::overflow_error& e) {
         DHT_LOG.ERR("Can't send value: buffer not large enough !");
@@ -155,7 +157,7 @@ NetworkEngine::requestStep(std::shared_ptr<Request> req)
 
     send((char*)req->msg.data(), req->msg.size(),
             (req->node->reply_time >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM,
-            (sockaddr*)&req->node->ss, req->node->sslen);
+            req->node->addr);
     ++req->attempt_count;
     req->last_try = now;
     std::weak_ptr<Request> wreq = req;
@@ -200,15 +202,15 @@ NetworkEngine::rateLimit()
 }
 
 bool
-NetworkEngine::isMartian(const sockaddr* sa, socklen_t len)
+NetworkEngine::isMartian(const SockAddr& addr)
 {
     // Check that sa_family can be accessed safely
-    if (!sa || len < sizeof(sockaddr_in))
+    if (addr.second < sizeof(sockaddr_in))
         return true;
 
-    switch(sa->sa_family) {
+    switch(addr.first.ss_family) {
     case AF_INET: {
-        sockaddr_in *sin = (sockaddr_in*)sa;
+        sockaddr_in *sin = (sockaddr_in*)&addr.first;
         const uint8_t *address = (const uint8_t*)&sin->sin_addr;
         return sin->sin_port == 0 ||
             (address[0] == 0) ||
@@ -216,9 +218,9 @@ NetworkEngine::isMartian(const sockaddr* sa, socklen_t len)
             ((address[0] & 0xE0) == 0xE0);
     }
     case AF_INET6: {
-        if (len < sizeof(sockaddr_in6))
+        if (addr.second < sizeof(sockaddr_in6))
             return true;
-        sockaddr_in6 *sin6 = (sockaddr_in6*)sa;
+        sockaddr_in6 *sin6 = (sockaddr_in6*)&addr.first;
         const uint8_t *address = (const uint8_t*)&sin6->sin6_addr;
         return sin6->sin6_port == 0 ||
             (address[0] == 0xFF) ||
@@ -247,37 +249,25 @@ NetworkEngine::blacklistNode(const std::shared_ptr<Node>& n)
             ++rit;
         }
     }
-    memcpy(&blacklist[next_blacklisted], &n->ss, n->sslen);
-    next_blacklisted = (next_blacklisted + 1) % BLACKLISTED_MAX;
+    blacklist.emplace(n->addr);
 }
 
 bool
-NetworkEngine::isNodeBlacklisted(const sockaddr *sa, socklen_t salen) const
+NetworkEngine::isNodeBlacklisted(const SockAddr& addr) const
 {
-    if (salen > sizeof(sockaddr_storage))
-        return true;
-
-    /*if (isBlacklisted(sa, salen))
-        return true;*/
-
-    for (unsigned i = 0; i < BLACKLISTED_MAX; i++) {
-        if (memcmp(&blacklist[i], sa, salen) == 0)
-            return true;
-    }
-
-    return false;
+    return blacklist.find(addr) != blacklist.end();
 }
 
 void
-NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen)
+NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& from)
 {
-    if (isMartian(from, fromlen)) {
-        DHT_LOG.WARN("Received packet from martian node %s", print_addr(from, fromlen).c_str());
+    if (isMartian(from)) {
+        DHT_LOG.WARN("Received packet from martian node %s", from.toString().c_str());
         return;
     }
 
-    if (isNodeBlacklisted(from, fromlen)) {
-        DHT_LOG.WARN("Received packet from blacklisted node %s", print_addr(from, fromlen).c_str());
+    if (isNodeBlacklisted(from)) {
+        DHT_LOG.WARN("Received packet from blacklisted node %s", from.toString().c_str());
         return;
     }
 
@@ -293,6 +283,11 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
         return;
     }
 
+    if (msg.network != network) {
+        DHT_LOG.DEBUG("Received message from other network %u.", msg.network);
+        return;
+    }
+
     if (msg.id == myid || msg.id == zeroes) {
         DHT_LOG.DEBUG("Received message from self.");
         return;
@@ -311,7 +306,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
     if (msg.tid.length != 4) {
         DHT_LOG.ERR("Broken node truncates transaction ids (len: %d): ", msg.tid.length);
         DHT_LOG.ERR.logPrintable(buf, buflen);
-        blacklistNode(cache.getNode(msg.id, from, fromlen, now, 1));
+        blacklistNode(cache.getNode(msg.id, from, now, 1));
         return;
     }
 
@@ -326,7 +321,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
         auto node = req->node;
         if (node->id != msg.id) {
             bool unknown_node = node->id == zeroes;
-            node = cache.getNode(msg.id, from, fromlen, now, 2);
+            node = cache.getNode(msg.id, from, now, 2);
             if (unknown_node) {
                 // received reply to a message sent when we didn't know the node ID.
                 req->node = node;
@@ -338,11 +333,11 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                 return;
             }
         } else
-            node->update(from, fromlen);
+            node->update(from);
         node->received(now, req);
 
         onNewNode(node, 2);
-        onReportedAddr(msg.id, (sockaddr*)&msg.addr.first, msg.addr.second);
+        onReportedAddr(msg.id, msg.addr);
 
         if (req->cancelled() or req->expired() or (req->completed() and not req->persistent)) {
             DHT_LOG.WARN("[node %s] response to expired, cancelled or completed request", node->toString().c_str());
@@ -362,7 +357,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                 onError(req, DhtProtocolException {DhtProtocolException::UNAUTHORIZED});
             } else {
                 DHT_LOG.WARN("[node %s %s] received unknown error message %u",
-                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.error_code);
+                        msg.id.toString().c_str(), from.toString().c_str(), msg.error_code);
                 DHT_LOG.WARN.logPrintable(buf, buflen);
             }
             break;
@@ -380,7 +375,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
             break;
         }
     } else {
-        auto node = cache.getNode(msg.id, from, fromlen, now, 1);
+        auto node = cache.getNode(msg.id, from, now, 1);
         node->received(now, {});
         onNewNode(node, 1);
         try {
@@ -389,29 +384,29 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                 ++in_stats.ping;
                 DHT_LOG.DEBUG("Sending pong.");
                 onPing(node);
-                sendPong(from, fromlen, msg.tid);
+                sendPong(from, msg.tid);
                 break;
             case MessageType::FindNode: {
                 DHT_LOG.DEBUG("[node %s %s] got 'find' request (%d).",
-                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.want);
+                        msg.id.toString().c_str(), from.toString().c_str(), msg.want);
                 ++in_stats.find;
                 RequestAnswer answer = onFindNode(node, msg.target, msg.want);
-                auto nnodes = bufferNodes(from->sa_family, msg.target, msg.want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, {}, answer.ntoken);
+                auto nnodes = bufferNodes(from.getFamily(), msg.target, msg.want, answer.nodes4, answer.nodes6);
+                sendNodesValues(from, msg.tid, nnodes.first, nnodes.second, {}, answer.ntoken);
                 break;
             }
             case MessageType::GetValues: {
                 DHT_LOG.DEBUG("[node %s %s] got 'get' request for %s.",
-                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.info_hash.toString().c_str());
+                        msg.id.toString().c_str(), from.toString().c_str(), msg.info_hash.toString().c_str());
                 ++in_stats.get;
                 RequestAnswer answer = onGetValues(node, msg.info_hash, msg.want);
-                auto nnodes = bufferNodes(from->sa_family, msg.info_hash, msg.want, answer.nodes4, answer.nodes6);
-                sendNodesValues(from, fromlen, msg.tid, nnodes.first, nnodes.second, answer.values, answer.ntoken);
+                auto nnodes = bufferNodes(from.getFamily(), msg.info_hash, msg.want, answer.nodes4, answer.nodes6);
+                sendNodesValues(from, msg.tid, nnodes.first, nnodes.second, answer.values, answer.ntoken);
                 break;
             }
             case MessageType::AnnounceValue: {
                 DHT_LOG.DEBUG("[node %s %s] got 'put' request for %s.",
-                    msg.id.toString().c_str(), print_addr(from, fromlen).c_str(),
+                    msg.id.toString().c_str(), from.toString().c_str(),
                     msg.info_hash.toString().c_str());
                 ++in_stats.put;
                 onAnnounce(node, msg.info_hash, msg.token, msg.values, msg.created);
@@ -420,16 +415,16 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
                    This is to prevent them from backtracking, and hence
                    polluting the DHT. */
                 for (auto& v : msg.values) {
-                   sendValueAnnounced(from, fromlen, msg.tid, v->id);
+                   sendValueAnnounced(from, msg.tid, v->id);
                 }
                 break;
             }
             case MessageType::Listen: {
                 DHT_LOG.DEBUG("[node %s %s] got 'listen' request for %s.",
-                        msg.id.toString().c_str(), print_addr(from, fromlen).c_str(), msg.info_hash.toString().c_str());
+                        msg.id.toString().c_str(), from.toString().c_str(), msg.info_hash.toString().c_str());
                 ++in_stats.listen;
                 RequestAnswer answer = onListen(node, msg.info_hash, msg.token, msg.tid.getTid());
-                sendListenConfirmation(from, fromlen, msg.tid);
+                sendListenConfirmation(from, msg.tid);
                 break;
             }
             default:
@@ -438,7 +433,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const sockaddr*
         } catch (const std::overflow_error& e) {
             DHT_LOG.ERR("Can't send value: buffer not large enough !");
         } catch (DhtProtocolException& e) {
-            sendError(from, fromlen, msg.tid, e.getCode(), e.getMsg().c_str(), true);
+            sendError(from, msg.tid, e.getCode(), e.getMsg().c_str(), true);
         }
     }
 }
@@ -451,33 +446,34 @@ packToken(msgpack::packer<msgpack::sbuffer>& pk, Blob token)
 }
 
 void
-insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const sockaddr *sa, socklen_t)
+insertAddr(msgpack::packer<msgpack::sbuffer>& pk, const SockAddr& addr)
 {
-    size_t addr_len = (sa->sa_family == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr);
-    void* addr_ptr = (sa->sa_family == AF_INET) ? (void*)&((sockaddr_in*)sa)->sin_addr
-                                                : (void*)&((sockaddr_in6*)sa)->sin6_addr;
+    size_t addr_len = std::min<size_t>(addr.second,
+                     (addr.getFamily() == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr));
+    void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&((sockaddr_in*)&addr.first)->sin_addr
+                                                : (void*)&((sockaddr_in6*)&addr.first)->sin6_addr;
     pk.pack("sa");
     pk.pack_bin(addr_len);
     pk.pack_bin_body((char*)addr_ptr, addr_len);
 }
 
 int
-NetworkEngine::send(const char *buf, size_t len, int flags, const sockaddr *sa, socklen_t salen)
+NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr)
 {
-    if (salen == 0)
+    if (addr.second == 0)
         return -1;
 
     int s;
-    if (sa->sa_family == AF_INET)
+    if (addr.getFamily() == AF_INET)
         s = dht_socket;
-    else if (sa->sa_family == AF_INET6)
+    else if (addr.getFamily() == AF_INET6)
         s = dht_socket6;
     else
         s = -1;
 
     if (s < 0)
         return -1;
-    return sendto(s, buf, len, flags, sa, salen);
+    return sendto(s, buf, len, flags, (const sockaddr*)&addr.first, addr.second);
 }
 
 std::shared_ptr<Request>
@@ -485,7 +481,7 @@ NetworkEngine::sendPing(std::shared_ptr<Node> node, RequestCb on_done, RequestEx
     auto tid = TransId {TransPrefix::PING, getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(1);
      pk.pack(std::string("id")); pk.pack(myid);
@@ -495,6 +491,9 @@ NetworkEngine::sendPing(std::shared_ptr<Node> node, RequestCb on_done, RequestEx
                               pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("q"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
     Blob b {buffer.data(), buffer.data() + buffer.size()};
     std::shared_ptr<Request> req(new Request {tid.getTid(), node, std::move(b),
@@ -516,21 +515,24 @@ NetworkEngine::sendPing(std::shared_ptr<Node> node, RequestCb on_done, RequestEx
 }
 
 void
-NetworkEngine::sendPong(const sockaddr* sa, socklen_t salen, TransId tid) {
+NetworkEngine::sendPong(const SockAddr& addr, TransId tid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(2);
       pk.pack(std::string("id")); pk.pack(myid);
-      insertAddr(pk, sa, salen);
+      insertAddr(pk, addr);
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("r"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
-    send(buffer.data(), buffer.size(), 0, sa, salen);
+    send(buffer.data(), buffer.size(), 0, addr);
 }
 
 std::shared_ptr<Request>
@@ -539,7 +541,7 @@ NetworkEngine::sendFindNode(std::shared_ptr<Node> n, const InfoHash& target, wan
     auto tid = TransId {TransPrefix::FIND_NODE, getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(2 + (want>0?1:0));
       pk.pack(std::string("id"));     pk.pack(myid);
@@ -556,7 +558,9 @@ NetworkEngine::sendFindNode(std::shared_ptr<Node> n, const InfoHash& target, wan
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("q"));
     pk.pack(std::string("v")); pk.pack(my_v);
-
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
     Blob b {buffer.data(), buffer.data() + buffer.size()};
     std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -583,7 +587,7 @@ NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash,
     auto tid = TransId {TransPrefix::GET_VALUES, getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a"));  pk.pack_map(2 + (want>0?1:0));
       pk.pack(std::string("id")); pk.pack(myid);
@@ -600,6 +604,9 @@ NetworkEngine::sendGetValues(std::shared_ptr<Node> n, const InfoHash& info_hash,
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("q"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
     Blob b {buffer.data(), buffer.data() + buffer.size()};
     std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -626,19 +633,21 @@ NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {
     } else {
         // deserialize nodes
         const auto& now = scheduler.time();
+        SockAddr addr;
         for (unsigned i = 0; i < msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i++) {
             uint8_t *ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN;
             const InfoHash& ni_id = *reinterpret_cast<InfoHash*>(ni);
             if (ni_id == myid)
                 continue;
-            sockaddr_in sin;
-            std::fill_n((uint8_t*)&sin, sizeof(sockaddr_in), 0);
-            sin.sin_family = AF_INET;
-            memcpy(&sin.sin_addr, ni + ni_id.size(), 4);
-            memcpy(&sin.sin_port, ni + ni_id.size() + 4, 2);
-            if (isMartian((sockaddr*)&sin, sizeof(sin)) || isNodeBlacklisted((sockaddr*)&sin, sizeof(sin)))
+            auto sin = (sockaddr_in*)&addr.first;
+            std::fill_n((uint8_t*)sin, sizeof(sockaddr_in), 0);
+            sin->sin_family = AF_INET;
+            memcpy(&sin->sin_addr, ni + ni_id.size(), 4);
+            memcpy(&sin->sin_port, ni + ni_id.size() + 4, 2);
+            addr.second = sizeof(sockaddr_in);
+            if (isMartian(addr) || isNodeBlacklisted(addr))
                 continue;
-            msg.nodes4.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin, sizeof(sin), now, 0));
+            msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, 0));
             onNewNode(msg.nodes4.back(), 0);
         }
         for (unsigned i = 0; i < msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i++) {
@@ -646,30 +655,31 @@ NetworkEngine::deserializeNodesValues(ParsedMessage& msg) {
             const InfoHash& ni_id = *reinterpret_cast<InfoHash*>(ni);
             if (ni_id == myid)
                 continue;
-            sockaddr_in6 sin6;
-            std::fill_n((uint8_t*)&sin6, sizeof(sockaddr_in6), 0);
-            sin6.sin6_family = AF_INET6;
-            memcpy(&sin6.sin6_addr, ni + HASH_LEN, 16);
-            memcpy(&sin6.sin6_port, ni + HASH_LEN + 16, 2);
-            if (isMartian((sockaddr*)&sin6, sizeof(sin6)) || isNodeBlacklisted((sockaddr*)&sin6, sizeof(sin6)))
+            auto sin6 = (sockaddr_in6*)&addr.first;
+            std::fill_n((uint8_t*)sin6, sizeof(sockaddr_in6), 0);
+            sin6->sin6_family = AF_INET6;
+            memcpy(&sin6->sin6_addr, ni + HASH_LEN, 16);
+            memcpy(&sin6->sin6_port, ni + HASH_LEN + 16, 2);
+            addr.second = sizeof(sockaddr_in6);
+            if (isMartian(addr) || isNodeBlacklisted(addr))
                 continue;
-            msg.nodes6.emplace_back(cache.getNode(ni_id, (sockaddr*)&sin6, sizeof(sin6), now, 0));
+            msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, 0));
             onNewNode(msg.nodes6.back(), 0);
         }
     }
 }
 
 void
-NetworkEngine::sendNodesValues(const sockaddr* sa, socklen_t salen, TransId tid, const Blob& nodes, const Blob& nodes6,
+NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& nodes, const Blob& nodes6,
         const std::vector<std::shared_ptr<Value>>& st, const Blob& token) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r"));
     pk.pack_map(2 + (not st.empty()?1:0) + (nodes.size()>0?1:0) + (nodes6.size()>0?1:0) + (not token.empty()?1:0));
     pk.pack(std::string("id")); pk.pack(myid);
-    insertAddr(pk, sa, salen);
+    insertAddr(pk, addr);
     if (nodes.size() > 0) {
         pk.pack(std::string("n4"));
         pk.pack_bin(nodes.size());
@@ -715,8 +725,11 @@ NetworkEngine::sendNodesValues(const sockaddr* sa, socklen_t salen, TransId tid,
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("r"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
-    send(buffer.data(), buffer.size(), 0, sa, salen);
+    send(buffer.data(), buffer.size(), 0, addr);
 }
 
 Blob
@@ -725,14 +738,14 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vector<std::
     std::sort(nodes.begin(), nodes.end(), [&](const std::shared_ptr<Node>& a, const std::shared_ptr<Node>& b){
         return id.xorCmp(a->id, b->id) < 0;
     });
-    size_t nnode = std::min<size_t>(TARGET_NODES, nodes.size());
+    size_t nnode = std::min<size_t>(SEND_NODES, nodes.size());
     Blob bnodes;
     if (af == AF_INET) {
         bnodes.resize(NODE4_INFO_BUF_LEN * nnode);
         const constexpr size_t size = HASH_LEN + sizeof(in_addr) + sizeof(in_port_t); // 26
         for (size_t i=0; i<nnode; i++) {
             const Node& n = *nodes[i];
-            sockaddr_in *sin = (sockaddr_in*)&n.ss;
+            sockaddr_in *sin = (sockaddr_in*)&n.addr.first;
             auto dest = bnodes.data() + size * i;
             memcpy(dest, n.id.data(), HASH_LEN);
             memcpy(dest + HASH_LEN, &sin->sin_addr, sizeof(in_addr));
@@ -743,7 +756,7 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vector<std::
         const constexpr size_t size = HASH_LEN + sizeof(in6_addr) + sizeof(in_port_t); // 38
         for (size_t i=0; i<nnode; i++) {
             const Node& n = *nodes[i];
-            sockaddr_in6 *sin6 = (sockaddr_in6*)&n.ss;
+            sockaddr_in6 *sin6 = (sockaddr_in6*)&n.addr.first;
             auto dest = bnodes.data() + size * i;
             memcpy(dest, n.id.data(), HASH_LEN);
             memcpy(dest + HASH_LEN, &sin6->sin6_addr, sizeof(in6_addr));
@@ -777,7 +790,7 @@ NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, con
     auto tid = TransId {TransPrefix::LISTEN, getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map(3);
       pk.pack(std::string("id"));    pk.pack(myid);
@@ -789,7 +802,9 @@ NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, con
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("q"));
     pk.pack(std::string("v")); pk.pack(my_v);
-
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
     Blob b {buffer.data(), buffer.data() + buffer.size()};
     std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -809,21 +824,24 @@ NetworkEngine::sendListen(std::shared_ptr<Node> n, const InfoHash& infohash, con
 }
 
 void
-NetworkEngine::sendListenConfirmation(const sockaddr* sa, socklen_t salen, TransId tid) {
+NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(2);
       pk.pack(std::string("id")); pk.pack(myid);
-      insertAddr(pk, sa, salen);
+      insertAddr(pk, addr);
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("r"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
-    send(buffer.data(), buffer.size(), 0, sa, salen);
+    send(buffer.data(), buffer.size(), 0, addr);
 }
 
 std::shared_ptr<Request>
@@ -832,7 +850,7 @@ NetworkEngine::sendAnnounceValue(std::shared_ptr<Node> n, const InfoHash& infoha
     auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, getNewTid()};
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(5);
+    pk.pack_map(5+(network?1:0));
 
     pk.pack(std::string("a")); pk.pack_map((created < scheduler.time() ? 5 : 4));
       pk.pack(std::string("id"));     pk.pack(myid);
@@ -849,6 +867,9 @@ NetworkEngine::sendAnnounceValue(std::shared_ptr<Node> n, const InfoHash& infoha
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("q"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
     Blob b {buffer.data(), buffer.data() + buffer.size()};
     std::shared_ptr<Request> req(new Request {tid.getTid(), n, std::move(b),
@@ -875,27 +896,29 @@ NetworkEngine::sendAnnounceValue(std::shared_ptr<Node> n, const InfoHash& infoha
 }
 
 void
-NetworkEngine::sendValueAnnounced(const sockaddr* sa, socklen_t salen, TransId tid, Value::Id vid) {
+NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id vid) {
     msgpack::sbuffer buffer;
     msgpack::packer<msgpack::sbuffer> pk(&buffer);
-    pk.pack_map(4);
+    pk.pack_map(4+(network?1:0));
 
     pk.pack(std::string("r")); pk.pack_map(3);
       pk.pack(std::string("id"));  pk.pack(myid);
       pk.pack(std::string("vid")); pk.pack(vid);
-      insertAddr(pk, sa, salen);
+      insertAddr(pk, addr);
 
     pk.pack(std::string("t")); pk.pack_bin(tid.size());
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("r"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
-    send(buffer.data(), buffer.size(), 0, sa, salen);
+    send(buffer.data(), buffer.size(), 0, addr);
 }
 
 void
-NetworkEngine::sendError(const sockaddr* sa,
-        socklen_t salen,
+NetworkEngine::sendError(const SockAddr& addr,
         TransId tid,
         uint16_t code,
         const std::string& message,
@@ -917,8 +940,11 @@ NetworkEngine::sendError(const sockaddr* sa,
                                pk.pack_bin_body((const char*)tid.data(), tid.size());
     pk.pack(std::string("y")); pk.pack(std::string("e"));
     pk.pack(std::string("v")); pk.pack(my_v);
+    if (network) {
+        pk.pack(std::string("n")); pk.pack(network);
+    }
 
-    send(buffer.data(), buffer.size(), 0, sa, salen);
+    send(buffer.data(), buffer.size(), 0, addr);
 }
 
 msgpack::object*
@@ -960,6 +986,9 @@ ParsedMessage::msgpack_unpack(msgpack::object msg)
         error_code = e->via.array.ptr[0].as<uint16_t>();
     }
 
+    if (auto netid = findMapValue(msg, "n"))
+        network = netid->as<NetId>();
+
     if (auto rid = findMapValue(req, "id"))
         id = {*rid};
 

src/node.cpp

@@ -65,10 +65,9 @@ Node::getPendingMessageCount() const
 }
 
 void
-Node::update(const sockaddr* sa, socklen_t salen)
+Node::update(const SockAddr& new_addr)
 {
-    std::copy_n((const uint8_t*)sa, salen, (uint8_t*)&ss);
-    sslen = salen;
+    addr = new_addr;
 }
 
 /** To be called when a message was sent to the node */
@@ -119,7 +118,7 @@ Node::toString() const
 
 std::ostream& operator<< (std::ostream& s, const Node& h)
 {
-    s << h.id << " " << print_addr(h.ss, h.sslen);
+    s << h.id << " " << h.addr.toString();
     return s;
 }
 

src/node_cache.cpp

@@ -27,10 +27,10 @@ NodeCache::getNode(const InfoHash& id, sa_family_t family) {
 }
 
 std::shared_ptr<Node>
-NodeCache::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirm) {
+NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, int confirm) {
     if (id == zeroes)
-        return std::make_shared<Node>(id, sa, sa_len);
-    return (sa->sa_family == AF_INET ? cache_4 : cache_6).getNode(id, sa, sa_len, now, confirm);
+        return std::make_shared<Node>(id, addr);
+    return (addr.getFamily() == AF_INET ? cache_4 : cache_6).getNode(id, addr, now, confirm);
 }
 
 void
@@ -57,15 +57,15 @@ NodeCache::NodeMap::getNode(const InfoHash& id)
 }
 
 std::shared_ptr<Node>
-NodeCache::NodeMap::getNode(const InfoHash& id, const sockaddr* sa, socklen_t sa_len, time_point now, int confirm)
+NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, int confirm)
 {
     auto it = emplace(id, std::weak_ptr<Node>{});
     auto node = it.first->second.lock();
     if (not node) {
-        node = std::make_shared<Node>(id, sa, sa_len);
+        node = std::make_shared<Node>(id, addr);
         it.first->second = node;
     } else if (confirm || node->time < now - Node::NODE_EXPIRE_TIME) {
-        node->update(sa, sa_len);
+        node->update(addr);
     }
     return node;
 }

src/securedht.cpp

@@ -141,7 +141,7 @@ SecureDht::getCertificate(const InfoHash& node) const
         return it->second;
 }
 
-const std::shared_ptr<crypto::PublicKey>
+const std::shared_ptr<const crypto::PublicKey>
 SecureDht::getPublicKey(const InfoHash& node) const
 {
     if (node == getId())
@@ -226,7 +226,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function<void(const std::s
 }
 
 void
-SecureDht::findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<crypto::PublicKey>)> cb)
+SecureDht::findPublicKey(const InfoHash& node, std::function<void(const std::shared_ptr<const crypto::PublicKey>)> cb)
 {
     auto pk = getPublicKey(node);
     if (pk && *pk) {
@@ -333,7 +333,7 @@ SecureDht::putSigned(const InfoHash& hash, std::shared_ptr<Value> val, DoneCallb
         },
         [hash,val,this,callback,permanent] (bool /* ok */) {
             sign(*val);
-            put(hash, val, callback, {}, permanent);
+            put(hash, val, callback, time_point::max(), permanent);
         },
         Value::IdFilter(val->id)
     );
@@ -342,7 +342,7 @@ SecureDht::putSigned(const InfoHash& hash, std::shared_ptr<Value> val, DoneCallb
 void
 SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, std::shared_ptr<Value> val, DoneCallback callback, bool permanent)
 {
-    findPublicKey(to, [=](const std::shared_ptr<crypto::PublicKey> pk) {
+    findPublicKey(to, [=](const std::shared_ptr<const crypto::PublicKey> pk) {
         if(!pk || !*pk) {
             if (callback)
                 callback(false, {});
@@ -350,7 +350,7 @@ SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, std::shared_pt
         }
         DHT_LOG.WARN("Encrypting data for PK: %s", pk->getId().toString().c_str());
         try {
-            put(hash, encrypt(*val, *pk), callback, {}, permanent);
+            put(hash, encrypt(*val, *pk), callback, time_point::max(), permanent);
         } catch (const std::exception& e) {
             DHT_LOG.ERR("Error putting encrypted data: %s", e.what());
             if (callback)
@@ -362,22 +362,13 @@ SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, std::shared_pt
 void
 SecureDht::sign(Value& v) const
 {
-    if (v.isEncrypted())
-        throw DhtException("Can't sign encrypted data.");
-    v.owner = std::make_shared<crypto::PublicKey>(key_->getPublicKey());
-    v.signature = key_->sign(v.getToSign());
+    v.sign(*key_);
 }
 
 Value
 SecureDht::encrypt(Value& v, const crypto::PublicKey& to) const
 {
-    if (v.isEncrypted())
-        throw DhtException("Data is already encrypted.");
-    v.setRecipient(to.getId());
-    sign(v);
-    Value nv {v.id};
-    nv.setCypher(to.encrypt(v.getToEncrypt()));
-    return nv;
+    return v.encrypt(*key_, to);
 }
 
 Value

src/utils.cpp

@@ -47,7 +47,7 @@ print_addr(const sockaddr_storage& ss, socklen_t sslen)
 }
 
 std::string
-printAddr(const Address& addr) {
+printAddr(const SockAddr& addr) {
     return print_addr((const sockaddr*)&addr.first, addr.second);
 }
 

src/value.cpp

@@ -142,8 +142,9 @@ Value::msgpack_unpack_body(const msgpack::object& o)
                 seq = rseq->as<decltype(seq)>();
             else
                 throw msgpack::type_error();
-            owner = std::make_shared<crypto::PublicKey>();
-            owner->msgpack_unpack(*rowner);
+            crypto::PublicKey new_owner;
+            new_owner.msgpack_unpack(*rowner);
+            owner = std::make_shared<const crypto::PublicKey>(std::move(new_owner));
             if (auto rrecipient = findMapValue(*rbody, "to")) {
                 recipient = rrecipient->as<InfoHash>();
             }

tools/Makefile.am

@@ -3,10 +3,10 @@ bin_PROGRAMS = dhtnode dhtchat dhtscanner
 AM_CPPFLAGS = -I../include
 
 dhtnode_SOURCES = dhtnode.cpp
-dhtnode_LDFLAGS = -lopendht -lreadline -L../src/.libs @GNUTLS_LIBS@
+dhtnode_LDFLAGS = -lopendht -lreadline -L../src/.libs @GnuTLS_LIBS@
 
 dhtchat_SOURCES = dhtchat.cpp
-dhtchat_LDFLAGS = -lopendht -lreadline -L../src/.libs  @GNUTLS_LIBS@
+dhtchat_LDFLAGS = -lopendht -lreadline -L../src/.libs @GnuTLS_LIBS@
 
 dhtscanner_SOURCES = dhtscanner.cpp
-dhtscanner_LDFLAGS = -lopendht -lreadline -L../src/.libs  @GNUTLS_LIBS@
+dhtscanner_LDFLAGS = -lopendht -lreadline -L../src/.libs @GnuTLS_LIBS@

tools/dhtchat.cpp

@@ -53,7 +53,7 @@ main(int argc, char **argv)
         throw std::runtime_error(std::string("Error initializing GnuTLS: ")+gnutls_strerror(rc));
 
     DhtRunner dht;
-    dht.run(params.port, dht::crypto::generateIdentity("DHT Chat Node"), true);
+    dht.run(params.port, dht::crypto::generateIdentity("DHT Chat Node"), true, params.network);
 
     if (not params.bootstrap.first.empty())
         dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());

tools/dhtnode.cpp

@@ -29,7 +29,7 @@ extern "C" {
 using namespace dht;
 
 void print_usage() {
-    std::cout << "Usage: dhtnode [-p local_port] [-b bootstrap_host:port]" << std::endl << std::endl;
+    std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl;
     std::cout << "dhtnode, a simple OpenDHT command line node runner." << std::endl;
     std::cout << "Report bugs to: http://opendht.net" << std::endl;
 }
@@ -38,12 +38,10 @@ void print_id_req() {
     std::cout << "An identity is required to perform this operation (run with -i)" << std::endl;
 }
 
-void print_node_info(const std::shared_ptr<DhtRunner>& dht, const dht_params& params) {
-    std::cout << "OpenDht node " << dht->getNodeId() << " running on port " <<  dht->getBoundPort() << std::endl;
-    if (params.is_bootstrap_node)
-        std::cout << "Running in bootstrap mode (discouraged)." << std::endl;
+void print_node_info(const DhtRunner& dht, const dht_params& params) {
+    std::cout << "OpenDht node " << dht.getNodeId() << " running on port " <<  dht.getBoundPort() << std::endl;
     if (params.generate_identity)
-        std::cout << "Public key ID " << dht->getId() << std::endl;
+        std::cout << "Public key ID " << dht.getId() << std::endl;
 }
 
 void print_help() {
@@ -60,20 +58,16 @@ void print_help() {
               << "  lr         Print the full current routing table of this node" << std::endl;
 
     std::cout << std::endl << "Operations on the DHT:" << std::endl
-              << "  b <ip:port>             Ping potential node at given IP address/port." << std::endl
-              << "  g <key>               Get values at <key>." << std::endl
-              << "  l <key>               Listen for value changes at <key>." << std::endl
-              << "  p <key> <str>         Put string value at <key>." << std::endl
-              << "  s <key> <str>         Put string value at <key>, signed with our generated private key." << std::endl
-              << "  e <key> <dest> <str>  Put string value at <key>, encrypted for <dest> with its public key (if found)." << std::endl;
-    std::cout << std::endl << "Indexation operations on the DHT:" << std::endl
-              << "  il <name> <key> [exact match]   Lookup the index named <name> with the key <key>." << std::endl
-              << "                                  Set [exact match] to 'false' for inexact match lookup." << std::endl
-              << "  ii <name> <key> <value>         Inserts the value <value> under the key <key> in the index named <name>." << std::endl
+              << "  b ip:port             Ping potential node at given IP address/port." << std::endl
+              << "  g [key]               Get values at [key]." << std::endl
+              << "  l [key]               Listen for value changes at [key]." << std::endl
+              << "  p [key] [str]         Put string value at [key]." << std::endl
+              << "  s [key] [str]         Put string value at [key], signed with our generated private key." << std::endl
+              << "  e [key] [dest] [str]  Put string value at [key], encrypted for [dest] with its public key (if found)." << std::endl
               << std::endl;
 }
 
-void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation::Pht> indexes, dht_params& params)
+void cmd_loop(DhtRunner& dht, dht_params& params)
 {
     print_node_info(dht, params);
     std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl << std::endl;
@@ -89,8 +83,8 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
             break;
 
         std::istringstream iss(line);
-        std::string op, idstr, value, index, keystr;
-        iss >> op;
+        std::string op, idstr, value;
+        iss >> op >> idstr;
 
         if (op == "x" || op == "q" || op == "exit" || op == "quit") {
             break;
@@ -101,32 +95,31 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
             print_node_info(dht, params);
             unsigned good4, dubious4, cached4, incoming4;
             unsigned good6, dubious6, cached6, incoming6;
-            dht->getNodesStats(AF_INET, &good4, &dubious4, &cached4, &incoming4);
-            dht->getNodesStats(AF_INET6, &good6, &dubious6, &cached6, &incoming6);
+            dht.getNodesStats(AF_INET, &good4, &dubious4, &cached4, &incoming4);
+            dht.getNodesStats(AF_INET6, &good6, &dubious6, &cached6, &incoming6);
             std::cout << "IPv4 nodes : " << good4 << " good, " << dubious4 << " dubious, " << incoming4 << " incoming." << std::endl;
             std::cout << "IPv6 nodes : " << good6 << " good, " << dubious6 << " dubious, " << incoming6 << " incoming." << std::endl;
             continue;
         } else if (op == "lr") {
             std::cout << "IPv4 routing table:" << std::endl;
-            std::cout << dht->getRoutingTablesLog(AF_INET) << std::endl;
+            std::cout << dht.getRoutingTablesLog(AF_INET) << std::endl;
             std::cout << "IPv6 routing table:" << std::endl;
-            std::cout << dht->getRoutingTablesLog(AF_INET6) << std::endl;
+            std::cout << dht.getRoutingTablesLog(AF_INET6) << std::endl;
             continue;
         } else if (op == "ld") {
-            std::cout << dht->getStorageLog() << std::endl;
+            std::cout << dht.getStorageLog() << std::endl;
             continue;
         } else if (op == "ls") {
             std::cout << "Searches:" << std::endl;
-            std::cout << dht->getSearchesLog() << std::endl;
+            std::cout << dht.getSearchesLog() << std::endl;
             continue;
         } else if (op == "la")  {
             std::cout << "Reported public addresses:" << std::endl;
-            auto addrs = dht->getPublicAddressStr();
+            auto addrs = dht.getPublicAddressStr();
             for (const auto& addr : addrs)
                 std::cout << addr << std::endl;
             continue;
         } else if (op == "b") {
-            iss >> idstr;
             try {
                 auto addr = splitPort(idstr);
                 if (not addr.first.empty() and addr.second.empty()){
@@ -134,7 +127,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
                     ss << DHT_DEFAULT_PORT;
                     addr.second = ss.str();
                 }
-                dht->bootstrap(addr.first.c_str(), addr.second.c_str());
+                dht.bootstrap(addr.first.c_str(), addr.second.c_str());
             } catch (const std::exception& e) {
                 std::cerr << e.what() << std::endl;
             }
@@ -142,57 +135,31 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
         } else if (op == "log") {
             params.log = !params.log;
             if (params.log)
-                log::enableLogging(*dht);
+                log::enableLogging(dht);
             else
-                log::disableLogging(*dht);
+                log::disableLogging(dht);
             continue;
         }
 
         if (op.empty())
             continue;
 
-        static const std::set<std::string> VALID_OPS {"g", "l", "il", "ii", "p", "s", "e", "a"};
+        dht::InfoHash id {idstr};
+        static const std::set<std::string> VALID_OPS {"g", "l", "p", "s", "e", "a"};
         if (VALID_OPS.find(op) == VALID_OPS.cend()) {
             std::cout << "Unknown command: " << op << std::endl;
             std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl;
             continue;
         }
-
-        if (op == "il" or op == "ii") {
-            // Pht syntax
-            iss >> index >> keystr;
-            auto new_index = std::find_if(indexes.begin(), indexes.end(),
-                    [&](std::pair<const std::string, indexation::Pht>& i) {
-                        return i.first == index;
-                    }) == indexes.end();
-            if (not index.size()) {
-                std::cout << "You must enter the index name." << std::endl;
-                continue;
-            } else if (new_index) {
-                using namespace dht::indexation;
-                try {
-                    auto key = createPhtKey(parseStringMap(keystr));
-                    Pht::KeySpec ks;
-                    std::transform(key.begin(), key.end(), std::inserter(ks, ks.end()), [](Pht::Key::value_type& f) {
-                        return std::make_pair(f.first, f.second.size());
-                    });
-                    indexes.emplace(index, Pht {index, std::move(ks), dht});
-                } catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
-            }
-        }
-        else {
-            // Dht syntax
-            iss >> idstr;
-            InfoHash h {idstr};
-            if (not isInfoHash(h))
-                continue;
+        static constexpr dht::InfoHash INVALID_ID {};
+        if (id == INVALID_ID) {
+            std::cout << "Syntax error: invalid InfoHash." << std::endl;
+            continue;
         }
 
-        dht::InfoHash id {idstr};
-        // Dht
         auto start = std::chrono::high_resolution_clock::now();
         if (op == "g") {
-            dht->get(id, [start](std::shared_ptr<Value> value) {
+            dht.get(id, [start](std::shared_ptr<Value> value) {
                 auto now = std::chrono::high_resolution_clock::now();
                 std::cout << "Get: found value (after " << print_dt(now-start) << "s)" << std::endl;
                 std::cout << "\t" << *value << std::endl;
@@ -203,7 +170,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
             });
         }
         else if (op == "l") {
-            dht->listen(id, [](std::shared_ptr<Value> value) {
+            dht.listen(id, [](std::shared_ptr<Value> value) {
                 std::cout << "Listen: found value:" << std::endl;
                 std::cout << "\t" << *value << std::endl;
                 return true;
@@ -212,7 +179,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
         else if (op == "p") {
             std::string v;
             iss >> v;
-            dht->put(id, dht::Value {
+            dht.put(id, dht::Value {
                 dht::ValueType::USER_DATA.id,
                 std::vector<uint8_t> {v.begin(), v.end()}
             }, [start](bool ok) {
@@ -227,7 +194,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
             }
             std::string v;
             iss >> v;
-            dht->putSigned(id, dht::Value {
+            dht.putSigned(id, dht::Value {
                 dht::ValueType::USER_DATA.id,
                 std::vector<uint8_t> {v.begin(), v.end()}
             }, [start](bool ok) {
@@ -243,7 +210,7 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
             std::string tostr;
             std::string v;
             iss >> tostr >> v;
-            dht->putEncrypted(id, InfoHash(tostr), dht::Value {
+            dht.putEncrypted(id, InfoHash(tostr), dht::Value {
                 dht::ValueType::USER_DATA.id,
                 std::vector<uint8_t> {v.begin(), v.end()}
             }, [start](bool ok) {
@@ -254,56 +221,11 @@ void cmd_loop(std::shared_ptr<DhtRunner>& dht, std::map<std::string, indexation:
         else if (op == "a") {
             in_port_t port;
             iss >> port;
-            dht->put(id, dht::Value {dht::IpServiceAnnouncement::TYPE.id, dht::IpServiceAnnouncement(port)}, [start](bool ok) {
+            dht.put(id, dht::Value {dht::IpServiceAnnouncement::TYPE.id, dht::IpServiceAnnouncement(port)}, [start](bool ok) {
                 auto end = std::chrono::high_resolution_clock::now();
                 std::cout << "Announce: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl;
             });
         }
-        else if (op == "il") {
-            std::string exact_match;
-            iss >> exact_match;
-            try {
-                auto key = createPhtKey(parseStringMap(keystr));
-                indexes.at(index).lookup(key,
-                    [=](std::vector<std::shared_ptr<indexation::Value>>& vals, indexation::Prefix p) {
-                        if (vals.empty())
-                            return;
-                        std::cout << "Pht::lookup: found entries!" << std::endl
-                                  << p.toString() << std::endl
-                                  << "   hash: " << p.hash() << std::endl;
-                        std::cout << "   entries:" << std::endl;
-                        for (auto v : vals)
-                             std::cout << "      " << v->first.toString() << "[vid: " << v->second << "]" << std::endl;
-                    },
-                    [start](bool ok) {
-                        auto end = std::chrono::high_resolution_clock::now();
-                        std::cout << "Pht::lookup: " << (ok ? "done." : "failed.")
-                                  << " took " << print_dt(end-start) << "s)" << std::endl;
-
-                    }, exact_match.size() != 0 and exact_match == "false" ? false : true
-                );
-            }
-            catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
-            catch (std::out_of_range& e) { }
-        }
-        else if (op == "ii") {
-            iss >> idstr;
-            InfoHash h {idstr};
-            if (not isInfoHash(h))
-                continue;
-
-            indexation::Value v {h, 0};
-            try {
-                auto key = createPhtKey(parseStringMap(keystr));
-                indexes.at(index).insert(key, v,
-                    [=](bool ok) {
-                        std::cout << "Pht::insert: " << (ok ? "done." : "failed.") << std::endl;
-                    }
-                );
-            }
-            catch (std::invalid_argument& e) { std::cout << e.what() << std::endl; }
-            catch (std::out_of_range& e) { }
-        }
     }
 
     std::cout << std::endl <<  "Stopping node..." << std::endl;
@@ -315,9 +237,7 @@ main(int argc, char **argv)
     if (int rc = gnutls_global_init())  // TODO: remove with GnuTLS >= 3.3
         throw std::runtime_error(std::string("Error initializing GnuTLS: ")+gnutls_strerror(rc));
 
-    auto dht = std::make_shared<DhtRunner>();
-    std::map<std::string, indexation::Pht> indexes;
-
+    DhtRunner dht;
     try {
         auto params = parseArgs(argc, argv);
         if (params.help) {
@@ -334,25 +254,25 @@ main(int argc, char **argv)
             crt = dht::crypto::generateIdentity("DHT Node", ca_tmp);
         }
 
-        dht->run(params.port, crt, true, params.is_bootstrap_node);
+        dht.run(params.port, crt, true, params.network);
 
         if (params.log) {
             if (not params.logfile.empty())
-                log::enableFileLogging(*dht, params.logfile);
+                log::enableFileLogging(dht, params.logfile);
             else
-                log::enableLogging(*dht);
+                log::enableLogging(dht);
         }
 
         if (not params.bootstrap.first.empty()) {
             //std::cout << "Bootstrap: " << params.bootstrap.first << ":" << params.bootstrap.second << std::endl;
-            dht->bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
+            dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());
         }
 
         if (params.daemonize) {
             while (true)
                 std::this_thread::sleep_for(std::chrono::seconds(30));
         } else {
-            cmd_loop(dht, indexes, params);
+            cmd_loop(dht, params);
         }
 
     } catch(const std::exception&e) {
@@ -363,7 +283,7 @@ main(int argc, char **argv)
     std::mutex m;
     std::atomic_bool done {false};
 
-    dht->shutdown([&]()
+    dht.shutdown([&]()
     {
         std::lock_guard<std::mutex> lk(m);
         done = true;
@@ -374,7 +294,7 @@ main(int argc, char **argv)
     std::unique_lock<std::mutex> lk(m);
     cv.wait(lk, [&](){ return done.load(); });
 
-    dht->join();
+    dht.join();
     gnutls_global_deinit();
 
     return 0;

tools/dhtscanner.cpp

@@ -80,7 +80,7 @@ main(int argc, char **argv)
     auto crt_tmp = dht::crypto::generateIdentity("Scanner node", ca_tmp);
 
     DhtRunner dht;
-    dht.run(params.port, crt_tmp, true, [](dht::NodeStatus /* ipv4 */, dht::NodeStatus /* ipv6 */) {});
+    dht.run(params.port, crt_tmp, true, params.network);
 
     if (not params.bootstrap.first.empty())
         dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str());

tools/tools_common.h

@@ -135,47 +135,6 @@ splitPort(const std::string& s) {
     return {s.substr(0,found), s.substr(found+1)};
 }
 
-/*
- * The mapString shall have the following format:
- *
- *      k1:v1[,k2:v2[,...]]
- */
-std::map<std::string, std::string> parseStringMap(std::string mapString) {
-    std::istringstream keySs(mapString);
-    std::string mapStr;
-    std::map<std::string, std::string> map;
-
-    while (std::getline(keySs, mapStr, ',')) {
-        std::istringstream mapSs(mapStr);
-        std::string key, value;
-
-        while (std::getline(mapSs, key, ':')) {
-            std::getline(mapSs, value, ':');
-            map[key] = { value };
-        }
-    }
-    return map;
-}
-
-dht::indexation::Pht::Key createPhtKey(std::map<std::string, std::string> pht_key_str_map) {
-    dht::indexation::Pht::Key pht_key;
-    for (auto f : pht_key_str_map) {
-        dht::Blob prefix {f.second.begin(), f.second.end()};
-        pht_key.emplace(f.first, std::move(prefix));
-    }
-    return pht_key;
-}
-
-bool isInfoHash(dht::InfoHash& h) {
-    static constexpr dht::InfoHash INVALID_ID {};
-
-    if (h == INVALID_ID) {
-        std::cout << "Syntax error: invalid InfoHash." << std::endl;
-        return false;
-    }
-    return true;
-}
-
 static const constexpr in_port_t DHT_DEFAULT_PORT = 4222;
 
 struct dht_params {
@@ -183,27 +142,29 @@ struct dht_params {
     bool log {false};
     std::string logfile {};
     in_port_t port {0};
-    bool is_bootstrap_node {false};
+    dht::NetId network {0};
     bool generate_identity {false};
     bool daemonize {false};
     std::pair<std::string, std::string> bootstrap {};
 };
 
 static const constexpr struct option long_options[] = {
-   {"help",       no_argument,       nullptr, 'h'},
+   {"help",       no_argument      , nullptr, 'h'},
    {"port",       required_argument, nullptr, 'p'},
-   {"bootstrap",  optional_argument, nullptr, 'b'},
+   {"net",        required_argument, nullptr, 'n'},
+   {"bootstrap",  required_argument, nullptr, 'b'},
    {"identity",   no_argument      , nullptr, 'i'},
    {"verbose",    no_argument      , nullptr, 'v'},
    {"daemonize",  no_argument      , nullptr, 'd'},
-   {nullptr,      0,                 nullptr,  0}
+   {"logfile",    required_argument, nullptr, 'l'},
+   {nullptr,      0                , nullptr,  0}
 };
 
 dht_params
 parseArgs(int argc, char **argv) {
     dht_params params;
     int opt;
-    while ((opt = getopt_long(argc, argv, ":hidv:p:b:", long_options, nullptr)) != -1) {
+    while ((opt = getopt_long(argc, argv, "hidvp:n:b:l:", long_options, nullptr)) != -1) {
         switch (opt) {
         case 'p': {
                 int port_arg = atoi(optarg);
@@ -213,24 +174,24 @@ parseArgs(int argc, char **argv) {
                     std::cout << "Invalid port: " << port_arg << std::endl;
             }
             break;
+        case 'n':
+            params.network = strtoul(optarg, nullptr, 0);
+            break;
         case 'b':
-            if (optarg) {
-                params.bootstrap = splitPort((optarg[0] == '=') ? optarg+1 : optarg);
-                if (not params.bootstrap.first.empty() and params.bootstrap.second.empty()) {
-                    std::stringstream ss;
-                    ss << DHT_DEFAULT_PORT;
-                    params.bootstrap.second = ss.str();
-                }
+            params.bootstrap = splitPort((optarg[0] == '=') ? optarg+1 : optarg);
+            if (not params.bootstrap.first.empty() and params.bootstrap.second.empty()) {
+                std::stringstream ss;
+                ss << DHT_DEFAULT_PORT;
+                params.bootstrap.second = ss.str();
             }
-            else
-                params.is_bootstrap_node = true;
             break;
         case 'h':
             params.help = true;
             break;
+        case 'l':
+            params.logfile = optarg;
+            break;
         case 'v':
-            if (optarg)
-                params.logfile = optarg;
             params.log = true;
             break;
         case 'i':
@@ -239,19 +200,6 @@ parseArgs(int argc, char **argv) {
         case 'd':
             params.daemonize = true;
             break;
-        case ':':
-            switch (optopt) {
-            case 'b':
-                params.is_bootstrap_node = true;
-                break;
-            case 'v':
-                params.log = true;
-                break;
-            default:
-                std::cout << "option requires an argument -- '" << optopt << '\'' << std::endl;
-                break;
-            }
-            break;
         default:
             break;
         }