move python to top directory

python/Makefile.am

@@ -0,0 +1,13 @@
+if USE_CYTHON
+
+pybuild.stamp:
+	LDFLAGS="-L$(top_srcdir)/src/.libs" $(PYTHON) setup.py build_ext --inplace
+	echo stamp > pybuild.stamp
+
+CLEANFILES = pybuild.stamp
+
+all-local: pybuild.stamp
+clean-local:
+	rm -rf $(builddir)/build $(builddir)/*.so
+
+endif

python/opendht.pyx

@@ -0,0 +1,316 @@
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
+# distutils: include_dirs = ../../include
+# distutils: library_dirs = ../../src
+# distutils: libraries = opendht gnutls
+# cython: language_level=3
+#
+# Copyright (c) 2015 Savoir-Faire Linux Inc. 
+# Author: Guillaume Roguez <guillaume.roguez@savoirfairelinux.com>
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+#
+# This wrapper is written for Cython 0.22
+# 
+# This file is part of OpenDHT Python Wrapper.
+#
+#    OpenDHT Python Wrapper is free software:  you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    OpenDHT Python Wrapper is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with OpenDHT Python Wrapper. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from libc.stdint cimport *
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.map cimport map as map
+from libcpp cimport bool
+from libcpp.utility cimport pair
+
+from cython.parallel import parallel, prange
+from cython.operator cimport dereference as deref, preincrement as inc, predecrement as dec
+from cpython cimport ref
+
+ctypedef uint16_t in_port_t
+ctypedef unsigned short int sa_family_t;
+
+cdef extern from "<memory>" namespace "std" nogil:
+    cdef cppclass shared_ptr[T]:
+        shared_ptr() except +
+        T* get()
+        T operator*()
+        void reset(T*);
+
+cdef extern from "opendht/infohash.h" namespace "dht":
+    cdef cppclass InfoHash:
+        InfoHash() except +
+        InfoHash(string s) except +
+        string toString() const
+        bool getBit(unsigned bit) const
+        void setBit(unsigned bit, bool b)
+        double toFloat() const
+        @staticmethod
+        unsigned commonBits(InfoHash a, InfoHash b)
+        @staticmethod
+        InfoHash get(string s)
+
+cdef extern from "opendht/value.h" namespace "dht":
+    cdef cppclass Value:
+        Value() except +
+        Value(vector[uint8_t]) except +
+        Value(const uint8_t* dat_ptr, size_t dat_len) except +
+        string toString() const
+
+cdef extern from "opendht/dht.h" namespace "dht":
+    cdef cppclass Node:
+        Node() except +
+        InfoHash getId() const
+        string getAddrStr() const
+        bool isExpired() const
+    ctypedef bool (*GetCallbackRaw)(shared_ptr[Value] values, void *user_data)
+    ctypedef void (*DoneCallbackRaw)(bool done, vector[shared_ptr[Node]]* nodes, void *user_data)
+    cdef cppclass Dht:
+        cppclass GetCallback:
+            GetCallback() except +
+            #GetCallback(GetCallbackRaw cb, void *user_data) except +
+        cppclass DoneCallback:
+            DoneCallback() except +
+            #DoneCallback(DoneCallbackRaw, void *user_data) except +
+        Dht() except +
+        InfoHash getNodeId() const
+        @staticmethod
+        GetCallback bindGetCb(GetCallbackRaw cb, void *user_data)
+        @staticmethod
+        DoneCallback bindDoneCb(DoneCallbackRaw cb, void *user_data)
+
+cdef extern from "opendht/crypto.h" namespace "dht::crypto":
+    ctypedef pair[shared_ptr[PrivateKey], shared_ptr[Certificate]] Identity
+    cdef Identity generateIdentity(string name, Identity ca, unsigned bits)
+
+    cdef cppclass PrivateKey:
+        PrivateKey()
+        PublicKey getPublicKey() const
+
+    cdef cppclass PublicKey:
+        PublicKey()
+        InfoHash getId() const
+
+    cdef cppclass Certificate:
+        Certificate()
+        InfoHash getId() const
+
+cdef class _WithID(object):
+    def __repr__(self):
+        return "<%s '%s'>" % (self.__class__.__name__, str(self))
+    def __str__(self):
+        return self.getId().toString().decode()
+
+cdef class PyInfoHash(_WithID):
+    cdef InfoHash _infohash
+    def __init__(self, bytes str=b''):
+        self._infohash = InfoHash(str)
+    def getBit(self, bit):
+        return self._infohash.getBit(bit)
+    def setBit(self, bit, b):
+        self._infohash.setBit(bit, b)
+    def getId(self):
+        return self
+    def toString(self):
+        return self._infohash.toString()
+    def toFloat(self):
+        return self._infohash.toFloat()
+    @staticmethod
+    def commonBits(PyInfoHash a, PyInfoHash b):
+        return InfoHash.commonBits(a._infohash, b._infohash)
+    @staticmethod
+    def get(str key):
+        h = PyInfoHash()
+        h._infohash = InfoHash.get(key.encode())
+        return h
+
+cdef class PyNode(_WithID):
+    cdef shared_ptr[Node] _node
+    def getId(self):
+        h = PyInfoHash()
+        h._infohash = self._node.get().getId()
+        return h
+    def getAddr(self):
+        return self._node.get().getAddrStr()
+    def isExpired(self):
+        return self._node.get().isExpired()
+
+cdef class PyNodeEntry(_WithID):
+    cdef pair[InfoHash, shared_ptr[Node]] _v
+    def getId(self):
+        h = PyInfoHash()
+        h._infohash = self._v.first
+        return h
+    def getNode(self):
+        n = PyNode()
+        n._node = self._v.second
+        return n
+
+cdef class PyValue(object):
+    cdef shared_ptr[Value] _value
+    def __init__(self, bytes val=b''):
+        self._value.reset(new Value(val, len(val)))
+    def __str__(self):
+        return self._value.get().toString().decode()
+
+cdef class PyNodeSetIter(object):
+    cdef map[InfoHash, shared_ptr[Node]]* _nodes
+    cdef map[InfoHash, shared_ptr[Node]].iterator _curIter
+    def __init__(self, PyNodeSet s):
+        self._nodes = &s._nodes
+        self._curIter = self._nodes.begin()
+    def __next__(self):
+        if self._curIter == self._nodes.end():
+            raise StopIteration
+        h = PyNodeEntry()
+        h._v = deref(self._curIter)
+        inc(self._curIter)
+        return h
+
+cdef class PyNodeSet(object):
+    cdef map[InfoHash, shared_ptr[Node]] _nodes
+    def size(self):
+        return self._nodes.size()
+    def insert(self, PyNodeEntry l):
+        self._nodes.insert(l._v)
+    def extend(self, li):
+        for n in li:
+            self.insert(n)
+    def first(self):
+        if self._nodes.empty():
+            raise IndexError()
+        h = PyInfoHash()
+        h._infohash = deref(self._nodes.begin()).first
+        return h
+    def last(self):
+        if self._nodes.empty():
+            raise IndexError()
+        h = PyInfoHash()
+        h._infohash = deref(dec(self._nodes.end())).first
+        return h
+    def __str__(self):
+        s = ''
+        cdef map[InfoHash, shared_ptr[Node]].iterator it = self._nodes.begin()
+        while it != self._nodes.end():
+            s += deref(it).first.toString().decode() + ' ' + deref(it).second.get().getAddrStr().decode() + '\n'
+            inc(it)
+        return s
+    def __iter__(self):
+        return PyNodeSetIter(self)
+
+cdef class PyPublicKey(_WithID):
+    cdef PublicKey _key
+    def getId(self):
+        h = PyInfoHash()
+        h._infohash = self._key.getId()
+        return h
+
+cdef class PySharedCertificate(_WithID):
+    cdef shared_ptr[Certificate] _cert
+    def getId(self):
+        h = PyInfoHash()
+        h._infohash = self._cert.get().getId()
+        return h
+
+cdef class PyIdentity(object):
+    cdef Identity _id;
+    def generate(self, str name = "pydht", PyIdentity ca = PyIdentity(), unsigned bits = 4096):
+        self._id = generateIdentity(name.encode(), ca._id, bits)
+    property PublicKey:
+        def __get__(self):
+            k = PyPublicKey()
+            k._key = self._id.first.get().getPublicKey()
+            return k
+    property Certificate:
+        def __get__(self):
+            c = PySharedCertificate()
+            c._cert = self._id.second
+            return c
+
+cdef extern from "opendht/dhtrunner.h" namespace "dht":
+    cdef cppclass DhtRunner:
+        DhtRunner() except +
+        InfoHash getId() const
+        InfoHash getNodeId() const
+        void bootstrap(const char*, const char*)
+        void run(in_port_t, const Identity, bool)
+        #void run(const sockaddr_in*, const sockaddr_in6*, const Identity, bool)
+        void run(const char*, const char*, const char*, const Identity, bool)
+        void join()
+        bool isRunning()
+        string getStorageLog() const
+        string getRoutingTablesLog(sa_family_t af) const
+        string getSearchesLog(sa_family_t af) const
+        void get(InfoHash key, Dht.GetCallback get_cb, Dht.DoneCallback done_cb)
+        void put(InfoHash key, shared_ptr[Value] val, Dht.DoneCallback done_cb)
+
+cdef bool py_get_callback(shared_ptr[Value] value, void *user_data) with gil:
+    cb = (<object>user_data)['get']
+    pv = PyValue()
+    pv._value = value
+    return cb(pv)
+
+cdef void py_done_callback(bool done, vector[shared_ptr[Node]]* nodes, void *user_data) with gil:
+    node_ids = []
+    for n in deref(nodes):
+        h = PyNodeEntry()
+        h._v.first = n.get().getId()
+        h._v.second = n
+        node_ids.append(h)
+    (<object>user_data)['done'](done, node_ids)
+    ref.Py_DECREF(<object>user_data)
+
+cdef class PyDhtRunner(_WithID):
+    cdef DhtRunner* thisptr;
+    def __cinit__(self):
+        self.thisptr = new DhtRunner()
+    def getId(self):
+        h = PyInfoHash()
+        h._infohash = self.thisptr.getId()
+        return h
+    def getNodeId(self):
+        return self.thisptr.getNodeId().toString()
+    def bootstrap(self, str host, str port):
+        self.thisptr.bootstrap(host.encode(), port.encode())
+    def run(self, PyIdentity id, bool threaded=True, in_port_t port=0, str ipv4="", str ipv6=""):
+        if ipv4 or ipv6:
+            self.thisptr.run(ipv4.encode(), ipv6.encode(), str(port).encode(), id._id, threaded)
+        else:
+            self.thisptr.run(port, id._id, threaded)
+    def join(self):
+        self.thisptr.join()
+    def isRunning(self):
+        return self.thisptr.isRunning()
+    def getStorageLog(self):
+        return self.thisptr.getStorageLog().decode()
+    def getRoutingTablesLog(self, sa_family_t af):
+        return self.thisptr.getRoutingTablesLog(af).decode()
+    def getSearchesLog(self, sa_family_t af):
+        return self.thisptr.getSearchesLog(af).decode()
+    def get(self, PyInfoHash key, get_cb, done_cb):
+        cb_obj = {'get':get_cb, 'done':done_cb}
+        ref.Py_INCREF(cb_obj)
+        self.thisptr.get(key._infohash, Dht.bindGetCb(py_get_callback, <void*>cb_obj), Dht.bindDoneCb(py_done_callback, <void*>cb_obj))
+    def get(self, str key, get_cb, done_cb):
+        cb_obj = {'get':get_cb, 'done':done_cb}
+        ref.Py_INCREF(cb_obj)
+        self.thisptr.get(InfoHash.get(key.encode()), Dht.bindGetCb(py_get_callback, <void*>cb_obj), Dht.bindDoneCb(py_done_callback, <void*>cb_obj))
+    def put(self, PyInfoHash key, PyValue val, done_cb):
+        cb_obj = {'done':done_cb}
+        ref.Py_INCREF(cb_obj)
+        self.thisptr.put(key._infohash, val._value, Dht.bindDoneCb(py_done_callback, <void*>cb_obj))
+    def put(self, str key, PyValue val, done_cb):
+        cb_obj = {'done':done_cb}
+        ref.Py_INCREF(cb_obj)
+        self.thisptr.put(InfoHash.get(key.encode()), val._value, Dht.bindDoneCb(py_done_callback, <void*>cb_obj))

python/setup.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2015 Savoir-Faire Linux Inc. 
+# Author: Guillaume Roguez <guillaume.roguez@savoirfairelinux.com>
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+#
+# A Python3 wrapper to access to OpenDHT API
+# This wrapper is written for Cython 0.22
+#
+# This file is part of OpenDHT Python Wrapper.
+#
+#    OpenDHT Python Wrapper is free software:  you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    OpenDHT Python Wrapper is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with OpenDHT Python Wrapper. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from distutils.core import setup, Extension
+from Cython.Build import cythonize
+from Cython.Distutils import build_ext
+
+setup(name="opendht",
+      version="0.1",
+      description="Cython generated wrapper for opendht",
+      author="Guillaume Roguez",
+      license="GPLv3",
+      cmdclass = { 'build_ext' : build_ext },
+      ext_modules = cythonize(Extension(
+          "opendht",
+          ["opendht.pyx"],
+          language="c++",
+          extra_compile_args=["-std=c++11"],
+          extra_link_args=["-std=c++11"],
+          libraries=["opendht"]
+      ))
+)

python/tools/benchmark.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+# Copyright (C) 2015 Savoir-Faire Linux Inc.
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
+import sys, subprocess, argparse, time, random, string, threading, signal
+from pyroute2.netns.process.proxy import NSPopen
+import numpy as np
+import matplotlib.pyplot as plt
+from dhtnetwork import DhtNetwork
+
+sys.path.append('..')
+from opendht import *
+
+def random_hash():
+    return PyInfoHash(''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(40)).encode())
+
+parser = argparse.ArgumentParser(description='Create a dummy network interface for testing')
+parser.add_argument('-i', '--ifname', help='interface name', default='ethdht')
+parser.add_argument('-n', '--node-num', help='number of dht nodes to run', type=int, default=32)
+parser.add_argument('-v', '--virtual-locs', help='number of virtual locations (node clusters)', type=int, default=8)
+parser.add_argument('-l', '--loss', help='simulated cluster packet loss (percent)', type=int, default=0)
+parser.add_argument('-d', '--delay', help='simulated cluster latency (ms)', type=int, default=0)
+parser.add_argument('-no4', '--disable-ipv4', help='Enable IPv4', action="store_true")
+parser.add_argument('-no6', '--disable-ipv6', help='Enable IPv6', action="store_true")
+
+args = parser.parse_args()
+
+clusters = min(args.virtual_locs, args.node_num)
+node_per_loc = int(args.node_num / clusters)
+
+print("Launching", args.node_num, "nodes (", clusters, "clusters of", node_per_loc, "nodes)")
+
+if args.virtual_locs > 1:
+    cmd = ["/usr/bin/sudo", "python3", "dummy_if.py", "-i", args.ifname, "-n", str(clusters), '-l', str(args.loss), '-d', str(args.delay)]
+    if not args.disable_ipv4:
+        cmd.append('-4')
+    if not args.disable_ipv6:
+        cmd.append('-6')
+    print(cmd)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    output, err = p.communicate()
+    print(output.decode())
+
+bootstrap = DhtNetwork(iface='br'+args.ifname)
+bootstrap.resize(1)
+
+procs = [None for _ in range(clusters)]
+
+def start_cluster(i):
+    global procs
+    cmd = ["python3", "dhtnetwork.py", "-n", str(node_per_loc), '-I', args.ifname+str(i)+'.1']
+    if not args.disable_ipv4 and bootstrap.ip4:
+        cmd.extend(['-b', bootstrap.ip4])
+    if not args.disable_ipv6 and bootstrap.ip6:
+        cmd.extend(['-b6', bootstrap.ip6])
+    procs[i] = NSPopen('node'+str(i), cmd)
+    plt.pause(2)
+
+def stop_cluster(i):
+    global procs
+    if procs[i]:
+        try:
+            procs[i].send_signal(signal.SIGINT);
+            procs[i].wait()
+            procs[i].release()
+        except Exception as e:
+            print(e)
+        procs[i] = None
+
+def replace_cluster():
+    n = random.randrange(0, clusters)
+    stop_cluster(n)
+    start_cluster(n)
+
+try:
+    for i in range(clusters):
+        start_cluster(i)
+
+    plt.ion()
+
+    lines = plt.plot([])
+    plt.ylabel('time (s)')
+    #plt.show()
+
+    # let the network stabilise
+    plt.pause(5)
+
+    start = time.time()
+    times = []
+    done = 0
+
+    lock = threading.Condition()
+
+    def getcb(v):
+        print("found", v)
+
+    def donecb(ok, nodes):
+        global lock, done, times
+        t = time.time()-start
+        with lock:
+            if not ok:
+                print("failed !")
+            times.append(t)
+            done -= 1
+            lock.notify()
+
+    def update_plot():
+        global lines
+        while lines:
+            l = lines.pop()
+            l.remove()
+            del l
+        lines = plt.plot(times, color='blue')
+        plt.draw()
+
+    plt.pause(5)
+
+    plt.show()
+    update_plot()
+
+    times = []
+    for n in range(10):
+        #replace_cluster()
+        plt.pause(2)
+        print("Getting 50 random hashes succesively.")
+        for i in range(50):
+            with lock:
+                done += 1
+                start = time.time()
+                bootstrap.front().get(random_hash(), getcb, donecb)
+                while done > 0:
+                    lock.wait()
+            update_plot()
+        print("Took", np.sum(times), "mean", np.mean(times), "std", np.std(times), "min", np.min(times), "max", np.max(times))
+
+    plt.ioff()
+    plt.show()
+except Exception as e:
+    print(e)
+finally:
+    for i in range(clusters):
+        if procs[i]:
+            procs[i].send_signal(signal.SIGINT);
+    bootstrap.resize(0)
+    subprocess.call(["/usr/bin/sudo", "python3", "dummy_if.py", "-i", args.ifname, "-n", str(clusters), "-r"])
+    for i in range(clusters):
+        if procs[i]:
+            try:
+                procs[i].wait()
+                procs[i].release()
+            except Exception as e:
+                print(e)

python/tools/dhtnetwork.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# Copyright (C) 2015 Savoir-Faire Linux Inc.
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
+import signal, os, sys, ipaddress, random
+from pyroute2 import IPDB
+
+sys.path.append('..')
+from opendht import *
+
+class DhtNetwork(object):
+    nodes = []
+
+    @staticmethod
+    def run_node(ip4, ip6, p, bootstrap=[]):
+        #print("run_node", ip4, ip6, p, bootstrap)
+        id = PyIdentity()
+        id.generate("dhtbench"+str(p), PyIdentity(), 1024)
+        n = PyDhtRunner()
+        n.run(id, ipv4=ip4 if ip4 else "", ipv6=ip6 if ip6 else "", port=p)
+        for b in bootstrap:
+            n.bootstrap(b[0], b[1])
+        #plt.pause(0.02)
+        return ((ip4, ip6, p), n, id)
+
+    @staticmethod
+    def find_ip(iface):
+        if not iface or iface == 'any':
+            return ('0.0.0.0','')
+        if_ip4 = None
+        if_ip6 = None
+        ipdb = IPDB()
+        try:
+            for ip in ipdb.interfaces[iface].ipaddr:
+                if_ip = ipaddress.ip_address(ip[0])
+                if isinstance(if_ip, ipaddress.IPv4Address):
+                    if_ip4 = ip[0]
+                elif isinstance(if_ip, ipaddress.IPv6Address):
+                    if not if_ip.is_link_local:
+                        if_ip6 = ip[0]
+                if if_ip4 and if_ip6:
+                    break
+        except Exception as e:
+            pass
+        finally:
+            ipdb.release()
+        return (if_ip4, if_ip6)
+
+    def __init__(self, iface=None, ip4=None, ip6=None, port=4000, bootstrap=[]):
+        self.port = port
+        ips = DhtNetwork.find_ip(iface)
+        self.ip4 = ip4 if ip4 else ips[0]
+        self.ip6 = ip6 if ip6 else ips[1]
+        self.bootstrap = bootstrap
+        #print(self.ip4, self.ip6, self.port)
+
+    def front(self):
+        if len(self.nodes) == 0:
+            return None
+        return self.nodes[0][1]
+
+    def launch_node(self):
+        n = DhtNetwork.run_node(self.ip4, self.ip6, self.port, self.bootstrap)
+        self.nodes.append(n)
+        self.port += 1
+        return n
+
+    def end_node(self):
+        if not self.nodes:
+            return
+        n = self.nodes.pop()
+        n[1].join()
+
+    def replace_node(self):
+        random.shuffle(self.nodes)
+        self.end_node()
+        self.launch_node()
+
+    def resize(self, n):
+        n = min(n, 500)
+        l = len(self.nodes)
+        if n == l:
+            return
+        if n > l:
+            print("Launching", n-l, "nodes")
+            for i in range(l, n):
+                self.launch_node()
+        else:
+            print("Ending", l-n, "nodes")
+            random.shuffle(self.nodes)
+            for i in range(n, l):
+                self.end_node()
+
+if __name__ == '__main__':
+    import argparse, threading
+
+    lock = threading.Condition()
+    quit = False
+
+    def handler(signum, frame):
+        global quit
+        with lock:
+            quit = True
+            lock.notify()
+
+    signal.signal(signal.SIGALRM, handler)
+    signal.signal(signal.SIGABRT, handler)
+    signal.signal(signal.SIGINT, handler)
+    signal.signal(signal.SIGTERM, handler)
+
+    net = None
+    try:
+        parser = argparse.ArgumentParser(description='Create a dht network of -n nodes')
+        parser.add_argument('-n', '--node-num', help='number of dht nodes to run', type=int, default=32)
+        parser.add_argument('-I', '--iface', help='local interface to bind', default='any')
+        parser.add_argument('-p', '--port', help='start of port range (port, port+node_num)', type=int, default=4000)
+        parser.add_argument('-b', '--bootstrap', help='bootstrap address')
+        parser.add_argument('-b6', '--bootstrap6', help='bootstrap address (IPv6)')
+        args = parser.parse_args()
+
+        bs = []
+        if args.bootstrap:
+            bs.append((args.bootstrap, str(4000)))
+        if args.bootstrap6:
+            bs.append((args.bootstrap6, str(4000)))
+
+        net = DhtNetwork(iface=args.iface, port=args.port, bootstrap=bs)
+        net.resize(args.node_num)
+
+        with lock:
+            while not quit:
+                lock.wait()
+    except Exception as e:
+        pass
+    finally:
+        if net:
+            net.resize(0)

python/tools/dummy_if.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# Copyright (C) 2015 Savoir-Faire Linux Inc.
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+
+import argparse, subprocess
+
+from pyroute2 import IPDB, NetNS
+from pyroute2.netns.process.proxy import NSPopen
+
+parser = argparse.ArgumentParser(description='Create a dummy network interface for testing')
+parser.add_argument('-i', '--ifname', help='interface name', default='ethdht')
+parser.add_argument('-n', '--ifnum', type=int, help='interface number', default=1)
+parser.add_argument('-r', '--remove', help='remove instead of adding the interface', action="store_true")
+parser.add_argument('-l', '--loss', help='simulated packet loss (percent)', type=int, default=0)
+parser.add_argument('-d', '--delay', help='simulated latency (ms)', type=int, default=0)
+parser.add_argument('-4', '--ipv4', help='Enable IPv4', action="store_true")
+parser.add_argument('-6', '--ipv6', help='Enable IPv6', action="store_true")
+
+args = parser.parse_args()
+
+local_addr4 = '10.0.42.'
+local_addr6 = '2001:db9::'
+brige_name = 'br'+args.ifname
+
+ip = None
+try:
+    ip = IPDB()
+    if args.remove:
+        for ifn in range(args.ifnum):
+            iface = args.ifname+str(ifn)
+            if iface in ip.interfaces:
+                with ip.interfaces[iface] as i:
+                    i.remove()
+        if 'tap'+args.ifname in ip.interfaces:
+            with ip.interfaces['tap'+args.ifname] as i:
+                i.remove()
+        if brige_name in ip.interfaces:
+            with ip.interfaces[brige_name] as i:
+                i.remove()
+        for ifn in range(args.ifnum):
+            netns = NetNS('node'+str(ifn))
+            netns.close()
+            netns.remove()
+    else:
+        for ifn in range(args.ifnum):
+            iface = args.ifname+str(ifn)
+            if not iface in ip.interfaces:
+                ip.create(kind='veth', ifname=iface, peer=iface+'.1').commit()
+
+        ip.create(kind='tuntap', ifname='tap'+args.ifname, mode='tap').commit()
+
+        with ip.create(kind='bridge', ifname=brige_name) as i:
+            for ifn in range(args.ifnum):
+                iface = args.ifname+str(ifn)
+                i.add_port(ip.interfaces[iface])
+            i.add_port(ip.interfaces['tap'+args.ifname])
+            if args.ipv4:
+                i.add_ip(local_addr4+'1/24')
+            if args.ipv6:
+                i.add_ip(local_addr6+'1/64')
+            i.up()
+
+        with ip.interfaces['tap'+args.ifname] as tap:
+            tap.up()
+
+        for ifn in range(args.ifnum):
+            iface = args.ifname+str(ifn)
+
+            nns = NetNS('node'+str(ifn))
+            iface1 = iface+'.1'
+            with ip.interfaces[iface1] as i:
+                i.net_ns_fd = nns.netns
+
+            with ip.interfaces[iface] as i:
+                i.up()
+
+            ip_ns = IPDB(nl=nns)
+            try:
+                with ip_ns.interfaces.lo as lo:
+                    lo.up()
+                with ip_ns.interfaces[iface1] as i:
+                    if args.ipv4:
+                        i.add_ip(local_addr4+str(ifn+8)+'/24')
+                    if args.ipv6:
+                        i.add_ip(local_addr6+str(ifn+8)+'/64')
+                    i.up()
+            finally:
+                ip_ns.release()
+
+            nsp = NSPopen(nns.netns, ["tc", "qdisc", "add", "dev", iface1, "root", "netem", "delay", str(args.delay)+"ms", str(int(args.delay/2))+"ms", "loss", str(args.loss)+"%", "25%"], stdout=subprocess.PIPE)
+            #print(nsp.communicate()[0].decode())
+            nsp.communicate()
+            nsp.wait()
+            nsp.release()
+
+        if args.ipv4:
+            subprocess.call(["sysctl", "-w", "net.ipv4.conf."+brige_name+".forwarding=1"])
+        if args.ipv6:
+            subprocess.call(["sysctl", "-w", "net.ipv6.conf."+brige_name+".forwarding=1"])
+
+except Exception as e:
+      print('Error',e)
+finally:
+    if ip:
+        ip.release()

python/tools/scanner.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015 Savoir-Faire Linux Inc.
+# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERWISE
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import time, sys
+from pprint import pprint
+from math import cos, sin, pi
+
+sys.path.append('..')
+from opendht import *
+
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+from matplotlib.colors import colorConverter
+from matplotlib.collections import RegularPolyCollection
+from matplotlib.widgets import Button
+from mpl_toolkits.basemap import Basemap
+
+import GeoIP
+
+done = 0
+all_nodes = PyNodeSet()
+
+plt.ion()
+plt.figaspect(2.)
+
+fig, axes = plt.subplots(2, 1)
+fig.set_size_inches(12,16,forward=True)
+fig.tight_layout()
+fig.canvas.set_window_title('OpenDHT scanner')
+
+mpx = axes[0]
+mpx.set_title("Node GeoIP")
+
+m = Basemap(projection='robin', resolution = 'l', area_thresh = 1000.0, lat_0=0, lon_0=0, ax=mpx)
+m.fillcontinents(color='#cccccc',lake_color='white')
+m.drawparallels(np.arange(-90.,120.,30.))
+m.drawmeridians(np.arange(0.,420.,60.))
+m.drawmapboundary(fill_color='white')
+plt.show()
+
+ringx = axes[1]
+ringx.set_title("Node IDs")
+ringx.set_autoscale_on(False)
+ringx.set_aspect('equal', 'datalim')
+ringx.set_xlim(-2.,2.)
+ringx.set_ylim(-1.5,1.5)
+
+exitax = plt.axes([0.92, 0.95, 0.07, 0.04])
+exitbtn = Button(exitax, 'Exit')
+reloadax = plt.axes([0.92, 0.90, 0.07, 0.04])
+button = Button(reloadax, 'Reload')
+
+gi = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
+gi6 = GeoIP.open("GeoLiteCityv6.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
+
+def gcb(v):
+    return True
+
+r = PyDhtRunner()
+i = PyIdentity()
+i.generate(bits = 1024)
+
+r.run(i, port=4112)
+r.bootstrap("bootstrap.ring.cx", "4222")
+
+all_lines = []
+
+plt.pause(2)
+
+def step(cur_h, cur_depth):
+    global done, all_nodes, all_lines
+    done += 1
+    a = 2.*pi*cur_h.toFloat()
+    b = a + 2.*pi/(2**(cur_depth))
+    print("step", cur_h, cur_depth)
+    arc = ringx.add_patch(mpatches.Wedge([0.,0,], 1., a*180/pi, b*180/pi, fill=True, color="blue", alpha=0.5))
+    lines = ringx.plot([0, cos(a)], [0, sin(a)], 'k-', lw=1.2)
+    all_lines.extend(lines)
+    r.get(cur_h, gcb, lambda d, nodes: nextstep(cur_h, cur_depth, d, nodes, arc=arc, lines=lines))
+
+def nextstep(cur_h, cur_depth, ok, nodes, arc=None, lines=[]):
+    global done, all_nodes
+    if arc:
+        arc.remove()
+        del arc
+    for l in lines:
+        l.set_color('#444444')
+    snodes = PyNodeSet()
+    snodes.extend(nodes)
+    all_nodes.extend(nodes)
+    depth = min(6, PyInfoHash.commonBits(snodes.first(), snodes.last())+4)
+    if cur_depth < depth:
+        for b in range(cur_depth, depth):
+            new_h = PyInfoHash(cur_h.toString());
+            new_h.setBit(b, 1);
+            step(new_h, b+1);
+    done -= 1
+
+run = True
+def exitcb(arg):
+    global run
+    run = False
+exitbtn.on_clicked(exitcb)
+
+def restart(arg):
+    global collection, all_lines, points
+    for l in all_lines:
+        l.remove()
+        del l
+    all_lines = []
+    if collection:
+        collection.remove()
+        del collection
+        collection = None
+    for p in points:
+        p.remove()
+        del p
+    points = []
+
+    print(arg)
+    start_h = PyInfoHash()
+    start_h.setBit(159, 1)
+    step(start_h, 0)
+    plt.draw()
+
+collection = None
+points = []
+not_found = []
+
+def generate_set():
+    node_ipv4 = {}
+    node_ipv6 = {}
+    for n in all_nodes:
+        addr = b''.join(n.getNode().getAddr().split(b':')[0:-1]).decode()
+        if addr[0] == '[':
+            addr = addr[1:-1]
+            if addr in node_ipv6:
+                node_ipv6[addr][1] = 1
+            else:
+                node_ipv6[addr] = [n, 1]
+        else:
+            if addr in node_ipv4:
+                node_ipv4[addr][1] += 1
+            else:
+                node_ipv4[addr] = [n, 1]
+    return node_ipv4, node_ipv6
+
+def update_plot():
+    global done, m, collection, not_found, points
+    for p in points:
+        p.remove()
+        del p
+    points = []
+    lats = []
+    lons = []
+    cities=[]
+    colors=[]
+    not_found.clear()
+    ip4s, ip6s = generate_set()
+    ares = []
+    for addr, n in ip4s.items():
+        ares.append((addr, n[0].getNode(), gi.record_by_name(addr)))
+    for addr, n in ip6s.items():
+        ares.append((addr, n[0].getNode(), gi6.record_by_name_v6(addr)))
+    for r in ares:
+        res = r[2]
+        n = r[1]
+        if res:
+            lats.append(res['latitude'])
+            lons.append(res['longitude'])
+            cities.append(res['city'] if res['city'] else (str(int(res['latitude']))+'-'+str(int(res['longitude']))))
+            colors.append('red' if n.isExpired() else 'blue')
+        else:
+            not_found.append(r[0])
+
+    x,y = m(lons,lats)
+    points.extend(m.plot(x,y,'bo'))
+    for name, xpt, ypt in zip(cities, x, y):
+        points.append(mpx.text(xpt+50000, ypt+50000, name))
+    node_val = [n.getId().toFloat() for n in all_nodes]
+    xys = [(cos(d*2*pi), sin(d*2*pi)) for d in node_val]
+    if collection:
+        collection.remove()
+        del collection
+        collection = None
+    collection = ringx.add_collection(RegularPolyCollection(
+                fig.dpi, 6, sizes=(10,), facecolors=colors,
+                offsets = xys, transOffset = ringx.transData))
+
+if run:
+    # start first step
+    start_h = PyInfoHash()
+    start_h.setBit(159, 1)
+    step(start_h, 0)
+
+def d(arg):
+   pass
+
+while run:
+    while run and done > 0:
+        update_plot()
+        plt.draw()
+        plt.pause(.5)
+
+    if not run:
+        break
+
+    button.on_clicked(restart)
+
+    node_ip4s, node_ip6s = generate_set()
+
+    print(all_nodes.size(), " nodes found")
+    print(all_nodes)
+    print(len(not_found), " nodes not geolocalized")
+    for n in not_found:
+        print(n)
+    print('')
+    print(len(node_ip4s), " different IPv4s :")
+    for ip in node_ip4s.items():
+        print(ip[0] + " : " + str(ip[1][1]) + " nodes")
+    print('')
+    print(len(node_ip6s), " different IPv6s :")
+    for ip in node_ip6s.items():
+        print(ip[0] + " : " + str(ip[1][1]) + " nodes")
+
+    while run and done == 0:
+        plt.pause(.5)
+    button.on_clicked(d)
+    plt.draw()
+
+all_nodes = []
+r.join()