move python to top directory

This commit is contained in:
Adrien Béraud committed 2015-07-27 15:18:23 -04:00
1 parent 3cbae5526a
commit c9ee6eca44
10 files changed
+39 -10

No files matched your search

+152
View File
@@ -0,0 +1,152 @@
#!/usr/bin/env python3
# Copyright (C) 2015 Savoir-Faire Linux Inc.
# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
import sys, subprocess, argparse, time, random, string, threading, signal
from pyroute2.netns.process.proxy import NSPopen
import numpy as np
import matplotlib.pyplot as plt
from dhtnetwork import DhtNetwork
sys.path.append('..')
from opendht import *
def random_hash():
return PyInfoHash(''.join(random.SystemRandom().choice(string.hexdigits) for _ in range(40)).encode())
parser = argparse.ArgumentParser(description='Create a dummy network interface for testing')
parser.add_argument('-i', '--ifname', help='interface name', default='ethdht')
parser.add_argument('-n', '--node-num', help='number of dht nodes to run', type=int, default=32)
parser.add_argument('-v', '--virtual-locs', help='number of virtual locations (node clusters)', type=int, default=8)
parser.add_argument('-l', '--loss', help='simulated cluster packet loss (percent)', type=int, default=0)
parser.add_argument('-d', '--delay', help='simulated cluster latency (ms)', type=int, default=0)
parser.add_argument('-no4', '--disable-ipv4', help='Enable IPv4', action="store_true")
parser.add_argument('-no6', '--disable-ipv6', help='Enable IPv6', action="store_true")
args = parser.parse_args()
clusters = min(args.virtual_locs, args.node_num)
node_per_loc = int(args.node_num / clusters)
print("Launching", args.node_num, "nodes (", clusters, "clusters of", node_per_loc, "nodes)")
if args.virtual_locs > 1:
cmd = ["/usr/bin/sudo", "python3", "dummy_if.py", "-i", args.ifname, "-n", str(clusters), '-l', str(args.loss), '-d', str(args.delay)]
if not args.disable_ipv4:
cmd.append('-4')
if not args.disable_ipv6:
cmd.append('-6')
print(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
output, err = p.communicate()
print(output.decode())
bootstrap = DhtNetwork(iface='br'+args.ifname)
bootstrap.resize(1)
procs = [None for _ in range(clusters)]
def start_cluster(i):
global procs
cmd = ["python3", "dhtnetwork.py", "-n", str(node_per_loc), '-I', args.ifname+str(i)+'.1']
if not args.disable_ipv4 and bootstrap.ip4:
cmd.extend(['-b', bootstrap.ip4])
if not args.disable_ipv6 and bootstrap.ip6:
cmd.extend(['-b6', bootstrap.ip6])
procs[i] = NSPopen('node'+str(i), cmd)
plt.pause(2)
def stop_cluster(i):
global procs
if procs[i]:
try:
procs[i].send_signal(signal.SIGINT);
procs[i].wait()
procs[i].release()
except Exception as e:
print(e)
procs[i] = None
def replace_cluster():
n = random.randrange(0, clusters)
stop_cluster(n)
start_cluster(n)
try:
for i in range(clusters):
start_cluster(i)
plt.ion()
lines = plt.plot([])
plt.ylabel('time (s)')
#plt.show()
# let the network stabilise
plt.pause(5)
start = time.time()
times = []
done = 0
lock = threading.Condition()
def getcb(v):
print("found", v)
def donecb(ok, nodes):
global lock, done, times
t = time.time()-start
with lock:
if not ok:
print("failed !")
times.append(t)
done -= 1
lock.notify()
def update_plot():
global lines
while lines:
l = lines.pop()
l.remove()
del l
lines = plt.plot(times, color='blue')
plt.draw()
plt.pause(5)
plt.show()
update_plot()
times = []
for n in range(10):
#replace_cluster()
plt.pause(2)
print("Getting 50 random hashes succesively.")
for i in range(50):
with lock:
done += 1
start = time.time()
bootstrap.front().get(random_hash(), getcb, donecb)
while done > 0:
lock.wait()
update_plot()
print("Took", np.sum(times), "mean", np.mean(times), "std", np.std(times), "min", np.min(times), "max", np.max(times))
plt.ioff()
plt.show()
except Exception as e:
print(e)
finally:
for i in range(clusters):
if procs[i]:
procs[i].send_signal(signal.SIGINT);
bootstrap.resize(0)
subprocess.call(["/usr/bin/sudo", "python3", "dummy_if.py", "-i", args.ifname, "-n", str(clusters), "-r"])
for i in range(clusters):
if procs[i]:
try:
procs[i].wait()
procs[i].release()
except Exception as e:
print(e)
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
# Copyright (C) 2015 Savoir-Faire Linux Inc.
# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
import signal, os, sys, ipaddress, random
from pyroute2 import IPDB
sys.path.append('..')
from opendht import *
class DhtNetwork(object):
nodes = []
@staticmethod
def run_node(ip4, ip6, p, bootstrap=[]):
#print("run_node", ip4, ip6, p, bootstrap)
id = PyIdentity()
id.generate("dhtbench"+str(p), PyIdentity(), 1024)
n = PyDhtRunner()
n.run(id, ipv4=ip4 if ip4 else "", ipv6=ip6 if ip6 else "", port=p)
for b in bootstrap:
n.bootstrap(b[0], b[1])
#plt.pause(0.02)
return ((ip4, ip6, p), n, id)
@staticmethod
def find_ip(iface):
if not iface or iface == 'any':
return ('0.0.0.0','')
if_ip4 = None
if_ip6 = None
ipdb = IPDB()
try:
for ip in ipdb.interfaces[iface].ipaddr:
if_ip = ipaddress.ip_address(ip[0])
if isinstance(if_ip, ipaddress.IPv4Address):
if_ip4 = ip[0]
elif isinstance(if_ip, ipaddress.IPv6Address):
if not if_ip.is_link_local:
if_ip6 = ip[0]
if if_ip4 and if_ip6:
break
except Exception as e:
pass
finally:
ipdb.release()
return (if_ip4, if_ip6)
def __init__(self, iface=None, ip4=None, ip6=None, port=4000, bootstrap=[]):
self.port = port
ips = DhtNetwork.find_ip(iface)
self.ip4 = ip4 if ip4 else ips[0]
self.ip6 = ip6 if ip6 else ips[1]
self.bootstrap = bootstrap
#print(self.ip4, self.ip6, self.port)
def front(self):
if len(self.nodes) == 0:
return None
return self.nodes[0][1]
def launch_node(self):
n = DhtNetwork.run_node(self.ip4, self.ip6, self.port, self.bootstrap)
self.nodes.append(n)
self.port += 1
return n
def end_node(self):
if not self.nodes:
return
n = self.nodes.pop()
n[1].join()
def replace_node(self):
random.shuffle(self.nodes)
self.end_node()
self.launch_node()
def resize(self, n):
n = min(n, 500)
l = len(self.nodes)
if n == l:
return
if n > l:
print("Launching", n-l, "nodes")
for i in range(l, n):
self.launch_node()
else:
print("Ending", l-n, "nodes")
random.shuffle(self.nodes)
for i in range(n, l):
self.end_node()
if __name__ == '__main__':
import argparse, threading
lock = threading.Condition()
quit = False
def handler(signum, frame):
global quit
with lock:
quit = True
lock.notify()
signal.signal(signal.SIGALRM, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
net = None
try:
parser = argparse.ArgumentParser(description='Create a dht network of -n nodes')
parser.add_argument('-n', '--node-num', help='number of dht nodes to run', type=int, default=32)
parser.add_argument('-I', '--iface', help='local interface to bind', default='any')
parser.add_argument('-p', '--port', help='start of port range (port, port+node_num)', type=int, default=4000)
parser.add_argument('-b', '--bootstrap', help='bootstrap address')
parser.add_argument('-b6', '--bootstrap6', help='bootstrap address (IPv6)')
args = parser.parse_args()
bs = []
if args.bootstrap:
bs.append((args.bootstrap, str(4000)))
if args.bootstrap6:
bs.append((args.bootstrap6, str(4000)))
net = DhtNetwork(iface=args.iface, port=args.port, bootstrap=bs)
net.resize(args.node_num)
with lock:
while not quit:
lock.wait()
except Exception as e:
pass
finally:
if net:
net.resize(0)
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
# Copyright (C) 2015 Savoir-Faire Linux Inc.
# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
import argparse, subprocess
from pyroute2 import IPDB, NetNS
from pyroute2.netns.process.proxy import NSPopen
parser = argparse.ArgumentParser(description='Create a dummy network interface for testing')
parser.add_argument('-i', '--ifname', help='interface name', default='ethdht')
parser.add_argument('-n', '--ifnum', type=int, help='interface number', default=1)
parser.add_argument('-r', '--remove', help='remove instead of adding the interface', action="store_true")
parser.add_argument('-l', '--loss', help='simulated packet loss (percent)', type=int, default=0)
parser.add_argument('-d', '--delay', help='simulated latency (ms)', type=int, default=0)
parser.add_argument('-4', '--ipv4', help='Enable IPv4', action="store_true")
parser.add_argument('-6', '--ipv6', help='Enable IPv6', action="store_true")
args = parser.parse_args()
local_addr4 = '10.0.42.'
local_addr6 = '2001:db9::'
brige_name = 'br'+args.ifname
ip = None
try:
ip = IPDB()
if args.remove:
for ifn in range(args.ifnum):
iface = args.ifname+str(ifn)
if iface in ip.interfaces:
with ip.interfaces[iface] as i:
i.remove()
if 'tap'+args.ifname in ip.interfaces:
with ip.interfaces['tap'+args.ifname] as i:
i.remove()
if brige_name in ip.interfaces:
with ip.interfaces[brige_name] as i:
i.remove()
for ifn in range(args.ifnum):
netns = NetNS('node'+str(ifn))
netns.close()
netns.remove()
else:
for ifn in range(args.ifnum):
iface = args.ifname+str(ifn)
if not iface in ip.interfaces:
ip.create(kind='veth', ifname=iface, peer=iface+'.1').commit()
ip.create(kind='tuntap', ifname='tap'+args.ifname, mode='tap').commit()
with ip.create(kind='bridge', ifname=brige_name) as i:
for ifn in range(args.ifnum):
iface = args.ifname+str(ifn)
i.add_port(ip.interfaces[iface])
i.add_port(ip.interfaces['tap'+args.ifname])
if args.ipv4:
i.add_ip(local_addr4+'1/24')
if args.ipv6:
i.add_ip(local_addr6+'1/64')
i.up()
with ip.interfaces['tap'+args.ifname] as tap:
tap.up()
for ifn in range(args.ifnum):
iface = args.ifname+str(ifn)
nns = NetNS('node'+str(ifn))
iface1 = iface+'.1'
with ip.interfaces[iface1] as i:
i.net_ns_fd = nns.netns
with ip.interfaces[iface] as i:
i.up()
ip_ns = IPDB(nl=nns)
try:
with ip_ns.interfaces.lo as lo:
lo.up()
with ip_ns.interfaces[iface1] as i:
if args.ipv4:
i.add_ip(local_addr4+str(ifn+8)+'/24')
if args.ipv6:
i.add_ip(local_addr6+str(ifn+8)+'/64')
i.up()
finally:
ip_ns.release()
nsp = NSPopen(nns.netns, ["tc", "qdisc", "add", "dev", iface1, "root", "netem", "delay", str(args.delay)+"ms", str(int(args.delay/2))+"ms", "loss", str(args.loss)+"%", "25%"], stdout=subprocess.PIPE)
#print(nsp.communicate()[0].decode())
nsp.communicate()
nsp.wait()
nsp.release()
if args.ipv4:
subprocess.call(["sysctl", "-w", "net.ipv4.conf."+brige_name+".forwarding=1"])
if args.ipv6:
subprocess.call(["sysctl", "-w", "net.ipv6.conf."+brige_name+".forwarding=1"])
except Exception as e:
print('Error',e)
finally:
if ip:
ip.release()
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env python3
# Copyright (c) 2015 Savoir-Faire Linux Inc.
# Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERWISE
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import time, sys
from pprint import pprint
from math import cos, sin, pi
sys.path.append('..')
from opendht import *
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.colors import colorConverter
from matplotlib.collections import RegularPolyCollection
from matplotlib.widgets import Button
from mpl_toolkits.basemap import Basemap
import GeoIP
done = 0
all_nodes = PyNodeSet()
plt.ion()
plt.figaspect(2.)
fig, axes = plt.subplots(2, 1)
fig.set_size_inches(12,16,forward=True)
fig.tight_layout()
fig.canvas.set_window_title('OpenDHT scanner')
mpx = axes[0]
mpx.set_title("Node GeoIP")
m = Basemap(projection='robin', resolution = 'l', area_thresh = 1000.0, lat_0=0, lon_0=0, ax=mpx)
m.fillcontinents(color='#cccccc',lake_color='white')
m.drawparallels(np.arange(-90.,120.,30.))
m.drawmeridians(np.arange(0.,420.,60.))
m.drawmapboundary(fill_color='white')
plt.show()
ringx = axes[1]
ringx.set_title("Node IDs")
ringx.set_autoscale_on(False)
ringx.set_aspect('equal', 'datalim')
ringx.set_xlim(-2.,2.)
ringx.set_ylim(-1.5,1.5)
exitax = plt.axes([0.92, 0.95, 0.07, 0.04])
exitbtn = Button(exitax, 'Exit')
reloadax = plt.axes([0.92, 0.90, 0.07, 0.04])
button = Button(reloadax, 'Reload')
gi = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
gi6 = GeoIP.open("GeoLiteCityv6.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
def gcb(v):
return True
r = PyDhtRunner()
i = PyIdentity()
i.generate(bits = 1024)
r.run(i, port=4112)
r.bootstrap("bootstrap.ring.cx", "4222")
all_lines = []
plt.pause(2)
def step(cur_h, cur_depth):
global done, all_nodes, all_lines
done += 1
a = 2.*pi*cur_h.toFloat()
b = a + 2.*pi/(2**(cur_depth))
print("step", cur_h, cur_depth)
arc = ringx.add_patch(mpatches.Wedge([0.,0,], 1., a*180/pi, b*180/pi, fill=True, color="blue", alpha=0.5))
lines = ringx.plot([0, cos(a)], [0, sin(a)], 'k-', lw=1.2)
all_lines.extend(lines)
r.get(cur_h, gcb, lambda d, nodes: nextstep(cur_h, cur_depth, d, nodes, arc=arc, lines=lines))
def nextstep(cur_h, cur_depth, ok, nodes, arc=None, lines=[]):
global done, all_nodes
if arc:
arc.remove()
del arc
for l in lines:
l.set_color('#444444')
snodes = PyNodeSet()
snodes.extend(nodes)
all_nodes.extend(nodes)
depth = min(6, PyInfoHash.commonBits(snodes.first(), snodes.last())+4)
if cur_depth < depth:
for b in range(cur_depth, depth):
new_h = PyInfoHash(cur_h.toString());
new_h.setBit(b, 1);
step(new_h, b+1);
done -= 1
run = True
def exitcb(arg):
global run
run = False
exitbtn.on_clicked(exitcb)
def restart(arg):
global collection, all_lines, points
for l in all_lines:
l.remove()
del l
all_lines = []
if collection:
collection.remove()
del collection
collection = None
for p in points:
p.remove()
del p
points = []
print(arg)
start_h = PyInfoHash()
start_h.setBit(159, 1)
step(start_h, 0)
plt.draw()
collection = None
points = []
not_found = []
def generate_set():
node_ipv4 = {}
node_ipv6 = {}
for n in all_nodes:
addr = b''.join(n.getNode().getAddr().split(b':')[0:-1]).decode()
if addr[0] == '[':
addr = addr[1:-1]
if addr in node_ipv6:
node_ipv6[addr][1] = 1
else:
node_ipv6[addr] = [n, 1]
else:
if addr in node_ipv4:
node_ipv4[addr][1] += 1
else:
node_ipv4[addr] = [n, 1]
return node_ipv4, node_ipv6
def update_plot():
global done, m, collection, not_found, points
for p in points:
p.remove()
del p
points = []
lats = []
lons = []
cities=[]
colors=[]
not_found.clear()
ip4s, ip6s = generate_set()
ares = []
for addr, n in ip4s.items():
ares.append((addr, n[0].getNode(), gi.record_by_name(addr)))
for addr, n in ip6s.items():
ares.append((addr, n[0].getNode(), gi6.record_by_name_v6(addr)))
for r in ares:
res = r[2]
n = r[1]
if res:
lats.append(res['latitude'])
lons.append(res['longitude'])
cities.append(res['city'] if res['city'] else (str(int(res['latitude']))+'-'+str(int(res['longitude']))))
colors.append('red' if n.isExpired() else 'blue')
else:
not_found.append(r[0])
x,y = m(lons,lats)
points.extend(m.plot(x,y,'bo'))
for name, xpt, ypt in zip(cities, x, y):
points.append(mpx.text(xpt+50000, ypt+50000, name))
node_val = [n.getId().toFloat() for n in all_nodes]
xys = [(cos(d*2*pi), sin(d*2*pi)) for d in node_val]
if collection:
collection.remove()
del collection
collection = None
collection = ringx.add_collection(RegularPolyCollection(
fig.dpi, 6, sizes=(10,), facecolors=colors,
offsets = xys, transOffset = ringx.transData))
if run:
# start first step
start_h = PyInfoHash()
start_h.setBit(159, 1)
step(start_h, 0)
def d(arg):
pass
while run:
while run and done > 0:
update_plot()
plt.draw()
plt.pause(.5)
if not run:
break
button.on_clicked(restart)
node_ip4s, node_ip6s = generate_set()
print(all_nodes.size(), " nodes found")
print(all_nodes)
print(len(not_found), " nodes not geolocalized")
for n in not_found:
print(n)
print('')
print(len(node_ip4s), " different IPv4s :")
for ip in node_ip4s.items():
print(ip[0] + " : " + str(ip[1][1]) + " nodes")
print('')
print(len(node_ip6s), " different IPv6s :")
for ip in node_ip6s.items():
print(ip[0] + " : " + str(ip[1][1]) + " nodes")
while run and done == 0:
plt.pause(.5)
button.on_clicked(d)
plt.draw()
all_nodes = []
r.join()