Compare commits

...
Author SHA1 Message Date
Adrien Béraud 961b96ca3d update to version 2.1.5 2020-09-23 15:00:48 -04:00
Adrien Béraud bcb29c3588 build/cmake: support new pkgconfig macros 2020-09-23 14:45:04 -04:00
Sébastien Blin 5d9395b16f stats: use /node/stats 2020-09-18 10:40:12 -04:00
Sébastien Blin 23a75a3f50 proxyclient: do not lock loopSignal_ 2020-09-15 10:54:44 -04:00
Adrien Béraud 7b87d32c0a python scanner: update for new APIs 2020-09-01 17:33:08 -04:00
Adrien Béraud d5f4f81566 http tester: adapt to API changes 2020-09-01 16:26:52 -04:00
Adrien Béraud a69863da4e search: avoid comparing invalidated iterators 2020-09-01 15:55:43 -04:00
Doron Behar 4d70c214b0 Enable to configure systemd unit installation dir 2020-08-22 10:24:37 -04:00
Doron Behar 11afc2b468 Enable to disable network tests 2020-08-02 11:32:28 -04:00
Adrien Béraud 879e4f4e02 tests/http: update for new API 2020-07-06 10:37:20 -04:00
Adrien Béraud b5b4eea253 dht: reschedule search step 2020-07-06 10:26:08 -04:00
Adrien Béraud 67adc09d52 http: add non-Linux exception for current libressl limitation 2020-07-03 16:33:12 -04:00
Adrien Béraud e2b39dd3a0 http/request: make terminate public 2020-06-30 10:42:49 -04:00
Adrien Béraud 9dc0ce7392 http: cleanup 2020-06-29 23:06:42 -04:00
Adrien Béraud 1a025c5cea http: log peer subject name during TLS certificate verification 2020-06-29 20:52:53 -04:00
Adrien Béraud 732e9af439 http: set SNI when using TLS 2020-06-29 19:17:16 -04:00
Adrien Béraud 4e5666fc3a http: add verify path for Android 2020-06-29 19:17:16 -04:00
Sébastien Blin b41f5b04f5 proxyclient: only restart listeners when connected
restartListeners() must be called only when the proxy is connected. Else, the
client will receive a lot of http errors
2020-06-29 15:18:58 -04:00
Adrien Béraud cfa21df8b3 http: cleanup, always verify host in TLS mode 2020-06-29 12:12:22 -04:00
Adrien Béraud 6ac451a5de update to version 2.1.4 2020-06-19 15:45:26 -04:00
Adrien Béraud 9efb8bb662 dockerfile: build with proxy 2020-06-19 15:44:47 -04:00
Adrien Béraud f0aee6f909 docker: update restinio 2020-06-19 15:44:35 -04:00
Adrien Béraud f72bfc9df2 http: use hostname for certificate authentication 2020-06-19 15:21:34 -04:00
Adrien Béraud c9ebfd1119 tests: cleanup 2020-06-15 18:33:04 -04:00
Adrien Béraud cf1874baaa crypto: free token before return 2020-06-15 18:25:23 -04:00
Adrien Béraud c221cfdafe http/Request: add get json constructor 2020-06-15 17:47:32 -04:00
Adrien Béraud 31b84e6d92 thread pool: unlock after notify 2020-06-15 17:47:32 -04:00
Adrien Béraud ae05bc8ea5 crypto: avoid goto 2020-06-15 17:47:32 -04:00
Seva acbf7cfcf0 crypto: cleanup ocsp api & tests 2020-06-15 17:47:32 -04:00
Seva 8250116946 crypto: embed ocsp response into class 2020-06-15 17:47:32 -04:00
Seva 45e9458373 crypto: add get ocsp response in readable format 2020-06-15 17:47:32 -04:00
Seva 2623b86274 crypto/cert: add generate ocsp request & verify response 2020-06-15 17:47:32 -04:00
Seva 2e1dda8189 crypto: enable test certificate serial number 2020-06-15 17:47:32 -04:00
Seva e26256b819 crypto: add get ocsp response certificate status 2020-06-15 17:47:32 -04:00
Seva 18e59ad7c8 crypto: test certificate serial number 2020-06-15 17:47:32 -04:00
Seva a7a3d23f50 crypto: add get serial number of cert. 2020-06-15 17:47:32 -04:00
Adrien Béraud 323a2b5093 dht: move socket to SearchNode 2020-06-15 17:46:43 -04:00
Adrien Béraud 563811ebf8 dht: add error handler for sendRefreshValue 2020-06-15 17:46:43 -04:00
Adrien Béraud 190c7f2063 network engine: complete request in case of error 2020-06-15 17:46:43 -04:00
27 changed files with 543 additions and 218 deletions

No files matched your search

+26 -20
View File
@@ -4,7 +4,7 @@ include(CMakeDependentOption)
project (opendht)
set (opendht_VERSION_MAJOR 2)
set (opendht_VERSION_MINOR 1.3)
set (opendht_VERSION_MINOR 1.5)
set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
set (PACKAGE_VERSION ${opendht_VERSION})
set (VERSION "${opendht_VERSION}")
@@ -16,6 +16,7 @@ option (OPENDHT_LOG "Build with logs" ON)
option (OPENDHT_PYTHON "Build Python bindings" OFF)
option (OPENDHT_TOOLS "Build DHT tools" ON)
option (OPENDHT_SYSTEMD "Install systemd module" OFF)
option (OPENDHT_SYSTEMD_UNIT_FILE_LOCATION "Where to install systemd unit file")
option (OPENDHT_ARGON2 "Use included argon2 sources" OFF)
option (OPENDHT_LTO "Build with LTO" OFF)
option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF)
@@ -29,6 +30,7 @@ CMAKE_DEPENDENT_OPTION(OPENDHT_HTTP "Build embedded http(s) client" OFF "NOT OPE
option (OPENDHT_PEER_DISCOVERY "Enable multicast peer discovery" ON)
option (OPENDHT_INDEX "Build DHT indexation feature" OFF)
option (OPENDHT_TESTS "Add unit tests executable" OFF)
option (OPENDHT_TESTS_NETWORK "Enable unit tests that require network access" ON)
option (OPENDHT_C "Build C bindings" OFF)
find_package(Doxygen)
@@ -49,6 +51,7 @@ if (NOT OPENDHT_ARGON2)
if (argon2_FOUND)
message("-- Found Argon2: " ${argon2_LIBRARY_DIRS} " (found version \"" ${argon2_VERSION} "\")")
link_directories (${argon2_LIBRARY_DIRS})
set (argon2_lib ", libargon2")
else ()
message("Argon2 not found, using included version.")
set(OPENDHT_ARGON2 ON)
@@ -62,16 +65,16 @@ if (Jsoncpp_FOUND)
src/base64.h
src/base64.cpp
)
set (jsoncpp_lib ", jsoncpp")
endif()
if (OPENDHT_HTTP)
find_package(Restinio REQUIRED)
if (Restinio_FOUND)
find_library(FMT_LIBRARY fmt)
add_library(fmt SHARED IMPORTED)
find_library(HTTP_PARSER_LIBRARY http_parser)
add_library(http_parser SHARED IMPORTED)
endif()
find_library(FMT_LIBRARY fmt)
add_library(fmt SHARED IMPORTED)
find_library(HTTP_PARSER_LIBRARY http_parser)
add_library(http_parser SHARED IMPORTED)
set (http_parser_lib "-lhttp-parser")
if (NOT Jsoncpp_FOUND)
message(SEND_ERROR "Jsoncpp is required for DHT proxy support")
endif()
@@ -81,6 +84,7 @@ if (OPENDHT_HTTP)
if (OPENSSL_FOUND)
message(STATUS "Found OpenSSL ${OPENSSL_VERSION}")
include_directories(${OPENSSL_INCLUDE_DIRS})
set(openssl_lib ", openssl")
else ()
message(SEND_ERROR "OpenSSL is required for DHT proxy as specified")
endif()
@@ -393,19 +397,21 @@ if (OPENDHT_TESTS)
tests/threadpooltester.h
tests/threadpooltester.cpp
)
if (OPENDHT_PROXY_SERVER AND OPENDHT_PROXY_CLIENT)
list (APPEND test_FILES
tests/httptester.h
tests/httptester.cpp
tests/dhtproxytester.h
tests/dhtproxytester.cpp
)
endif()
if (OPENDHT_PEER_DISCOVERY)
list (APPEND test_FILES
tests/peerdiscoverytester.h
tests/peerdiscoverytester.cpp
)
if (OPENDHT_TESTS_NETWORK)
if (OPENDHT_PROXY_SERVER AND OPENDHT_PROXY_CLIENT)
list (APPEND test_FILES
tests/httptester.h
tests/httptester.cpp
tests/dhtproxytester.h
tests/dhtproxytester.cpp
)
endif()
if (OPENDHT_PEER_DISCOVERY)
list (APPEND test_FILES
tests/peerdiscoverytester.h
tests/peerdiscoverytester.cpp
)
endif()
endif()
add_executable(opendht_unit_tests
tests/tests_runner.cpp
+1 -1
View File
@@ -1,7 +1,7 @@
dnl define macros
m4_define([opendht_major_version], 2)
m4_define([opendht_minor_version], 1)
m4_define([opendht_patch_version], 3)
m4_define([opendht_patch_version], 5)
m4_define([opendht_version],
[opendht_major_version.opendht_minor_version.opendht_patch_version])
+1 -1
View File
@@ -2,5 +2,5 @@ FROM aberaud/opendht-deps
MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
RUN git clone https://github.com/savoirfairelinux/opendht.git \
&& cd opendht && mkdir build && cd build \
&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On && make -j8 && make install \
&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On -DOPENDHT_PROXY_SERVER=On -DOPENDHT_PROXY_CLIENT=On && make -j8 && make install \
&& cd ../.. && rm -rf opendht
+1 -1
View File
@@ -1,4 +1,4 @@
FROM docker.pkg.github.com/savoirfairelinux/opendht/opendht-deps-bionic:2.0.0
FROM docker.pkg.github.com/savoirfairelinux/opendht/opendht-deps-bionic:2.1.3
MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
RUN git clone https://github.com/savoirfairelinux/opendht.git \
+4 -4
View File
@@ -17,11 +17,11 @@ RUN apt-get update && apt-get install -y \
RUN echo "*** Downloading RESTinio ***" \
&& mkdir restinio && cd restinio \
&& wget https://github.com/aberaud/restinio/archive/a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
&& ls -l && tar -xzf a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
&& cd restinio-a7a10e419d9089c5b8ee63f5e3098c892f22fae4/dev \
&& wget https://github.com/aberaud/restinio/archive/2c0b6f5e5ba04d7a74e8406a3df1fd433680599d.tar.gz \
&& ls -l && tar -xzf 2c0b6f5e5ba04d7a74e8406a3df1fd433680599d.tar.gz \
&& cd restinio-2c0b6f5e5ba04d7a74e8406a3df1fd433680599d/dev \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DRESTINIO_TEST=OFF -DRESTINIO_SAMPLE=OFF \
-DRESTINIO_INSTALL_SAMPLES=OFF -DRESTINIO_BENCH=OFF -DRESTINIO_INSTALL_BENCHES=OFF \
-DRESTINIO_FIND_DEPS=ON -DRESTINIO_ALLOW_SOBJECTIZER=Off -DRESTINIO_USE_BOOST_ASIO=none . \
&& make -j8 && make install \
&& cd ../../ && rm -rf restinio*
&& cd ../../.. && rm -rf restinio
+3 -3
View File
@@ -18,9 +18,9 @@ ENV CXX c++
RUN echo "*** Downloading RESTinio ***" \
&& mkdir restinio && cd restinio \
&& wget https://github.com/aberaud/restinio/archive/a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
&& ls -l && tar -xzf a7a10e419d9089c5b8ee63f5e3098c892f22fae4.tar.gz \
&& cd restinio-a7a10e419d9089c5b8ee63f5e3098c892f22fae4/dev \
&& wget https://github.com/aberaud/restinio/archive/2c0b6f5e5ba04d7a74e8406a3df1fd433680599d.tar.gz \
&& ls -l && tar -xzf 2c0b6f5e5ba04d7a74e8406a3df1fd433680599d.tar.gz \
&& cd restinio-2c0b6f5e5ba04d7a74e8406a3df1fd433680599d/dev \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr -DRESTINIO_TEST=OFF -DRESTINIO_SAMPLE=OFF \
-DRESTINIO_INSTALL_SAMPLES=OFF -DRESTINIO_BENCH=OFF -DRESTINIO_INSTALL_BENCHES=OFF \
-DRESTINIO_FIND_DEPS=ON -DRESTINIO_ALLOW_SOBJECTIZER=Off -DRESTINIO_USE_BOOST_ASIO=none . \
+1 -1
View File
@@ -2,5 +2,5 @@ FROM aberaud/opendht-deps-llvm
MAINTAINER Adrien Béraud <adrien.beraud@savoirfairelinux.com>
RUN git clone https://github.com/savoirfairelinux/opendht.git \
&& cd opendht && mkdir build && cd build \
&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On && make -j8 && make install \
&& cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DOPENDHT_PYTHON=On -DOPENDHT_LTO=On -DOPENDHT_PROXY_SERVER=On -DOPENDHT_PROXY_CLIENT=On && make -j8 && make install \
&& cd ../.. && rm -rf opendht
+42 -1
View File
@@ -1,6 +1,7 @@
/*
* Copyright (C) 2014-2020 Savoir-faire Linux Inc.
* Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -26,6 +27,7 @@ extern "C" {
#include <gnutls/gnutls.h>
#include <gnutls/abstract.h>
#include <gnutls/x509.h>
#include <gnutls/ocsp.h>
}
#include <vector>
@@ -196,7 +198,6 @@ private:
//friend dht::crypto::Identity dht::crypto::generateIdentity(const std::string&, dht::crypto::Identity, unsigned key_length);
};
class OPENDHT_PUBLIC RevocationList
{
using clock = std::chrono::system_clock;
@@ -309,6 +310,36 @@ private:
gnutls_x509_crq_t request {nullptr};
};
class OPENDHT_PUBLIC OcspResponse
{
public:
OcspResponse(const uint8_t* dat_ptr, size_t dat_size);
~OcspResponse();
Blob pack() const;
/*
* Get OCSP Response in readable format.
*/
std::string toString(const bool compact = true) const;
/*
* Get OCSP response certificate status.
* Return certificate status.
* http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-cert-status-t
*/
gnutls_ocsp_cert_status_t getCertificateStatus() const;
/*
* Verify OCSP response.
* Return OCSP verify reason.
* http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-verify-reason-t
*/
gnutls_ocsp_verify_reason_t verifyDirect(const Certificate& crt, const Blob& nonce);
private:
gnutls_ocsp_resp_t response;
};
struct OPENDHT_PUBLIC Certificate {
Certificate() noexcept {}
@@ -434,6 +465,8 @@ struct OPENDHT_PUBLIC Certificate {
/** Same as getPublicKey().getLongId() */
PkId getLongId() const;
Blob getSerialNumber() const;
/** Read certificate Common Name (CN) */
std::string getName() const;
@@ -527,8 +560,16 @@ struct OPENDHT_PUBLIC Certificate {
gnutls_digest_algorithm_t getPreferredDigest() const;
/*
* Generate OCSP request.
* Return GnuTLS error code.
* https://www.gnutls.org/manual/html_node/Error-codes.html
*/
std::pair<Blob,Blob> generateOcspRequest(gnutls_x509_crt_t& issuer);
gnutls_x509_crt_t cert {nullptr};
std::shared_ptr<Certificate> issuer {};
std::shared_ptr<OcspResponse> ocspResponse;
private:
Certificate(const Certificate&) = delete;
Certificate& operator=(const Certificate&) = delete;
+2 -3
View File
@@ -264,11 +264,10 @@ public:
std::vector<unsigned> getNodeMessageStats(bool) override { return {}; }
void setStorageLimit(size_t) override {}
void connectivityChanged(sa_family_t) override {
restartListeners();
getProxyInfos();
}
void connectivityChanged() override {
getProxyInfos();
restartListeners();
loopSignal_();
}
@@ -385,7 +384,7 @@ private:
/**
* Relaunch LISTEN requests if the client disconnect/reconnect.
*/
void restartListeners();
void restartListeners(const asio::error_code &ec);
/**
* Refresh a listen via a token
+13 -7
View File
@@ -72,15 +72,17 @@ using socket_t = asio::ip::tcp::socket;
class OPENDHT_PUBLIC Url
{
public:
Url(){};
Url() = default;
Url(const std::string& url);
std::string url;
std::string protocol {"http"};
std::string host;
std::string service {"80"};
std::string target {"/"};
std::string service;
std::string target;
std::string query;
std::string fragment;
std::string toString() const;
};
class OPENDHT_PUBLIC Connection : public std::enable_shared_from_this<Connection>
@@ -95,7 +97,7 @@ public:
bool is_open() const;
bool is_ssl() const;
void set_ssl_verification(const asio::ip::tcp::endpoint& endpoint, const asio::ssl::verify_mode verify_mode);
void set_ssl_verification(const std::string& hostname, const asio::ssl::verify_mode verify_mode);
asio::streambuf& input();
std::istream& data() { return istream_; }
@@ -226,12 +228,15 @@ public:
using OnStatusCb = std::function<void(unsigned status_code)>;
using OnDataCb = std::function<void(const char* at, size_t length)>;
using OnStateChangeCb = std::function<void(State state, const Response& response)>;
using OnJsonCb = std::function<void(Json::Value value, unsigned status_code)>;
using OnJsonCb = std::function<void(Json::Value value, const Response& response)>;
using OnDoneCb = std::function<void(const Response& response)>;
// resolves implicitly
Request(asio::io_context& ctx, const std::string& url, const Json::Value& json, OnJsonCb jsoncb,
std::shared_ptr<dht::Logger> logger = {});
Request(asio::io_context& ctx, const std::string& url, OnJsonCb jsoncb,
std::shared_ptr<dht::Logger> logger = {});
Request(asio::io_context& ctx, const std::string& url, std::shared_ptr<dht::Logger> logger = {});
Request(asio::io_context& ctx, const std::string& host, const std::string& service,
const bool ssl = false, std::shared_ptr<dht::Logger> logger = {});
@@ -289,6 +294,7 @@ public:
* User action to cancel the Request and call the completion callbacks.
*/
void cancel();
void terminate(const asio::error_code& ec);
private:
using OnCompleteCb = std::function<void()>;
@@ -301,6 +307,8 @@ private:
OnStateChangeCb on_state_change;
};
static std::string getRelativePath(const Url& origin, const std::string& path);
void notify_state_change(State state);
void build();
@@ -313,8 +321,6 @@ private:
void connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, HandlerCb cb = {});
void terminate(const asio::error_code& ec);
void post();
void handle_request(const asio::error_code& ec);
+3 -3
View File
@@ -82,13 +82,13 @@ private:
};
struct Logger {
LogMethod DBG = {};
LogMethod WARN = {};
LogMethod ERR = {};
LogMethod WARN = {};
LogMethod DBG = {};
Logger() = default;
Logger(LogMethod&& err, LogMethod&& warn, LogMethod&& dbg)
: DBG(std::move(dbg)), WARN(std::move(warn)), ERR(std::move(err)) {}
: ERR(std::move(err)), WARN(std::move(warn)), DBG(std::move(dbg)) {}
void setFilter(const InfoHash& f) {
filter_ = f;
filterEnable_ = static_cast<bool>(filter_);
+4 -3
View File
@@ -211,6 +211,7 @@ private:
public:
using RequestCb = std::function<void(const Request&, RequestAnswer&&)>;
using RequestErrorCb = std::function<bool(const Request&, DhtProtocolException&&)>;
using RequestExpiredCb = std::function<void(const Request&, bool)>;
NetworkEngine(const Sp<Logger>& log, std::mt19937_64& rd, Scheduler& scheduler, std::unique_ptr<DatagramSocket>&& sock);
@@ -361,10 +362,9 @@ public:
const InfoHash& hash,
const Query& query,
const Blob& token,
Sp<Request> previous,
Tid socketId,
RequestCb&& on_done,
RequestExpiredCb&& on_expired,
SocketCb&& socket_cb);
RequestExpiredCb&& on_expired);
/**
* Send a "announce" request to a given node.
*
@@ -403,6 +403,7 @@ public:
const Value::Id& vid,
const Blob& token,
RequestCb&& on_done,
RequestErrorCb&& on_error,
RequestExpiredCb&& on_expired);
/**
* Send a "update" request to a given node. Used for Listen operations
+11 -26
View File
@@ -21,6 +21,7 @@ from math import cos, sin, pi
import urllib3
import gzip
import queue
from geoip import geolite2
sys.path.append('..')
from opendht import *
@@ -55,7 +56,7 @@ plt.ion()
plt.figaspect(2.)
fig, axes = plt.subplots(2, 1)
fig.set_size_inches(8,16,forward=True)
#fig.set_size_inches(8,16,forward=True)
fig.tight_layout()
fig.canvas.set_window_title('OpenDHT scanner')
@@ -90,27 +91,12 @@ def exitcb(arg):
run = False
exitbtn.on_clicked(exitcb)
def check_dl(fname, url):
if os.path.isfile(fname):
return
print('downloading', url)
ghandle = gzip.GzipFile(fileobj=http.request('GET', url, headers={'User-Agent': 'Mozilla/5.0'}, preload_content=False))
with open(fname, 'wb') as out:
for line in ghandle:
out.write(line)
check_dl("GeoLiteCity.dat", "http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz")
check_dl("GeoLiteCityv6.dat", "http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz")
gi = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
gi6 = GeoIP.open("GeoLiteCityv6.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
def gcb(v):
return True
r = DhtRunner()
r.run(port=4112)
r.bootstrap("bootstrap.ring.cx", "4222")
r.bootstrap("bootstrap.jami.net", "4222")
plt.pause(1)
@@ -143,9 +129,9 @@ def nextstep(cur_h, cur_depth, ok, nodes):
depth = min(8, commonBits+6)
if cur_depth < depth:
for b in range(cur_depth, depth):
new_h = InfoHash(cur_h.toString());
new_h.setBit(b, 1);
step(new_h, b+1);
new_h = InfoHash(cur_h.toString())
new_h.setBit(b, 1)
step(new_h, b+1)
else:
print("step done with no nodes", ok, cur_h.toString().decode())
done -= 1
@@ -186,23 +172,22 @@ def appendNewNode(n):
if addr in nodes_ip6s:
nodes_ip6s[addr][1] += 1
else:
georecord = gi6.record_by_name_v6(addr)
georecord = geolite2.lookup(addr)
nodes_ip6s[addr] = [n, 1, georecord]
else:
if addr in nodes_ip4s:
nodes_ip4s[addr][1] += 1
else:
georecord = gi.record_by_name(addr)
georecord = geolite2.lookup(addr)
nodes_ip4s[addr] = [n, 1, georecord]
if georecord:
appendMapPoint(georecord)
def appendMapPoint(res):
global lons, lats, cities
lons.append(res['longitude'])
lats.append(res['latitude'])
cities.append(res['city'] if res['city'] else (str(int(res['latitude']))+'-'+str(int(res['longitude']))))
lons.append(res.location[1])
lats.append(res.location[0])
cities.append(res.city if res.city else (str(int(res.location[0]))+'-'+str(int(res.location[1]))))
def restart(arg):
global collection, all_lines, all_nodes, points, done, nodes_ip4s, nodes_ip6s, lats, lons, cities, xys, colors
+161
View File
@@ -1,6 +1,7 @@
/*
* Copyright (C) 2014-2020 Savoir-faire Linux Inc.
* Author : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -25,6 +26,7 @@ extern "C" {
#include <gnutls/x509.h>
#include <nettle/gcm.h>
#include <nettle/aes.h>
#include <gnutls/crypto.h>
#include <argon2.h>
}
@@ -875,6 +877,18 @@ Certificate::getLongId() const
#endif
}
Blob
Certificate::getSerialNumber() const
{
if (not cert)
return {};
// extract from certificate
unsigned char serial[64];
auto size = sizeof(serial);
gnutls_x509_crt_get_serial(cert, &serial, &size);
return {serial, serial + size};
}
std::string
Certificate::getName() const
{
@@ -1020,6 +1034,32 @@ Certificate::getPreferredDigest() const
return getPublicKey().getPreferredDigest();
}
std::pair<Blob,Blob>
Certificate::generateOcspRequest(gnutls_x509_crt_t& issuer)
{
gnutls_ocsp_req_t rreq;
int err = gnutls_ocsp_req_init(&rreq);
if (err < 0)
throw CryptoException(gnutls_strerror(err));
std::unique_ptr<struct gnutls_ocsp_req_int, decltype(&gnutls_ocsp_req_deinit)> req(rreq, &gnutls_ocsp_req_deinit);
err = gnutls_ocsp_req_add_cert(req.get(), GNUTLS_DIG_SHA512, issuer, cert);
if (err < 0)
throw CryptoException(gnutls_strerror(err));
Blob noncebuf(64);
gnutls_datum_t nonce = { noncebuf.data(), (unsigned)noncebuf.size() };
err = gnutls_rnd(GNUTLS_RND_NONCE, nonce.data, nonce.size);
err = gnutls_ocsp_req_set_nonce(req.get(), 0, &nonce);
if (err < 0)
throw CryptoException(gnutls_strerror(err));
gnutls_datum_t rdata;
err = gnutls_ocsp_req_export(req.get(), &rdata);
if (err != 0)
throw CryptoException(gnutls_strerror(err));
Blob ret(rdata.data, rdata.data + rdata.size);
gnutls_free(rdata.data);
return std::make_pair<Blob,Blob>(std::move(ret), std::move(noncebuf));
}
// PrivateKey
PrivateKey
@@ -1206,6 +1246,127 @@ Certificate::getRevocationLists() const
return ret;
}
// OcspResponse
OcspResponse::OcspResponse(const uint8_t* dat_ptr, size_t dat_size)
{
int ret = gnutls_ocsp_resp_init(&response);
if (ret < 0)
throw CryptoException(gnutls_strerror(ret));
gnutls_datum_t dat = {(unsigned char*)dat_ptr,(unsigned int)dat_size};
ret = gnutls_ocsp_resp_import(response, &dat);
if (ret < 0){
gnutls_ocsp_resp_deinit(response);
throw CryptoException(gnutls_strerror(ret));
}
}
OcspResponse::~OcspResponse()
{
gnutls_ocsp_resp_deinit(response);
}
Blob
OcspResponse::pack() const
{
gnutls_datum_t dat;
int ret = gnutls_ocsp_resp_export(response, &dat);
if (ret < 0)
throw CryptoException(gnutls_strerror(ret));
return {dat.data, dat.data + dat.size};
}
std::string
OcspResponse::toString(const bool compact) const
{
int ret;
std::string str;
gnutls_datum_t dat;
ret = gnutls_ocsp_resp_print(response, compact ? GNUTLS_OCSP_PRINT_COMPACT : GNUTLS_OCSP_PRINT_FULL, &dat);
if (ret == 0)
str = std::string((const char*)dat.data, (size_t)dat.size);
gnutls_free(dat.data);
if (ret < 0)
throw CryptoException(gnutls_strerror(ret));
return str;
}
gnutls_ocsp_cert_status_t
OcspResponse::getCertificateStatus() const
{
int ret;
unsigned int status;
ret = gnutls_ocsp_resp_get_single(response, 0, NULL, NULL, NULL, NULL, &status, NULL, NULL, NULL, NULL);
if (ret < 0)
throw CryptoException(gnutls_strerror(ret));
return (gnutls_ocsp_cert_status_t) status;
}
gnutls_ocsp_verify_reason_t
OcspResponse::verifyDirect(const Certificate& crt, const Blob& nonce)
{
// https://www.gnutls.org/manual/html_node/Error-codes.html
int ret = -1;
// http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-verify-reason-t
unsigned int verify = 0;
// http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-cert-status-t
unsigned int status = 6; // by default assume GNUTLS_OCSP_RESP_UNAUTHORIZED
#if GNUTLS_VERSION_NUMBER >= 0x030103
/*
* This function will check whether the OCSP response is about the provided certificate.
* index: Specifies response number to get. Use (0) to get the first one.
*/
if ((ret = gnutls_ocsp_resp_check_crt(response, 0, crt.cert)) < 0)
goto end;
#endif
/*
* Verify signature of the Basic OCSP Response against the public key in the issuer certificate.
* http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-resp-verify-direct
*/
ret = gnutls_ocsp_resp_verify_direct(response, crt.issuer->cert /*signer*/, &verify, 0);
if (ret < 0)
goto end;
ret = gnutls_ocsp_resp_check_crt(response, 0 /*index*/, crt.cert);
if (ret < 0)
goto end;
/*
* This function will return the certificate information of the indx'ed response in the
* Basic OCSP Response resp. The information returned corresponds to the OCSP SingleResponse
* structure except the final singleExtensions.
* http://www.gnu.org/software/gnutls/reference/gnutls-ocsp.html#gnutls-ocsp-resp-get-single
*/
ret = gnutls_ocsp_resp_get_single(response, 0, NULL, NULL, NULL, NULL, &status, NULL, NULL, NULL, NULL);
if (ret < 0)
goto end;
gnutls_datum_t rnonce;
/*
* This function will return the Basic OCSP Response nonce extension data.
*/
ret = gnutls_ocsp_resp_get_nonce(response, NULL, &rnonce);
if (ret < 0)
goto end;
// Ensure no replay attack has been done
if (rnonce.size != nonce.size() || memcmp(nonce.data(), rnonce.data, nonce.size()) != 0){
ret = GNUTLS_E_OCSP_RESPONSE_ERROR;
goto end;
}
end:
/*
* OCSP validation outcome based on interogative aspects
*/
if (ret != GNUTLS_E_SUCCESS)
throw CryptoException(gnutls_strerror(ret));
else if (status != GNUTLS_OCSP_CERT_GOOD)
if (status == GNUTLS_OCSP_CERT_REVOKED)
throw CryptoException("Certificate was revoked");
else
throw CryptoException("Certificate is unknown");
else
return (gnutls_ocsp_verify_reason_t) verify;
}
// RevocationList
RevocationList::RevocationList()
+39 -32
View File
@@ -432,7 +432,7 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
{ /* when put done */
if (auto sr = ws.lock()) {
onAnnounceDone(req.node, answer, sr);
searchStep(sr);
scheduler.edit(sr->nextSearchStep, scheduler.time());
}
};
@@ -490,7 +490,29 @@ void Dht::searchSendAnnounceValue(const Sp<Search>& sr) {
logger_->w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)",
sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id);
sn->acked[a.value->id] = {
network_engine.sendRefreshValue(sn->node, sr->id, a.value->id, sn->token, onDone, onExpired),
network_engine.sendRefreshValue(sn->node, sr->id, a.value->id, sn->token, onDone,
[this, ws, node=sn->node, v=a.value,
onDone,
onExpired,
created = a.permanent ? time_point::max() : a.created,
next_refresh_time
](const net::Request& /*req*/, net::DhtProtocolException&& e){
if (e.getCode() == net::DhtProtocolException::NOT_FOUND) {
if (logger_)
logger_->e(node->id, "[node %s] returned error 404: storage not found", node->toString().c_str());
if (auto sr = ws.lock()) {
if (auto sn = sr->getNode(node)) {
sn->acked[v->id] = {
network_engine.sendAnnounceValue(sn->node, sr->id, v, created, sn->token, onDone, onExpired),
next_refresh_time
};
scheduler.edit(sr->nextSearchStep, scheduler.time());
return true;
}
}
}
return false;
}, onExpired),
next_refresh_time
};
} else {
@@ -569,33 +591,30 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
std::weak_ptr<Search> ws = sr;
for (const auto& l : sr->listeners) {
const auto& query = l.second.query;
auto list_token = l.first;
if (n.getListenTime(query, listenExp) > scheduler.time())
auto r = n.listenStatus.find(query);
if (n.getListenTime(r, listenExp) > scheduler.time())
continue;
// if (logger_)
// logger_->d(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'",
// sr->id.toString().c_str(), n.node->toString().c_str());
auto r = n.listenStatus.find(query);
if (r == n.listenStatus.end()) {
r = n.listenStatus.emplace(std::piecewise_construct,
std::forward_as_tuple(query),
std::forward_as_tuple(
[ws,list_token](const std::vector<Sp<Value>>& values, bool expired){
if (auto sr = ws.lock()) {
auto l = sr->listeners.find(list_token);
if (l != sr->listeners.end()) {
l->second.get_cb(values, expired);
l.second.get_cb,
l.second.sync_cb,
n.node->openSocket([this,ws,query](const Sp<Node>& node, net::RequestAnswer&& answer) mutable {
/* on new values */
if (auto sr = ws.lock()) {
scheduler.edit(sr->nextSearchStep, scheduler.time());
sr->insertNode(node, scheduler.time(), answer.ntoken);
if (auto sn = sr->getNode(node)) {
sn->onValues(query, std::move(answer), types, scheduler);
}
}
}
}, [ws,list_token] (ListenSyncStatus status) {
if (auto sr = ws.lock()) {
auto l = sr->listeners.find(list_token);
if (l != sr->listeners.end()) {
l->second.sync_cb(status);
}
}
})).first;
}))).first;
r->second.cacheExpirationJob = scheduler.add(time_point::max(), [this,ws,query,node=n.node]{
if (auto sr = ws.lock()) {
if (auto sn = sr->getNode(node)) {
@@ -604,8 +623,7 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
}
});
}
auto prev_req = r != n.listenStatus.end() ? r->second.req : nullptr;
auto new_req = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req,
auto new_req = network_engine.sendListen(n.node, sr->id, *query, n.token, r->second.socketId,
[this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable
{ /* on done */
if (auto sr = ws.lock()) {
@@ -625,16 +643,6 @@ Dht::searchSynchedNodeListen(const Sp<Search>& sr, SearchNode& n)
if (auto sn = sr->getNode(req.node))
sn->listenStatus.erase(query);
}
},
[this,ws,query](const Sp<Node>& node, net::RequestAnswer&& answer) mutable
{ /* on new values */
if (auto sr = ws.lock()) {
scheduler.edit(sr->nextSearchStep, scheduler.time());
sr->insertNode(node, scheduler.time(), answer.ntoken);
if (auto sn = sr->getNode(node)) {
sn->onValues(query, std::move(answer), types, scheduler);
}
}
}
);
// Here the request may have failed and the CachedListenStatus removed
@@ -2175,7 +2183,6 @@ Dht::onError(Sp<net::Request> req, net::DhtProtocolException e) {
if (logger_)
logger_->e(node->id, "[node %s] token flush", node->toString().c_str());
node->authError();
node->cancelRequest(req);
for (auto& srp : searches(node->getFamily())) {
auto& sr = srp.second;
for (auto& n : sr->nodes) {
+12 -3
View File
@@ -683,7 +683,7 @@ DhtProxyClient::onProxyInfos(const Json::Value& proxyInfos, const sa_family_t fa
{
if (isDestroying_)
return;
std::lock_guard<std::mutex> l(lockCurrentProxyInfos_);
std::unique_lock<std::mutex> l(lockCurrentProxyInfos_);
auto oldStatus = std::max(statusIpv4_, statusIpv6_);
auto& status = family == AF_INET ? statusIpv4_ : statusIpv6_;
if (not proxyInfos.isMember("node_id")) {
@@ -720,7 +720,7 @@ DhtProxyClient::onProxyInfos(const Json::Value& proxyInfos, const sa_family_t fa
if (newStatus == NodeStatus::Connected) {
if (oldStatus == NodeStatus::Disconnected || oldStatus == NodeStatus::Connecting) {
listenerRestartTimer_->expires_at(std::chrono::steady_clock::now());
listenerRestartTimer_->async_wait(std::bind(&DhtProxyClient::restartListeners, this));
listenerRestartTimer_->async_wait(std::bind(&DhtProxyClient::restartListeners, this, std::placeholders::_1));
}
nextProxyConfirmationTimer_->expires_at(std::chrono::steady_clock::now() + std::chrono::minutes(15));
nextProxyConfirmationTimer_->async_wait(std::bind(&DhtProxyClient::handleProxyConfirm, this, std::placeholders::_1));
@@ -729,6 +729,7 @@ DhtProxyClient::onProxyInfos(const Json::Value& proxyInfos, const sa_family_t fa
nextProxyConfirmationTimer_->expires_at(std::chrono::steady_clock::now() + std::chrono::minutes(1));
nextProxyConfirmationTimer_->async_wait(std::bind(&DhtProxyClient::handleProxyConfirm, this, std::placeholders::_1));
}
l.unlock();
loopSignal_();
}
@@ -1078,8 +1079,16 @@ DhtProxyClient::getConnectivityStatus()
}
void
DhtProxyClient::restartListeners()
DhtProxyClient::restartListeners(const asio::error_code &ec)
{
if (ec == asio::error::operation_aborted)
return;
else if (ec){
if (logger_)
logger_->e("[proxy:client] restart error: %s", ec.message().c_str());
return;
}
if (isDestroying_)
return;
if (logger_)
+130 -52
View File
@@ -29,8 +29,11 @@
namespace dht {
namespace http {
constexpr char HTTP_HEADER_CONTENT_TYPE_JSON[] = "application/json";
constexpr char HTTP_HEADER_DELIM[] = "\r\n\r\n";
constexpr const char HTTP_HEADER_CONTENT_TYPE_JSON[] = "application/json";
constexpr const char HTTP_HEADER_DELIM[] = "\r\n\r\n";
constexpr const char HTTP_PROTOCOL[] = "http://";
constexpr const char HTTPS_PROTOCOL[] = "https://";
constexpr const char ORIGIN_PROTOCOL[] = "//";
constexpr unsigned MAX_REDIRECTS {5};
Url::Url(const std::string& url): url(url)
@@ -42,7 +45,6 @@ Url::Url(const std::string& url): url(url)
addr_begin = proto_end + 3;
if (url.substr(0, proto_end) == "https"){
protocol = "https";
service = protocol;
}
}
// host and service
@@ -68,6 +70,21 @@ Url::Url(const std::string& url): url(url)
}
}
std::string
Url::toString() const
{
std::stringstream ss;
if (not protocol.empty()) {
ss << protocol << "://";
}
ss << host;
if (not service.empty()) {
ss << ':' << service;
}
ss << target;
return ss.str();
}
// connection
std::atomic_uint Connection::ids_ {1};
@@ -75,10 +92,14 @@ std::atomic_uint Connection::ids_ {1};
Connection::Connection(asio::io_context& ctx, const bool ssl, std::shared_ptr<dht::Logger> l)
: id_(Connection::ids_++), ctx_(ctx), istream_(&read_buf_), logger_(l)
{
if (ssl){
ssl_ctx_ = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23);
if (ssl) {
ssl_ctx_ = std::make_shared<asio::ssl::context>(asio::ssl::context::tls_client);
ssl_ctx_->set_verify_mode(asio::ssl::verify_peer | asio::ssl::verify_fail_if_no_peer_cert);
#ifdef __ANDROID__
ssl_ctx_->add_verify_path("/system/etc/security/cacerts");
#else
ssl_ctx_->set_default_verify_paths();
ssl_ctx_->set_verify_mode(asio::ssl::verify_none);
#endif
ssl_socket_ = std::make_unique<ssl_socket_t>(ctx_, ssl_ctx_);
if (logger_)
logger_->d("[connection:%i] start https session", id_);
@@ -94,8 +115,13 @@ Connection::Connection(asio::io_context& ctx, std::shared_ptr<dht::crypto::Certi
const dht::crypto::Identity& identity, std::shared_ptr<dht::Logger> l)
: id_(Connection::ids_++), ctx_(ctx), istream_(&read_buf_), logger_(l)
{
ssl_ctx_ = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23);
ssl_ctx_->set_default_verify_paths();
ssl_ctx_ = std::make_shared<asio::ssl::context>(asio::ssl::context::tls_client);
ssl_ctx_->set_verify_mode(asio::ssl::verify_peer | asio::ssl::verify_fail_if_no_peer_cert);
#ifdef __ANDROID__
ssl_ctx_->add_verify_path("/system/etc/security/cacerts");
#else
ssl_ctx_->set_default_verify_paths();
#endif
asio::error_code ec;
if (server_ca){
auto ca = server_ca->toString(false/*chain*/);
@@ -120,7 +146,6 @@ Connection::Connection(asio::io_context& ctx, std::shared_ptr<dht::crypto::Certi
else if (logger_)
logger_->d("[connection:%i] client certificate %s", id_, identity.second->getUID().c_str());
}
ssl_ctx_->set_verify_mode(asio::ssl::verify_peer | asio::ssl::verify_fail_if_no_peer_cert);
ssl_socket_ = std::make_unique<ssl_socket_t>(ctx_, ssl_ctx_);
}
@@ -160,29 +185,33 @@ Connection::is_ssl() const
}
void
Connection::set_ssl_verification(const asio::ip::tcp::endpoint& endpoint, const asio::ssl::verify_mode verify_mode)
Connection::set_ssl_verification(const std::string& hostname, const asio::ssl::verify_mode verify_mode)
{
std::lock_guard<std::mutex> lock(mutex_);
if (ssl_socket_ and verify_mode != asio::ssl::verify_none) {
if (ssl_socket_) {
// Set SNI Hostname (many hosts need this to handshake successfully)
SSL_set_tlsext_host_name(ssl_socket_->asio_ssl_stream().native_handle(), hostname.c_str());
ssl_socket_->asio_ssl_stream().set_verify_mode(verify_mode);
ssl_socket_->asio_ssl_stream().set_verify_callback([
id = id_,
logger = logger_,
hostname = endpoint.address().to_string()
] (bool preverified, asio::ssl::verify_context& ctx) -> bool {
if (logger)
logger->d("[connection:%i] verify %s compliance to RFC 2818", id, hostname.c_str());
if (preverified)
return preverified;
// starts from CA and goes down the presented chain
auto verifier = asio::ssl::rfc2818_verification(hostname);
bool verified = verifier(preverified, ctx);
auto verify_ec = X509_STORE_CTX_get_error(ctx.native_handle());
if (verified != 0 /*X509_V_OK*/ and logger)
logger->e("[http::connection:%i] ssl verification error=%i", id, verify_ec);
return verified;
}
);
if (verify_mode != asio::ssl::verify_none) {
ssl_socket_->asio_ssl_stream().set_verify_callback([
id = id_, logger = logger_, hostname
] (bool preverified, asio::ssl::verify_context& ctx) -> bool {
if (logger) {
char subject_name[1024];
X509* cert = X509_STORE_CTX_get_current_cert(ctx.native_handle());
X509_NAME_oneline(X509_get_subject_name(cert), subject_name, 1024);
logger->d("[connection:%i] verify %s compliance to RFC 2818:\n%s", id, hostname.c_str(), subject_name);
}
// starts from CA and goes down the presented chain
auto verifier = asio::ssl::rfc2818_verification(hostname);
bool verified = verifier(preverified, ctx);
auto verify_ec = X509_STORE_CTX_get_error(ctx.native_handle());
if (verify_ec != 0 /*X509_V_OK*/ and logger)
logger->e("[http::connection:%i] ssl verification error=%i %d", id, verify_ec, verified);
return verified;
}
);
}
}
}
@@ -242,7 +271,7 @@ Connection::async_handshake(HandlerCb cb)
if (this_.logger_) {
if (verify_ec == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT /*18*/
|| verify_ec == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN /*19*/)
this_.logger_->d("[connection:%i] allow self-signed certificate in handshake", this_.id_);
this_.logger_->d("[connection:%i] self-signed certificate in handshake: %i", this_.id_, verify_ec);
else if (verify_ec != X509_V_OK)
this_.logger_->e("[connection:%i] verify handshake error: %i", this_.id_, verify_ec);
}
@@ -356,7 +385,7 @@ Connection::timeout(const std::chrono::seconds timeout, HandlerCb cb)
Resolver::Resolver(asio::io_context& ctx, const std::string& url, std::shared_ptr<dht::Logger> logger)
: url_(url), resolver_(ctx), destroyed_(std::make_shared<bool>(false)), logger_(logger)
{
resolve(url_.host, url_.service);
resolve(url_.host, url_.service.empty() ? url_.protocol : url_.service);
}
Resolver::Resolver(asio::io_context& ctx, const std::string& host, const std::string& service,
@@ -366,7 +395,7 @@ Resolver::Resolver(asio::io_context& ctx, const std::string& host, const std::st
url_.host = host;
url_.service = service;
url_.protocol = (ssl ? "https" : "http");
resolve(url_.host, url_.service);
resolve(url_.host, url_.service.empty() ? url_.protocol : url_.service);
}
Resolver::Resolver(asio::io_context& ctx, std::vector<asio::ip::tcp::endpoint> endpoints, const bool ssl,
@@ -429,13 +458,12 @@ void
Resolver::resolve(const std::string& host, const std::string& service)
{
asio::ip::tcp::resolver::query query_(host, service);
resolver_.async_resolve(query_, [this, host, service, destroyed = destroyed_]
(const asio::error_code& ec, asio::ip::tcp::resolver::results_type endpoints)
{
if (ec == asio::error::operation_aborted or *destroyed)
return;
if (logger_){
if (logger_) {
if (ec)
logger_->e("[http:client] [resolver] error for %s:%s: %s",
host.c_str(), service.c_str(), ec.message().c_str());
@@ -483,7 +511,29 @@ Request::Request(asio::io_context& ctx, const std::string& url, const Json::Valu
logger_->e("[http:request:%i] can't parse response to json", id_, err.c_str());
}
if (jsoncb)
jsoncb(std::move(json), response.status_code);
jsoncb(std::move(json), response);
});
}
Request::Request(asio::io_context& ctx, const std::string& url, OnJsonCb jsoncb, std::shared_ptr<dht::Logger> logger)
: logger_(std::move(logger)), id_(Request::ids_++), ctx_(ctx),
resolver_(std::make_shared<Resolver>(ctx, url, logger))
{
init_default_headers();
set_header_field(restinio::http_field_t::accept, HTTP_HEADER_CONTENT_TYPE_JSON);
Json::StreamWriterBuilder wbuilder;
set_method(restinio::http_method_get());
add_on_done_callback([this, jsoncb](const Response& response) {
Json::Value json;
if (response.status_code != 0) {
std::string err;
Json::CharReaderBuilder rbuilder;
auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
if (!reader->parse(response.body.data(), response.body.data() + response.body.size(), &json, &err) and logger_)
logger_->e("[http:request:%i] can't parse response to json", id_, err.c_str());
}
if (jsoncb)
jsoncb(std::move(json), response);
});
}
@@ -590,7 +640,7 @@ Request::set_method(restinio::http_method_id_t method) {
void
Request::set_target(std::string target) {
header_.request_target(std::move(target));
header_.request_target(target.empty() ? "/" : std::move(target));
}
void
@@ -762,18 +812,25 @@ Request::connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, HandlerCb cb)
eps.append(endpoint.address().to_string() + ":" + std::to_string(endpoint.port()) + " ");
logger_->d("[http:request:%i] connect begin: %s", id_, eps.c_str());
}
if (get_url().protocol == "https"){
if (server_ca_)
bool isHttps = get_url().protocol == "https";
if (isHttps) {
if (server_ca_ or client_identity_.first)
conn_ = std::make_shared<Connection>(ctx_, server_ca_, client_identity_, logger_);
else
conn_ = std::make_shared<Connection>(ctx_, true/*ssl*/, logger_);
#if !defined(LINUX) || defined(__ANDROID__)
conn_->set_ssl_verification(get_url().host, asio::ssl::verify_none);
#else
conn_->set_ssl_verification(get_url().host, asio::ssl::verify_peer
| asio::ssl::verify_fail_if_no_peer_cert);
#endif
}
else
conn_ = std::make_shared<Connection>(ctx_, false/*ssl*/, logger_);
// try to connect to any until one works
std::weak_ptr<Request> wthis = shared_from_this();
conn_->async_connect(std::move(endpoints), [wthis, cb]
conn_->async_connect(std::move(endpoints), [wthis, cb, isHttps]
(const asio::error_code& ec, const asio::ip::tcp::endpoint& endpoint){
auto sthis = wthis.lock();
if (not sthis)
@@ -788,23 +845,16 @@ Request::connect(std::vector<asio::ip::tcp::endpoint>&& endpoints, HandlerCb cb)
if (this_.logger_)
this_.logger_->e("[http:request:%i] connect failed with all endpoints: %s", this_.id_, ec.message().c_str());
} else {
// if (this_.logger_)
// this_.logger_->d("[http:request:%i] connect success", this_.id_);
const auto& url = this_.get_url();
auto port = endpoint.port();
if ((url.protocol == "http" && port == (in_port_t)80)
|| (url.protocol == "https" && port == (in_port_t)443))
if ((!isHttps && port == (in_port_t)80)
|| (isHttps && port == (in_port_t)443))
this_.set_header_field(restinio::http_field_t::host, url.host);
else
this_.set_header_field(restinio::http_field_t::host, url.host + ":" + std::to_string(port));
if (url.protocol == "https") {
if (this_.server_ca_)
this_.conn_->set_ssl_verification(endpoint, asio::ssl::verify_peer
| asio::ssl::verify_fail_if_no_peer_cert);
if (this_.conn_ and this_.conn_->is_open() and this_.conn_->is_ssl()){
if (isHttps) {
if (this_.conn_ and this_.conn_->is_open() and this_.conn_->is_ssl()) {
this_.conn_->async_handshake([id = this_.id_, cb, logger = this_.logger_](const asio::error_code& ec){
if (ec == asio::error::operation_aborted)
return;
@@ -931,7 +981,7 @@ Request::handle_request(const asio::error_code& ec)
}
void
Request::handle_response(const asio::error_code& ec, size_t n_bytes)
Request::handle_response(const asio::error_code& ec, size_t /* n_bytes */)
{
std::lock_guard<std::mutex> lock(mutex_);
if (ec && ec != asio::error::eof){
@@ -988,7 +1038,10 @@ Request::onHeadersComplete() {
}
if (follow_redirect and num_redirect < MAX_REDIRECTS) {
auto next = std::make_shared<Request>(ctx_, location_it->second, logger_);
auto newUrl = getRelativePath(get_url(), location_it->second);
if (logger_)
logger_->w("[http:client] [request:%i] redirect to %s", id_, newUrl.c_str());
auto next = std::make_shared<Request>(ctx_, newUrl, logger_);
next->set_method(header_.method());
next->headers_ = std::move(headers_);
next->body_ = std::move(body_);
@@ -1018,5 +1071,30 @@ Request::onHeadersComplete() {
}
}
bool startsWith(const std::string& haystack, const std::string& needle) {
return needle.length() <= haystack.length()
&& std::equal(needle.begin(), needle.end(), haystack.begin());
}
std::string
Request::getRelativePath(const Url& origin, const std::string& path)
{
if (startsWith(path, HTTP_PROTOCOL)
|| startsWith(path, HTTPS_PROTOCOL)
|| startsWith(path, ORIGIN_PROTOCOL)) {
// Absolute path
return path;
}
Url newPath = origin;
if (not path.empty() and path[0] == '/') {
newPath.target = path;
} else {
if (newPath.target.empty())
newPath.target.push_back('/');
newPath.target += path;
}
return newPath.toString();
}
} // namespace http
} // namespace dht
+8 -22
View File
@@ -560,7 +560,8 @@ NetworkEngine::process(std::unique_ptr<ParsedMessage>&& msg, const SockAddr& fro
{
req->last_try = time_point::min();
req->reply_time = time_point::min();
onError(req, DhtProtocolException {msg->error_code});
if (not req->setError(DhtProtocolException {msg->error_code}))
onError(req, DhtProtocolException {msg->error_code});
} else {
if (logIncoming_)
if (logger_)
@@ -1072,27 +1073,11 @@ NetworkEngine::sendListen(Sp<Node> n,
const InfoHash& hash,
const Query& query,
const Blob& token,
Sp<Request> previous,
Tid socketId,
RequestCb&& on_done,
RequestExpiredCb&& on_expired,
SocketCb&& socket_cb)
RequestExpiredCb&& on_expired)
{
Tid socket;
Tid tid (n->getNewTid());
if (previous and previous->node == n) {
socket = previous->getSocket();
} else {
if (previous)
if (logger_)
logger_->e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str());
socket = n->openSocket(std::move(socket_cb));
}
if (not socket) {
if (logger_)
logger_->e(hash, "[node %s] unable to get a valid socket for listen. Aborting listen", n->toString().c_str());
return {};
}
msgpack::sbuffer buffer;
msgpack::packer<msgpack::sbuffer> pk(&buffer);
pk.pack_map(5+(config.network?1:0));
@@ -1103,7 +1088,7 @@ NetworkEngine::sendListen(Sp<Node> n,
pk.pack(KEY_VERSION); pk.pack(1);
pk.pack(KEY_REQ_H); pk.pack(hash);
pk.pack(KEY_REQ_TOKEN); packToken(pk, token);
pk.pack(KEY_REQ_SID); pk.pack(socket);
pk.pack(KEY_REQ_SID); pk.pack(socketId);
if (has_query) {
pk.pack(KEY_REQ_QUERY); pk.pack(query);
}
@@ -1125,8 +1110,7 @@ NetworkEngine::sendListen(Sp<Node> n,
[=](const Request& req_status, bool done) { /* on expired */
if (on_expired)
on_expired(req_status, done);
},
socket
}
);
sendRequest(req);
++out_stats.listen;
@@ -1263,6 +1247,7 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
const Value::Id& vid,
const Blob& token,
RequestCb&& on_done,
RequestErrorCb&& on_error,
RequestExpiredCb&& on_expired)
{
Tid tid (n->getNewTid());
@@ -1298,6 +1283,7 @@ NetworkEngine::sendRefreshValue(Sp<Node> n,
}
}
},
on_error,
[=](const Request& req_status, bool done) { /* on expired */
if (on_expired) {
on_expired(req_status, done);
-1
View File
@@ -116,7 +116,6 @@ Node::cancelRequest(const Sp<net::Request>& req)
{
if (req) {
req->cancel();
closeSocket(req->closeSocket());
requests_.erase(req->getTid());
}
}
+21 -7
View File
@@ -27,6 +27,7 @@ struct Node;
namespace net {
class NetworkEngine;
class DhtProtocolException;
struct ParsedMessage;
/*!
@@ -72,16 +73,19 @@ struct Request {
Sp<Node> node,
Blob&& msg,
std::function<void(const Request&, ParsedMessage&&)> on_done,
std::function<void(const Request&, bool)> on_expired,
Tid socket = 0) :
node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)), socket(socket) { }
std::function<void(const Request&, bool)> on_expired) :
node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)) { }
Request(MessageType type, Tid tid,
Sp<Node> node,
Blob&& msg,
std::function<void(const Request&, ParsedMessage&&)> on_done,
std::function<bool(const Request&, DhtProtocolException&&)> on_error,
std::function<void(const Request&, bool)> on_expired) :
node(node), tid(tid), type(type), on_done(on_done), on_error(on_error), on_expired(on_expired), msg(std::move(msg)) { }
Tid getTid() const { return tid; }
MessageType getType() const { return type; }
Tid getSocket() { return socket; }
Tid closeSocket() { auto ret = socket; socket = 0; return ret; }
void setExpired() {
if (pending()) {
state_ = Request::State::EXPIRED;
@@ -96,6 +100,15 @@ struct Request {
clear();
}
}
bool setError(DhtProtocolException&& e) {
if (pending()) {
state_ = Request::State::EXPIRED;
bool handled = on_error and on_error(*this, std::forward<DhtProtocolException>(e));
clear();
return handled;
}
return true;
}
void cancel() {
if (pending()) {
@@ -113,6 +126,7 @@ private:
void clear() {
on_done = {};
on_error = {};
on_expired = {};
msg = {};
parts = {};
@@ -128,11 +142,11 @@ private:
time_point last_try {time_point::min()}; /* time of the last attempt to process the request. */
std::function<void(const Request&, ParsedMessage&&)> on_done {};
std::function<bool(const Request&, DhtProtocolException&&)> on_error {};
std::function<void(const Request&, bool)> on_expired {};
Blob msg {}; /* the serialized message. */
std::vector<Blob> parts;
Tid socket; /* the socket used for further reponses. */
};
} /* namespace net */
+17 -10
View File
@@ -65,10 +65,16 @@ struct Dht::SearchNode {
ValueCache cache;
Sp<Scheduler::Job> cacheExpirationJob {};
Sp<net::Request> req {};
CachedListenStatus(ValueStateCallback&& cb, SyncCallback&& scb)
: cache(std::forward<ValueStateCallback>(cb), std::forward<SyncCallback>(scb)) {}
Tid socketId {0};
CachedListenStatus(ValueStateCallback&& cb, SyncCallback scb, Tid sid)
: cache(std::forward<ValueStateCallback>(cb), std::forward<SyncCallback>(scb)), socketId(sid) {}
CachedListenStatus(CachedListenStatus&&) = delete;
CachedListenStatus(const CachedListenStatus&) = delete;
~CachedListenStatus() {
if (socketId and req and req->node) {
req->node->closeSocket(socketId);
}
}
};
using NodeListenerStatus = std::map<Sp<Query>, CachedListenStatus>;
@@ -344,13 +350,15 @@ struct Dht::SearchNode {
* Assuming the node is synced, should the "listen" request with Query q be
* sent to this node now ?
*/
time_point getListenTime(const Sp<Query>& q, duration listen_expire) const {
auto listen_status = listenStatus.find(q);
if (listen_status == listenStatus.end() or not listen_status->second.req)
time_point getListenTime(const decltype(listenStatus)::const_iterator listen_status, duration listen_expire) const {
if (listen_status == listenStatus.cend() or not listen_status->second.req)
return time_point::min();
return listen_status->second.req->pending() ? time_point::max() :
listen_status->second.req->reply_time + listen_expire - REANNOUNCE_MARGIN;
}
time_point getListenTime(const Sp<Query>& q, duration listen_expire) const {
return getListenTime(listenStatus.find(q), listen_expire);
}
/**
* Is this node expired or candidate
@@ -759,7 +767,6 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
}
}
bool new_search_node = false;
if (not found) {
// find if and where to trim excessive nodes
auto t = nodes.cend();
@@ -783,9 +790,10 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
}
if (full) {
bool addNode = n < t;
if (t != nodes.cend())
nodes.resize(std::distance(nodes.cbegin(), t));
if (n >= t)
if (not addNode)
return false;
}
@@ -795,7 +803,6 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
}
n = nodes.emplace(n, std::make_unique<SearchNode>(snode));
node.setTime(now);
new_search_node = true;
if (node.isExpired()) {
if (not expired)
bad++;
@@ -817,9 +824,9 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
(*n)->token = token;
expired = false;
}
if (new_search_node)
if (not found)
removeExpiredNode(now);
return new_search_node;
return not found;
}
std::vector<Sp<Node>>
-1
View File
@@ -107,7 +107,6 @@ ThreadPool::run(std::function<void()>&& cb)
tasks_.emplace(std::move(cb));
// notify thread
l.unlock();
cv_.notify_one();
}
+26
View File
@@ -2,6 +2,7 @@
* Copyright (C) 2014-2020 Savoir-faire Linux Inc.
*
* Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -135,6 +136,31 @@ CryptoTester::testCertificateRequest()
CPPUNIT_ASSERT_MESSAGE(v.toString(), v);
}
void CryptoTester::testCertificateSerialNumber()
{
static const std::string cert_pem = "-----BEGIN CERTIFICATE-----"
"MIICDjCCAZSgAwIBAgIIS90uAKp+u/swCgYIKoZIzj0EAwMwTDEQMA4GA1UEAxMH"
"ZGh0bm9kZTE4MDYGCgmSJomT8ixkAQETKDBlNDQxZTA4YWJmYTQzYTc3ZTVjZDBm"
"Y2QzMzAzMTc4MjYxMTk0MzIwHhcNMTkxMTA3MDA0MTMwWhcNMjkxMTA0MDA0MTMw"
"WjBMMRAwDgYDVQQDEwdkaHRub2RlMTgwNgYKCZImiZPyLGQBARMoMGU0NDFlMDhh"
"YmZhNDNhNzdlNWNkMGZjZDMzMDMxNzgyNjExOTQzMjB2MBAGByqGSM49AgEGBSuB"
"BAAiA2IABCZ7sBp0Pu+b5yIifoNXchU9crv9won0in++COWynvM4GCLF2Gk6QGhh"
"YLDxNGsyQjGR7z5AGibvYhNLU0JA4RbmxYWHw4g3wBrPA1jm9hGZ8y5Y8R97d0Hl"
"VpyreEMjRKNDMEEwHQYDVR0OBBYEFA5EHgir+kOnflzQ/NMwMXgmEZQyMA8GA1Ud"
"EwEB/wQFMAMBAf8wDwYDVR0PAQH/BAUDAwcGADAKBggqhkjOPQQDAwNoADBlAjEA"
"kKF/6WReNytwSrJ8JSTToc7xWS5SvAa23Fnla4mywZUxUFS4VSxCMQTjQCknO3iZ"
"AjBgxXyXYqn0d7vz7S6oAY5TdaD5YFT5MD2c1MAAp8pxQSwdPa9k0ZSoGIEn31Z0"
"GxU="
"-----END CERTIFICATE-----";
/*
* $ openssl x509 -in cert.pem -noout -serial
* serial=4BDD2E00AA7EBBFB
*/
static constexpr std::array<uint8_t, 8> SERIAL {{0x4b,0xdd,0x2e,0x00,0xaa,0x7e,0xbb,0xfb}};
auto serial = dht::crypto::Certificate(cert_pem).getSerialNumber();
CPPUNIT_ASSERT(std::equal(SERIAL.begin(), SERIAL.end(), serial.begin(), serial.end()));
}
void
CryptoTester::tearDown() {
+6
View File
@@ -2,6 +2,7 @@
* Copyright (C) 2014-2020 Savoir-faire Linux Inc.
*
* Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Vsevolod Ivanov <vsevolod.ivanov@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -30,6 +31,7 @@ class CryptoTester : public CppUnit::TestFixture {
CPPUNIT_TEST(testSignatureEncryption);
CPPUNIT_TEST(testCertificateRevocation);
CPPUNIT_TEST(testCertificateRequest);
CPPUNIT_TEST(testCertificateSerialNumber);
CPPUNIT_TEST_SUITE_END();
public:
@@ -53,6 +55,10 @@ class CryptoTester : public CppUnit::TestFixture {
* Test certificate requests
*/
void testCertificateRequest();
/**
* Test certificate serial number extraction
*/
void testCertificateSerialNumber();
};
} // namespace test
+4 -15
View File
@@ -61,7 +61,6 @@ HttpTester::test_parse_url() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "google.com");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
}
@@ -75,7 +74,6 @@ HttpTester::test_parse_https_url_no_service() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "https");
CPPUNIT_ASSERT(parsed.host == "jami.net");
CPPUNIT_ASSERT(parsed.service == "https");
CPPUNIT_ASSERT(parsed.target == "/");
}
@@ -89,8 +87,7 @@ HttpTester::test_parse_url_no_prefix_no_target() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "google.com");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
CPPUNIT_ASSERT(parsed.target == "");
}
void
@@ -117,7 +114,6 @@ HttpTester::test_parse_url_query() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "google.com");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/?key=1");
CPPUNIT_ASSERT(parsed.query == "key=1");
}
@@ -132,7 +128,6 @@ HttpTester::test_parse_url_fragment() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "google.com");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/?key=1");
CPPUNIT_ASSERT(parsed.query == "key=1");
CPPUNIT_ASSERT(parsed.fragment == "#some-important-id");
@@ -148,7 +143,6 @@ HttpTester::test_parse_url_ipv4() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "172.217.13.132");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
}
@@ -162,8 +156,6 @@ HttpTester::test_parse_url_no_prefix_no_target_ipv4() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "172.217.13.132");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
}
void
@@ -190,7 +182,6 @@ HttpTester::test_parse_url_ipv6() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "2607:f8b0:4006:804::2004");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
}
@@ -204,8 +195,6 @@ HttpTester::test_parse_url_no_prefix_no_target_ipv6() {
CPPUNIT_ASSERT(parsed.url == url);
CPPUNIT_ASSERT(parsed.protocol == "http");
CPPUNIT_ASSERT(parsed.host == "2607:f8b0:4006:804::2004");
CPPUNIT_ASSERT(parsed.service == "80");
CPPUNIT_ASSERT(parsed.target == "/");
}
void
@@ -235,13 +224,13 @@ HttpTester::test_send_json() {
Json::Value resp_val;
// Act
auto request = std::make_shared<dht::http::Request>(serverProxy->io_context(),
auto request = std::make_shared<dht::http::Request>(serverProxy->io_context(),
"http://127.0.0.1:8080/key",
json,
[&](Json::Value value, unsigned int status_code) {
[&](Json::Value value, const dht::http::Response& response) {
std::lock_guard<std::mutex> lk(cv_m);
resp_val = std::move(value);
status = status_code;
status = response.status_code;
done = true;
cv.notify_all();
});
+6
View File
@@ -42,8 +42,14 @@ endif ()
install (TARGETS dhtnode dhtscanner dhtchat RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
if (OPENDHT_SYSTEMD)
if (NOT DEFINED OPENDHT_SYSTEMD_UNIT_FILE_LOCATION)
execute_process(COMMAND ${PKG_CONFIG_EXECUTABLE} systemd --variable=systemdsystemunitdir
OUTPUT_VARIABLE SYSTEMD_UNIT_INSTALL_DIR)
message("-- Using Systemd unit installation directory by pkg-config: " ${SYSTEMD_UNIT_INSTALL_DIR})
else()
message("-- Using Systemd unit installation directory requested: " ${OPENDHT_SYSTEMD_UNIT_FILE_LOCATION})
set(SYSTEMD_UNIT_INSTALL_DIR ${OPENDHT_SYSTEMD_UNIT_FILE_LOCATION})
endif()
string(REGEX REPLACE "[ \t\n]+" "" SYSTEMD_UNIT_INSTALL_DIR "${SYSTEMD_UNIT_INSTALL_DIR}")
set (systemdunitdir "${SYSTEMD_UNIT_INSTALL_DIR}")
+1 -1
View File
@@ -7,7 +7,7 @@ stats_total = {"users":0, "pushListenersCount":0, "listenCount":0, "totalListene
for i in range(80,101):
print("Collecting stats for proxy " + str(i))
response = requests.request('STATS', 'http://127.0.0.1:' + str(i))
response = requests.request('GET', 'http://127.0.0.1:' + str(i) + '/node/stats')
if response.status_code == 200:
result = response.json()