diff --git a/.gitignore b/.gitignore index 9d50e4d5..c79e0d15 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,11 @@ Makefile.in # Python *.pyc +python/opendht.cpp +python/setup.py + +# Doxygen +doc/Doxyfile # git backup files *.orig diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f607dbb..4332712d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required (VERSION 3.1) project (opendht) set (opendht_VERSION_MAJOR 1) -set (opendht_VERSION_MINOR 3.5) +set (opendht_VERSION_MINOR 6.0) set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR}) set (PACKAGE_VERSION ${opendht_VERSION}) set (VERSION "${opendht_VERSION}") @@ -15,9 +15,18 @@ option (OPENDHT_TOOLS "Build DHT tools" ON) option (OPENDHT_SYSTEMD "Install systemd module" OFF) option (OPENDHT_ARGON2 "Use included argon2 sources" OFF) option (OPENDHT_LTO "Build with LTO" OFF) +option (OPENDHT_SANITIZE "Build with address sanitizer and stack protector" OFF) +option (OPENDHT_PROXY_SERVER "Enable DHT proxy server, use Restbed and jsoncpp" OFF) +option (OPENDHT_PUSH_NOTIFICATIONS "Enable push notifications support" OFF) +option (OPENDHT_PROXY_SERVER_IDENTITY "Allow clients to use the node identity" OFF) +option (OPENDHT_PROXY_CLIENT "Enable DHT proxy client, use Restbed and jsoncpp" OFF) + +find_package(Doxygen) +option (OPENDHT_DOCUMENTATION "Create and install the HTML based API documentation (requires Doxygen)" ${DOXYGEN_FOUND}) # Dependencies list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +find_package (Threads) find_package (PkgConfig) find_package (GnuTLS 3.3 REQUIRED) pkg_search_module (Nettle nettle) @@ -36,11 +45,18 @@ if (NOT OPENDHT_ARGON2) set(OPENDHT_ARGON2 ON) endif() endif () +if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT) + find_package(Restbed REQUIRED) + pkg_search_module(Jsoncpp jsoncpp) +endif() # Build flags set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD_REQUIRED on) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type -Wall -Wextra -Wnon-virtual-dtor -pedantic-errors -fvisibility=hidden") +if (OPENDHT_SANITIZE) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fstack-protector-strong") +endif () set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMSGPACK_DISABLE_LEGACY_NIL -DMSGPACK_DISABLE_LEGACY_CONVERT") if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) @@ -69,7 +85,14 @@ endif () if (Nettle_INCLUDE_DIRS) include_directories (SYSTEM "${Nettle_INCLUDE_DIRS}") endif () +if (RESTBED_INCLUDE) + include_directories (SYSTEM "${RESTBED_INCLUDE}") +endif () +if (Jsoncpp_INCLUDE_DIRS) + include_directories (SYSTEM "${Jsoncpp_INCLUDE_DIRS}") +endif () link_directories (${Nettle_LIBRARY_DIRS}) +link_directories (${Jsoncpp_LIBRARY_DIRS}) include_directories ( ./ include/ @@ -85,6 +108,7 @@ set (libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}") set (includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") set (bindir "${CMAKE_INSTALL_FULL_BINDIR}") set (sysconfdir "${CMAKE_INSTALL_FULL_SYSCONFDIR}") +set (top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}") # Sources list (APPEND opendht_SOURCES @@ -98,6 +122,8 @@ list (APPEND opendht_SOURCES src/storage.h src/listener.h src/search.h + src/net.h + src/parsed_message.h src/request.h src/callbacks.cpp src/routing_table.cpp @@ -120,11 +146,11 @@ list (APPEND opendht_HEADERS include/opendht/node.h include/opendht/value.h include/opendht/dht.h + include/opendht/dht_interface.h include/opendht/callbacks.h include/opendht/routing_table.h include/opendht/node_cache.h include/opendht/network_engine.h - include/opendht/net.h include/opendht/scheduler.h include/opendht/rate_limiter.h include/opendht/securedht.h @@ -134,6 +160,48 @@ list (APPEND opendht_HEADERS include/opendht/indexation/pht.h ) +if (OPENDHT_PROXY_SERVER) + add_definitions(-DOPENDHT_PROXY_SERVER=true) + if (OPENDHT_PROXY_SERVER_IDENTITY) + add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=true) + else () + add_definitions(-DOPENDHT_PROXY_SERVER_IDENTITY=false) + endif() + list (APPEND opendht_HEADERS + include/opendht/dht_proxy_server.h + ) + list (APPEND opendht_SOURCES + src/dht_proxy_server.cpp + ) + if (OPENDHT_PUSH_NOTIFICATIONS) + add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=true) + else () + add_definitions(-DOPENDHT_PUSH_NOTIFICATIONS=false) + endif () +else () + add_definitions(-DENABLE_PROXY_SERVER=false) +endif () + +if (OPENDHT_PROXY_CLIENT) + add_definitions(-DOPENDHT_PROXY_CLIENT=true) + list (APPEND opendht_HEADERS + include/opendht/dht_proxy_client.h + ) + list (APPEND opendht_SOURCES + src/dht_proxy_client.cpp + ) +else () + add_definitions(-DOPENDHT_PROXY_CLIENT=false) +endif () + +if (OPENDHT_PROXY_SERVER OR OPENDHT_PROXY_CLIENT) + list (APPEND opendht_SOURCES + src/base64.h + src/base64.cpp + ) +endif () + + if(OPENDHT_ARGON2) # make sure argon2 submodule is up to date and initialized message("Initializing Argon2 submodule") @@ -167,7 +235,7 @@ if (OPENDHT_STATIC) target_link_libraries(opendht-static ${argon2_LIBRARIES}) target_include_directories(opendht-static SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht-static ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + target_link_libraries(opendht-static ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES}) install (TARGETS opendht-static DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () @@ -180,15 +248,15 @@ if (OPENDHT_SHARED) set_target_properties (opendht PROPERTIES SOVERSION ${opendht_VERSION_MAJOR} VERSION ${opendht_VERSION}) target_compile_definitions(opendht PRIVATE OPENDHT_BUILD) if (OPENDHT_LTO) - target_link_libraries(opendht -flto -fuse-linker-plugin) + target_link_libraries(opendht PRIVATE -flto -fuse-linker-plugin) endif () if (OPENDHT_ARGON2) target_include_directories(opendht SYSTEM PRIVATE argon2) else () - target_link_libraries(opendht ${argon2_LIBRARIES}) + target_link_libraries(opendht PRIVATE ${argon2_LIBRARIES}) target_include_directories(opendht SYSTEM PRIVATE ${argon2_INCLUDE_DIRS}) endif () - target_link_libraries(opendht ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES}) + target_link_libraries(opendht PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${GNUTLS_LIBRARIES} ${Nettle_LIBRARIES} ${Restbed_LIBRARIES} ${Jsoncpp_LIBRARIES}) install (TARGETS opendht DESTINATION ${CMAKE_INSTALL_LIBDIR} EXPORT opendht) endif () diff --git a/MSVC/opendht.vcxproj b/MSVC/opendht.vcxproj index ce94bdd5..38158f9c 100644 --- a/MSVC/opendht.vcxproj +++ b/MSVC/opendht.vcxproj @@ -19,11 +19,14 @@ + + + @@ -36,12 +39,16 @@ + + + + @@ -50,14 +57,18 @@ - + + + + + @@ -211,9 +222,9 @@ copy ..\include\opendht.h $(OutDir)\include\ true true true - $(ProjectDir)contrib\build\include;$(ProjectDir)..\include;$(ProjectDir)..\include\opendht;$(ProjectDir)contrib\build\msgpack-c\include;$(ProjectDir)contrib\build\argon2\include + $(ProjectDir)contrib\build\include;$(ProjectDir)..\include;$(ProjectDir)..\include\opendht;$(ProjectDir)contrib\build\msgpack-c\include;$(ProjectDir)contrib\build\argon2\include;$(ProjectDir)..\..\include;$(ProjectDir)..\..\argon2\include _CRT_SECURE_NO_WARNINGS;WIN32_NATIVE;WIN32_LEAN_AND_MEAN;_MBCS;%(PreprocessorDefinitions) - 4804;4800;4101;4267;4244;4503; + 4804;4800;4101;4267;4244;4503;4273; -D_SCL_SECURE_NO_WARNINGS %(AdditionalOptions) $(OutDir)\lib\x64\$(TargetName).pdb diff --git a/MSVC/opendht.vcxproj.filters b/MSVC/opendht.vcxproj.filters index 3cec5eab..c0c9bb86 100644 --- a/MSVC/opendht.vcxproj.filters +++ b/MSVC/opendht.vcxproj.filters @@ -31,9 +31,6 @@ Source Files - - Source Files - Source Files @@ -46,80 +43,102 @@ Source Files + + Source Files + + + Source Files + + + Source Files\indexation + + - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files + + + + + + Source Files - Header Files + Source Files - - Header Files + + Source Files - - Header Files + + Source Files - - Header Files - - - Header Files - - - Header Files + + Source Files Header Files - - Header Files + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht + + + Header Files\opendht - Header Files - - - Header Files + Header Files\opendht - Header Files + Header Files\opendht - Header Files + Header Files\opendht - Header Files - - - Header Files + Header Files\opendht - Header Files - - - Header Files + Header Files\opendht - Header Files + Header Files\opendht - Header Files + Header Files\opendht + + Header Files\opendht\indexation + + + @@ -128,5 +147,14 @@ {d1ab5bfe-3ab1-45ee-9324-b4b071887668} + + {939134d3-d5a2-4458-bca8-a10ea9042ea2} + + + {0b46e2d0-4874-404c-bae3-fae8d49b85b8} + + + {03bd5139-2ae5-41b0-a77b-944814e1b2b6} + \ No newline at end of file diff --git a/README.md b/README.md index c556cdba..3402b065 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

- OpenDHT + OpenDHT

A lightweight C++11 Distributed Hash Table implementation originally based on https://github.com/jech/dht by Juliusz Chroboczek. @@ -18,7 +18,7 @@ See the wiki: #### How-to build and install -Build instructions : +Build instructions: #### How-to build a simple client app ```bash @@ -91,7 +91,9 @@ for r in results: - msgpack-c 1.2+, used for data serialization. - GnuTLS 3.3+, used for cryptographic operations. - Nettle 2.4+, a GnuTLS dependency for crypto. -- Build tested with GCC 5.4+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, macOS). +- (optional) restbed used for the REST API. commit fb84213e170bc171fecd825a8e47ed9f881a12cd (https://github.com/AmarOk1412/restbed/tree/async_read_until) +- (optional) jsoncpp 1.7.4-3+, used for the REST API. +- Build tested with GCC 5.2+ (GNU/Linux, Android, Windows with MinGW), Clang/LLVM (Linux, macOS). - Build tested with Microsoft Visual Studio 2015 ## Contact diff --git a/cmake/FindRestbed.cmake b/cmake/FindRestbed.cmake new file mode 100644 index 00000000..05f1e814 --- /dev/null +++ b/cmake/FindRestbed.cmake @@ -0,0 +1,17 @@ +find_path (RESTBED_INCLUDE restbed + HINTS + "/usr/include" + "/usr/local/include" + "/opt/local/include" +) + +if (RESTBED_INCLUDE) + message(STATUS "${green}Found Restbed: ${RESTBED_INCLUDE}") +else() + message(FATAL_ERROR "${red}Failed to locate Restbed.}") +endif() + +if (RESTBED_INCLUDE) + set(RESTBED_FOUND TRUE) + set(Restbed_LIBRARIES restbed) +endif() diff --git a/configure.ac b/configure.ac index dbc5359c..7d051d72 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl define macros m4_define([opendht_major_version], 1) -m4_define([opendht_minor_version], 3) -m4_define([opendht_patch_version], 5) +m4_define([opendht_minor_version], 6) +m4_define([opendht_patch_version], 0) m4_define([opendht_version], [opendht_major_version.opendht_minor_version.opendht_patch_version]) @@ -128,6 +128,54 @@ AM_COND_IF([ENABLE_TOOLS], [ ]) ]) +AC_ARG_ENABLE([proxy_server], AS_HELP_STRING([--enable-proxy-server], [Enable proxy server ability]), proxy_server=yes, proxy_server=no) +AM_CONDITIONAL(ENABLE_PROXY_SERVER, test x$proxy_server == xyes) +AC_ARG_ENABLE([push_notifications], AS_HELP_STRING([--enable-push-notifications], [Enable push notifications support]), push_notifications=yes, push_notifications=no) +AM_CONDITIONAL(ENABLE_PUSH_NOTIFICATIONS, test x$push_notifications == xyes) +AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity], + [Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no) +AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes) +AC_ARG_ENABLE([proxy_client], AS_HELP_STRING([--enable-proxy-client], [Enable proxy client ability]), proxy_client=yes, proxy_client=no) +AM_CONDITIONAL(ENABLE_PROXY_CLIENT, test x$proxy_client == xyes) + +AM_COND_IF([ENABLE_PROXY_SERVER], [ + AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files])) + PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.2]) + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=true -ljsoncpp -lrestbed" +], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER=false" +]) + +AM_COND_IF([ENABLE_PUSH_NOTIFICATIONS], [ + CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=true" +], [ + CPPFLAGS+=" -DOPENDHT_PUSH_NOTIFICATIONS=false" +]) + +AC_ARG_ENABLE([proxy_server_identity], AS_HELP_STRING([--enable-proxy-server-identity], + [Enable proxy server ability]), proxy_server_identity=yes, proxy_server_identity=no) +AM_CONDITIONAL(ENABLE_PROXY_SERVER_IDENTITY, test x$proxy_server_identity == xyes -a x$proxy_server == xyes) +AM_COND_IF([ENABLE_PROXY_SERVER_IDENTITY], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=true" +], [ + CPPFLAGS+=" -DOPENDHT_PROXY_SERVER_IDENTITY=false" +]) + +AM_COND_IF([ENABLE_PROXY_CLIENT], [ + CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=true" +], [ + CPPFLAGS+=" -DOPENDHT_PROXY_CLIENT=false" +]) + +AM_CONDITIONAL(PROXY_CLIENT_OR_SERVER, test x$proxy_client == xyes | test x$proxy_server == xyes) +AM_COND_IF([PROXY_CLIENT_OR_SERVER], [ + AC_CHECK_LIB(restbed, exit,, AC_MSG_ERROR([Missing restbed files])) + PKG_CHECK_MODULES([Jsoncpp], [jsoncpp >= 1.7.4]) + CPPFLAGS="${CPPFLAGS} ${Jsoncpp_CFLAGS}" + LDFLAGS="${LDFLAGS} ${Jsoncpp_LIBS} -lrestbed" +], []) + + AC_CONFIG_FILES([doc/Doxyfile doc/Makefile]) dnl Configure setup.py if we build the python module diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 3a204fe5..681cb281 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,3 +1,16 @@ if (OPENDHT_TOOLS) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dhtnode.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1) endif () + +if (OPENDHT_DOCUMENTATION) + if (NOT DOXYGEN_FOUND) + message(FATAL_ERROR "Doxygen is needed to build the documentation.") + endif() + configure_file (Doxyfile.in Doxyfile @ONLY) + add_custom_target(doc ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" + VERBATIM) + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html DESTINATION share/doc/opendht) +endif() diff --git a/include/opendht.h b/include/opendht.h index 98d3ece9..e003a216 100644 --- a/include/opendht.h +++ b/include/opendht.h @@ -19,6 +19,9 @@ #pragma once #include "opendht/dhtrunner.h" +#if OPENDHT_PROXY_SERVER +#include "opendht/dht_proxy_server.h" +#endif #include "opendht/log.h" #include "opendht/default_types.h" #include "opendht/indexation/pht.h" diff --git a/include/opendht/callbacks.h b/include/opendht/callbacks.h index 21f2dd0c..fd067c6b 100644 --- a/include/opendht/callbacks.h +++ b/include/opendht/callbacks.h @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author : Adrien Béraud + * Authors: Adrien Béraud * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,6 +27,10 @@ #include #include +#if OPENDHT_PROXY_SERVER +#include +#endif //OPENDHT_PROXY_SERVER + namespace dht { struct Node; @@ -40,13 +45,21 @@ enum class NodeStatus { }; struct OPENDHT_PUBLIC NodeStats { - unsigned good_nodes, - dubious_nodes, - cached_nodes, - incoming_nodes; - unsigned table_depth; + unsigned good_nodes {0}, + dubious_nodes {0}, + cached_nodes {0}, + incoming_nodes {0}; + unsigned table_depth {0}; unsigned getKnownNodes() const { return good_nodes + dubious_nodes; } std::string toString() const; +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT + /** + * Build a json object from a NodeStats + */ + Json::Value toJson() const; + NodeStats() {}; + explicit NodeStats(const Json::Value& v); +#endif //OPENDHT_PROXY_SERVER }; /** diff --git a/include/opendht/crypto.h b/include/opendht/crypto.h index 81985c66..a840576b 100644 --- a/include/opendht/crypto.h +++ b/include/opendht/crypto.h @@ -36,6 +36,10 @@ extern "C" { #endif namespace dht { + +/** + * Contains all crypto primitives + */ namespace crypto { class OPENDHT_PUBLIC CryptoException : public std::runtime_error { @@ -583,6 +587,9 @@ using SecureBlob = secure_vector; OPENDHT_PUBLIC Identity generateIdentity(const std::string& name, Identity ca, unsigned key_length, bool is_ca); OPENDHT_PUBLIC Identity generateIdentity(const std::string& name = "dhtnode", Identity ca = {}, unsigned key_length = 4096); +OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name, Identity ca, bool is_ca); +OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name = "dhtnode", Identity ca = {}); + /** * Performs SHA512, SHA256 or SHA1, depending on hash_length. @@ -594,6 +601,8 @@ OPENDHT_PUBLIC Identity generateIdentity(const std::string& name = "dhtnode", Id */ OPENDHT_PUBLIC Blob hash(const Blob& data, size_t hash_length = 512/8); +OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length); + /** * Generates an encryption key from a text password, * making the key longer to bruteforce. diff --git a/include/opendht/default_types.h b/include/opendht/default_types.h index cbfc7d25..58595163 100644 --- a/include/opendht/default_types.h +++ b/include/opendht/default_types.h @@ -19,6 +19,7 @@ #pragma once #include "value.h" +#include "sockaddr.h" namespace dht { enum class ImStatus : uint8_t { @@ -45,7 +46,7 @@ public: static Value::Filter getFilter() { return {}; } - static bool storePolicy(InfoHash key, std::shared_ptr& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len); + static bool storePolicy(InfoHash key, std::shared_ptr& value, const InfoHash& from, const SockAddr&); static Value::Filter ServiceFilter(std::string s); @@ -92,7 +93,7 @@ public: static Value::Filter getFilter() { return Value::Filter::chain( BaseClass::getFilter(), - [](const Value& v){ return v.recipient != InfoHash(); } + [](const Value& v) { return static_cast(v.recipient); } ); } @@ -204,16 +205,12 @@ private: public: static const ValueType TYPE; - IpServiceAnnouncement(in_port_t p = 0) { - ss.ss_family = 0; - setPort(p); + IpServiceAnnouncement(sa_family_t family = AF_UNSPEC, in_port_t p = 0) { + addr.setFamily(family); + addr.setPort(p); } - IpServiceAnnouncement(const sockaddr* sa, socklen_t sa_len) { - ss.ss_family = 0; - if (sa) - std::copy_n((const uint8_t*)sa, sa_len, (uint8_t*)&ss); - } + IpServiceAnnouncement(const SockAddr& sa) : addr(sa) {} IpServiceAnnouncement(const Blob& b) { msgpack_unpack(unpackMsg(b).get()); @@ -222,53 +219,40 @@ public: template void msgpack_pack(Packer& pk) const { - pk.pack_array(2); - pk.pack(reinterpret_cast(&ss)->sin_port); - if (ss.ss_family == AF_INET) { - pk.pack_bin(sizeof(in_addr)); - pk.pack_bin_body((const char*)&reinterpret_cast(&ss)->sin_addr, sizeof(in_addr)); - } else if (ss.ss_family == AF_INET6) { - pk.pack_bin(sizeof(in6_addr)); - pk.pack_bin_body((const char*)&reinterpret_cast(&ss)->sin6_addr, sizeof(in6_addr)); - } + pk.pack_bin(addr.getLength()); + pk.pack_bin_body((const char*)addr.get(), addr.getLength()); } virtual void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::ARRAY) throw msgpack::type_error(); - if (o.via.array.size < 2) throw msgpack::type_error(); - reinterpret_cast(&ss)->sin_port = o.via.array.ptr[0].as(); - auto ip_dat = o.via.array.ptr[1].as(); - if (ip_dat.size() == sizeof(in_addr)) - std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast(&ss)->sin_addr); - else if (ip_dat.size() == sizeof(in6_addr)) - std::copy(ip_dat.begin(), ip_dat.end(), (char*)&reinterpret_cast(&ss)->sin6_addr); + if (o.type == msgpack::type::BIN) + addr = {(sockaddr*)o.via.bin.ptr, (socklen_t)o.via.bin.size}; else throw msgpack::type_error(); } in_port_t getPort() const { - return ntohs(reinterpret_cast(&ss)->sin_port); + return addr.getPort(); } void setPort(in_port_t p) { - reinterpret_cast(&ss)->sin_port = htons(p); + addr.setPort(p); } - sockaddr_storage getPeerAddr() const { - return ss; + const SockAddr& getPeerAddr() const { + return addr; } virtual const ValueType& getType() const { return TYPE; } - static bool storePolicy(InfoHash, std::shared_ptr&, InfoHash, const sockaddr*, socklen_t); + static bool storePolicy(InfoHash, std::shared_ptr&, const InfoHash&, const SockAddr&); /** print value for debugging */ friend std::ostream& operator<< (std::ostream&, const IpServiceAnnouncement&); private: - sockaddr_storage ss; + SockAddr addr; }; diff --git a/include/opendht/dht.h b/include/opendht/dht.h index 7167fa1f..274a34cf 100644 --- a/include/opendht/dht.h +++ b/include/opendht/dht.h @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author(s) : Adrien Béraud - * Simon Désaulniers + * Authors: Adrien Béraud + * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,7 +27,7 @@ #include "scheduler.h" #include "routing_table.h" #include "callbacks.h" -#include "log_enable.h" +#include "dht_interface.h" #include #include @@ -58,22 +59,16 @@ struct LocalListener; * Must be given open UDP sockets and ::periodic must be * called regularly. */ -class OPENDHT_PUBLIC Dht { +class OPENDHT_PUBLIC Dht final : public DhtInterface { public: - // [[deprecated]] - using NodeExport = dht::NodeExport; - - // [[deprecated]] - using Status = NodeStatus; - Dht(); /** * Initialise the Dht with two open sockets (for IPv4 and IP6) * and an ID for the node. */ - Dht(int s, int s6, Config config); + Dht(const int& s, const int& s6, Config config); virtual ~Dht(); /** @@ -103,18 +98,6 @@ public: */ bool isRunning(sa_family_t af = 0) const; - /** - * Enable or disable logging of DHT internal messages - */ - void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG); - - /** - * Only print logs related to the given InfoHash (if given), or disable filter (if zeroes). - */ - void setLogFilter(const InfoHash& f) { - DHT_LOG.setFilter(f); - } - virtual void registerType(const ValueType& type) { types[type.id] = type; } @@ -168,8 +151,8 @@ public: * @param key the key for which to query data for. * @param cb a function called when new values are found on the network. * It should return false to stop the operation. - * @param donecb a function called when the operation is complete. - cb and donecb won't be called again afterward. + * @param done_cb a function called when the operation is complete. + cb and done_cb won't be called again afterward. * @param q a query used to filter values on the remotes before they send a * response. */ @@ -248,7 +231,7 @@ public: * * @return a token to cancel the listener later. */ - virtual size_t listen(const InfoHash&, GetCallback, Value::Filter&&={}, Where&& w = {}); + virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where w = {}); virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) { return listen(key, bindGetCb(cb), std::forward(f), std::forward(w)); } @@ -307,10 +290,8 @@ public: std::vector getPublicAddress(sa_family_t family = 0); -protected: - Logger DHT_LOG; - bool logFilerEnable_ {}; - InfoHash logFiler_ {}; + void pushNotificationReceived(const std::map&) {} + void resubscribe(unsigned) {} private: @@ -339,7 +320,7 @@ private: static constexpr unsigned MAX_HASHES {16384}; /* The maximum number of searches we keep data about. */ - static constexpr unsigned MAX_SEARCHES {2048}; + static constexpr unsigned MAX_SEARCHES {16384}; static constexpr std::chrono::minutes MAX_STORAGE_MAINTENANCE_EXPIRE_TIME {10}; @@ -402,7 +383,6 @@ private: Scheduler scheduler; Sp nextNodesConfirmation {}; Sp nextStorageMaintenance {}; - time_point mybucket_grow_time {time_point::min()}, mybucket6_grow_time {time_point::min()}; net::NetworkEngine network_engine; unsigned pending_pings4 {0}; @@ -422,14 +402,13 @@ private: // Storage void storageAddListener(const InfoHash& id, const Sp& node, size_t tid, Query&& = {}); - bool storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr* sa = nullptr); + bool storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr& sa = {}); void expireStore(); void expireStorage(InfoHash h); void expireStore(decltype(store)::iterator); void storageChanged(const InfoHash& id, Storage& st, ValueStorage&); std::string printStorageLog(const decltype(store)::value_type&) const; - void printStorageQuota(std::ostream& out, const decltype(store_quota)::value_type& ip) const; /** * For a given storage, if values don't belong there anymore because this @@ -505,7 +484,7 @@ private: * @param query The query sent to the node. */ void searchNodeGetDone(const net::Request& status, - net::NetworkEngine::RequestAnswer&& answer, + net::RequestAnswer&& answer, std::weak_ptr ws, Sp query); @@ -554,48 +533,46 @@ private: bool neighbourhoodMaintenance(RoutingTable&); - void processMessage(const uint8_t *buf, size_t buflen, const SockAddr&); - void onError(Sp node, net::DhtProtocolException e); /* when our address is reported by a distant peer. */ void onReportedAddr(const InfoHash& id, const SockAddr&); /* when we receive a ping request */ - net::NetworkEngine::RequestAnswer onPing(Sp node); + net::RequestAnswer onPing(Sp node); /* when we receive a "find node" request */ - net::NetworkEngine::RequestAnswer onFindNode(Sp node, const InfoHash& hash, want_t want); + net::RequestAnswer onFindNode(Sp node, const InfoHash& hash, want_t want); void onFindNodeDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp sr); /* when we receive a "get values" request */ - net::NetworkEngine::RequestAnswer onGetValues(Sp node, + net::RequestAnswer onGetValues(Sp node, const InfoHash& hash, want_t want, const Query& q); void onGetValuesDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr, const Sp& orig_query); /* when we receive a listen request */ - net::NetworkEngine::RequestAnswer onListen(Sp node, + net::RequestAnswer onListen(Sp node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query); void onListenDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr); /* when we receive an announce request */ - net::NetworkEngine::RequestAnswer onAnnounce(Sp node, + net::RequestAnswer onAnnounce(Sp node, const InfoHash& hash, const Blob& token, const std::vector>& v, const time_point& created); - net::NetworkEngine::RequestAnswer onRefresh(Sp node, + net::RequestAnswer onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Value::Id& vid); void onAnnounceDone(const Sp& status, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr); }; diff --git a/include/opendht/dht_interface.h b/include/opendht/dht_interface.h new file mode 100644 index 00000000..7f364196 --- /dev/null +++ b/include/opendht/dht_interface.h @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2014-2017 Savoir-faire Linux Inc. + * Author: Sébastien Blin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "infohash.h" +#include "log_enable.h" + +namespace dht { + +class OPENDHT_PUBLIC DhtInterface { +public: + DhtInterface() = default; + virtual ~DhtInterface() = default; + + // [[deprecated]] + using Status = NodeStatus; + // [[deprecated]] + using NodeExport = dht::NodeExport; + + /** + * Get the current status of the node for the given family. + */ + virtual NodeStatus getStatus(sa_family_t af) const = 0; + virtual NodeStatus getStatus() const = 0; + + /** + * Get the ID of the node. + */ + virtual const InfoHash& getNodeId() const = 0; + + /** + * Performs final operations before quitting. + */ + virtual void shutdown(ShutdownCallback cb) = 0; + + /** + * Returns true if the node is running (have access to an open socket). + * + * af: address family. If non-zero, will return true if the node + * is running for the provided family. + */ + virtual bool isRunning(sa_family_t af = 0) const = 0; + + virtual void registerType(const ValueType& type) = 0; + + virtual const ValueType& getType(ValueType::Id type_id) const = 0; + + /** + * Insert a node in the main routing table. + * The node is not pinged, so this should be + * used to bootstrap efficiently from previously known nodes. + */ + virtual void insertNode(const InfoHash& id, const SockAddr&) = 0; + virtual void insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) = 0; + virtual void insertNode(const NodeExport& n) = 0; + + virtual void pingNode(const sockaddr*, socklen_t, DoneCallbackSimple&& cb={}) = 0; + + virtual time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr&) = 0; + virtual time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) = 0; + + /** + * Get a value by searching on all available protocols (IPv4, IPv6), + * and call the provided get callback when values are found at key. + * The operation will start as soon as the node is connected to the network. + * @param cb a function called when new values are found on the network. + * It should return false to stop the operation. + * @param donecb a function called when the operation is complete. + cb and donecb won't be called again afterward. + * @param f a filter function used to prefilter values. + */ + virtual void get(const InfoHash& key, GetCallback cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) = 0; + virtual void get(const InfoHash& key, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f={}, Where&& w = {}) = 0; + virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) = 0; + virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) = 0; + + /** + * Similar to Dht::get, but sends a Query to filter data remotely. + * @param key the key for which to query data for. + * @param cb a function called when new values are found on the network. + * It should return false to stop the operation. + * @param done_cb a function called when the operation is complete. + cb and done_cb won't be called again afterward. + * @param q a query used to filter values on the remotes before they send a + * response. + */ + virtual void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {}) = 0; + virtual void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) = 0; + + /** + * Get locally stored data for the given hash. + */ + virtual std::vector> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const = 0; + + /** + * Get locally stored data for the given key and value id. + */ + virtual Sp getLocalById(const InfoHash& key, Value::Id vid) const = 0; + + /** + * Announce a value on all available protocols (IPv4, IPv6). + * + * The operation will start as soon as the node is connected to the network. + * The done callback will be called once, when the first announce succeeds, or fails. + */ + virtual void put(const InfoHash& key, + Sp, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false) = 0; + virtual void put(const InfoHash& key, + const Sp& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) = 0; + virtual void put(const InfoHash& key, + Value&& v, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false) = 0; + virtual void put(const InfoHash& key, + Value&& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) = 0; + + /** + * Get data currently being put at the given hash. + */ + virtual std::vector> getPut(const InfoHash&) = 0; + + /** + * Get data currently being put at the given hash with the given id. + */ + virtual Sp getPut(const InfoHash&, const Value::Id&) = 0; + + /** + * Stop any put/announce operation at the given location, + * for the value with the given id. + */ + virtual bool cancelPut(const InfoHash&, const Value::Id&) = 0; + + /** + * Listen on the network for any changes involving a specified hash. + * The node will register to receive updates from relevent nodes when + * new values are added or removed. + * + * @return a token to cancel the listener later. + */ + virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where w = {}) = 0; + virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) = 0; + + virtual bool cancelListen(const InfoHash&, size_t token) = 0; + + /** + * Inform the DHT of lower-layer connectivity changes. + * This will cause the DHT to assume a public IP address change. + * The DHT will recontact neighbor nodes, re-register for listen ops etc. + */ + virtual void connectivityChanged(sa_family_t) = 0; + virtual void connectivityChanged() = 0; + + /** + * Get the list of good nodes for local storage saving purposes + * The list is ordered to minimize the back-to-work delay. + */ + virtual std::vector exportNodes() = 0; + + virtual std::vector exportValues() const = 0; + virtual void importValues(const std::vector&) = 0; + + virtual NodeStats getNodesStats(sa_family_t af) const = 0; + + virtual std::string getStorageLog() const = 0; + virtual std::string getStorageLog(const InfoHash&) const = 0; + + virtual std::string getRoutingTablesLog(sa_family_t) const = 0; + virtual std::string getSearchesLog(sa_family_t) const = 0; + virtual std::string getSearchLog(const InfoHash&, sa_family_t af = AF_UNSPEC) const = 0; + + virtual void dumpTables() const = 0; + virtual std::vector getNodeMessageStats(bool in = false) = 0; + + /** + * Set the in-memory storage limit in bytes + */ + virtual void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) = 0; + + /** + * Returns the total memory usage of stored values and the number + * of stored values. + */ + virtual std::pair getStoreSize() const = 0; + + virtual std::vector getPublicAddress(sa_family_t family = 0) = 0; + + /** + * Enable or disable logging of DHT internal messages + */ + virtual void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG) + { + DHT_LOG.DEBUG = debug; + DHT_LOG.WARN = warn; + DHT_LOG.ERR = error; + } + + /** + * Only print logs related to the given InfoHash (if given), or disable filter (if zeroes). + */ + virtual void setLogFilter(const InfoHash& f) + { + DHT_LOG.setFilter(f); + } + + virtual void setPushNotificationToken(const std::string&) {}; + + /** + * Call linked callback with a push notification + * @param notification to process + */ + virtual void pushNotificationReceived(const std::map& data) = 0; + /** + * Refresh a listen via a token + * @param token + */ + virtual void resubscribe(unsigned token) = 0; + +protected: + bool logFilerEnable_ {}; + InfoHash logFiler_ {}; + Logger DHT_LOG; +}; + +} // namespace dht diff --git a/include/opendht/dht_proxy_client.h b/include/opendht/dht_proxy_client.h new file mode 100644 index 00000000..8fd8f5ba --- /dev/null +++ b/include/opendht/dht_proxy_client.h @@ -0,0 +1,356 @@ +/* + * Copyright (C) 2016-2018 Savoir-faire Linux Inc. + * Author: Sébastien Blin + * Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if OPENDHT_PROXY_CLIENT + +#pragma once + +#include +#include +#include + +#include "callbacks.h" +#include "def.h" +#include "dht_interface.h" +#include "scheduler.h" + +namespace restbed { + class Request; +} + +namespace Json { + class Value; +} + +namespace dht { + +class OPENDHT_PUBLIC DhtProxyClient final : public DhtInterface { +public: + + DhtProxyClient() : scheduler(DHT_LOG) {} + + explicit DhtProxyClient(std::function loopSignal, const std::string& serverHost, const std::string& pushClientId = ""); + +#if OPENDHT_PUSH_NOTIFICATIONS + virtual void setPushNotificationToken(const std::string& token) { + deviceKey_ = token; + } +#endif + + virtual ~DhtProxyClient(); + + /** + * Get the ID of the node. + */ + inline const InfoHash& getNodeId() const { return myid; } + + /** + * Get the current status of the node for the given family. + */ + NodeStatus getStatus(sa_family_t af) const; + NodeStatus getStatus() const { + return std::max(getStatus(AF_INET), getStatus(AF_INET6)); + } + + /** + * Performs final operations before quitting. + */ + void shutdown(ShutdownCallback cb); + + /** + * Returns true if the node is running (have access to an open socket). + * + * af: address family. If non-zero, will return true if the node + * is running for the provided family. + */ + bool isRunning(sa_family_t af = 0) const; + + /** + * Get a value by asking the proxy and call the provided get callback when + * values are found at key. + * The operation will start as soon as the node is connected to the network. + * @param cb a function called when new values are found on the network. + * It should return false to stop the operation. + * @param donecb a function called when the operation is complete. + cb and donecb won't be called again afterward. + * @param f a filter function used to prefilter values. + */ + virtual void get(const InfoHash& key, GetCallback cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}); + virtual void get(const InfoHash& key, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f={}, Where&& w = {}) { + get(key, cb, bindDoneCb(donecb), std::forward(f), std::forward(w)); + } + virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) { + get(key, bindGetCb(cb), donecb, std::forward(f), std::forward(w)); + } + virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) { + get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward(f), std::forward(w)); + } + + void get(const InfoHash& key, const GetCallback& cb, DoneCallback donecb, const Value::Filter& filterChain); + + /** + * Announce a value on all available protocols (IPv4, IPv6). + * + * The operation will start as soon as the node is connected to the network. + * The done callback will be called once, when the first announce succeeds, or fails. + * NOTE: For now, created parameter is ignored. + */ + void put(const InfoHash& key, + Sp, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false); + void put(const InfoHash& key, + const Sp& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) + { + put(key, v, bindDoneCb(cb), created, permanent); + } + + void put(const InfoHash& key, + Value&& v, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false) + { + put(key, std::make_shared(std::move(v)), cb, created, permanent); + } + void put(const InfoHash& key, + Value&& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) + { + put(key, std::forward(v), bindDoneCb(cb), created, permanent); + } + + /** + * @param af the socket family + * @return node stats from the proxy + */ + NodeStats getNodesStats(sa_family_t af) const; + + /** + * @param family the socket family + * @return public address + */ + std::vector getPublicAddress(sa_family_t family = 0); + + /** + * Listen on the network for any changes involving a specified hash. + * The node will register to receive updates from relevent nodes when + * new values are added or removed. + * + * @return a token to cancel the listener later. + */ + virtual size_t listen(const InfoHash&, GetCallback, Value::Filter={}, Where={}); + virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) { + return listen(key, bindGetCb(cb), std::forward(f), std::forward(w)); + } + virtual bool cancelListen(const InfoHash&, size_t token); + +#if OPENDHT_PUSH_NOTIFICATIONS + /** + * Call linked callback with a push notification + * @param notification to process + */ + void pushNotificationReceived(const std::map& notification); + /** + * Refresh a listen via a token + * @param token + */ + void resubscribe(const unsigned token); +#endif // OPENDHT_PUSH_NOTIFICATIONS + + time_point periodic(const uint8_t*, size_t, const SockAddr&); + time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) { + return periodic(buf, buflen, SockAddr(from, fromlen)); + } + + + /** + * Similar to Dht::get, but sends a Query to filter data remotely. + * @param key the key for which to query data for. + * @param cb a function called when new values are found on the network. + * It should return false to stop the operation. + * @param done_cb a function called when the operation is complete. + cb and done_cb won't be called again afterward. + * @param q a query used to filter values on the remotes before they send a + * response. + */ + virtual void query(const InfoHash& /*key*/, QueryCallback /*cb*/, DoneCallback /*done_cb*/ = {}, Query&& /*q*/ = {}) { } + virtual void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) { + query(key, cb, bindDoneCb(done_cb), std::forward(q)); + } + + /** + * Get data currently being put at the given hash. + */ + std::vector> getPut(const InfoHash&) { return {}; } + + /** + * Get data currently being put at the given hash with the given id. + */ + Sp getPut(const InfoHash&, const Value::Id&) { return {}; } + + /** + * Stop any put/announce operation at the given location, + * for the value with the given id. + */ + bool cancelPut(const InfoHash&, const Value::Id&) { return false; } + + void pingNode(const sockaddr*, socklen_t, DoneCallbackSimple&& /*cb*/={}) { } + + /** + * NOTE: The following methods will not be implemented because the + * DhtProxyClient doesn't have any storage nor synchronization process + */ + void insertNode(const InfoHash&, const SockAddr&) { } + void insertNode(const InfoHash&, const sockaddr*, socklen_t) { } + void insertNode(const NodeExport&) { } + std::pair getStoreSize() const { return {}; } + virtual void registerType(const ValueType&) { } + const ValueType& getType(ValueType::Id) const { return NO_VALUE; } + std::vector> getLocal(const InfoHash&, Value::Filter) const { return {}; } + Sp getLocalById(const InfoHash&, Value::Id) const { return {}; } + std::vector exportNodes() { return {}; } + std::vector exportValues() const { return {}; } + void importValues(const std::vector&) {} + std::string getStorageLog() const { return {}; } + std::string getStorageLog(const InfoHash&) const { return {}; } + std::string getRoutingTablesLog(sa_family_t) const { return {}; } + std::string getSearchesLog(sa_family_t) const { return {}; } + std::string getSearchLog(const InfoHash&, sa_family_t) const { return {}; } + void dumpTables() const {} + std::vector getNodeMessageStats(bool) { return {}; } + void setStorageLimit(size_t) {} + void connectivityChanged(sa_family_t) {} + void connectivityChanged() { } + +private: + const ValueType NO_VALUE; + + /** + * Start the connection with a server. + */ + void startProxy(); + + /** + * Get informations from the proxy node + * @return the JSON returned by the proxy + */ + void getProxyInfos(); + void onProxyInfos(const Json::Value& val); + SockAddr parsePublicAddress(const Json::Value& val); + + void opFailed(); + + /** + * Initialize statusIpvX_ + */ + void getConnectivityStatus(); + /** + * cancel all Listeners + */ + void cancelAllListeners(); + /** + * cancel all Operations + */ + void cancelAllOperations(); + std::string serverHost_; + std::string pushClientId_; + + std::atomic_flag ongoingStatusUpdate_ = ATOMIC_FLAG_INIT; + NodeStatus statusIpv4_ {NodeStatus::Disconnected}; + NodeStatus statusIpv6_ {NodeStatus::Disconnected}; + NodeStats stats4_ {}; + NodeStats stats6_ {}; + SockAddr publicAddress_; + + InfoHash myid {}; + + /** + * Store listen requests. + */ + struct Listener + { + size_t token; + std::shared_ptr req; + std::string key; + GetCallback cb; + Value::Filter filterChain; + std::thread thread; + std::shared_ptr pushNotifToken; // NOTE: unused if not using push notifications + }; + std::vector listeners_; + size_t listener_token_ {0}; + std::mutex lockListener_; + + /** + * Store current put and get requests. + */ + struct Operation + { + std::shared_ptr req; + std::thread thread; + std::shared_ptr finished; + }; + std::vector operations_; + std::mutex lockOperations_; + /** + * Callbacks should be executed in the main thread. + */ + std::vector> callbacks_; + std::mutex lockCallbacks; + + std::thread statusThread_; + mutable std::mutex lockCurrentProxyInfos_; + + Scheduler scheduler; + /** + * Retrieve if we can connect to the proxy (update statusIpvX_) + */ + void confirmProxy(); + Sp nextProxyConfirmation {}; + /** + * Relaunch LISTEN requests if the client disconnect/reconnect. + */ + void restartListeners(); + + /** + * If we want to use push notifications by default. + * NOTE: empty by default to avoid to use services like FCM or APN. + */ + std::string deviceKey_ {}; + unsigned callbackId_ {0}; + std::mutex lockCallback_; + + const std::function loopSignal_; + +#if OPENDHT_PUSH_NOTIFICATIONS + void fillBodyToGetToken(std::shared_ptr request); +#endif // OPENDHT_PUSH_NOTIFICATIONS + +}; + +} + +#endif // OPENDHT_PROXY_CLIENT diff --git a/include/opendht/dht_proxy_server.h b/include/opendht/dht_proxy_server.h new file mode 100644 index 00000000..51dc56aa --- /dev/null +++ b/include/opendht/dht_proxy_server.h @@ -0,0 +1,237 @@ +/* + * Copyright (C) 2017-2018 Savoir-faire Linux Inc. + * Author: Sébastien Blin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if OPENDHT_PROXY_SERVER + +#pragma once + +#include "def.h" +#include "sockaddr.h" +#include "infohash.h" + +#include +#include +#include +#include + +namespace Json { + class Value; +} + +namespace dht { + +class DhtRunner; + +/** + * Describes the REST API + */ +class OPENDHT_PUBLIC DhtProxyServer +{ +public: + /** + * Start the Http server for OpenDHT + * @param dht the DhtRunner linked to this proxy server + * @param port to listen + * @param pushServer where to push notifications + * @note if the server fails to start (if port is already used or reserved), + * it will fails silently + */ + DhtProxyServer(std::shared_ptr dht, in_port_t port = 8000, const std::string& pushServer = ""); + virtual ~DhtProxyServer(); + + DhtProxyServer(const DhtProxyServer& other) = delete; + DhtProxyServer(DhtProxyServer&& other) = delete; + DhtProxyServer& operator=(const DhtProxyServer& other) = delete; + DhtProxyServer& operator=(DhtProxyServer&& other) = delete; + + /** + * Stop the DhtProxyServer + */ + void stop(); + +private: + /** + * Return the PublicKey id, the node id and node stats + * Method: GET "/" + * Result: HTTP 200, body: Node infos in JSON format + * On error: HTTP 503, body: {"err":"xxxx"} + * @param session + */ + void getNodeInfo(const std::shared_ptr& session) const; + + /** + * Return Values of an infoHash + * Method: GET "/{InfoHash: .*}" + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format + * + * On error: HTTP 503, body: {"err":"xxxx"} + * @param session + */ + void get(const std::shared_ptr& session) const; + + /** + * Listen incoming Values of an infoHash. + * Method: LISTEN "/{InfoHash: .*}" + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format + * + * On error: HTTP 503, body: {"err":"xxxx"} + * @param session + */ + void listen(const std::shared_ptr& session) const; + + /** + * Put a value on the DHT + * Method: POST "/{InfoHash: .*}" + * body = Value to put in JSON + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json or HTTP 502 if put fails + * @param session + */ + void put(const std::shared_ptr& session) const; + +#if OPENDHT_PROXY_SERVER_IDENTITY + /** + * Put a value to sign by the proxy on the DHT + * Method: SIGN "/{InfoHash: .*}" + * body = Value to put in JSON + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json + * @param session + */ + void putSigned(const std::shared_ptr& session) const; + + /** + * Put a value to encrypt by the proxy on the DHT + * Method: ENCRYPT "/{hash: .*}" + * body = Value to put in JSON + "to":"infoHash" + * Return: HTTP 200 if success and the value put in JSON + * On error: HTTP 503, body: {"err":"xxxx"} if no dht + * HTTP 400, body: {"err":"xxxx"} if bad json + * @param session + */ + void putEncrypted(const std::shared_ptr& session) const; +#endif // OPENDHT_PROXY_SERVER_IDENTITY + + /** + * Return Values of an infoHash filtered by a value id + * Method: GET "/{InfoHash: .*}/{ValueId: .*}" + * Return: Multiple JSON object in parts. Example: + * Value in JSON format\n + * Value in JSON format + * + * On error: HTTP 503, body: {"err":"xxxx"} + * @param session + */ + void getFiltered(const std::shared_ptr& session) const; + + /** + * Respond allowed Methods + * Method: OPTIONS "/{hash: .*}" + * Return: HTTP 200 + Allow: allowed methods + * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS + * @param session + */ + void handleOptionsMethod(const std::shared_ptr& session) const; + +#if OPENDHT_PUSH_NOTIFICATIONS + /** + * Subscribe to push notifications for an iOS or Android device. + * Method: SUBSCRIBE "/{InfoHash: .*}" + * Body: {"key": "device_key", (optional)"callback_id":y, + * (optional)"isAndroid":false (default true)}" + * Return: {"token": x}" where x if a token to save + * @note: the listen will timeout after six hours (and send a push notification). + * so you need to refresh the operation each six hours. + * @note: callback_id is used to add the possibility to have multiple listen + * on same hash for same device and must be > 0 + * @param session + */ + void subscribe(const std::shared_ptr& session) const; + /** + * Unsubscribe to push notifications for an iOS or Android device. + * Method: UNSUBSCRIBE "/{InfoHash: .*}" + * Body: {"key": "device_key", "token": x, (optional)"callback_id":y" + * where x if the token to cancel + * Return: nothing + * @note: callback id is used to add the possibility to have multiple listen + * on same hash for same device + * @param session + */ + void unsubscribe(const std::shared_ptr& session) const; + /** + * Send a push notification via a gorush push gateway + * @param key of the device + * @param json, the content to send + */ + void sendPushNotification(const std::string& key, const Json::Value& json, bool isAndroid) const; +#endif //OPENDHT_PUSH_NOTIFICATIONS + + std::thread server_thread {}; + std::unique_ptr service_; + std::shared_ptr dht_; + + // Handle client quit for listen. + // NOTE: can be simplified when we will supports restbed 5.0 + std::thread listenThread_; + struct SessionToHashToken { + std::shared_ptr session; + InfoHash hash; + std::future token; + }; + mutable std::vector currentListeners_; + mutable std::mutex lockListener_; + std::atomic_bool stopListeners {false}; + +#if OPENDHT_PUSH_NOTIFICATIONS + struct PushListener { + std::string pushToken; + InfoHash hash; + unsigned token; + std::future internalToken; + std::chrono::steady_clock::time_point deadline; + bool started {false}; + unsigned callbackId {0}; + std::string clientId {}; + bool isAndroid {true}; + }; + mutable std::mutex lockPushListeners_; + mutable std::vector pushListeners_; + mutable unsigned tokenPushNotif_ {0}; + const std::string pushServer_; +#endif //OPENDHT_PUSH_NOTIFICATIONS + + /** + * Remove finished listeners + * @param testSession if we remove the listener only if the session is closed + */ + void removeClosedListeners(bool testSession = true); + /** + * Launch or remove push listeners if needed + */ + void handlePushListeners(); +}; + +} + +#endif //OPENDHT_PROXY_SERVER diff --git a/include/opendht/dhtrunner.h b/include/opendht/dhtrunner.h index 0007c31a..6814f517 100644 --- a/include/opendht/dhtrunner.h +++ b/include/opendht/dhtrunner.h @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author(s) : Adrien Béraud - * Simon Désaulniers + * Authors: Adrien Béraud + * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +20,6 @@ #pragma once -//#include "securedht.h" #include "infohash.h" #include "value.h" #include "callbacks.h" @@ -53,6 +53,13 @@ class OPENDHT_PUBLIC DhtRunner { public: typedef std::function StatusCallback; + struct Config { + SecureDhtConfig dht_config; + bool threaded; + std::string proxy_server; + std::string push_node_id; + }; + DhtRunner(); virtual ~DhtRunner(); @@ -202,7 +209,7 @@ public: * Insert known nodes to the routing table, without necessarly ping them. * Usefull to restart a node and get things running fast without putting load on the network. */ - void bootstrap(const std::vector>& nodes, DoneCallbackSimple&& cb={}); + void bootstrap(const std::vector& nodes, DoneCallbackSimple&& cb={}); void bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb={}); /** @@ -250,7 +257,7 @@ public: * @param f: address family of the bound port to retreive. */ in_port_t getBoundPort(sa_family_t f = AF_INET) const { - return ntohs(((sockaddr_in*)&getBound(f).first)->sin_port); + return getBound(f).getPort(); } std::pair getStoreSize() const; @@ -294,11 +301,6 @@ public: void registerCertificate(std::shared_ptr cert); void setLocalCertificateStore(CertificateStoreQuery&& query_method); - struct Config { - SecureDhtConfig dht_config; - bool threaded; - }; - /** * @param port: Local port to bind. Both IPv4 and IPv6 will be tried (ANY). * @param identity: RSA key pair to use for cryptographic operations. @@ -316,7 +318,9 @@ public: }, /*.id = */identity }, - /*.threaded = */threaded + /*.threaded = */threaded, + /*.proxy_server = */"", + /*.push_node_id = */"" }); } void run(in_port_t port, Config config); @@ -329,7 +333,7 @@ public: * @param threaded: If false, loop() must be called periodically. Otherwise a thread is launched. * @param cb: Optional callback to receive general state information. */ - void run(const sockaddr_in* local4, const sockaddr_in6* local6, Config config); + void run(const SockAddr& local4, const SockAddr& local6, Config config); /** * Same as @run(sockaddr_in, sockaddr_in6, Identity, bool, StatusCallback), but with string IP addresses and service (port). @@ -347,7 +351,13 @@ public: */ time_point loop() { std::lock_guard lck(dht_mtx); - return loop_(); + time_point wakeup = time_point::min(); + try { + wakeup = loop_(); + } catch (const dht::SocketException& e) { + startNetwork(bound4, bound6); + } + return wakeup; } /** @@ -362,6 +372,43 @@ public: */ void join(); + void setProxyServer(const std::string& proxy, const std::string& pushNodeId = "") { +#if OPENDHT_PROXY_CLIENT + if (config_.proxy_server == proxy and config_.push_node_id == pushNodeId) + return; + config_.proxy_server = proxy; + config_.push_node_id = pushNodeId; + enableProxy(use_proxy and not config_.proxy_server.empty()); +#endif + } + + /** + * Start or stop the proxy + * @param proxify if we want to use the proxy + * @param deviceKey non empty to enable push notifications + */ + void enableProxy(bool proxify); + + /* Push notification methods */ + + /** + * Updates the push notification device token + */ + void setPushNotificationToken(const std::string& token); + + /** + * Insert a push notification to process for OpenDHT + */ + void pushNotificationReceived(const std::map& data) const; + /** + * Refresh a listen via a token + * @param token + */ + void resubscribe(unsigned token); + + /* Proxy server mothods */ + void forwardAllMessages(bool forward); + private: static constexpr std::chrono::seconds BOOTSTRAP_PERIOD {10}; @@ -373,16 +420,48 @@ private: */ void tryBootstrapContinuously(); - void doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDhtConfig config); + void startNetwork(const SockAddr sin4, const SockAddr sin6); time_point loop_(); - static std::vector> getAddrInfo(const std::string& host, const std::string& service); - NodeStatus getStatus() const { return std::max(status4, status6); } + /** Local DHT instance */ std::unique_ptr dht_; + + /** Proxy client instance */ + std::unique_ptr dht_via_proxy_; + + /** true if we are currently using a proxy */ + std::atomic_bool use_proxy {false}; + + /** Current configuration */ + Config config_; + + /** + * reset dht clients + */ + void resetDht(); + /** + * @return the current active DHT + */ + SecureDht* activeDht() const; + + /** + * Store current listeners and translates global tokens for each client. + */ + struct Listener { + size_t tokenClassicDht; + size_t tokenProxyDht; + GetCallback gcb; + InfoHash hash; + Value::Filter f; + Where w; + }; + std::map listeners_ {}; + size_t listener_token_ {1}; + mutable std::mutex dht_mtx {}; std::thread dht_thread {}; std::condition_variable cv {}; @@ -405,14 +484,19 @@ private: std::queue> pending_ops {}; std::mutex storage_mtx {}; - std::atomic running {false}; + std::atomic_bool running {false}; + std::atomic_bool running_network {false}; NodeStatus status4 {NodeStatus::Disconnected}, status6 {NodeStatus::Disconnected}; StatusCallback statusCb {nullptr}; + int s4 {-1}, s6 {-1}; SockAddr bound4 {}; SockAddr bound6 {}; + + /** Push notification token */ + std::string pushToken_; }; } diff --git a/include/opendht/infohash.h b/include/opendht/infohash.h index 74186ed2..ade90dc3 100644 --- a/include/opendht/infohash.h +++ b/include/opendht/infohash.h @@ -19,6 +19,7 @@ #pragma once #include "def.h" +#include "rng.h" #include @@ -44,133 +45,99 @@ typedef uint16_t in_port_t; #include #include - -// bytes -#define HASH_LEN 20u - namespace dht { using byte = uint8_t; -template -class OPENDHT_PUBLIC Hash -{ -public: - Hash() { - data_.fill(0); - } - Hash(const msgpack::object& o) { - msgpack_unpack(o); - } - byte* data() { - return data_.data(); - } - const byte* data() const { - return data_.data(); - } - constexpr size_t size() const { return data_.size(); } - explicit operator bool() const { return std::any_of(data_.begin(), data_.end(), [](byte _b) { return _b != 0; }); } - bool operator==(Hash const& _c) const { return data_ == _c.data_; } - bool operator!=(Hash const& _c) const { return data_ != _c.data_; } - bool operator<(Hash const& _c) const { for (unsigned i = 0; i < N; ++i) if (data_[i] < _c.data_[i]) return true; else if (data_[i] > _c.data_[i]) return false; return false; } - bool operator>=(Hash const& _c) const { return !operator<(_c); } - bool operator<=(Hash const& _c) const { return operator==(_c) || operator<(_c); } - bool operator>(Hash const& _c) const { return !operator<=(_c); } - template - void msgpack_pack(Packer& pk) const - { - pk.pack_bin(N); - pk.pack_bin_body((char*)data_.data(), N); - } - void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::BIN or o.via.bin.size != N) - throw msgpack::type_error(); - std::copy_n(o.via.bin.ptr, N, data_.data()); - } - - template - OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Hash& h); - - template - OPENDHT_PUBLIC friend std::istream& operator>> (std::istream& s, Hash& h); - - std::string toString() const { - std::stringstream ss; - ss << *this; - return ss.str(); - } - -private: - std::array data_; -}; - -template -std::ostream& operator<< (std::ostream& s, const Hash& h) -{ - s << std::hex; - for (unsigned i=0; i -std::istream& operator>> (std::istream& s, Hash& h) -{ - std::array dat; - s.exceptions(std::istream::eofbit | std::istream::failbit); - s.read(&(*dat.begin()), dat.size()); - for (size_t i = 0; i < h.size(); i++) { - unsigned res = 0; - sscanf(dat.data() + 2*i, "%02x", &res); - h[i] = res; - } - return s; -} - -using h256 = Hash<32>; -using PkId = h256; - /** * Represents an InfoHash. * An InfoHash is a byte array of HASH_LEN bytes. * InfoHashes identify nodes and values in the Dht. */ -class OPENDHT_PUBLIC InfoHash final : public std::array { +template +class OPENDHT_PUBLIC Hash { public: - constexpr InfoHash() : std::array() {} - constexpr InfoHash(const std::array& h) : std::array(h) {} - InfoHash(const uint8_t* h, size_t h_len) : std::array() { - if (h_len < HASH_LEN) - fill(0); - else - std::copy_n(h, HASH_LEN, begin()); - } + using T = std::array; + typedef typename T::iterator iterator; + typedef typename T::const_iterator const_iterator; + Hash () { + data_.fill(0); + } + Hash (const uint8_t* h, size_t data_len) { + if (data_len < N) + data_.fill(0); + else + std::copy_n(h, N, data_.begin()); + } /** * Constructor from an hexadecimal string (without "0x"). * hex must be at least 2.HASH_LEN characters long. * If too long, only the first 2.HASH_LEN characters are read. */ - explicit InfoHash(const std::string& hex); + explicit Hash(const std::string& hex); - InfoHash(const msgpack::object& o) { + Hash(const msgpack::object& o) { msgpack_unpack(o); } + size_t size() const { return data_.size(); } + const uint8_t* data() const { return data_.data(); } + uint8_t* data() { return data_.data(); } + iterator begin() { return data_.begin(); } + const_iterator cbegin() const { return data_.cbegin(); } + iterator end() { return data_.end(); } + const_iterator cend() const { return data_.cend(); } + + bool operator==(const Hash& h) const { + auto a = reinterpret_cast(data_.data()); + auto b = reinterpret_cast(h.data_.data()); + constexpr unsigned n = N / sizeof(uint32_t); + for (unsigned i=0; i < n; i++) + if (a[i] != b[i]) + return false; + return true; + } + bool operator!=(const Hash& h) const { return !(*this == h); } + + bool operator<(const Hash& o) const { + for(unsigned i = 0; i < N; i++) { + if(data_[i] != o.data_[i]) + return data_[i] < o.data_[i]; + } + return false; + } + + explicit operator bool() const { + auto a = reinterpret_cast(data_.data()); + auto b = reinterpret_cast(data_.data() + N); + for (; a != b; a++) { + if (*a) + return true; + } + return false; + } + + uint8_t& operator[](size_t index) { return data_[index]; } + const uint8_t& operator[](size_t index) const { return data_[index]; } + /** * Find the lowest 1 bit in an id. - * Result will allways be lower than 8*HASH_LEN + * Result will allways be lower than 8*N */ inline unsigned lowbit() const { int i, j; - for(i = HASH_LEN-1; i >= 0; i--) - if((*this)[i] != 0) + for(i = N-1; i >= 0; i--) + if(data_[i] != 0) break; if(i < 0) return -1; for(j = 7; j >= 0; j--) - if(((*this)[i] & (0x80 >> j)) != 0) + if((data_[i] & (0x80 >> j)) != 0) break; return 8 * i + j; } @@ -179,25 +146,25 @@ public: * Forget about the ``XOR-metric''. An id is just a path from the * root of the tree, so bits are numbered from the start. */ - static inline int cmp(const InfoHash& id1, const InfoHash& id2) { - return std::memcmp(id1.data(), id2.data(), HASH_LEN); + static inline int cmp(const Hash& id1, const Hash& id2) { + return std::memcmp(id1.data_.data(), id2.data_.data(), N); } /** Find how many bits two ids have in common. */ static inline unsigned - commonBits(const InfoHash& id1, const InfoHash& id2) + commonBits(const Hash& id1, const Hash& id2) { unsigned i, j; uint8_t x; - for(i = 0; i < HASH_LEN; i++) { - if(id1[i] != id2[i]) + for(i = 0; i < N; i++) { + if(id1.data_[i] != id2.data_[i]) break; } - if(i == HASH_LEN) - return 8*HASH_LEN; + if(i == N) + return 8*N; - x = id1[i] ^ id2[i]; + x = id1.data_[i] ^ id2.data_[i]; j = 0; while((x & 0x80) == 0) { @@ -210,14 +177,14 @@ public: /** Determine whether id1 or id2 is closer to this */ int - xorCmp(const InfoHash& id1, const InfoHash& id2) const + xorCmp(const Hash& id1, const Hash& id2) const { - for(unsigned i = 0; i < HASH_LEN; i++) { + for(unsigned i = 0; i < N; i++) { uint8_t xor1, xor2; - if(id1[i] == id2[i]) + if(id1.data_[i] == id2.data_[i]) continue; - xor1 = id1[i] ^ (*this)[i]; - xor2 = id2[i] ^ (*this)[i]; + xor1 = id1.data_[i] ^ data_[i]; + xor2 = id2.data_[i] ^ data_[i]; if(xor1 < xor2) return -1; else @@ -229,7 +196,7 @@ public: bool getBit(unsigned nbit) const { - auto& num = *(cbegin()+(nbit/8)); + auto& num = *(data_.cbegin()+(nbit/8)); unsigned bit = 7 - (nbit % 8); return (num >> bit) & 1; } @@ -237,46 +204,44 @@ public: void setBit(unsigned nbit, bool b) { - auto& num = (*this)[nbit/8]; + auto& num = data_[nbit/8]; unsigned bit = 7 - (nbit % 8); num ^= (-b ^ num) & (1 << bit); } - double - toFloat() const - { + double toFloat() const { + using D = size_t; double v = 0.; - for (unsigned i = 0; i < std::min(HASH_LEN, sizeof(unsigned)-1); i++) - v += *(cbegin()+i)/(double)(1<<(8*(i+1))); + for (size_t i = 0; i < std::min(N, sizeof(D)-1); i++) + v += *(data_.cbegin()+i) / (double)((D)1 << 8*(i+1)); return v; } - bool - operator<(const InfoHash& o) const { - for(unsigned i = 0; i < HASH_LEN; i++) { - if((*this)[i] != o[i]) - return (*this)[i] < o[i]; - } - return false; - } - - static inline InfoHash get(const std::string& data) { + static inline Hash get(const std::string& data) { return get((const uint8_t*)data.data(), data.size()); } - static inline InfoHash get(const std::vector& data) { + static inline Hash get(const std::vector& data) { return get(data.data(), data.size()); } /** * Computes the hash from a given data buffer of size data_len. */ - static InfoHash get(const uint8_t* data, size_t data_len); + static Hash get(const uint8_t* data, size_t data_len) + { + Hash ret; + crypto::hash(data, data_len, ret.data(), N); + return ret; + } - static InfoHash getRandom(); + static Hash getRandom(); - OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const InfoHash& h); - OPENDHT_PUBLIC friend std::istream& operator>> (std::istream& s, InfoHash& h); + template + OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Hash& h); + + template + OPENDHT_PUBLIC friend std::istream& operator>> (std::istream& s, Hash& h); const char* to_c_str() const; @@ -285,24 +250,135 @@ public: template void msgpack_pack(Packer& pk) const { - pk.pack_bin(HASH_LEN); - pk.pack_bin_body((char*)data(), HASH_LEN); + pk.pack_bin(N); + pk.pack_bin_body((char*)data_.data(), N); } void msgpack_unpack(msgpack::object o) { - if (o.type != msgpack::type::BIN or o.via.bin.size != HASH_LEN) + if (o.type != msgpack::type::BIN or o.via.bin.size != N) throw msgpack::type_error(); - std::copy_n(o.via.bin.ptr, HASH_LEN, data()); + std::copy_n(o.via.bin.ptr, N, data_.data()); } - +private: + T data_; + void fromString(const char*); }; -static constexpr const InfoHash zeroes {}; +#define HASH_LEN 20u +using InfoHash = Hash; +using h256 = Hash<32>; +using PkId = h256; -struct NodeExport { +template +std::ostream& operator<< (std::ostream& s, const Hash& h) +{ + s.write(h.to_c_str(), N*2); + return s; +} + +template +std::istream& operator>> (std::istream& s, Hash& h) +{ + std::array dat; + s.exceptions(std::istream::eofbit | std::istream::failbit); + s.read(&(*dat.begin()), dat.size()); + fromString(dat.data()); + return s; +} + +template +Hash::Hash(const std::string& hex) { + if (hex.size() < 2*N) + data_.fill(0); + else + fromString(hex.c_str()); +} + +template +void +Hash::fromString(const char* in) { + auto hex2bin = [](char c) -> uint8_t { + if (c >= 'a' and c <= 'f') return 10 + c - 'a'; + else if (c >= 'A' and c <= 'F') return 10 + c - 'A'; + else if (c >= '0' and c <= '9') return c - '0'; + else throw std::domain_error("not an hex character"); + }; + try { + for (size_t i=0; i +Hash +Hash::getRandom() +{ + Hash h; + crypto::random_device rdev; + std::uniform_int_distribution rand_int; + auto a = reinterpret_cast(h.data()); + auto b = reinterpret_cast(h.data() + h.size()); + std::generate(a, b, std::bind(rand_int, std::ref(rdev))); + return h; +} + +struct HexMap : public std::array, 256> { + HexMap() { + for (size_t i=0; i> 4) & 0x0F]; + e[1] = hex_digits[i & 0x0F]; + } + } +private: + static constexpr const char* hex_digits = "0123456789abcdef"; +}; + +OPENDHT_PUBLIC extern const HexMap hex_map; + +template +const char* +Hash::to_c_str() const +{ + thread_local std::array buf; + for (size_t i=0; i +std::string +Hash::toString() const +{ + return std::string(to_c_str(), N*2); +} + +const InfoHash zeroes {}; + +struct OPENDHT_PUBLIC NodeExport { InfoHash id; sockaddr_storage ss; socklen_t sslen; + + template + void msgpack_pack(Packer& pk) const + { + pk.pack_map(2); + pk.pack(std::string("id")); + pk.pack(id); + pk.pack(std::string("addr")); + pk.pack_bin(sslen); + pk.pack_bin_body((char*)&ss, sslen); + } + + void msgpack_unpack(msgpack::object o); + + OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const NodeExport& h); }; } diff --git a/include/opendht/log.h b/include/opendht/log.h index 220eaa70..9fda9eb9 100644 --- a/include/opendht/log.h +++ b/include/opendht/log.h @@ -24,6 +24,10 @@ #include namespace dht { + +/** + * Logging-related functions + */ namespace log { /** diff --git a/include/opendht/log_enable.h b/include/opendht/log_enable.h index b55f5556..7310de54 100644 --- a/include/opendht/log_enable.h +++ b/include/opendht/log_enable.h @@ -75,7 +75,7 @@ struct Logger { LogMethod ERR = NOLOG; void setFilter(const InfoHash& f) { filter_ = f; - filterEnable_ = filter_ != InfoHash{}; + filterEnable_ = static_cast(filter_); } inline void log0(const LogMethod& logger, char const* format, va_list args) const { #if OPENDHT_LOG diff --git a/include/opendht/net.h b/include/opendht/net.h deleted file mode 100644 index 180c4cf9..00000000 --- a/include/opendht/net.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author(s) : Adrien Béraud - * Simon Désaulniers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include -#include - -namespace dht { -namespace net { - -struct TransPrefix : public std::array { - TransPrefix(const std::string& str) : std::array({{(uint8_t)str[0], (uint8_t)str[1]}}) {} - static const TransPrefix PING; - static const TransPrefix FIND_NODE; - static const TransPrefix GET_VALUES; - static const TransPrefix ANNOUNCE_VALUES; - static const TransPrefix REFRESH; - static const TransPrefix LISTEN; -}; - -/* Transaction-ids are 4-bytes long, with the first two bytes identifying - * the kind of request, and the remaining two a sequence number in - * host order. - */ -struct TransId final : public std::array { - static const constexpr uint16_t INVALID {0}; - - TransId() { std::fill_n(begin(), 4, 0); } - TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } - TransId(const TransPrefix prefix, uint16_t seqno = 0) { - std::copy_n(prefix.begin(), prefix.size(), begin()); - *reinterpret_cast(data()+prefix.size()) = seqno; - } - - TransId(const char* q, size_t l) : array() { - if (l > 4) { - length = 0; - } else { - std::copy_n(q, l, begin()); - length = l; - } - } - - uint16_t getTid() const { - return *reinterpret_cast(&(*this)[2]); - } - - uint32_t toInt() const { - return *reinterpret_cast(&(*this)[0]); - } - - bool matches(const TransPrefix prefix, uint16_t* tid = nullptr) const { - if (std::equal(begin(), begin()+2, prefix.begin())) { - if (tid) - *tid = getTid(); - return true; - } else - return false; - } - - unsigned length {4}; -}; - -} /* namespace net */ -} /* dht */ diff --git a/include/opendht/network_engine.h b/include/opendht/network_engine.h index 8cf6827a..2a75e9a3 100644 --- a/include/opendht/network_engine.h +++ b/include/opendht/network_engine.h @@ -27,14 +27,12 @@ #include "utils.h" #include "rng.h" #include "rate_limiter.h" -#include "net.h" #include #include #include #include #include -#include #include namespace dht { @@ -42,6 +40,7 @@ namespace net { struct Request; struct Socket; +struct TransId; #ifndef MSG_CONFIRM #define MSG_CONFIRM 0 @@ -81,6 +80,20 @@ private: struct ParsedMessage; +/** + * Answer for a request. + */ +struct RequestAnswer { + Blob ntoken {}; + Value::Id vid {}; + std::vector> values {}; + std::vector> fields {}; + std::vector> nodes4 {}; + std::vector> nodes6 {}; + RequestAnswer() {} + RequestAnswer(ParsedMessage&& msg); +}; + /*! * @class NetworkEngine * @brief An abstraction of communication protocol on the network. @@ -101,26 +114,6 @@ struct ParsedMessage; */ class NetworkEngine final { -public: - /*! - * @class RequestAnswer - * @brief Answer for a request. - * @details - * Answer for a request to be (de)serialized. Used for reponding to a node - * and looking up the response from a node. - */ - struct RequestAnswer { - Blob ntoken {}; - Value::Id vid {}; - std::vector> values {}; - std::vector> fields {}; - std::vector> nodes4 {}; - std::vector> nodes6 {}; - RequestAnswer() {} - RequestAnswer(ParsedMessage&& msg); - }; - - private: /** * Called when we receive an error message. @@ -137,7 +130,7 @@ private: /** * Called when an addres is reported from a requested node. * - * @param h: id + * @param h: id * @param saddr_len (type: socklen_t) lenght of the sockaddr struct. */ std::function onReportedAddr; @@ -176,7 +169,7 @@ private: std::function, const InfoHash&, const Blob&, - uint32_t, + Tid, const Query&)> onListen {}; /** * Called on announce request. @@ -206,12 +199,11 @@ private: const Value::Id&)> onRefresh {}; public: - using SocketCb = std::function&, RequestAnswer&&)>; using RequestCb = std::function; using RequestExpiredCb = std::function; - NetworkEngine(Logger& log, Scheduler& scheduler); - NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler, + NetworkEngine(Logger& log, Scheduler& scheduler, const int& s = -1, const int& s6 = -1); + NetworkEngine(InfoHash& myid, NetId net, const int& s, const int& s6, Logger& log, Scheduler& scheduler, decltype(NetworkEngine::onError) onError, decltype(NetworkEngine::onNewNode) onNewNode, decltype(NetworkEngine::onReportedAddr) onReportedAddr, @@ -239,19 +231,13 @@ public: * @param nodes6 The ipv6 closest nodes. * @param values The values to send. */ - void tellListener(Sp n, uint32_t socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, + void tellListener(Sp n, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, std::vector>&& nodes, std::vector>&& nodes6, std::vector>&& values, const Query& q); bool isRunning(sa_family_t af) const; inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; } - /** - * Cancel a request. Setting req->cancelled = true is not enough in the case - * a request is "persistent". - */ - void cancelRequest(Sp& req); - void connectivityChanged(sa_family_t); /************** @@ -392,23 +378,6 @@ public: const Blob& token, RequestCb&& on_done, RequestExpiredCb&& on_expired); - /** - * Opens a socket on which a node will be able allowed to write for further - * additionnal updates following the response to a previous request. - * - * @param node The node which will be allowed to write on this socket. - * @param cb The callback to execute once updates arrive on the socket. - * - * @return the socket. - */ - Sp openSocket(const Sp& node, TransPrefix tp, SocketCb&& cb); - - /** - * Closes a socket so that no further data will be red on that socket. - * - * @param socket The socket to close. - */ - void closeSocket(Sp socket); /** * Parses a message and calls appropriate callbacks. @@ -447,7 +416,7 @@ private: /*************** * Constants * ***************/ - static constexpr long unsigned MAX_REQUESTS_PER_SEC {1600}; + static constexpr size_t MAX_REQUESTS_PER_SEC {1600}; /* the length of a node info buffer in ipv4 format */ static const constexpr size_t NODE4_INFO_BUF_LEN {HASH_LEN + sizeof(in_addr) + sizeof(in_port_t)}; /* the length of a node info buffer in ipv6 format */ @@ -467,7 +436,6 @@ private: static constexpr size_t MAX_PACKET_VALUE_SIZE {600}; static const std::string my_v; - static std::mt19937 rd_device; void process(std::unique_ptr&&, const SockAddr& from); @@ -482,17 +450,7 @@ private: * Sends a request to a node. Request::MAX_ATTEMPT_COUNT attempts will * be made before the request expires. */ - void sendRequest(Sp& request); - - /** - * Generates a new request id, skipping the invalid id. - * - * @return the new id. - */ - uint16_t getNewTid() { - ++transaction_id; - return transaction_id == TransId::INVALID ? ++transaction_id : transaction_id; - } + void sendRequest(const Sp& request); struct MessageStats { unsigned ping {0}; @@ -507,18 +465,18 @@ private: // basic wrapper for socket sendto function int send(const char *buf, size_t len, int flags, const SockAddr& addr); - void sendValueParts(TransId tid, const std::vector& svals, const SockAddr& addr); + void sendValueParts(const TransId& tid, const std::vector& svals, const SockAddr& addr); std::vector packValueHeader(msgpack::sbuffer&, const std::vector>&); - void maintainRxBuffer(const TransId& tid); + void maintainRxBuffer(Tid tid); /************* * Answers * *************/ /* answer to a ping request */ - void sendPong(const SockAddr& addr, TransId tid); + void sendPong(const SockAddr& addr, Tid tid); /* answer to findnodes/getvalues request */ void sendNodesValues(const SockAddr& addr, - TransId tid, + Tid tid, const Blob& nodes, const Blob& nodes6, const std::vector>& st, @@ -532,12 +490,12 @@ private: std::vector>& nodes, std::vector>& nodes6); /* answer to a listen request */ - void sendListenConfirmation(const SockAddr& addr, TransId tid); + void sendListenConfirmation(const SockAddr& addr, Tid tid); /* answer to put request */ - void sendValueAnnounced(const SockAddr& addr, TransId, Value::Id); + void sendValueAnnounced(const SockAddr& addr, Tid, Value::Id); /* answer in case of error */ void sendError(const SockAddr& addr, - TransId tid, + Tid tid, uint16_t code, const std::string& message, bool include_id=false); @@ -547,51 +505,22 @@ private: /* DHT info */ const InfoHash& myid; const NetId network {0}; - const int dht_socket {-1}; - const int dht_socket6 {-1}; + const int& dht_socket; + const int& dht_socket6; const Logger& DHT_LOG; NodeCache cache {}; - /** - * A comparator to classify IP addresses, only considering the - * first 64 bits in IPv6. - */ - struct cmpSockAddr { - bool operator()(const SockAddr& a, const SockAddr& b) const { - if (a.second != b.second) - return a.second < b.second; - socklen_t start, len; - switch(a.getFamily()) { - case AF_INET: - start = offsetof(sockaddr_in, sin_addr); - len = sizeof(in_addr); - break; - case AF_INET6: - start = offsetof(sockaddr_in6, sin6_addr); - // don't consider more than 64 bits (IPv6) - len = 8; - break; - default: - start = 0; - len = a.second; - break; - } - - return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0; - } - }; // global limiting should be triggered by at least 8 different IPs using IpLimiter = RateLimiter; using IpLimiterMap = std::map; IpLimiterMap address_rate_limiter {}; RateLimiter rate_limiter {}; + size_t limiter_maintenance {0}; // requests handling - uint16_t transaction_id {1}; - std::map> requests {}; - std::map partial_messages; - std::map> opened_sockets {}; + std::map> requests {}; + std::map partial_messages; MessageStats in_stats {}, out_stats {}; std::set blacklist {}; diff --git a/include/opendht/node.h b/include/opendht/node.h index 7a2c9256..e7845b4c 100644 --- a/include/opendht/node.h +++ b/include/opendht/node.h @@ -24,33 +24,46 @@ #include "sockaddr.h" #include +#include namespace dht { +struct Node; namespace net { struct Request; +struct Socket; +struct RequestAnswer; } /* namespace net */ +using Tid = uint32_t; +using SocketCb = std::function&, net::RequestAnswer&&)>; +struct Socket { + Socket() {} + Socket(SocketCb&& on_receive) : + on_receive(std::move(on_receive)) {} + SocketCb on_receive {}; +}; + struct Node { - InfoHash id; - SockAddr addr; - - time_point time {time_point::min()}; /* last time eared about */ - time_point reply_time {time_point::min()}; /* time of last correct reply received */ + const InfoHash id; + Node(const InfoHash& id, const SockAddr& addr, bool client=false); Node(const InfoHash& id, const sockaddr* sa, socklen_t salen) - : id(id), addr(sa, salen) {} - Node(const InfoHash& id, const SockAddr& addr) : id(id), addr(addr) {} + : Node(id, SockAddr(sa, salen)) {} InfoHash getId() const { return id; } - std::pair getAddr() const { - return {(const sockaddr*)&addr.first, addr.second}; - } + const SockAddr& getAddr() const { return addr; } std::string getAddrStr() const { return addr.toString(); } + bool isClient() const { return is_client; } + bool isIncoming() { return time > reply_time; } + + const time_point& getTime() const { return time; } + const time_point& getReplyTime() const { return reply_time; } + void setTime(const time_point& t) { time = t; } /** * Makes notice about an additionnal authentication error with this node. Up @@ -68,21 +81,66 @@ struct Node { bool isPendingMessage() const; size_t getPendingMessageCount() const; - NodeExport exportNode() const { return NodeExport {id, addr.first, addr.second}; } + bool isOld(const time_point& now) const { + return time + NODE_EXPIRE_TIME < now; + } + bool isRemovable(const time_point& now) const { + return isExpired() and isOld(now); + } + + NodeExport exportNode() const { + NodeExport ne; + ne.id = id; + ne.sslen = addr.getLength(); + std::memcpy(&ne.ss, addr.get(), ne.sslen); + return ne; + } sa_family_t getFamily() const { return addr.getFamily(); } void update(const SockAddr&); - void requested(std::shared_ptr& req); - void received(time_point now, std::shared_ptr req); + void requested(const Sp& req); + void received(time_point now, const Sp& req); + Sp getRequest(Tid tid); + void cancelRequest(const Sp& req); void setExpired(); + /** + * Opens a socket on which a node will be able allowed to write for further + * additionnal updates following the response to a previous request. + * + * @param node The node which will be allowed to write on this socket. + * @param cb The callback to execute once updates arrive on the socket. + * + * @return the socket. + */ + Tid openSocket(SocketCb&& cb); + + Sp getSocket(Tid id); + + /** + * Closes a socket so that no further data will be red on that socket. + * + * @param socket The socket to close. + */ + void closeSocket(Tid id); + /** * Resets the state of the node so it's not expired anymore. */ void reset() { expired_ = false; reply_time = time_point::min(); } + /** + * Generates a new request id, skipping the invalid id. + * + * @return the new id. + */ + Tid getNewTid() { + ++transaction_id; + return transaction_id ? ++transaction_id : transaction_id; + } + std::string toString() const; OPENDHT_PUBLIC friend std::ostream& operator<< (std::ostream& s, const Node& h); @@ -99,15 +157,17 @@ private: /* Number of times we accept authentication errors from this node. */ static const constexpr unsigned MAX_AUTH_ERRORS {3}; - std::list> requests_ {}; + SockAddr addr; + bool is_client {false}; + time_point time {time_point::min()}; /* last time eared about */ + time_point reply_time {time_point::min()}; /* time of last correct reply received */ unsigned auth_errors {0}; bool expired_ {false}; + Tid transaction_id; + using TransactionDist = std::uniform_int_distribution; - void clearPendingQueue() { - requests_.remove_if([](std::weak_ptr& w) { - return w.expired(); - }); - } + std::map> requests_ {}; + std::map> sockets_; }; } diff --git a/include/opendht/node_cache.h b/include/opendht/node_cache.h index d18b7664..9b1e2a06 100644 --- a/include/opendht/node_cache.h +++ b/include/opendht/node_cache.h @@ -26,9 +26,9 @@ namespace dht { struct NodeCache { - std::shared_ptr getNode(const InfoHash& id, sa_family_t family); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed); - std::vector> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count); + Sp getNode(const InfoHash& id, sa_family_t family); + Sp getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client=false); + std::vector> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const; /** * Reset the connectivity state of every node, @@ -37,14 +37,19 @@ struct NodeCache { */ void clearBadNodes(sa_family_t family = 0); + ~NodeCache(); + private: class NodeMap : public std::map> { public: - std::shared_ptr getNode(const InfoHash& id); - std::shared_ptr getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed); + Sp getNode(const InfoHash& id); + Sp getNode(const InfoHash& id, const SockAddr&, time_point now, bool confirmed, bool client); void clearBadNodes(); + void setExpired(); }; + const NodeMap& cache(sa_family_t af) const { return af == AF_INET ? cache_4 : cache_6; } + NodeMap& cache(sa_family_t af) { return af == AF_INET ? cache_4 : cache_6; } NodeMap cache_4; NodeMap cache_6; }; diff --git a/include/opendht/rate_limiter.h b/include/opendht/rate_limiter.h index 504c8934..17ba4bbe 100644 --- a/include/opendht/rate_limiter.h +++ b/include/opendht/rate_limiter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Savoir-faire Linux Inc. + * Copyright (C) 2017 Savoir-faire Linux Inc. * Author : Adrien Béraud * * This program is free software; you can redistribute it and/or modify @@ -23,13 +23,13 @@ namespace dht { -template +template class RateLimiter { public: - /** Clear outdated records and return current quota usage */ + /** Clear outdated records and return current quota usage */ size_t maintain(const time_point& now) { - using namespace std::chrono; - while (not records.empty() and duration_cast(now - records.front()) >= seconds(Perdiod)) + auto limit = now - std::chrono::seconds(Period); + while (not records.empty() and records.front() < limit) records.pop(); return records.size(); } diff --git a/include/opendht/routing_table.h b/include/opendht/routing_table.h index e71149b6..6873f395 100644 --- a/include/opendht/routing_table.h +++ b/include/opendht/routing_table.h @@ -24,6 +24,9 @@ namespace dht { static constexpr unsigned TARGET_NODES {8}; +namespace net { +class NetworkEngine; +} struct Bucket { Bucket() : cached() {} @@ -37,12 +40,17 @@ struct Bucket { /** Return a random node in a bucket. */ Sp randomNode(); + + void sendCachedPing(net::NetworkEngine& ne); }; class RoutingTable : public std::list { public: using std::list::list; + time_point grow_time {time_point::min()}; + bool is_client {false}; + InfoHash middle(const RoutingTable::const_iterator&) const; std::vector> findClosestNodes(const InfoHash id, time_point now, size_t count = TARGET_NODES) const; @@ -65,6 +73,14 @@ public: return empty() || (size() == 1 && front().nodes.empty()); } + void connectivityChanged(const time_point& now) { + grow_time = now; + for (auto& b : *this) + b.time = time_point::min(); + } + + bool onNewNode(const Sp& node, int comfirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne); + /** * Return a random id in the bucket's range. */ diff --git a/include/opendht/securedht.h b/include/opendht/securedht.h index acae729f..7c61c983 100644 --- a/include/opendht/securedht.h +++ b/include/opendht/securedht.h @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author : Adrien Béraud + * Authors: Adrien Béraud * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,13 +30,21 @@ namespace dht { -class OPENDHT_PUBLIC SecureDht : public Dht { +class OPENDHT_PUBLIC SecureDht final : public DhtInterface { public: typedef std::function SignatureCheckCallback; using Config = SecureDhtConfig; + static dht::Config& getConfig(SecureDht::Config& conf) + { + auto& c = conf.node_config; + if (not c.node_id and conf.id.second) + c.node_id = InfoHash::get("node:"+conf.id.second->getId().toString()); + return c; + } + SecureDht() {} /** @@ -44,7 +53,7 @@ public: * id: the identity to use for the crypto layer and to compute * our own hash on the Dht. */ - SecureDht(int s, int s6, Config config); + SecureDht(std::unique_ptr dht, Config config); virtual ~SecureDht(); @@ -62,14 +71,17 @@ public: return secureType(std::move(tmp_type)); } - virtual void registerType(const ValueType& type) override { - Dht::registerType(secureType(type)); + void registerType(const ValueType& type) { + if (dht_) + dht_->registerType(secureType(type)); } - virtual void registerType(ValueType&& type) { - Dht::registerType(secureType(std::forward(type))); + void registerType(ValueType&& type) { + if (dht_) + dht_->registerType(secureType(std::forward(type))); } - virtual void registerInsecureType(const ValueType& type) { - Dht::registerType(type); + void registerInsecureType(const ValueType& type) { + if (dht_) + dht_->registerType(type); } /** @@ -77,18 +89,18 @@ public: * If the signature can't be checked, or if the data can't be decrypted, it is not returned. * Public, non-signed & non-encrypted data is retransmitted as-is. */ - virtual void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}, Where&& w = {}) override; - virtual void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}, Where&& w = {}) override { + void get(const InfoHash& id, GetCallback cb, DoneCallback donecb={}, Value::Filter&& = {}, Where&& w = {}); + void get(const InfoHash& id, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f = {}, Where&& w = {}) { get(id, cb, bindDoneCb(donecb), std::forward(f), std::forward(w)); } - virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) override { + void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) { get(key, bindGetCb(cb), donecb, std::forward(f), std::forward(w)); } - virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) override { + void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) { get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward(f), std::forward(w)); } - virtual size_t listen(const InfoHash& id, GetCallback cb, Value::Filter&& = {}, Where&& w = {}) override; + size_t listen(const InfoHash& id, GetCallback cb, Value::Filter = {}, Where w = {}); /** * Will take ownership of the value, sign it using our private key and put it in the DHT. @@ -135,7 +147,193 @@ public: localQueryMethod_ = std::move(query_method); } + /** + * SecureDht to Dht proxy + */ + void shutdown(ShutdownCallback cb) { + dht_->shutdown(cb); + } + void dumpTables() const { + dht_->dumpTables(); + } + inline const InfoHash& getNodeId() const { return dht_->getNodeId(); } + std::pair getStoreSize() const { + return dht_->getStoreSize(); + } + std::string getStorageLog() const { + return dht_->getStorageLog(); + } + std::string getStorageLog(const InfoHash& h) const { + return dht_->getStorageLog(h); + } + void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) { + dht_->setStorageLimit(limit); + } + std::vector exportNodes() { + return dht_->exportNodes(); + } + std::vector exportValues() const { + return dht_->exportValues(); + } + void importValues(const std::vector& v) { + dht_->importValues(v); + } + NodeStats getNodesStats(sa_family_t af) const { + return dht_->getNodesStats(af); + } + std::vector getNodeMessageStats(bool in = false) { + return dht_->getNodeMessageStats(in); + } + std::string getRoutingTablesLog(sa_family_t af) const { + return dht_->getRoutingTablesLog(af); + } + std::string getSearchesLog(sa_family_t af) const { + return dht_->getSearchesLog(af); + } + std::string getSearchLog(const InfoHash& h, sa_family_t af = AF_UNSPEC) const { + return dht_->getSearchLog(h, af); + } + std::vector getPublicAddress(sa_family_t family = 0) { + return dht_->getPublicAddress(family); + } + time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr& sa) { + return dht_->periodic(buf, buflen, sa); + } + time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) { + return dht_->periodic(buf, buflen, from, fromlen); + } + NodeStatus getStatus(sa_family_t af) const { + return dht_->getStatus(af); + } + NodeStatus getStatus() const { + return dht_->getStatus(); + } + bool isRunning(sa_family_t af = 0) const { + return dht_->isRunning(af); + } + const ValueType& getType(ValueType::Id type_id) const { + return dht_->getType(type_id); + } + void insertNode(const InfoHash& id, const SockAddr& sa) { + dht_->insertNode(id, sa); + } + void insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) { + dht_->insertNode(id, sa, salen); + } + void insertNode(const NodeExport& n) { + dht_->insertNode(n); + } + void pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb={}) { + dht_->pingNode(sa, salen, std::move(cb)); + } + void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {}) { + dht_->query(key, cb, done_cb, std::move(q)); + } + void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) { + dht_->query(key, cb, done_cb, std::move(q)); + } + std::vector> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const { + return dht_->getLocal(key, f); + } + Sp getLocalById(const InfoHash& key, Value::Id vid) const { + return dht_->getLocalById(key, vid); + } + void put(const InfoHash& key, + Sp v, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false) + { + dht_->put(key, v, cb, created, permanent); + } + void put(const InfoHash& key, + const Sp& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) + { + dht_->put(key, v, cb, created, permanent); + } + + void put(const InfoHash& key, + Value&& v, + DoneCallback cb=nullptr, + time_point created=time_point::max(), + bool permanent = false) + { + dht_->put(key, std::move(v), cb, created, permanent); + } + void put(const InfoHash& key, + Value&& v, + DoneCallbackSimple cb, + time_point created=time_point::max(), + bool permanent = false) + { + dht_->put(key, std::move(v), cb, created, permanent); + } + std::vector> getPut(const InfoHash& h) { + return dht_->getPut(h); + } + Sp getPut(const InfoHash& h, const Value::Id& vid) { + return dht_->getPut(h, vid); + } + bool cancelPut(const InfoHash& h, const Value::Id& vid) { + return dht_->cancelPut(h, vid); + } + size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w = {}) { + return dht_->listen(key, cb, f, w); + } + bool cancelListen(const InfoHash& h, size_t token) { + return dht_->cancelListen(h, token); + } + void connectivityChanged(sa_family_t af) { + dht_->connectivityChanged(af); + } + void connectivityChanged() { + dht_->connectivityChanged(); + } + + void setPushNotificationToken(const std::string& token = "") { + dht_->setPushNotificationToken(token); + } + + void forwardAllMessages(bool forward) { + forward_all_ = forward; + } + + /** + * Call linked callback with push_notification + * @param notification to process + */ + void pushNotificationReceived(const std::map& notification) { + dht_->pushNotificationReceived(notification); + } + /** + * Refresh a listen via a token + * @param token + */ + void resubscribe(const unsigned token) { + dht_->resubscribe(token); + } + + void setLoggers(LogMethod error = NOLOG, LogMethod warn = NOLOG, LogMethod debug = NOLOG) + { + DHT_LOG.DEBUG = debug; + DHT_LOG.WARN = warn; + DHT_LOG.ERR = error; + dht_->setLoggers(std::forward(error), std::forward(warn), std::forward(debug)); + } + + /** + * Only print logs related to the given InfoHash (if given), or disable filter (if zeroes). + */ + void setLogFilter(const InfoHash& f) { + DHT_LOG.setFilter(f); + dht_->setLogFilter(f); + } + private: + std::unique_ptr dht_; // prevent copy SecureDht(const SecureDht&) = delete; SecureDht& operator=(const SecureDht&) = delete; @@ -153,12 +351,14 @@ private: std::map> nodesPubKeys_ {}; std::uniform_int_distribution rand_id {}; + + std::atomic_bool forward_all_ {false}; }; const ValueType CERTIFICATE_TYPE = { 8, "Certificate", std::chrono::hours(24 * 7), // A certificate can only be stored at its public key ID. - [](InfoHash id, Sp& v, InfoHash, const sockaddr*, socklen_t) { + [](InfoHash id, Sp& v, const InfoHash&, const SockAddr&) { try { crypto::Certificate crt(v->data); // TODO check certificate signature @@ -166,7 +366,7 @@ const ValueType CERTIFICATE_TYPE = { } catch (const std::exception& e) {} return false; }, - [](InfoHash, const Sp& o, Sp& n, InfoHash, const sockaddr*, socklen_t) { + [](InfoHash, const Sp& o, Sp& n, const InfoHash&, const SockAddr&) { try { return crypto::Certificate(o->data).getPublicKey().getId() == crypto::Certificate(n->data).getPublicKey().getId(); } catch (const std::exception& e) {} diff --git a/include/opendht/sockaddr.h b/include/opendht/sockaddr.h index d2b814d1..995579e4 100644 --- a/include/opendht/sockaddr.h +++ b/include/opendht/sockaddr.h @@ -28,62 +28,128 @@ typedef uint16_t in_port_t; #endif #else #include +#include +#include #include #include typedef uint16_t sa_family_t; typedef uint16_t in_port_t; #endif +#include +#include +#include +#include + +#include +#include + namespace dht { OPENDHT_PUBLIC std::string print_addr(const sockaddr* sa, socklen_t slen); OPENDHT_PUBLIC std::string print_addr(const sockaddr_storage& ss, socklen_t sslen); -struct OPENDHT_PUBLIC SockAddr : public std::pair { +/** + * A Socket Address (sockaddr*), with abstraction for IPv4, IPv6 address families. + */ +class OPENDHT_PUBLIC SockAddr { public: - SockAddr() : pair::pair({},0) {} - SockAddr(const SockAddr& o) : pair::pair({},o.second) { - std::copy_n((uint8_t*)&o.first, o.second, (uint8_t*)&first); + SockAddr() {} + SockAddr(const SockAddr& o) { + set(o.get(), o.getLength()); } - SockAddr(const sockaddr* sa, socklen_t len) : pair::pair({},len) { - if (len > sizeof(sockaddr_storage)) + SockAddr(SockAddr&& o) : len(o.len), addr(std::move(o.addr)) { + o.len = 0; + } + + /** + * Build from existing address. + */ + SockAddr(const sockaddr* sa, socklen_t length) { + if (length > sizeof(sockaddr_storage)) throw std::runtime_error("Socket address length is too large"); - std::copy_n((uint8_t*)sa, len, (uint8_t*)&first); + set(sa, length); } + SockAddr(const sockaddr* sa) { + socklen_t len = 0; + if (sa) { + if (sa->sa_family == AF_INET) + len = sizeof(sockaddr_in); + else if(sa->sa_family == AF_INET6) + len = sizeof(sockaddr_in6); + else + throw std::runtime_error("Unknown address family"); + } + set(sa, len); + } + + /** + * Build from an existing sockaddr_storage structure. + */ SockAddr(const sockaddr_storage& ss, socklen_t len) : SockAddr((const sockaddr*)&ss, len) {} + static std::vector resolve(const std::string& host, const std::string& service = {}); + bool operator<(const SockAddr& o) const { - if (second != o.second) - return second < o.second; - return std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) < 0; + if (len != o.len) + return len < o.len; + return std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) < 0; } bool equals(const SockAddr& o) const { - return second == o.second - && std::memcmp((uint8_t*)&first, (uint8_t*)&o.first, second) == 0; + return len == o.len + && std::memcmp((const uint8_t*)get(), (const uint8_t*)o.get(), len) == 0; } SockAddr& operator=(const SockAddr& o) { - std::copy_n((const uint8_t*)&o.first, o.second, (uint8_t*)&first); - second = o.second; + set(o.get(), o.getLength()); + return *this; + } + SockAddr& operator=(SockAddr&& o) { + len = o.len; + o.len = 0; + addr = std::move(o.addr); return *this; } std::string toString() const { - return print_addr(first, second); - } - sa_family_t getFamily() const { return second > sizeof(sa_family_t) ? first.ss_family : AF_UNSPEC; } - void setFamily(sa_family_t af) { - first.ss_family = af; - switch(af) { - case AF_INET: - second = sizeof(sockaddr_in); - break; - case AF_INET6: - second = sizeof(sockaddr_in6); - break; - } + return print_addr(get(), getLength()); } + /** + * Returns the address family or AF_UNSPEC if the address is not set. + */ + sa_family_t getFamily() const { return len > sizeof(sa_family_t) ? addr->sa_family : AF_UNSPEC; } + + /** + * Resize the managed structure to the appropriate size (if needed), + * in which case the sockaddr structure is cleared to zero, + * and set the address family field (sa_family). + */ + void setFamily(sa_family_t af) { + socklen_t new_length; + switch(af) { + case AF_INET: + new_length = sizeof(sockaddr_in); + break; + case AF_INET6: + new_length = sizeof(sockaddr_in6); + break; + default: + new_length = 0; + } + if (new_length != len) { + len = new_length; + if (len) addr.reset((sockaddr*)::calloc(len, 1)); + else addr.reset(); + } + if (len > sizeof(sa_family_t)) + addr->sa_family = af; + } + + /** + * Retreive the port (in host byte order) or 0 if the address is not + * of a supported family. + */ in_port_t getPort() const { switch(getFamily()) { case AF_INET: @@ -94,6 +160,10 @@ public: return 0; } } + /** + * Set the port. The address must be of a supported family. + * @param p The port in host byte order. + */ void setPort(in_port_t p) { switch(getFamily()) { case AF_INET: @@ -105,17 +175,42 @@ public: } } + /** + * Returns the accessible byte length at the pointer returned by #get(). + * If zero, #get() returns null. + */ + socklen_t getLength() const { return len; } + + /** + * An address is defined to be true if its length is not zero. + */ + explicit operator bool() const noexcept { + return len; + } + + /** + * Returns the address to the managed sockaddr structure. + * The accessible length is returned by #getLength(). + */ + const sockaddr* get() const { return addr.get(); } + + /** + * Returns the address to the managed sockaddr structure. + * The accessible length is returned by #getLength(). + */ + sockaddr* get() { return addr.get(); } + const sockaddr_in& getIPv4() const { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } const sockaddr_in6& getIPv6() const { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } sockaddr_in& getIPv4() { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } sockaddr_in6& getIPv6() { - return *reinterpret_cast(&first); + return *reinterpret_cast(get()); } /** @@ -130,14 +225,17 @@ public: bool isUnspecified() const; + bool isMappedIPv4() const; + SockAddr getMappedIPv4() const; + /** * A comparator to classify IP addresses, only considering the * first 64 bits in IPv6. */ struct ipCmp { bool operator()(const SockAddr& a, const SockAddr& b) const { - if (a.second != b.second) - return a.second < b.second; + if (a.len != b.len) + return a.len < b.len; socklen_t start, len; switch(a.getFamily()) { case AF_INET: @@ -151,18 +249,30 @@ public: break; default: start = 0; - len = a.second; + len = a.len; break; } - - return std::memcmp((uint8_t*)&a.first+start, (uint8_t*)&b.first+start, len) < 0; + return std::memcmp((uint8_t*)a.get()+start, + (uint8_t*)b.get()+start, len) < 0; } }; +private: + socklen_t len {0}; + struct free_delete { void operator()(void* p) { ::free(p); } }; + std::unique_ptr addr {}; + + void set(const sockaddr* sa, socklen_t length) { + if (len != length) { + len = length; + if (len) addr.reset((sockaddr*)::malloc(len)); + else addr.reset(); + } + if (len) + std::memcpy((uint8_t*)get(), (const uint8_t*)sa, len); + } }; OPENDHT_PUBLIC bool operator==(const SockAddr& a, const SockAddr& b); -OPENDHT_PUBLIC std::string printAddr(const SockAddr& addr); - } diff --git a/include/opendht/utils.h b/include/opendht/utils.h index 054b7be4..a7621296 100644 --- a/include/opendht/utils.h +++ b/include/opendht/utils.h @@ -32,6 +32,9 @@ #define WANT4 1 #define WANT6 2 +/** + * OpenDHT C++ namespace + */ namespace dht { using NetId = uint32_t; @@ -52,9 +55,15 @@ void erase_if(std::map& map, const Condition& condition) } class OPENDHT_PUBLIC DhtException : public std::runtime_error { - public: - DhtException(const std::string &str = "") : - std::runtime_error("DhtException occurred: " + str) {} +public: + DhtException(const std::string &str = "") : + std::runtime_error("DhtException occurred: " + str) {} +}; + +class OPENDHT_PUBLIC SocketException : public DhtException { +public: + SocketException(int err) : + DhtException(strerror(err)) {} }; // Time related definitions and utility functions @@ -93,6 +102,9 @@ public: // Serialization related definitions and utility functions +/** + * Arbitrary binary data. + */ using Blob = std::vector; /** diff --git a/include/opendht/value.h b/include/opendht/value.h index 407f480f..9c8b155a 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -22,6 +22,7 @@ #include "infohash.h" #include "crypto.h" #include "utils.h" +#include "sockaddr.h" #include @@ -36,6 +37,10 @@ #include #include +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT +#include +#endif //OPENDHT_PROXY_SERVER + namespace dht { struct Value; @@ -51,7 +56,7 @@ struct Query; * @param form_addr: network address of the incoming request. * @param from_len: network address lendth of the incoming request. */ -using StorePolicy = std::function& value, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>; +using StorePolicy = std::function& value, const InfoHash& from, const SockAddr& addr)>; /** * An edition policy is applied once to every incoming value storage requests, @@ -67,15 +72,15 @@ using StorePolicy = std::function& val * @param form_addr: network address of the incoming request. * @param from_len: network address lendth of the incoming request. */ -using EditPolicy = std::function& old_val, std::shared_ptr& new_val, InfoHash from, const sockaddr* from_addr, socklen_t from_len)>; +using EditPolicy = std::function& old_val, std::shared_ptr& new_val, const InfoHash& from, const SockAddr& addr)>; static constexpr const size_t MAX_VALUE_SIZE {1024 * 64}; struct OPENDHT_PUBLIC ValueType { typedef uint16_t Id; - static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, InfoHash, const sockaddr*, socklen_t); - static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr&, std::shared_ptr&, InfoHash, const sockaddr*, socklen_t) { + static bool DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, const InfoHash&, const SockAddr&); + static bool DEFAULT_EDIT_POLICY(InfoHash, const std::shared_ptr&, std::shared_ptr&, const InfoHash&, const SockAddr&) { return false; } @@ -345,6 +350,14 @@ struct OPENDHT_PUBLIC Value Value(ValueType::Id t, const uint8_t* dat_ptr, size_t dat_len, Id id = INVALID_ID) : id(id), type(t), data(dat_ptr, dat_ptr+dat_len) {} +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT + /** + * Build a value from a json object + * @param json + */ + Value(Json::Value& json); +#endif //OPENDHT_PROXY_SERVER + template Value(ValueType::Id t, const Type& d, Id id = INVALID_ID) : id(id), type(t), data(packMsg(d)) {} @@ -416,6 +429,18 @@ struct OPENDHT_PUBLIC Value return ss.str(); } +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT + /** + * Build a json object from a value + * Example: + * { + * "data":"base64ofdata", + * id":"0", "seq":0,"type":3 + * } + */ + Json::Value toJson() const; +#endif //OPENDHT_PROXY_SERVER + /** Return the size in bytes used by this value in memory (minimum). */ size_t size() const; @@ -423,11 +448,11 @@ struct OPENDHT_PUBLIC Value void msgpack_pack_to_sign(Packer& pk) const { bool has_owner = owner && *owner; - pk.pack_map((user_type.empty()?0:1) + (has_owner?(recipient == InfoHash() ? 4 : 5):2)); + pk.pack_map((user_type.empty()?0:1) + (has_owner?(recipient ? 5 : 4):2)); if (has_owner) { // isSigned pk.pack(std::string("seq")); pk.pack(seq); pk.pack(std::string("owner")); owner->msgpack_pack(pk); - if (recipient != InfoHash()) { + if (recipient) { pk.pack(std::string("to")); pk.pack(recipient); } } diff --git a/python/opendht.pyx b/python/opendht.pyx index 870e10ed..790b3e66 100644 --- a/python/opendht.pyx +++ b/python/opendht.pyx @@ -92,9 +92,9 @@ cdef class _WithID(object): cdef class InfoHash(_WithID): cdef cpp.InfoHash _infohash def __cinit__(self, bytes str=b''): - self._infohash = cpp.InfoHash(str) + self._infohash = cpp.InfoHash(str) if str else cpp.InfoHash() def __bool__(InfoHash self): - return not (self._infohash == cpp.InfoHash()) + return self._infohash def __richcmp__(InfoHash self, InfoHash other, int op): if op == 0: return self._infohash < other._infohash @@ -179,8 +179,11 @@ cdef class Query(object): cdef class Select(object): cdef cpp.Select _select - def __cinit__(self, str q_str=''): - self._select = cpp.Select(q_str.encode()) + def __cinit__(self, str q_str=None): + if q_str: + self._select = cpp.Select(q_str.encode()) + else: + self._select = cpp.Select() def __str__(self): return self._select.toString().decode() def isSatisfiedBy(self, Select os): @@ -191,8 +194,11 @@ cdef class Select(object): cdef class Where(object): cdef cpp.Where _where - def __cinit__(self, str q_str=''): - self._where = cpp.Where(q_str.encode()) + def __cinit__(self, str q_str=None): + if q_str: + self._where = cpp.Where(q_str.encode()) + else: + self._where = cpp.Where() def __str__(self): return self._where.toString().decode() def isSatisfiedBy(self, Where where): @@ -527,6 +533,8 @@ cdef class DhtRunner(_WithID): if get_cb: cb_obj = {'get':get_cb, 'done':done_cb, 'filter':filter} ref.Py_INCREF(cb_obj) + if where is None: + where = Where() self.thisptr.get().get(key._infohash, cpp.bindGetCb(get_callback, cb_obj), cpp.bindDoneCb(done_callback, cb_obj), cpp.nullptr, #filter implemented in the get_callback @@ -654,4 +662,3 @@ cdef class Pht(object): val, cpp.bindDoneCbSimple(done_callback_simple, cb_obj) ) - diff --git a/python/opendht_cpp.pxd b/python/opendht_cpp.pxd index 1ad79b56..3b889971 100644 --- a/python/opendht_cpp.pxd +++ b/python/opendht_cpp.pxd @@ -65,8 +65,9 @@ cdef extern from "opendht/infohash.h" namespace "dht": InfoHash get(string s) @staticmethod InfoHash getRandom() - bool operator==(InfoHash) - bool operator<(InfoHash) + bool operator==(InfoHash) const + bool operator<(InfoHash) const + bool operator bool() const cdef extern from "opendht/sockaddr.h" namespace "dht": cdef cppclass SockAddr: @@ -259,4 +260,3 @@ cdef extern from "opendht/indexation/pht.h" namespace "dht::indexation": void insert(IndexKey k, IndexValue v, DoneCallbackSimple cb) @staticmethod LookupCallback bindLookupCb(LookupCallbackRaw cb, void *user_data) - diff --git a/src/Makefile.am b/src/Makefile.am index 60536d99..d21212a3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,6 +9,8 @@ libopendht_la_SOURCES = \ listener.h \ request.h \ search.h \ + net.h \ + parsed_message.h \ node_cache.cpp \ callbacks.cpp \ routing_table.cpp \ @@ -36,7 +38,6 @@ nobase_include_HEADERS = \ ../include/opendht/node_cache.h \ ../include/opendht/routing_table.h \ ../include/opendht/network_engine.h \ - ../include/opendht/net.h \ ../include/opendht/scheduler.h \ ../include/opendht/rate_limiter.h \ ../include/opendht/utils.h \ @@ -53,6 +54,16 @@ nobase_include_HEADERS = \ ../include/opendht/rng.h \ ../include/opendht/indexation/pht.h +if ENABLE_PROXY_SERVER +libopendht_la_SOURCES += base64.h base64.cpp dht_proxy_server.cpp +nobase_include_HEADERS += ../include/opendht/dht_proxy_server.h +endif + +if ENABLE_PROXY_CLIENT +libopendht_la_SOURCES += dht_proxy_client.cpp +nobase_include_HEADERS += ../include/opendht/dht_proxy_client.h ../include/opendht/dht_interface.h +endif + clean-local: rm -rf libargon2.la @@ -82,4 +93,3 @@ noinst_HEADERS = \ @top_builddir@/argon2/src/encoding.h \ @top_builddir@/argon2/src/thread.h endif - diff --git a/src/base64.cpp b/src/base64.cpp new file mode 100644 index 00000000..7e255582 --- /dev/null +++ b/src/base64.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "base64.h" + +#include +#include + +/* Mainly based on the following stackoverflow question: + * http://stackoverflow.com/questions/342409/how-do-i-base64-encode-decode-in-c + */ +static const char encoding_table[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', + '3', '4', '5', '6', '7', '8', '9', '+', '/' +}; + +static const size_t mod_table[] = { 0, 2, 1 }; + +char *base64_encode(const uint8_t *input, size_t input_length, + char *output, size_t *output_length) +{ + size_t i, j; + size_t out_sz = *output_length; + *output_length = 4 * ((input_length + 2) / 3); + if (out_sz < *output_length || output == nullptr) + return nullptr; + + for (i = 0, j = 0; i < input_length; ) { + uint8_t octet_a = i < input_length ? input[i++] : 0; + uint8_t octet_b = i < input_length ? input[i++] : 0; + uint8_t octet_c = i < input_length ? input[i++] : 0; + + uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + output[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 1 * 6) & 0x3F]; + output[j++] = encoding_table[(triple >> 0 * 6) & 0x3F]; + } + + for (i = 0; i < mod_table[input_length % 3]; i++) + output[*output_length - 1 - i] = '='; + + return output; +} + +uint8_t *base64_decode(const char *input, size_t input_length, + uint8_t *output, size_t *output_length) +{ + size_t i, j; + uint8_t decoding_table[256]; + + uint8_t c; + for (c = 0; c < 64; c++) + decoding_table[static_cast(encoding_table[c])] = c; + + if (input_length % 4 != 0 || input_length < 2) + return nullptr; + + size_t out_sz = *output_length; + *output_length = input_length / 4 * 3; + if (input[input_length - 1] == '=') + (*output_length)--; + if (input[input_length - 2] == '=') + (*output_length)--; + + if (out_sz < *output_length || output == nullptr) + return nullptr; + + for (i = 0, j = 0; i < input_length;) { + uint8_t sextet_a = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_b = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_c = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + uint8_t sextet_d = input[i] == '=' ? 0 & i++ + : decoding_table[static_cast(input[i++])]; + + uint32_t triple = (sextet_a << 3 * 6) + + (sextet_b << 2 * 6) + + (sextet_c << 1 * 6) + + (sextet_d << 0 * 6); + + if (j < *output_length) + output[j++] = (triple >> 2 * 8) & 0xFF; + if (j < *output_length) + output[j++] = (triple >> 1 * 8) & 0xFF; + if (j < *output_length) + output[j++] = (triple >> 0 * 8) & 0xFF; + } + + return output; +} + +std::string +base64_encode(const std::vector::const_iterator begin, + const std::vector::const_iterator end) +{ + size_t output_length = 4 * ((std::distance(begin, end) + 2) / 3); + std::string out; + out.resize(output_length); + base64_encode(&(*begin), std::distance(begin, end), + &(*out.begin()), &output_length); + out.resize(output_length); + return out; +} + + +std::string +base64_encode(const std::vector& str) +{ + return base64_encode(str.cbegin(), str.cend()); +} + +std::string +base64_decode(const std::string& str) +{ + size_t output_length = str.length() / 4 * 3 + 2; + std::vector output; + output.resize(output_length); + base64_decode(str.data(), str.size(), output.data(), &output_length); + output.resize(output_length); + return std::string(output.begin(), output.end()); +} diff --git a/src/base64.h b/src/base64.h new file mode 100644 index 00000000..3f5f39e1 --- /dev/null +++ b/src/base64.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2004-2017 Savoir-faire Linux Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include + +/** + * Encode a buffer in base64. + * + * @param str the input buffer + * @return a base64-encoded buffer + */ +std::string base64_encode(const std::vector& str); +/** + * Decode a buffer in base64. + * + * @param str the input buffer + * @return a base64-decoded buffer + */ +std::string base64_decode(const std::string& str); diff --git a/src/callbacks.cpp b/src/callbacks.cpp index 8908a2df..2cac5e68 100644 --- a/src/callbacks.cpp +++ b/src/callbacks.cpp @@ -68,4 +68,36 @@ NodeStats::toString() const return ss.str(); } +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT +/** + * Build a json object from a NodeStats + */ +Json::Value +NodeStats::toJson() const +{ + Json::Value val; + val["good"] = static_cast(good_nodes); + val["dubious"] = static_cast(dubious_nodes); + val["incoming"] = static_cast(incoming_nodes); + if (table_depth > 1) { + val["table_depth"] = static_cast(table_depth); + unsigned long tot_nodes = 8 * std::exp2(table_depth); + val["network_size_estimation"] = static_cast(tot_nodes); + } + return val; +} + +NodeStats::NodeStats(const Json::Value& val) +{ + if (val.isMember("good")) + good_nodes = static_cast(val["good"].asLargestUInt()); + if (val.isMember("dubious")) + dubious_nodes = static_cast(val["dubious"].asLargestUInt()); + if (val.isMember("incoming")) + incoming_nodes = static_cast(val["incoming"].asLargestUInt()); + if (val.isMember("table_depth")) + table_depth = static_cast(val["table_depth"].asLargestUInt()); +} +#endif //OPENDHT_PROXY_SERVER + } diff --git a/src/crypto.cpp b/src/crypto.cpp index 8b162138..6b5b561f 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -84,7 +84,7 @@ static constexpr size_t PASSWORD_SALT_LENGTH {16}; constexpr gnutls_digest_algorithm_t gnutlsHashAlgo(size_t min_res) { return (min_res > 256/8) ? GNUTLS_DIG_SHA512 : ( - (min_res > 128/8) ? GNUTLS_DIG_SHA256 : ( + (min_res > 160/8) ? GNUTLS_DIG_SHA256 : ( GNUTLS_DIG_SHA1)); } @@ -212,12 +212,21 @@ Blob hash(const Blob& data, size_t hash_len) Blob res; res.resize(res_size); const gnutls_datum_t gdat {(uint8_t*)data.data(), (unsigned)data.size()}; - if (gnutls_fingerprint(algo, &gdat, res.data(), &res_size)) - throw CryptoException("Can't compute hash !"); + if (auto err = gnutls_fingerprint(algo, &gdat, res.data(), &res_size)) + throw CryptoException(std::string("Can't compute hash: ") + gnutls_strerror(err)); res.resize(std::min(hash_len, res_size)); return res; } +void hash(const uint8_t* data, size_t data_length, uint8_t* hash, size_t hash_length) +{ + auto algo = gnutlsHashAlgo(hash_length); + size_t res_size = hash_length; + const gnutls_datum_t gdat {(uint8_t*)data, (unsigned)data_length}; + if (auto err = gnutls_fingerprint(algo, &gdat, hash, &res_size)) + throw CryptoException(std::string("Can't compute hash: ") + gnutls_strerror(err)); +} + PrivateKey::PrivateKey() {} @@ -529,7 +538,12 @@ PublicKey::getId() const return {}; InfoHash id; size_t sz = id.size(); - if (auto err = gnutls_pubkey_get_key_id(pk, 0, id.data(), &sz)) +#if GNUTLS_VERSION_NUMBER < 0x030401 + const int flags = 0; +#else + const int flags = (id.size() == 32) ? GNUTLS_KEYID_USE_SHA256 : 0; +#endif + if (auto err = gnutls_pubkey_get_key_id(pk, flags, id.data(), &sz)) throw CryptoException(std::string("Can't get public key ID: ") + gnutls_strerror(err)); if (sz != id.size()) throw CryptoException("Can't get public key ID: wrong output length."); @@ -882,6 +896,19 @@ generateIdentity(const std::string& name, Identity ca, unsigned key_length) { return generateIdentity(name, ca, key_length, !ca.first || !ca.second); } +Identity +generateEcIdentity(const std::string& name, crypto::Identity ca, bool is_ca) +{ + auto key = std::make_shared(PrivateKey::generateEC()); + auto cert = std::make_shared(Certificate::generate(*key, name, ca, is_ca)); + return {std::move(key), std::move(cert)}; +} + +Identity +generateEcIdentity(const std::string& name, Identity ca) { + return generateEcIdentity(name, ca, !ca.first || !ca.second); +} + Certificate Certificate::generate(const PrivateKey& key, const std::string& name, Identity ca, bool is_ca) { @@ -916,10 +943,12 @@ Certificate::generate(const PrivateKey& key, const std::string& name, Identity c gnutls_x509_crt_set_serial(cert, &cert_serial, sizeof(cert_serial)); } - unsigned key_usage = GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_DATA_ENCIPHERMENT; + unsigned key_usage = 0; if (is_ca) { gnutls_x509_crt_set_ca_status(cert, 1); key_usage |= GNUTLS_KEY_KEY_CERT_SIGN | GNUTLS_KEY_CRL_SIGN; + } else { + key_usage |= GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_DATA_ENCIPHERMENT; } gnutls_x509_crt_set_key_usage(cert, key_usage); @@ -1262,8 +1291,10 @@ std::ostream& operator<< (std::ostream& o, const TrustList::VerifyResult& h) o << "* Certificate has expired" << std::endl; if (h.result & GNUTLS_CERT_UNEXPECTED_OWNER) o << "* The owner is not the expected one" << std::endl; +#if GNUTLS_VERSION_NUMBER >= 0x030401 if (h.result & GNUTLS_CERT_PURPOSE_MISMATCH) o << "* Certificate or an intermediate does not match the intended purpose" << std::endl; +#endif if (h.result & GNUTLS_CERT_MISMATCH) o << "* Certificate presented isn't the expected one" << std::endl; } else { diff --git a/src/default_types.cpp b/src/default_types.cpp index 9d59907e..1e7bfd13 100644 --- a/src/default_types.cpp +++ b/src/default_types.cpp @@ -27,14 +27,14 @@ std::ostream& operator<< (std::ostream& s, const DhtMessage& v) } bool -DhtMessage::storePolicy(InfoHash h, std::shared_ptr& v, InfoHash f, const sockaddr* fa, socklen_t fas) +DhtMessage::storePolicy(InfoHash h, std::shared_ptr& v, const InfoHash& f, const SockAddr& sa) { try { auto msg = unpackMsg(v->data); if (msg.service.empty()) return false; } catch (const std::exception& e) {} - return ValueType::DEFAULT_STORE_POLICY(h, v, f, fa, fas); + return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa); } Value::Filter @@ -54,12 +54,11 @@ DhtMessage::ServiceFilter(std::string s) std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v) { - s << "Peer: "; - s << "port " << v.getPort(); - - if (v.ss.ss_family == AF_INET || v.ss.ss_family == AF_INET6) { + if (v.addr) { + s << "Peer: "; + s << "port " << v.getPort(); char hbuf[NI_MAXHOST]; - if (getnameinfo((sockaddr*)&v.ss, sizeof(v.ss), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) { + if (getnameinfo(v.addr.get(), v.addr.getLength(), hbuf, sizeof(hbuf), nullptr, 0, NI_NUMERICHOST) == 0) { s << " addr " << std::string(hbuf, strlen(hbuf)); } } @@ -67,17 +66,17 @@ std::ostream& operator<< (std::ostream& s, const IpServiceAnnouncement& v) } bool -IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr& v, InfoHash f, const sockaddr* from, socklen_t fromlen) +IpServiceAnnouncement::storePolicy(InfoHash h, std::shared_ptr& v, const InfoHash& f, const SockAddr& sa) { try { auto msg = unpackMsg(v->data); if (msg.getPort() == 0) return false; - IpServiceAnnouncement sa_addr {from, fromlen}; + IpServiceAnnouncement sa_addr {sa}; sa_addr.setPort(msg.getPort()); // argument v is modified (not the value). v = std::make_shared(IpServiceAnnouncement::TYPE, sa_addr, v->id); - return ValueType::DEFAULT_STORE_POLICY(h, v, f, from, fromlen); + return ValueType::DEFAULT_STORE_POLICY(h, v, f, sa); } catch (const std::exception& e) {} return false; } diff --git a/src/dht.cpp b/src/dht.cpp index 901655be..f75dc479 100644 --- a/src/dht.cpp +++ b/src/dht.cpp @@ -39,21 +39,13 @@ constexpr std::chrono::minutes Dht::SEARCH_EXPIRE_TIME; constexpr std::chrono::seconds Dht::LISTEN_EXPIRE_TIME; constexpr std::chrono::seconds Dht::REANNOUNCE_MARGIN; -void -Dht::setLoggers(LogMethod error, LogMethod warn, LogMethod debug) -{ - DHT_LOG.DEBUG = debug; - DHT_LOG.WARN = warn; - DHT_LOG.ERR = error; -} - NodeStatus Dht::getStatus(sa_family_t af) const { const auto& stats = getNodesStats(af); - auto& ping = af == AF_INET ? pending_pings4 : pending_pings6; if (stats.good_nodes) return NodeStatus::Connected; + auto& ping = af == AF_INET ? pending_pings4 : pending_pings6; if (ping or stats.getKnownNodes()) return NodeStatus::Connecting; return NodeStatus::Disconnected; @@ -93,11 +85,9 @@ Dht::isRunning(sa_family_t af) const { return network_engine.isRunning(af); } const Sp Dht::findNode(const InfoHash& id, sa_family_t af) const { - const Bucket* b = findBucket(id, af); - if (!b) - return {}; - for (const auto& n : b->nodes) - if (n->id == id) return n; + if (const Bucket* b = findBucket(id, af)) + for (const auto& n : b->nodes) + if (n->id == id) return n; return {}; } @@ -105,11 +95,9 @@ Dht::findNode(const InfoHash& id, sa_family_t af) const void Dht::sendCachedPing(Bucket& b) { - if (not b.cached) - return; - DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str()); - network_engine.sendPing(b.cached, nullptr, nullptr); - b.cached = {}; + if (b.cached) + DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", b.cached->toString().c_str()); + b.sendCachedPing(network_engine); } std::vector @@ -119,8 +107,9 @@ Dht::getPublicAddress(sa_family_t family) return a.first > b.first; }); std::vector ret; + ret.reserve(!family ? reported_addr.size() : reported_addr.size()/2); for (const auto& addr : reported_addr) - if (!family || family == addr.second.first.ss_family) + if (!family || family == addr.second.getFamily()) ret.emplace_back(addr.second); return ret; } @@ -178,71 +167,8 @@ Dht::reportedAddr(const SockAddr& addr) void Dht::onNewNode(const Sp& node, int confirm) { - auto& list = buckets(node->getFamily()); - auto b = list.findBucket(node->id); - if (b == list.end()) - return; - - for (auto& n : b->nodes) { - if (n == node) { - if (confirm) - trySearchInsert(node); - return; - } - } - - /* New node. */ - /* Try adding the node to searches */ - trySearchInsert(node); - - const auto& now = scheduler.time(); - bool mybucket = list.contains(b, myid); - if (mybucket) { - if (node->getFamily() == AF_INET) - mybucket_grow_time = now; - else - mybucket6_grow_time = now; - //scheduler.edit(nextNodesConfirmation, now); - } - - if (b->nodes.size() >= TARGET_NODES) { - /* Try to get rid of an expired node. */ - for (auto& n : b->nodes) - if (n->isExpired()) { - n = node; - return; - } - /* Bucket full. Ping a dubious node */ - bool dubious = false; - for (auto& n : b->nodes) { - /* Pick the first dubious node that we haven't pinged in the - last 9 seconds. This gives nodes the time to reply, but - tends to concentrate on the same nodes, so that we get rid - of bad nodes fast. */ - if (not n->isGood(now)) { - dubious = true; - if (not n->isPendingMessage()) { - DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str()); - network_engine.sendPing(n, nullptr, nullptr); - break; - } - } - } - - if ((mybucket || (is_bootstrap and list.depth(b) < 6)) && (!dubious || list.size() == 1)) { - DHT_LOG.d("Splitting from depth %u", list.depth(b)); - sendCachedPing(*b); - list.split(b); - onNewNode(node, 0); - return; - } - - /* No space for this node. Cache it away for later. */ - if (confirm or not b->cached) - b->cached = node; - } else { - /* Create a new node. */ - b->nodes.emplace_front(node); + if (buckets(node->getFamily()).onNewNode(node, confirm, scheduler.time(), myid, network_engine) or confirm) { + trySearchInsert(node); } } @@ -285,7 +211,7 @@ Dht::expireSearches() void Dht::searchNodeGetDone(const net::Request& req, - net::NetworkEngine::RequestAnswer&& answer, + net::RequestAnswer&& answer, std::weak_ptr ws, Sp query) { @@ -329,7 +255,7 @@ void Dht::paginate(std::weak_ptr ws, Sp query, SearchNode* n) { if (not sr) return; auto select_q = std::make_shared(Select {}.field(Value::Field::Id), query ? query->where : Where {}); auto onSelectDone = [this,ws,query](const net::Request& status, - net::NetworkEngine::RequestAnswer&& answer) mutable { + net::RequestAnswer&& answer) mutable { // retreive search auto sr = ws.lock(); if (not sr) return; @@ -453,14 +379,15 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { std::weak_ptr ws = sr; const auto& now = scheduler.time(); for (auto& n : sr->nodes) { - auto something_to_announce = std::find_if(sr->announce.cbegin(), sr->announce.cend(), - [this,&now,&sr,&n](const Announce& a) { - return n.isSynced(now) and n.getAnnounceTime(a.value->id) <= now; - }) != sr->announce.cend(); - if (not something_to_announce) + if (not n.isSynced(now)) + continue; + if (std::find_if(sr->announce.cbegin(), sr->announce.cend(), + [this,&now,&n](const Announce& a) { + return n.getAnnounceTime(a.value->id) <= now; + }) == sr->announce.cend()) continue; - auto onDone = [this,ws](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) + auto onDone = [this,ws](const net::Request& req, net::RequestAnswer&& answer) { /* when put done */ if (auto sr = ws.lock()) { onAnnounceDone(req.node, answer, sr); @@ -474,64 +401,65 @@ void Dht::searchSendAnnounceValue(const Sp& sr) { scheduler.edit(sr->nextSearchStep, scheduler.time()); }; auto onSelectDone = - [this,ws,onDone,onExpired](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable + [this,ws,onDone,onExpired](const net::Request& req, net::RequestAnswer&& answer) mutable { /* on probing done */ + auto sr = ws.lock(); + if (not sr) return; + auto sn = sr->getNode(req.node); + if (not sn) return; + const auto& now = scheduler.time(); - if (auto sr = ws.lock()) { - if (auto sn = sr->getNode(req.node)) { - for (auto ait = sr->announce.begin(); ait != sr->announce.end();) { - auto& a = *ait; - if (sn->isSynced(now) and sn->getAnnounceTime(a.value->id) <= now) { - bool hasValue {false}; - uint16_t seq_no = 0; - try { - const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(), - [&a](const Sp& i){ - return i->index.at(Value::Field::Id).getInt() == a.value->id; - }); - if (f != answer.fields.cend() and *f) { - hasValue = true; - seq_no = static_cast((*f)->index.at(Value::Field::SeqNum).getInt()); - } - } catch (std::out_of_range&) { } - - auto next_refresh_time = now + getType(a.value->type).expiration; - /* only put the value if the node doesn't already have it */ - if (not hasValue or seq_no < a.value->seq) { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node, - sr->id, - a.value, - a.permanent ? time_point::max() : a.created, - sn->token, - onDone, - onExpired), next_refresh_time); - } else if (hasValue and a.permanent) { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node, - sr->id, - a.value->id, - sn->token, - onDone, - onExpired), next_refresh_time); - } else { - DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.", - sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); - auto ack_req = std::make_shared(net::Request::State::COMPLETED); - ack_req->reply_time = now; - sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time); - - /* step to clear announces */ - scheduler.edit(sr->nextSearchStep, now); - } - } else { - /* Search is now unsynced. Let's call searchStep to sync again. */ - scheduler.edit(sr->nextSearchStep, sr->getNextStepTime(now)); - } - ++ait; + if (not sn->isSynced(now)) { + /* Search is now unsynced. Let's call searchStep to sync again. */ + scheduler.edit(sr->nextSearchStep, sr->getNextStepTime(now)); + return; + } + for (auto& a : sr->announce) { + if (sn->getAnnounceTime(a.value->id) > now) + continue; + bool hasValue {false}; + uint16_t seq_no = 0; + try { + const auto& f = std::find_if(answer.fields.cbegin(), answer.fields.cend(), + [&a](const Sp& i){ + return i->index.at(Value::Field::Id).getInt() == a.value->id; + }); + if (f != answer.fields.cend() and *f) { + hasValue = true; + seq_no = static_cast((*f)->index.at(Value::Field::SeqNum).getInt()); } + } catch (std::out_of_range&) { } + + auto next_refresh_time = now + getType(a.value->type).expiration; + /* only put the value if the node doesn't already have it */ + if (not hasValue or seq_no < a.value->seq) { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'put' (vid: %d)", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + sn->acked[a.value->id] = std::make_pair(network_engine.sendAnnounceValue(sn->node, + sr->id, + a.value, + a.permanent ? time_point::max() : a.created, + sn->token, + onDone, + onExpired), next_refresh_time); + } else if (hasValue and a.permanent) { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] sending 'refresh' (vid: %d)", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + sn->acked[a.value->id] = std::make_pair(network_engine.sendRefreshValue(sn->node, + sr->id, + a.value->id, + sn->token, + onDone, + onExpired), next_refresh_time); + } else { + DHT_LOG.w(sr->id, sn->node->id, "[search %s] [node %s] already has value (vid: %d). Aborting.", + sr->id.toString().c_str(), sn->node->toString().c_str(), a.value->id); + auto ack_req = std::make_shared(net::Request::State::COMPLETED); + ack_req->reply_time = now; + sn->acked[a.value->id] = std::make_pair(std::move(ack_req), next_refresh_time); + + /* step to clear announces */ + scheduler.edit(sr->nextSearchStep, now); } } }; @@ -607,40 +535,40 @@ Dht::searchStep(Sp sr) continue; for (const auto& l : sr->listeners) { const auto& query = l.second.query; - if (n.getListenTime(query) <= now) { - DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'", - sr->id.toString().c_str(), n.node->toString().c_str()); + if (n.getListenTime(query) > now) + continue; + DHT_LOG.w(sr->id, n.node->id, "[search %s] [node %s] sending 'listen'", + sr->id.toString().c_str(), n.node->toString().c_str()); - const auto& r = n.listenStatus.find(query); - auto prev_req = r != n.listenStatus.end() ? r->second : nullptr; + const auto& r = n.listenStatus.find(query); + auto prev_req = r != n.listenStatus.end() ? r->second : nullptr; - std::weak_ptr ws = sr; - n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req, - [this,ws,query](const net::Request& req, net::NetworkEngine::RequestAnswer&& answer) mutable - { /* on done */ - if (auto sr = ws.lock()) { - onListenDone(req.node, answer, sr); - scheduler.edit(sr->nextSearchStep, scheduler.time()); - } - }, - [this,ws,query](const net::Request& req, bool over) mutable - { /* on expired */ - if (auto sr = ws.lock()) { - scheduler.edit(sr->nextSearchStep, scheduler.time()); - if (over) - if (auto sn = sr->getNode(req.node)) - sn->listenStatus.erase(query); - } - }, - [this,ws,query](const Sp& node, net::NetworkEngine::RequestAnswer&& answer) mutable - { /* on new values */ - if (auto sr = ws.lock()) { - onGetValuesDone(node, answer, sr, query); - scheduler.edit(sr->nextSearchStep, scheduler.time()); - } + std::weak_ptr ws = sr; + n.listenStatus[query] = network_engine.sendListen(n.node, sr->id, *query, n.token, prev_req, + [this,ws,query](const net::Request& req, net::RequestAnswer&& answer) mutable + { /* on done */ + if (auto sr = ws.lock()) { + scheduler.edit(sr->nextSearchStep, scheduler.time()); + onListenDone(req.node, answer, sr); } - ); - } + }, + [this,ws,query](const net::Request& req, bool over) mutable + { /* on expired */ + if (auto sr = ws.lock()) { + scheduler.edit(sr->nextSearchStep, scheduler.time()); + if (over) + if (auto sn = sr->getNode(req.node)) + sn->listenStatus.erase(query); + } + }, + [this,ws,query](const Sp& node, net::RequestAnswer&& answer) mutable + { /* on new values */ + if (auto sr = ws.lock()) { + scheduler.edit(sr->nextSearchStep, scheduler.time()); + onGetValuesDone(node, answer, sr, query); + } + } + ); } if (not n.candidate and ++i == LISTEN_NODES) break; @@ -737,8 +665,10 @@ Dht::search(const InfoHash& id, sa_family_t af, GetCallback gcb, QueryCallback q break; } } - if (not sr) - throw DhtException("Can't create search"); + if (not sr) { + DHT_LOG.e(id, "[search %s IPv%c] maximum number of searches reached !", id.toString().c_str(), (af == AF_INET) ? '4' : '6'); + return {}; + } } sr->af = af; sr->tid = search_id++; @@ -808,8 +738,9 @@ Dht::announce(const InfoHash& id, n.probe_query.reset(); n.acked[value->id].first.reset(); } - } - else { + } else { + a_sr->permanent = permanent; + a_sr->created = created; if (a_sr->value != value) { a_sr->value = value; for (auto& n : sr->nodes) { @@ -821,9 +752,8 @@ Dht::announce(const InfoHash& id, if (a_sr->callback) a_sr->callback(true, {}); a_sr->callback = {}; - if (callback) { + if (callback) callback(true, {}); - } return; } else { if (a_sr->callback) @@ -857,7 +787,7 @@ Dht::listenTo(const InfoHash& id, sa_family_t af, GetCallback cb, Value::Filter } size_t -Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f, Where&& where) +Dht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where where) { scheduler.syncTime(); @@ -933,39 +863,34 @@ Dht::cancelListen(const InfoHash& id, size_t token) if (st != store.end() && tokenlocal) st->second.local_listeners.erase(tokenlocal); - auto searches_cancel_listen = [&](std::map> srs) { - for (auto& sp : srs) { - auto& s = sp.second; - if (s->id != id) continue; - auto af_token = s->af == AF_INET ? std::get<1>(it->second) : std::get<2>(it->second); - if (af_token == 0) - continue; - Sp query; - const auto& ll = s->listeners.find(af_token); - if (ll != s->listeners.cend()) - query = ll->second.query; - for (auto& sn : s->nodes) { - if (s->listeners.empty()) { /* also erase requests for all searchnodes. */ - for (auto& ls : sn.listenStatus) - network_engine.cancelRequest(ls.second); - sn.listenStatus.clear(); - } else if (query) { - auto it = sn.listenStatus.find(query); - if (it != sn.listenStatus.end()) { - network_engine.cancelRequest(it->second); - sn.listenStatus.erase(it); - } - } - } - s->listeners.erase(af_token); - } + auto searches_cancel_listen = [&id](std::map>& srs, size_t token) { + auto srp = srs.find(id); + if (srp != srs.end() and token) + srp->second->cancelListen(token); }; - searches_cancel_listen(searches4); - searches_cancel_listen(searches6); + searches_cancel_listen(searches4, std::get<1>(it->second)); + searches_cancel_listen(searches6, std::get<2>(it->second)); listeners.erase(it); return true; } +struct OpStatus { + struct Status { + bool done {false}; + bool ok {false}; + Status(bool done=false, bool ok=false) : done(done), ok(ok) {} + }; + Status status; + Status status4; + Status status6; +}; + +template +struct GetStatus : public OpStatus { + std::vector> values; + std::vector> nodes; +}; + void Dht::put(const InfoHash& id, Sp val, DoneCallback callback, time_point created, bool permanent) { @@ -979,54 +904,38 @@ Dht::put(const InfoHash& id, Sp val, DoneCallback callback, time_point cr DHT_LOG.d(id, "put: adding %s -> %s", id.toString().c_str(), val->toString().c_str()); - auto ok = std::make_shared(false); - auto done = std::make_shared(false); - auto done4 = std::make_shared(false); - auto done6 = std::make_shared(false); - auto donecb = [=](const std::vector>& nodes) { + auto op = std::make_shared(); + auto donecb = [callback](const std::vector>& nodes, OpStatus& op) { // Callback as soon as the value is announced on one of the available networks - if (callback && !*done && (*done4 && *done6)) { - callback(*ok, nodes); - *done = true; + if (callback and not op.status.done and (op.status4.done && op.status6.done)) { + callback(op.status4.ok or op.status6.ok, nodes); + op.status.done = true; } }; announce(id, AF_INET, val, [=](bool ok4, const std::vector>& nodes) { DHT_LOG.d(id, "Announce done IPv4 %d", ok4); - *done4 = true; - *ok |= ok4; - donecb(nodes); + auto& o = *op; + o.status4 = {true, ok4}; + donecb(nodes, o); }, created, permanent); announce(id, AF_INET6, val, [=](bool ok6, const std::vector>& nodes) { DHT_LOG.d(id, "Announce done IPv6 %d", ok6); - *done6 = true; - *ok |= ok6; - donecb(nodes); + auto& o = *op; + o.status6 = {true, ok6}; + donecb(nodes, o); }, created, permanent); } template -struct OpStatus { - struct Status { - bool done {false}; - bool ok {false}; - }; - Status status; - Status status4; - Status status6; - std::vector> values; - std::vector> nodes; -}; - -template -void doneCallbackWrapper(DoneCallback dcb, const std::vector>& nodes, Sp> op) { - if (op->status.done) +void doneCallbackWrapper(DoneCallback dcb, const std::vector>& nodes, GetStatus& op) { + if (op.status.done) return; - op->nodes.insert(op->nodes.end(), nodes.begin(), nodes.end()); - if (op->status.ok || (op->status4.done and op->status6.done)) { - bool ok = op->status.ok || op->status4.ok || op->status6.ok; - op->status.done = true; + op.nodes.insert(op.nodes.end(), nodes.begin(), nodes.end()); + if (op.status.ok or (op.status4.done and op.status6.done)) { + bool ok = op.status.ok or op.status4.ok or op.status6.ok; + op.status.done = true; if (dcb) - dcb(ok, op->nodes); + dcb(ok, op.nodes); } } @@ -1035,17 +944,18 @@ bool callbackWrapper(Cb get_cb, DoneCallback done_cb, const std::vector>& values, std::function>(const std::vector>&)> add_values, - Sp> op) + Sp> o) { - if (op->status.done) + auto& op = *o; + if (op.status.done) return false; auto newvals = add_values(values); if (not newvals.empty()) { - op->status.ok = !get_cb(newvals); - op->values.insert(op->values.end(), newvals.begin(), newvals.end()); + op.status.ok = !get_cb(newvals); + op.values.insert(op.values.end(), newvals.begin(), newvals.end()); } doneCallbackWrapper(done_cb, {}, op); - return !op->status.ok; + return !op.status.ok; } void @@ -1054,7 +964,7 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt scheduler.syncTime(); Query q {{}, where}; - auto op = std::make_shared>(); + auto op = std::make_shared>(); auto f = filter.chain(q.where.getFilter()); auto add_values = [op,f](const std::vector>& values) { @@ -1077,22 +987,20 @@ Dht::get(const InfoHash& id, GetCallback getcb, DoneCallback donecb, Value::Filt Dht::search(id, AF_INET, gcb, {}, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv4"); - op->status4.done = true; - op->status4.ok = ok; - doneCallbackWrapper(donecb, nodes, op); + op->status4 = {true, ok}; + doneCallbackWrapper(donecb, nodes, *op); }, f, q); Dht::search(id, AF_INET6, gcb, {}, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv6"); - op->status6.done = true; - op->status6.ok = ok; - doneCallbackWrapper(donecb, nodes, op); + op->status6 = {true, ok}; + doneCallbackWrapper(donecb, nodes, *op); }, f, q); } void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Query&& q) { scheduler.syncTime(); - auto op = std::make_shared>(); + auto op = std::make_shared>(); auto f = q.where.getFilter(); auto values = getLocal(id, f); @@ -1126,15 +1034,13 @@ void Dht::query(const InfoHash& id, QueryCallback cb, DoneCallback done_cb, Quer Dht::search(id, AF_INET, {}, qcb, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv4"); - op->status4.done = true; - op->status4.ok = ok; - doneCallbackWrapper(done_cb, nodes, op); + op->status4 = {true, ok}; + doneCallbackWrapper(done_cb, nodes, *op); }, f, q); Dht::search(id, AF_INET6, {}, qcb, [=](bool ok, const std::vector>& nodes) { //DHT_LOG_WARN("DHT done IPv6"); - op->status6.done = true; - op->status6.ok = ok; - doneCallbackWrapper(done_cb, nodes, op); + op->status6 = {true, ok}; + doneCallbackWrapper(done_cb, nodes, *op); }, f, q); } @@ -1187,10 +1093,10 @@ Dht::getPut(const InfoHash& id, const Value::Id& vid) } return Sp {}; }; - auto v4 = find_value(searches4); - if (v4) return v4; - auto v6 = find_value(searches6); - if (v6) return v6; + if (auto v4 = find_value(searches4)) + return v4; + if (auto v6 = find_value(searches6)) + return v6; return {}; } @@ -1255,7 +1161,7 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v) node_listeners.first->toString().c_str()); std::vector> vals {}; vals.push_back(v.data); - Blob ntoken = makeToken(node_listeners.first->addr, false); + Blob ntoken = makeToken(node_listeners.first->getAddr(), false); network_engine.tellListener(node_listeners.first, l.second.sid, id, 0, ntoken, {}, {}, std::move(vals), l.second.query); } @@ -1263,13 +1169,12 @@ Dht::storageChanged(const InfoHash& id, Storage& st, ValueStorage& v) } bool -Dht::storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr* sa) +Dht::storageStore(const InfoHash& id, const Sp& value, time_point created, const SockAddr& sa) { const auto& now = scheduler.time(); created = std::min(created, now); - auto expiration = created + getType(value->id).expiration; - - if ( expiration < now ) + auto expiration = created + getType(value->type).expiration; + if (expiration < now) return false; auto st = store.find(id); @@ -1284,7 +1189,7 @@ Dht::storageStore(const InfoHash& id, const Sp& value, time_point created StorageBucket* store_bucket {nullptr}; if (sa) - store_bucket = &store_quota.emplace(*sa, StorageBucket{}).first->second; + store_bucket = &store_quota.emplace(sa, StorageBucket{}).first->second; auto store = st->second.store(id, value, created, expiration, store_bucket); if (auto vs = store.first) { @@ -1315,7 +1220,7 @@ Dht::storageAddListener(const InfoHash& id, const Sp& node, size_t socket_ if (l == node_listeners->second.end()) { auto vals = st->second.get(query.where.getFilter()); if (not vals.empty()) { - network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->addr, false), + network_engine.tellListener(node, socket_id, id, WANT4 | WANT6, makeToken(node->getAddr(), false), buckets4.findClosestNodes(id, now, TARGET_NODES), buckets6.findClosestNodes(id, now, TARGET_NODES), std::move(vals), query); } @@ -1348,8 +1253,7 @@ void Dht::expireStore() { // removing expired values - auto i = store.begin(); - while (i != store.end()) { + for (auto i = store.begin(); i != store.end();) { expireStore(i); if (i->second.empty() && i->second.listeners.empty() && i->second.local_listeners.empty()) { @@ -1383,10 +1287,17 @@ Dht::expireStore() DHT_LOG.w("Discarded %ld bytes, still %ld used", largest->first.toString().c_str(), total_store_size); if (ret.values_diff) break; - } else - std::cout << "exp_value not found " << exp_value.first << std::endl; + } } } + + // remove unused quota entires + for (auto i = store_quota.begin(); i != store_quota.end();) { + if (i->second.size() == 0) + i = store_quota.erase(i); + else + ++i; + } } void @@ -1394,15 +1305,12 @@ Dht::connectivityChanged(sa_family_t af) { const auto& now = scheduler.time(); scheduler.edit(nextNodesConfirmation, now); - auto& bucket_grow_time = (af == AF_INET) ? mybucket_grow_time : mybucket6_grow_time; - bucket_grow_time = now; - for (auto& b : buckets(af)) - b.time = time_point::min(); + buckets(af).connectivityChanged(now); network_engine.connectivityChanged(af); for (auto& sp : searches(af)) for (auto& sn : sp.second->nodes) { for (auto& ls : sn.listenStatus) - network_engine.cancelRequest(ls.second); + sn.node->cancelRequest(ls.second); sn.listenStatus.clear(); } reported_addr.erase(std::remove_if(reported_addr.begin(), reported_addr.end(), [&](const ReportedAddr& addr){ @@ -1457,7 +1365,7 @@ Dht::makeToken(const SockAddr& addr, bool old) const bool Dht::tokenMatch(const Blob& token, const SockAddr& addr) const { - if (!addr.second || token.size() != TOKEN_SIZE) + if (not addr or token.size() != TOKEN_SIZE) return false; if (token == makeToken(addr, false)) return true; @@ -1476,7 +1384,7 @@ Dht::getNodesStats(sa_family_t af) const for (auto& n : b.nodes) { if (n->isGood(now)) { stats.good_nodes++; - if (n->time > n->reply_time) + if (n->isIncoming()) stats.incoming_nodes++; } else if (not n->isExpired()) stats.dubious_nodes++; @@ -1499,10 +1407,12 @@ Dht::dumpBucket(const Bucket& b, std::ostream& out) const out << std::endl; for (auto& n : b.nodes) { out << " Node " << n->toString(); - if (n->time != n->reply_time) - out << " age " << duration_cast(now - n->time).count() << ", reply: " << duration_cast(now - n->reply_time).count(); + const auto& t = n->getTime(); + const auto& r = n->getReplyTime(); + if (t != r) + out << " age " << duration_cast(now - t).count() << ", reply: " << duration_cast(now - r).count(); else - out << " age " << duration_cast(now - n->time).count(); + out << " age " << duration_cast(now - t).count(); if (n->isExpired()) out << " [expired]"; else if (n->isGood(now)) @@ -1597,7 +1507,7 @@ Dht::dumpSearch(const Search& sr, std::ostream& out) const out << "] "; } } - out << n.node->addr.toString() << std::endl; + out << n.node->getAddrStr() << std::endl; } } @@ -1627,12 +1537,6 @@ Dht::dumpTables() const DHT_LOG.d("%s", out.str().c_str()); } -void -Dht::printStorageQuota(std::ostream& out, const decltype(store_quota)::value_type& ip) const -{ - out << "IP " << ip.first.toString() << " uses " << ip.second.size() << " bytes" << std::endl; -} - std::string Dht::getStorageLog() const { @@ -1645,9 +1549,12 @@ Dht::getStorageLog() const else out << (total_store_size/1024) << " KB)"; out << std::endl << std::endl; - for (const auto& ip : store_quota) { - printStorageQuota(out, ip); - } + std::multimap q_map; + for (const auto& ip : store_quota) + if (ip.second.size()) + q_map.emplace(ip.second.size(), &ip.first); + for (auto ip = q_map.rbegin(); ip != q_map.rend(); ++ip) + out << "IP " << ip->second->toString() << " uses " << ip->first << " bytes" << std::endl; out << std::endl; return out.str(); } @@ -1676,15 +1583,9 @@ Dht::printStorageLog(const decltype(store)::value_type& s) const << st.totalSize() << " bytes)" << std::endl; if (not st.local_listeners.empty()) out << " " << st.local_listeners.size() << " local listeners" << std::endl; - const auto& now = scheduler.time(); for (const auto& node_listeners : st.listeners) { const auto& node = node_listeners.first; - for (const auto& l : node_listeners.second) { - out << " " << "Listener " << node->toString(); - auto since = duration_cast(now - l.second.time); - auto expires = duration_cast(l.second.time + Node::NODE_EXPIRE_TIME - now); - out << " (since " << since.count() << "s, exp in " << expires.count() << "s)" << std::endl; - } + out << " " << "Listener " << node->toString() << " : " << node_listeners.second.size() << " entries" << std::endl; } return out.str(); } @@ -1748,8 +1649,8 @@ Dht::~Dht() Dht::Dht() : store(), scheduler(DHT_LOG), network_engine(DHT_LOG, scheduler) {} -Dht::Dht(int s, int s6, Config config) - : myid(config.node_id != zeroes ? config.node_id : InfoHash::getRandom()), +Dht::Dht(const int& s, const int& s6, Config config) + : myid(config.node_id ? config.node_id : InfoHash::getRandom()), is_bootstrap(config.is_bootstrap), maintain_storage(config.maintain_storage), store(), store_quota(), scheduler(DHT_LOG), @@ -1767,10 +1668,14 @@ Dht::Dht(int s, int s6, Config config) scheduler.syncTime(); if (s < 0 && s6 < 0) return; - if (s >= 0) + if (s >= 0) { buckets4 = {Bucket {AF_INET}}; - if (s6 >= 0) + buckets4.is_client = config.is_bootstrap; + } + if (s6 >= 0) { buckets6 = {Bucket {AF_INET6}}; + buckets6.is_client = config.is_bootstrap; + } search_id = std::uniform_int_distribution{}(rd); @@ -1952,32 +1857,17 @@ Dht::maintainStorage(decltype(store)::value_type& storage, bool force, DoneCallb return announce_per_af; } -void -Dht::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& from) -{ - if (buflen == 0) - return; - - try { - network_engine.processMessage(buf, buflen, from); - } catch (const std::exception& e) { - DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what()); - //auto code = e.getCode(); - //if (code == net::DhtProtocolException::INVALID_TID_SIZE or code == net::DhtProtocolException::WRONG_NODE_INFO_BUF_LEN) { - /* This is really annoying, as it means that we will - time-out all our searches that go through this node. - Kill it. */ - //const auto& id = e.getNodeId(); - //blacklistNode(&id, from, fromlen); - ///} - } -} - time_point Dht::periodic(const uint8_t *buf, size_t buflen, const SockAddr& from) { scheduler.syncTime(); - processMessage(buf, buflen, from); + if (buflen) { + try { + network_engine.processMessage(buf, buflen, from); + } catch (const std::exception& e) { + DHT_LOG.e("Can't parse message from %s: %s", from.toString().c_str(), e.what()); + } + } return scheduler.run(); } @@ -2014,9 +1904,9 @@ Dht::confirmNodes() soon |= bucketMaintenance(buckets6); if (!soon) { - if (mybucket_grow_time >= now - seconds(150)) + if (buckets4.grow_time >= now - seconds(150)) soon |= neighbourhoodMaintenance(buckets4); - if (mybucket6_grow_time >= now - seconds(150)) + if (buckets6.grow_time >= now - seconds(150)) soon |= neighbourhoodMaintenance(buckets6); } @@ -2072,7 +1962,7 @@ Dht::importValues(const std::vector& import) throw msgpack::type_error(); for (unsigned i = 0; i < valarr.via.array.size; i++) { auto& valel = valarr.via.array.ptr[i]; - if (valel.via.array.size < 2) + if (valel.type != msgpack::type::ARRAY or valel.via.array.size < 2) throw msgpack::type_error(); time_point val_time; Value tmp_val; @@ -2142,7 +2032,7 @@ Dht::pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb) DHT_LOG.d("Sending ping to %s", print_addr(sa, salen).c_str()); auto& count = sa->sa_family == AF_INET ? pending_pings4 : pending_pings6; count++; - network_engine.sendPing(sa, salen, [&count,cb](const net::Request&, net::NetworkEngine::RequestAnswer&&) { + network_engine.sendPing(sa, salen, [&count,cb](const net::Request&, net::RequestAnswer&&) { count--; if (cb) cb(true); @@ -2157,14 +2047,15 @@ Dht::pingNode(const sockaddr* sa, socklen_t salen, DoneCallbackSimple&& cb) void Dht::onError(Sp req, net::DhtProtocolException e) { + const auto& node = req->node; if (e.getCode() == net::DhtProtocolException::UNAUTHORIZED) { - DHT_LOG.e(req->node->id, "[node %s] token flush", req->node->toString().c_str()); - req->node->authError(); - network_engine.cancelRequest(req); - for (auto& srp : searches(req->node->getFamily())) { + DHT_LOG.e(node->id, "[node %s] token flush", node->toString().c_str()); + node->authError(); + node->cancelRequest(req); + for (auto& srp : searches(node->getFamily())) { auto& sr = srp.second; for (auto& n : sr->nodes) { - if (n.node != req->node) continue; + if (n.node != node) continue; n.token.clear(); n.last_get_reply = time_point::min(); searchSendGetValues(sr); @@ -2172,32 +2063,30 @@ Dht::onError(Sp req, net::DhtProtocolException e) { } } } else if (e.getCode() == net::DhtProtocolException::NOT_FOUND) { - DHT_LOG.e(req->node->id, "[node %s] returned error 404: storage not found", req->node->toString().c_str()); - network_engine.cancelRequest(req); + DHT_LOG.e(node->id, "[node %s] returned error 404: storage not found", node->toString().c_str()); + node->cancelRequest(req); } } void -Dht::onReportedAddr(const InfoHash& id, const SockAddr& addr) +Dht::onReportedAddr(const InfoHash& /*id*/, const SockAddr& addr) { - const auto& b = buckets(addr.getFamily()).findBucket(id); - b->time = scheduler.time(); - if (addr.second) + if (addr) reportedAddr(addr); } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onPing(Sp) { return {}; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onFindNode(Sp node, const InfoHash& target, want_t want) { const auto& now = scheduler.time(); - net::NetworkEngine::RequestAnswer answer; - answer.ntoken = makeToken(node->addr, false); + net::RequestAnswer answer; + answer.ntoken = makeToken(node->getAddr(), false); if (want & WANT4) answer.nodes4 = buckets4.findClosestNodes(target, now, TARGET_NODES); if (want & WANT6) @@ -2205,10 +2094,10 @@ Dht::onFindNode(Sp node, const InfoHash& target, want_t want) return answer; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query) { - if (hash == zeroes) { + if (not hash) { DHT_LOG.w("[node %s] Eek! Got get_values with no info_hash", node->toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, @@ -2216,9 +2105,9 @@ Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query }; } const auto& now = scheduler.time(); - net::NetworkEngine::RequestAnswer answer {}; + net::RequestAnswer answer {}; auto st = store.find(hash); - answer.ntoken = makeToken(node->addr, false); + answer.ntoken = makeToken(node->getAddr(), false); answer.nodes4 = buckets4.findClosestNodes(hash, now, TARGET_NODES); answer.nodes6 = buckets6.findClosestNodes(hash, now, TARGET_NODES); if (st != store.end() && not st->second.empty()) { @@ -2231,7 +2120,7 @@ Dht::onGetValues(Sp node, const InfoHash& hash, want_t, const Query& query } void Dht::onGetValuesDone(const Sp& node, - net::NetworkEngine::RequestAnswer& a, + net::RequestAnswer& a, Sp& sr, const Sp& orig_query) { @@ -2241,7 +2130,7 @@ void Dht::onGetValuesDone(const Sp& node, } DHT_LOG.d(sr->id, "[search %s] [node %s] got reply to 'get' with %u nodes", - sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size()); + sr->id.toString().c_str(), node->toString().c_str(), a.nodes4.size()+a.nodes6.size()); if (not a.ntoken.empty()) { if (not a.values.empty() or not a.fields.empty()) { @@ -2302,17 +2191,17 @@ void Dht::onGetValuesDone(const Sp& node, } } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t socket_id, const Query& query) { - if (hash == zeroes) { + if (not hash) { DHT_LOG.w(node->id, "[node %s] listen with no info_hash", node->toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, net::DhtProtocolException::LISTEN_NO_INFOHASH }; } - if (!tokenMatch(token, node->addr)) { + if (not tokenMatch(token, node->getAddr())) { DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'listen'", node->toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::LISTEN_WRONG_TOKEN}; } @@ -2323,7 +2212,7 @@ Dht::onListen(Sp node, const InfoHash& hash, const Blob& token, size_t soc void Dht::onListenDone(const Sp& node, - net::NetworkEngine::RequestAnswer& answer, + net::RequestAnswer& answer, Sp& sr) { DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got listen confirmation", @@ -2336,30 +2225,31 @@ Dht::onListenDone(const Sp& node, } } -net::NetworkEngine::RequestAnswer -Dht::onAnnounce(Sp node, +net::RequestAnswer +Dht::onAnnounce(Sp n, const InfoHash& hash, const Blob& token, const std::vector>& values, const time_point& creation_date) { - if (hash == zeroes) { - DHT_LOG.w(node->id, "put with no info_hash"); + auto& node = *n; + if (not hash) { + DHT_LOG.w(node.id, "put with no info_hash"); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, net::DhtProtocolException::PUT_NO_INFOHASH }; } - if (!tokenMatch(token, node->addr)) { - DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str()); + if (!tokenMatch(token, node.getAddr())) { + DHT_LOG.w(hash, node.id, "[node %s] incorrect token %s for 'put'", node.toString().c_str(), hash.toString().c_str()); throw net::DhtProtocolException {net::DhtProtocolException::UNAUTHORIZED, net::DhtProtocolException::PUT_WRONG_TOKEN}; } { // We store a value only if we think we're part of the // SEARCH_NODES nodes around the target id. - auto closest_nodes = buckets(node->getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES); + auto closest_nodes = buckets(node.getFamily()).findClosestNodes(hash, scheduler.time(), SEARCH_NODES); if (closest_nodes.size() >= TARGET_NODES and hash.xorCmp(closest_nodes.back()->id, myid) < 0) { - DHT_LOG.w(hash, node->id, "[node %s] announce too far from the target. Dropping value.", node->toString().c_str()); + DHT_LOG.w(hash, node.id, "[node %s] announce too far from the target. Dropping value.", node.toString().c_str()); return {}; } } @@ -2367,7 +2257,7 @@ Dht::onAnnounce(Sp node, auto created = std::min(creation_date, scheduler.time()); for (const auto& v : values) { if (v->id == Value::INVALID_ID) { - DHT_LOG.w(hash, node->id, "[value %s] incorrect value id", hash.toString().c_str()); + DHT_LOG.w(hash, node.id, "[value %s] incorrect value id", hash.toString().c_str()); throw net::DhtProtocolException { net::DhtProtocolException::NON_AUTHORITATIVE_INFORMATION, net::DhtProtocolException::PUT_INVALID_ID @@ -2377,26 +2267,26 @@ Dht::onAnnounce(Sp node, Sp vc = v; if (lv) { if (*lv == *vc) { - DHT_LOG.w(hash, node->id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); + DHT_LOG.d(hash, node.id, "[store %s] nothing to do for %s", hash.toString().c_str(), lv->toString().c_str()); } else { const auto& type = getType(lv->type); - if (type.editPolicy(hash, lv, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) { - DHT_LOG.d(hash, node->id, "[store %s] editing %s", + if (type.editPolicy(hash, lv, vc, node.id, node.getAddr())) { + DHT_LOG.d(hash, node.id, "[store %s] editing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node->addr); + storageStore(hash, vc, created, node.getAddr()); } else { - DHT_LOG.d(hash, node->id, "[store %s] rejecting edition of %s because of storage policy", + DHT_LOG.d(hash, node.id, "[store %s] rejecting edition of %s because of storage policy", hash.toString().c_str(), vc->toString().c_str()); } } } else { // Allow the value to be edited by the storage policy const auto& type = getType(vc->type); - if (type.storePolicy(hash, vc, node->id, (sockaddr*)&node->addr.first, node->addr.second)) { - DHT_LOG.d(hash, node->id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); - storageStore(hash, vc, created, &node->addr); + if (type.storePolicy(hash, vc, node.id, node.getAddr())) { + DHT_LOG.d(hash, node.id, "[store %s] storing %s", hash.toString().c_str(), vc->toString().c_str()); + storageStore(hash, vc, created, node.getAddr()); } else { - DHT_LOG.d(hash, node->id, "[store %s] rejecting storage of %s", + DHT_LOG.d(hash, node.id, "[store %s] rejecting storage of %s", hash.toString().c_str(), vc->toString().c_str()); } } @@ -2404,13 +2294,13 @@ Dht::onAnnounce(Sp node, return {}; } -net::NetworkEngine::RequestAnswer +net::RequestAnswer Dht::onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Value::Id& vid) { using namespace net; const auto& now = scheduler.time(); - if (not tokenMatch(token, node->addr)) { + if (not tokenMatch(token, node->getAddr())) { DHT_LOG.w(hash, node->id, "[node %s] incorrect token %s for 'put'", node->toString().c_str(), hash.toString().c_str()); throw DhtProtocolException {DhtProtocolException::UNAUTHORIZED, DhtProtocolException::PUT_WRONG_TOKEN}; } @@ -2427,7 +2317,7 @@ Dht::onRefresh(Sp node, const InfoHash& hash, const Blob& token, const Val } void -Dht::onAnnounceDone(const Sp& node, net::NetworkEngine::RequestAnswer& answer, Sp& sr) +Dht::onAnnounceDone(const Sp& node, net::RequestAnswer& answer, Sp& sr) { DHT_LOG.d(sr->id, node->id, "[search %s] [node %s] got reply to put!", sr->id.toString().c_str(), node->toString().c_str()); diff --git a/src/dht_proxy_client.cpp b/src/dht_proxy_client.cpp new file mode 100644 index 00000000..8794c985 --- /dev/null +++ b/src/dht_proxy_client.cpp @@ -0,0 +1,773 @@ +/* + * Copyright (C) 2016-2018 Savoir-faire Linux Inc. + * Author: Sébastien Blin + * Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if OPENDHT_PROXY_CLIENT + +#include "dht_proxy_client.h" + +#include +#include +#include +#include + +#include "dhtrunner.h" + +constexpr const char* const HTTP_PROTO {"http://"}; + +namespace dht { + +DhtProxyClient::DhtProxyClient(std::function signal, const std::string& serverHost, const std::string& pushClientId) +: serverHost_(serverHost), pushClientId_(pushClientId), scheduler(DHT_LOG), loopSignal_(signal) +{ + if (!serverHost_.empty()) + startProxy(); +} + +void +DhtProxyClient::confirmProxy() +{ + if (serverHost_.empty()) return; + getConnectivityStatus(); +} + +void +DhtProxyClient::startProxy() +{ + if (serverHost_.empty()) return; + DHT_LOG.WARN("Staring proxy client to %s", serverHost_.c_str()); + nextProxyConfirmation = scheduler.add(scheduler.time(), std::bind(&DhtProxyClient::confirmProxy, this)); +} + +DhtProxyClient::~DhtProxyClient() +{ + cancelAllOperations(); + cancelAllListeners(); +} + +void +DhtProxyClient::cancelAllOperations() +{ + std::lock_guard lock(lockOperations_); + auto operation = operations_.begin(); + while (operation != operations_.end()) { + if (operation->thread.joinable()) { + // Close connection to stop operation? + restbed::Http::close(operation->req); + operation->thread.join(); + operation = operations_.erase(operation); + } else { + ++operation; + } + } +} + +void +DhtProxyClient::cancelAllListeners() +{ + std::lock_guard lock(lockListener_); + for (auto& listener: listeners_) { + if (listener.thread.joinable()) { + // Close connection to stop listener? + if (listener.req) + restbed::Http::close(listener.req); + listener.thread.join(); + } + } +} + +void +DhtProxyClient::shutdown(ShutdownCallback cb) +{ + cancelAllOperations(); + cancelAllListeners(); + if (cb) + cb(); +} + +NodeStatus +DhtProxyClient::getStatus(sa_family_t af) const +{ + std::lock_guard l(lockCurrentProxyInfos_); + switch (af) + { + case AF_INET: + return statusIpv4_; + case AF_INET6: + return statusIpv6_; + default: + return NodeStatus::Disconnected; + } +} + +bool +DhtProxyClient::isRunning(sa_family_t af) const +{ + std::lock_guard l(lockCurrentProxyInfos_); + switch (af) + { + case AF_INET: + return statusIpv4_ != NodeStatus::Disconnected; + case AF_INET6: + return statusIpv6_ != NodeStatus::Disconnected; + default: + return false; + } +} + +time_point +DhtProxyClient::periodic(const uint8_t*, size_t, const SockAddr&) +{ + // Exec all currently stored callbacks + scheduler.syncTime(); + if (!callbacks_.empty()) { + std::lock_guard lock(lockCallbacks); + for (auto& callback : callbacks_) + callback(); + callbacks_.clear(); + } + // Remove finished operations + { + std::lock_guard lock(lockOperations_); + auto operation = operations_.begin(); + while (operation != operations_.end()) { + if (*(operation->finished)) { + if (operation->thread.joinable()) { + // Close connection to stop operation? + restbed::Http::close(operation->req); + operation->thread.join(); + } + operation = operations_.erase(operation); + } else { + ++operation; + } + } + } + return scheduler.run(); +} + +void +DhtProxyClient::get(const InfoHash& key, const GetCallback& cb, DoneCallback donecb, const Value::Filter& filterChain) +{ + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + key.toString()); + auto req = std::make_shared(uri); + + auto finished = std::make_shared(false); + Operation o; + o.req = req; + o.finished = finished; + o.thread = std::thread([=](){ + // Try to contact the proxy and set the status to connected when done. + // will change the connectivity status + auto ok = std::make_shared(true); + restbed::Http::async(req, + [=](const std::shared_ptr& req, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + + if (code == 200) { + try { + while (restbed::Http::is_open(req) and not *finished) { + restbed::Http::fetch("\n", reply); + if (*finished) + break; + std::string body; + reply->get_body(body); + reply->set_body(""); // Reset the body for the next fetch + + std::string err; + Json::Value json; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&body[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (reader->parse(char_data, char_data + body.size(), &json, &err)) { + auto value = std::make_shared(json); + if ((not filterChain or filterChain(*value)) && cb) { + std::lock_guard lock(lockCallbacks); + callbacks_.emplace_back([cb, value, finished]() { + if (not cb({value})) + *finished = true; + }); + loopSignal_(); + } + } else { + *ok = false; + } + } + } catch (std::runtime_error& e) { } + } else { + *ok = false; + } + }).wait(); + if (donecb) { + std::lock_guard lock(lockCallbacks); + callbacks_.emplace_back([=](){ + donecb(*ok, {}); + }); + loopSignal_(); + } + if (!ok) { + // Connection failed, update connectivity + opFailed(); + } + *finished = true; + }); + { + std::lock_guard lock(lockOperations_); + operations_.emplace_back(std::move(o)); + } +} + +void +DhtProxyClient::get(const InfoHash& key, GetCallback cb, DoneCallback donecb, + Value::Filter&& filter, Where&& where) +{ + Query query {{}, where}; + auto filterChain = filter.chain(query.where.getFilter()); + get(key, cb, donecb, filterChain); +} + +void +DhtProxyClient::put(const InfoHash& key, Sp val, DoneCallback cb, time_point, bool permanent) +{ + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + key.toString()); + auto req = std::make_shared(uri); + req->set_method("POST"); + auto json = val->toJson(); + if (permanent) + json["permanent"] = true; + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto body = Json::writeString(wbuilder, json) + "\n"; + req->set_body(body); + req->set_header("Content-Length", std::to_string(body.size())); + + auto finished = std::make_shared(false); + Operation o; + o.req = req; + o.finished = finished; + o.thread = std::thread([=](){ + auto ok = std::make_shared(true); + restbed::Http::async(req, + [this, ok](const std::shared_ptr& /*req*/, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + + if (code == 200) { + restbed::Http::fetch("\n", reply); + std::string body; + reply->get_body(body); + reply->set_body(""); // Reset the body for the next fetch + + try { + std::string err; + Json::Value json; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&body[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (not reader->parse(char_data, char_data + body.size(), &json, &err)) + *ok = false; + } catch (...) { + *ok = false; + } + } else { + *ok = false; + } + }).wait(); + if (cb) { + std::lock_guard lock(lockCallbacks); + callbacks_.emplace_back([=](){ + cb(*ok, {}); + }); + loopSignal_(); + } + if (!ok) { + // Connection failed, update connectivity + opFailed(); + } + *finished = true; + }); + { + std::lock_guard lock(lockOperations_); + operations_.emplace_back(std::move(o)); + } +} + +NodeStats +DhtProxyClient::getNodesStats(sa_family_t af) const +{ + return af == AF_INET ? stats4_ : stats6_; +} + +void +DhtProxyClient::getProxyInfos() +{ + DHT_LOG.DEBUG("Requesting proxy server node information"); + + if (ongoingStatusUpdate_.test_and_set()) + return; + + { + std::lock_guard l(lockCurrentProxyInfos_); + if (statusIpv4_ == NodeStatus::Disconnected) + statusIpv4_ = NodeStatus::Connecting; + if (statusIpv6_ == NodeStatus::Disconnected) + statusIpv6_ = NodeStatus::Connecting; + } + + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/"); + auto req = std::make_shared(uri); + + // Try to contact the proxy and set the status to connected when done. + // will change the connectivity status + statusThread_ = std::thread([this, req]{ + restbed::Http::async(req, + [this](const std::shared_ptr&, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + Json::Value proxyInfos; + if (code == 200) { + restbed::Http::fetch("\n", reply); + std::string body; + reply->get_body(body); + + std::string err; + Json::CharReaderBuilder rbuilder; + auto reader = std::unique_ptr(rbuilder.newCharReader()); + try { + reader->parse(body.data(), body.data() + body.size(), &proxyInfos, &err); + } catch (...) { + } + } + onProxyInfos(proxyInfos); + ongoingStatusUpdate_.clear(); + }); + }); + statusThread_.detach(); +} + +void +DhtProxyClient::onProxyInfos(const Json::Value& proxyInfos) +{ + std::lock_guard l(lockCurrentProxyInfos_); + + auto oldStatus = std::max(statusIpv4_, statusIpv6_); + + try { + myid = InfoHash(proxyInfos["node_id"].asString()); + + stats4_ = NodeStats(proxyInfos["ipv4"]); + if (stats4_.good_nodes) + statusIpv4_ = NodeStatus::Connected; + else if (stats4_.dubious_nodes) + statusIpv4_ = NodeStatus::Connecting; + else + statusIpv4_ = NodeStatus::Disconnected; + + stats6_ = NodeStats(proxyInfos["ipv6"]); + if (stats6_.good_nodes) + statusIpv6_ = NodeStatus::Connected; + else if (stats6_.dubious_nodes) + statusIpv6_ = NodeStatus::Connecting; + else + statusIpv6_ = NodeStatus::Disconnected; + + publicAddress_ = parsePublicAddress(proxyInfos["public_ip"]); + } catch (...) {} + + auto newStatus = std::max(statusIpv4_, statusIpv6_); + + if (newStatus == NodeStatus::Connecting || newStatus == NodeStatus::Connected) { + if (oldStatus == NodeStatus::Disconnected) { + restartListeners(); + } + scheduler.edit(nextProxyConfirmation, scheduler.time() + std::chrono::minutes(15)); + } + else if (newStatus == NodeStatus::Disconnected) { + scheduler.edit(nextProxyConfirmation, scheduler.time() + std::chrono::minutes(1)); + } + loopSignal_(); +} + +SockAddr +DhtProxyClient::parsePublicAddress(const Json::Value& val) +{ + auto public_ip = val.asString(); + auto endIp = public_ip.find_last_of(':'); + std::string service = public_ip.substr(endIp + 1); + std::string address = public_ip.substr(0, endIp - 1); + auto sa = SockAddr::resolve(address, service); + if (sa.empty()) return {}; + return sa.front().getMappedIPv4(); +} + +std::vector +DhtProxyClient::getPublicAddress(sa_family_t family) +{ + std::lock_guard l(lockCurrentProxyInfos_); + if (not publicAddress_) return {}; + return publicAddress_.getFamily() == family ? std::vector{publicAddress_} : std::vector{}; +} + +size_t +DhtProxyClient::listen(const InfoHash& key, GetCallback cb, Value::Filter filter, Where where) +{ + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + key.toString()); + auto req = std::make_shared(uri); + req->set_method(deviceKey_.empty() ? "LISTEN" : "SUBSCRIBE"); + + Query query {{}, where}; + auto filterChain = filter.chain(query.where.getFilter()); + auto pushNotifToken = std::make_shared(0); + + Listener l; + ++listener_token_; + l.key = key.toString(); + l.token = listener_token_; + l.req = req; + l.cb = cb; + l.pushNotifToken = pushNotifToken; + l.filterChain = std::move(filterChain); + l.thread = std::thread([=]() + { + auto settings = std::make_shared(); + if (deviceKey_.empty()) { + std::chrono::milliseconds timeout(std::numeric_limits::max()); + settings->set_connection_timeout(timeout); // Avoid the client to close the socket after 5 seconds. + } +#if OPENDHT_PUSH_NOTIFICATIONS + else + fillBodyToGetToken(req); +#endif + + struct State { + std::atomic_bool ok {true}; + std::atomic_bool cancel {false}; + }; + auto state = std::make_shared(); + restbed::Http::async(req, + [this, filterChain, cb, pushNotifToken, state](const std::shared_ptr& req, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + if (code == 200) { + try { + std::string err; + Json::Value json; + Json::CharReaderBuilder rbuilder; + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (!deviceKey_.empty()) { + restbed::Http::fetch("\n", reply); + if (state->cancel) + return; + std::string body; + reply->get_body(body); + + auto* char_data = reinterpret_cast(&body[0]); + if (reader->parse(char_data, char_data + body.size(), &json, &err)) { + if (!json.isMember("token")) return; + *pushNotifToken = json["token"].asLargestUInt(); + } else { + state->ok = false; + } + } else { + while (restbed::Http::is_open(req) and not state->cancel) { + restbed::Http::fetch("\n", reply); + if (state->cancel) + break; + std::string body; + reply->get_body(body); + reply->set_body(""); // Reset the body for the next fetch + + auto* char_data = reinterpret_cast(&body[0]); + if (reader->parse(char_data, char_data + body.size(), &json, &err)) { + auto value = std::make_shared(json); + if ((not filterChain or filterChain(*value)) && cb) { + std::lock_guard lock(lockCallbacks); + callbacks_.emplace_back([cb, value, state]() { + if (not state->cancel and not cb({value})) { + state->cancel = true; + } + }); + loopSignal_(); + } + } else { + state->ok = false; + } + } + } + } catch (std::runtime_error&) { + state->ok = false; + } + } else { + state->ok = false; + } + }, settings).get(); + if (not state->ok) { + opFailed(); + } + } + ); + { + std::lock_guard lock(lockListener_); + listeners_.emplace_back(std::move(l)); + } + return listener_token_; +} + +bool +DhtProxyClient::cancelListen(const InfoHash&, size_t token) +{ + std::lock_guard lock(lockListener_); + for (auto it = listeners_.begin(); it != listeners_.end(); ++it) { + auto& listener = *it; + if (listener.token == token) { + if (!deviceKey_.empty()) { + // First, be sure to have a token + if (listener.thread.joinable()) { + listener.thread.join(); + } + // UNSUBSCRIBE + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + listener.key); + auto req = std::make_shared(uri); + req->set_method("UNSUBSCRIBE"); + restbed::Http::async(req, + [](const std::shared_ptr&, + const std::shared_ptr&){} + ); + // And remove + listeners_.erase(it); + return true; + } else { + // Just stop the request + if (listener.thread.joinable()) { + // Close connection to stop listener? + if (listener.req) + restbed::Http::close(listener.req); + listener.thread.join(); + listeners_.erase(it); + return true; + } + } + } + } + return false; +} + +void +DhtProxyClient::opFailed() +{ + DHT_LOG.ERR("Proxy request failed"); + { + std::lock_guard l(lockCurrentProxyInfos_); + statusIpv4_ = NodeStatus::Disconnected; + statusIpv6_ = NodeStatus::Disconnected; + } + getConnectivityStatus(); + loopSignal_(); +} + +void +DhtProxyClient::getConnectivityStatus() +{ + getProxyInfos(); +} + +void +DhtProxyClient::restartListeners() +{ + std::lock_guard lock(lockListener_); + for (auto& listener: listeners_) { + if (listener.thread.joinable()) + listener.thread.join(); + // Redo listen + auto filterChain = listener.filterChain; + auto cb = listener.cb; + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + listener.key); + auto req = std::make_shared(uri); + req->set_method("LISTEN"); + listener.req = req; + listener.thread = std::thread([this, filterChain, cb, req]() + { + auto settings = std::make_shared(); + std::chrono::milliseconds timeout(std::numeric_limits::max()); + settings->set_connection_timeout(timeout); // Avoid the client to close the socket after 5 seconds. + + auto ok = std::make_shared(true); + restbed::Http::async(req, + [this, filterChain, cb, ok](const std::shared_ptr& req, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + + if (code == 200) { + try { + while (restbed::Http::is_open(req)) { + restbed::Http::fetch("\n", reply); + std::string body; + reply->get_body(body); + reply->set_body(""); // Reset the body for the next fetch + + Json::Value json; + std::string err; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&body[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (reader->parse(char_data, char_data + body.size(), &json, &err)) { + auto value = std::make_shared(json); + if ((not filterChain or filterChain(*value)) && cb) { + auto okCb = std::make_shared>(); + auto futureCb = okCb->get_future(); + { + std::lock_guard lock(lockCallbacks); + callbacks_.emplace_back([cb, value, okCb](){ + okCb->set_value(cb({value})); + }); + loopSignal_(); + } + futureCb.wait(); + if (!futureCb.get()) { + return; + } + } + } + } + } catch (std::runtime_error&) { + // NOTE: Http::close() can occurs here. Ignore this. + } + } else { + *ok = false; + } + }, settings).get(); + if (!ok) opFailed(); + } + ); + } +} + +#if OPENDHT_PUSH_NOTIFICATIONS +void +DhtProxyClient::pushNotificationReceived(const std::map& notification) +{ + try { + auto token = std::stoul(notification.at("token")); + for (const auto& listener: listeners_) { + if (*(listener.pushNotifToken) != token) + continue; + if (notification.find("timeout") == notification.cend()) { + // Wake up daemon and get values + get(InfoHash(listener.key), listener.cb, {}, listener.filterChain); + } else { + // A timeout has occured, we need to relaunch the listener + resubscribe(token); + } + + } + } catch (...) { + + } +} + +void +DhtProxyClient::resubscribe(const unsigned token) +{ + if (deviceKey_.empty()) return; + for (auto& listener: listeners_) { + if (*(listener.pushNotifToken) == token) { + // Subscribe + restbed::Uri uri(HTTP_PROTO + serverHost_ + "/" + listener.key); + auto req = std::make_shared(uri); + req->set_method("SUBSCRIBE"); + + auto pushNotifToken = std::make_shared(0); + + if (listener.thread.joinable()) + listener.thread.join(); + listener.req = req; + listener.pushNotifToken = pushNotifToken; + listener.thread = std::thread([=]() + { + fillBodyToGetToken(req); + auto settings = std::make_shared(); + auto ok = std::make_shared(true); + restbed::Http::async(req, + [this, pushNotifToken, ok](const std::shared_ptr&, + const std::shared_ptr& reply) { + auto code = reply->get_status_code(); + if (code == 200) { + try { + restbed::Http::fetch("\n", reply); + std::string body; + reply->get_body(body); + + std::string err; + Json::Value json; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&body[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (reader->parse(char_data, char_data + body.size(), &json, &err)) { + if (!json.isMember("token")) return; + *pushNotifToken = json["token"].asLargestUInt(); + } + } catch (std::runtime_error&) { + // NOTE: Http::close() can occurs here. Ignore this. + } + } else { + *ok = false; + } + }, settings).get(); + if (!ok) opFailed(); + }); + } + } +} + +void +DhtProxyClient::fillBodyToGetToken(std::shared_ptr req) +{ + // Fill body with + // { + // "key":"device_key", + // "callback_id": xxx + // } + Json::Value body; + body["key"] = deviceKey_; + body["client_id"] = pushClientId_; + { + std::lock_guard lock(lockCallback_); + callbackId_ += 1; + body["callback_id"] = callbackId_; + } +#ifdef __ANDROID__ + body["platform"] = "android"; +#endif +#ifdef __APPLE__ + body["platform"] = "apple"; +#endif + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto content = Json::writeString(wbuilder, body) + "\n"; + std::replace(content.begin(), content.end(), '\n', ' '); + req->set_body(content); + req->set_header("Content-Length", std::to_string(content.size())); +} +#endif // OPENDHT_PUSH_NOTIFICATIONS + +} // namespace dht + +#endif // OPENDHT_PROXY_CLIENT diff --git a/src/dht_proxy_server.cpp b/src/dht_proxy_server.cpp new file mode 100644 index 00000000..06582e02 --- /dev/null +++ b/src/dht_proxy_server.cpp @@ -0,0 +1,682 @@ +/* + * Copyright (C) 2017-2018 Savoir-faire Linux Inc. + * Author: Sébastien Blin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if OPENDHT_PROXY_SERVER +#include "dht_proxy_server.h" + +#include "default_types.h" +#include "dhtrunner.h" +#include "msgpack.hpp" + +#include +#include +#include +#include + +#if OPENDHT_PUSH_NOTIFICATIONS +constexpr int const TIMEOUT {6 * 60 * 60}; // in seconds (so six hours) +constexpr const char* const HTTP_PROTO {"http://"}; // TODO, https for prod +#endif //OPENDHT_PUSH_NOTIFICATIONS + +using namespace std::placeholders; + +namespace dht { + +DhtProxyServer::DhtProxyServer(std::shared_ptr dht, in_port_t port , const std::string& pushServer) +: dht_(dht) , pushServer_(pushServer) +{ + // NOTE in c++14, use make_unique + service_ = std::unique_ptr(new restbed::Service()); + +#if !OPENDHT_PUSH_NOTIFICATIONS + if (not pushServer.empty()) + std::cerr << "Push server defined but built OpenDHT built without push notification support" << std::endl; +#endif + + server_thread = std::thread([this, port]() { + // Create endpoints + auto resource = std::make_shared(); + resource->set_path("/"); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::getNodeInfo, this, _1)); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/{hash: .*}"); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::get, this, _1)); + resource->set_method_handler("LISTEN", std::bind(&DhtProxyServer::listen, this, _1)); +#if OPENDHT_PUSH_NOTIFICATIONS + resource->set_method_handler("SUBSCRIBE", std::bind(&DhtProxyServer::subscribe, this, _1)); + resource->set_method_handler("UNSUBSCRIBE", std::bind(&DhtProxyServer::unsubscribe, this, _1)); +#endif //OPENDHT_PUSH_NOTIFICATIONS + resource->set_method_handler("POST", std::bind(&DhtProxyServer::put, this, _1)); +#if OPENDHT_PROXY_SERVER_IDENTITY + resource->set_method_handler("SIGN", std::bind(&DhtProxyServer::putSigned, this, _1)); + resource->set_method_handler("ENCRYPT", std::bind(&DhtProxyServer::putEncrypted, this, _1)); +#endif // OPENDHT_PROXY_SERVER_IDENTITY + resource->set_method_handler("OPTIONS", std::bind(&DhtProxyServer::handleOptionsMethod, this, _1)); + service_->publish(resource); + resource = std::make_shared(); + resource->set_path("/{hash: .*}/{value: .*}"); + resource->set_method_handler("GET", std::bind(&DhtProxyServer::getFiltered, this, _1)); + service_->publish(resource); + + // Start server + auto settings = std::make_shared(); + settings->set_default_header("Content-Type", "application/json"); + settings->set_default_header("Connection", "keep-alive"); + settings->set_default_header("Access-Control-Allow-Origin", "*"); + std::chrono::milliseconds timeout(std::numeric_limits::max()); + settings->set_connection_timeout(timeout); // there is a timeout, but really huge + settings->set_port(port); + auto maxThreads = std::thread::hardware_concurrency() - 1; + settings->set_worker_limit(maxThreads > 1 ? maxThreads : 1); + try { + service_->start(settings); + } catch(std::system_error& e) { + std::cerr << "Error running server on port " << port << ": " << e.what() << std::endl; + } + }); + + listenThread_ = std::thread([this]() { + while (!service_->is_up() && !stopListeners) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + while (service_->is_up() && !stopListeners) { + removeClosedListeners(); + // add listener from push notifs +#if OPENDHT_PUSH_NOTIFICATIONS + handlePushListeners(); +#endif //OPENDHT_PUSH_NOTIFICATIONS + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + // Remove last listeners + removeClosedListeners(false); + }); + + dht->forwardAllMessages(true); +} + +DhtProxyServer::~DhtProxyServer() +{ + stop(); +} + +void +DhtProxyServer::stop() +{ + service_->stop(); + { + std::lock_guard lock(lockListener_); + auto listener = currentListeners_.begin(); + while (listener != currentListeners_.end()) { + listener->session->close(); + ++ listener; + } + } + stopListeners = true; + // listenThreads_ will stop because there is no more sessions + if (listenThread_.joinable()) + listenThread_.join(); + if (server_thread.joinable()) + server_thread.join(); +} + +void +DhtProxyServer::getNodeInfo(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& /*b*/) + { + if (dht_) { + Json::Value result; + auto id = dht_->getId(); + if (id) + result["id"] = id.toString(); + result["node_id"] = dht_->getNodeId().toString(); + result["ipv4"] = dht_->getNodesStats(AF_INET).toJson(); + result["ipv6"] = dht_->getNodesStats(AF_INET6).toJson(); + result["public_ip"] = s->get_origin(); // [ipv6:ipv4]:port or ipv4:port + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, result) + "\n"; + s->close(restbed::OK, output); + } + else + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + ); +} + +void +DhtProxyServer::get(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) + { + if (dht_) { + InfoHash infoHash(hash); + if (!infoHash) { + infoHash = InfoHash::get(hash); + } + s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { + auto cacheSession = std::weak_ptr(s); + + dht_->get(infoHash, [cacheSession](std::shared_ptr value) { + auto s = cacheSession.lock(); + if (!s) return false; + // Send values as soon as we get them + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, value->toJson()) + "\n"; + s->yield(output, [](const std::shared_ptr /*session*/){ }); + return true; + }, [s](bool /*ok* */) { + // Communication is finished + s->close(); + }); + }); + } else { + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); + +} + +void +DhtProxyServer::listen(const std::shared_ptr& session) const +{ + + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (!infoHash) + infoHash = InfoHash::get(hash); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) + { + if (dht_) { + InfoHash infoHash(hash); + if (!infoHash) { + infoHash = InfoHash::get(hash); + } + s->yield(restbed::OK); + // Handle client deconnection + // NOTE: for now, there is no handler, so we test the session in a thread + // will be the case in restbed 5.0 + SessionToHashToken listener; + listener.session = session; + listener.hash = infoHash; + // cache the session to avoid an incrementation of the shared_ptr's counter + // else, the session->close() will not close the socket. + auto cacheSession = std::weak_ptr(s); + listener.token = dht_->listen(infoHash, [cacheSession](std::shared_ptr value) { + auto s = cacheSession.lock(); + if (!s) return false; + // Send values as soon as we get them + if (!s->is_closed()) { + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, value->toJson()) + "\n"; + s->yield(output, [](const std::shared_ptr){ }); + } + return !s->is_closed(); + }); + { + std::lock_guard lock(lockListener_); + currentListeners_.emplace_back(std::move(listener)); + } + } else { + session->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} + +#if OPENDHT_PUSH_NOTIFICATIONS +void +DhtProxyServer::subscribe(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (!infoHash) + infoHash = InfoHash::get(hash); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& b) + { + try { + restbed::Bytes buf(b); + std::string strJson(buf.begin(), buf.end()); + + std::string err; + Json::Value root; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&strJson[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (!reader->parse(char_data, char_data + strJson.size(), &root, &err)) { + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); + return; + } + auto pushToken = root["key"].asString(); + if (pushToken.empty()) return; + auto callbackId = root.isMember("callback_id") ? root["callback_id"].asLargestUInt() : 0; + auto platform = root["platform"].asString(); + auto isAndroid = platform == "android"; + auto clientId = root.isMember("client_id") ? root["client_id"].asString() : std::string(); + + auto token = 0; + { + std::lock_guard lock(lockListener_); + // Check if listener is already present and refresh timeout if launched + for(auto& listener: pushListeners_) { + if (listener.pushToken == pushToken && listener.hash == infoHash + && listener.callbackId == callbackId) { + if (listener.started) + listener.deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(TIMEOUT); + s->close(restbed::OK, "{\"token\": " + std::to_string(listener.token) + "}\n"); + return; + } + } + // The listener is not found, so add it. + ++tokenPushNotif_; + token = tokenPushNotif_; + PushListener listener; + listener.pushToken = pushToken; + listener.hash = std::move(infoHash); + listener.token = token; + listener.started = false; + listener.callbackId = callbackId; + listener.clientId = clientId; + listener.isAndroid = isAndroid; + pushListeners_.emplace_back(std::move(listener)); + } + s->close(restbed::OK, "{\"token\": " + std::to_string(token) + "}\n"); + } catch (...) { + // do nothing + } + } + ); +} + +void +DhtProxyServer::unsubscribe(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (!infoHash) + infoHash = InfoHash::get(hash); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& b) + { + try { + restbed::Bytes buf(b); + std::string strJson(buf.begin(), buf.end()); + + std::string err; + Json::Value root; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&strJson[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (!reader->parse(char_data, char_data + strJson.size(), &root, &err)) { + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); + return; + } + auto pushToken = root["key"].asString(); + if (pushToken.empty()) return; + auto token = root["token"].asLargestUInt(); + if (token == 0) return; + auto callbackId = root.isMember("callback_id") ? root["callback_id"].asLargestUInt() : 0; + + std::lock_guard lock(lockListener_); + // Check if listener is already present and refresh timeout if launched + auto listener = pushListeners_.begin(); + while (listener != pushListeners_.end()) { + if (listener->pushToken == pushToken && listener->token == token + && listener->hash == infoHash && listener->callbackId == callbackId) { + if (dht_ && listener->started) + dht_->cancelListen(listener->hash, std::move(listener->internalToken.get())); + listener = pushListeners_.erase(listener); + } else { + ++listener; + } + } + } catch (...) { + // do nothing + } + } + ); +} + +void +DhtProxyServer::sendPushNotification(const std::string& token, const Json::Value& json, bool isAndroid) const +{ + restbed::Uri uri(HTTP_PROTO + pushServer_ + "/api/push"); + auto req = std::make_shared(uri); + req->set_method("POST"); + + // NOTE: see https://github.com/appleboy/gorush + Json::Value notification(Json::objectValue); + Json::Value tokens(Json::arrayValue); + tokens[0] = token; + notification["tokens"] = tokens; + notification["platform"] = isAndroid ? 2 : 1; + notification["data"] = json; + + Json::Value notifications(Json::arrayValue); + notifications[0] = notification; + + Json::Value content; + content["notifications"] = notifications; + + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto valueStr = Json::writeString(wbuilder, content); + + req->set_header("Content-Type", "application/json"); + req->set_header("Accept", "*/*"); + req->set_header("Host", pushServer_); + req->set_header("Content-Length", std::to_string(valueStr.length())); + req->set_body(valueStr); + // Send request. + restbed::Http::async(req, {}); +} + +void +DhtProxyServer::handlePushListeners() +{ + std::lock_guard lock(lockListener_); + auto pushListener = pushListeners_.begin(); + while (pushListener != pushListeners_.end()) { + if (dht_ && !pushListener->started) { + // Try to start unstarted listeners + auto key = pushListener->pushToken; + auto token = pushListener->token; + auto callbackId = pushListener->callbackId; + auto isAndroid = pushListener->isAndroid; + auto clientId = pushListener->clientId; + pushListener->internalToken = dht_->listen(pushListener->hash, + [this, key, callbackId, token, isAndroid, clientId](std::shared_ptr /*value*/) { + // Build message content. + Json::Value json; + if (callbackId > 0) { + json["callback_id"] = callbackId; + } + json["to"] = clientId; + json["token"] = token; + sendPushNotification(key, json, isAndroid); + return true; + } + ); + pushListener->deadline = std::chrono::steady_clock::now() + std::chrono::seconds(TIMEOUT); + pushListener->started = true; + pushListener++; + } else if (dht_ && pushListener->started && pushListener->deadline < std::chrono::steady_clock::now()) { + // Cancel listen if deadline has been reached + dht_->cancelListen(pushListener->hash, std::move(pushListener->internalToken.get())); + // Send a push notification to inform the client that this listen has timeout + Json::Value json; + json["timeout"] = pushListener->hash.toString(); + if (pushListener->callbackId > 0) { + json["callback_id"] = pushListener->callbackId; + } + json["to"] = pushListener->clientId; + json["token"] = pushListener->token; + sendPushNotification(pushListener->pushToken, json, pushListener->isAndroid); + pushListener = pushListeners_.erase(pushListener); + } else { + pushListener++; + } + } +} +#endif //OPENDHT_PUSH_NOTIFICATIONS + +void +DhtProxyServer::put(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (!infoHash) + infoHash = InfoHash::get(hash); + + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("{\"err\":\"Missing parameters\"}"); + s->close(restbed::BAD_REQUEST, response); + } else { + restbed::Bytes buf(b); + std::string strJson(buf.begin(), buf.end()); + + std::string err; + Json::Value root; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&strJson[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (reader->parse(char_data, char_data + strJson.size(), &root, &err)) { + // Build the Value from json + auto value = std::make_shared(root); + auto permanent = root.isMember("permanent") ? root["permanent"].asBool() : false; + + dht_->put(infoHash, value, [s, value](bool ok) { + if (ok) { + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + if (s->is_open()) + s->close(restbed::OK, Json::writeString(wbuilder, value->toJson()) + "\n"); + } else { + if (s->is_open()) + s->close(restbed::BAD_GATEWAY, "{\"err\":\"put failed\"}"); + } + }, time_point::max(), permanent); + } else { + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); + } + } + } else { + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} + +#if OPENDHT_PROXY_SERVER_IDENTITY +void +DhtProxyServer::putSigned(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash infoHash(hash); + if (!infoHash) + infoHash = InfoHash::get(hash); + + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("{\"err\":\"Missing parameters\"}"); + s->close(restbed::BAD_REQUEST, response); + } else { + restbed::Bytes buf(b); + std::string strJson(buf.begin(), buf.end()); + + std::string err; + Json::Value root; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&strJson[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + if (reader->parse(char_data, char_data + strJson.size(), &root, &err)) { + auto value = std::make_shared(root); + + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, value->toJson()) + "\n"; + dht_->putSigned(infoHash, value); + s->close(restbed::OK, output); + } else { + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); + } + } + } else { + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} + +void +DhtProxyServer::putEncrypted(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + InfoHash key(hash); + if (!key) + key = InfoHash::get(hash); + + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& b) + { + if (dht_) { + if(b.empty()) { + std::string response("{\"err\":\"Missing parameters\"}"); + s->close(restbed::BAD_REQUEST, response); + } else { + restbed::Bytes buf(b); + std::string strJson(buf.begin(), buf.end()); + + std::string err; + Json::Value root; + Json::CharReaderBuilder rbuilder; + auto* char_data = reinterpret_cast(&strJson[0]); + auto reader = std::unique_ptr(rbuilder.newCharReader()); + bool parsingSuccessful = reader->parse(char_data, char_data + strJson.size(), &root, &err); + InfoHash to(root["to"].asString()); + if (parsingSuccessful && to) { + auto value = std::make_shared(root); + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, value->toJson()) + "\n"; + dht_->putEncrypted(key, to, value); + s->close(restbed::OK, output); + } else { + if(!parsingSuccessful) + s->close(restbed::BAD_REQUEST, "{\"err\":\"Incorrect JSON\"}"); + else + s->close(restbed::BAD_REQUEST, "{\"err\":\"No destination found\"}"); + } + } + } else { + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} +#endif // OPENDHT_PROXY_SERVER_IDENTITY + +void +DhtProxyServer::handleOptionsMethod(const std::shared_ptr& session) const +{ +#if OPENDHT_PROXY_SERVER_IDENTITY + const auto allowed = "OPTIONS, GET, POST, LISTEN, SIGN, ENCRYPT"; +#else + const auto allowed = "OPTIONS, GET, POST, LISTEN"; +#endif //OPENDHT_PROXY_SERVER_IDENTITY + session->close(restbed::OK, {{"Access-Control-Allow-Methods", allowed}, + {"Access-Control-Allow-Headers", "content-type"}, + {"Access-Control-Max-Age", "86400"}}); +} + +void +DhtProxyServer::getFiltered(const std::shared_ptr& session) const +{ + const auto request = session->get_request(); + int content_length = std::stoi(request->get_header("Content-Length", "0")); + auto hash = request->get_path_parameter("hash"); + auto value = request->get_path_parameter("value"); + session->fetch(content_length, + [=](const std::shared_ptr s, const restbed::Bytes& /*b* */) + { + if (dht_) { + InfoHash infoHash(hash); + if (!infoHash) { + infoHash = InfoHash::get(hash); + } + s->yield(restbed::OK, "", [=]( const std::shared_ptr< restbed::Session > s) { + dht_->get(infoHash, [s](std::shared_ptr v) { + // Send values as soon as we get them + Json::StreamWriterBuilder wbuilder; + wbuilder["commentStyle"] = "None"; + wbuilder["indentation"] = ""; + auto output = Json::writeString(wbuilder, v->toJson()) + "\n"; + s->yield(output, [](const std::shared_ptr /*session*/){ }); + return true; + }, [s](bool /*ok* */) { + // Communication is finished + s->close(); + }, {}, value); + }); + } else { + s->close(restbed::SERVICE_UNAVAILABLE, "{\"err\":\"Incorrect DhtRunner\"}"); + } + } + ); +} + +void +DhtProxyServer::removeClosedListeners(bool testSession) +{ + // clean useless listeners + std::lock_guard lock(lockListener_); + auto listener = currentListeners_.begin(); + while (listener != currentListeners_.end()) { + auto cancel = testSession ? dht_ && listener->session->is_closed() : static_cast(dht_); + if (cancel) { + dht_->cancelListen(listener->hash, std::move(listener->token.get())); + // Remove listener if unused + listener = currentListeners_.erase(listener); + } else { + ++listener; + } + } +} + +} +#endif //OPENDHT_PROXY_SERVER diff --git a/src/dhtrunner.cpp b/src/dhtrunner.cpp index d8903274..e511a1c2 100644 --- a/src/dhtrunner.cpp +++ b/src/dhtrunner.cpp @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author(s) : Adrien Béraud - * Simon Désaulniers + * Authors: Adrien Béraud + * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,6 +21,10 @@ #include "dhtrunner.h" #include "securedht.h" +#if OPENDHT_PROXY_CLIENT +#include "dht_proxy_client.h" +#endif + #ifndef _WIN32 #include #else @@ -39,6 +44,9 @@ namespace dht { constexpr std::chrono::seconds DhtRunner::BOOTSTRAP_PERIOD; DhtRunner::DhtRunner() : dht_() +#if OPENDHT_PROXY_CLIENT +, dht_via_proxy_() +#endif //OPENDHT_PROXY_CLIENT { #ifdef _WIN32 WSADATA wsd; @@ -58,41 +66,51 @@ DhtRunner::~DhtRunner() void DhtRunner::run(in_port_t port, DhtRunner::Config config) { - sockaddr_in sin4; - std::fill_n((uint8_t*)&sin4, sizeof(sin4), 0); - sin4.sin_family = AF_INET; - sin4.sin_port = htons(port); - sockaddr_in6 sin6; - std::fill_n((uint8_t*)&sin6, sizeof(sin6), 0); - sin6.sin6_family = AF_INET6; - sin6.sin6_port = htons(port); - run(&sin4, &sin6, config); + SockAddr sin4; + sin4.setFamily(AF_INET); + sin4.setPort(port); + SockAddr sin6; + sin6.setFamily(AF_INET6); + sin6.setPort(port); + run(sin4, sin6, config); } void DhtRunner::run(const char* ip4, const char* ip6, const char* service, DhtRunner::Config config) { - auto res4 = getAddrInfo(ip4, service); - auto res6 = getAddrInfo(ip6, service); - run(res4.empty() ? nullptr : (sockaddr_in*) &res4.front().first, - res6.empty() ? nullptr : (sockaddr_in6*)&res6.front().first, config); + auto res4 = SockAddr::resolve(ip4, service); + auto res6 = SockAddr::resolve(ip6, service); + run(res4.empty() ? SockAddr() : res4.front(), + res6.empty() ? SockAddr() : res6.front(), config); } void -DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner::Config config) +DhtRunner::run(const SockAddr& local4, const SockAddr& local6, DhtRunner::Config config) { if (running) return; - if (rcv_thread.joinable()) - rcv_thread.join(); + startNetwork(local4, local6); + + auto dht = std::unique_ptr(new Dht(s4, s6, SecureDht::getConfig(config.dht_config))); + dht_ = std::unique_ptr(new SecureDht(std::move(dht), config.dht_config)); + +#if OPENDHT_PROXY_CLIENT + config_ = config; +#endif + enableProxy(not config.proxy_server.empty()); + running = true; - doRun(local4, local6, config.dht_config); if (not config.threaded) return; - dht_thread = std::thread([this]() { + dht_thread = std::thread([this, local4, local6]() { while (running) { std::unique_lock lk(dht_mtx); - auto wakeup = loop_(); + time_point wakeup; + try { + wakeup = loop_(); + } catch (const dht::SocketException& e) { + startNetwork(local4, local6); + } cv.wait_until(lk, wakeup, [this]() { if (not running) return true; @@ -117,6 +135,10 @@ DhtRunner::run(const sockaddr_in* local4, const sockaddr_in6* local6, DhtRunner: void DhtRunner::shutdown(ShutdownCallback cb) { +#if OPENDHT_PROXY_CLIENT + if (dht_via_proxy_) + dht_via_proxy_->shutdown(cb); +#endif std::lock_guard lck(storage_mtx); pending_ops_prio.emplace([=](SecureDht& dht) mutable { dht.shutdown(cb); @@ -127,15 +149,16 @@ DhtRunner::shutdown(ShutdownCallback cb) { void DhtRunner::join() { + running_network = false; running = false; cv.notify_all(); bootstrap_cv.notify_all(); if (dht_thread.joinable()) dht_thread.join(); - if (rcv_thread.joinable()) - rcv_thread.join(); if (bootstrap_thread.joinable()) bootstrap_thread.join(); + if (rcv_thread.joinable()) + rcv_thread.join(); { std::lock_guard lck(storage_mtx); @@ -144,11 +167,9 @@ DhtRunner::join() } { std::lock_guard lck(dht_mtx); - dht_.reset(); + resetDht(); status4 = NodeStatus::Disconnected; status6 = NodeStatus::Disconnected; - bound4 = {}; - bound6 = {}; } } @@ -156,23 +177,23 @@ void DhtRunner::dumpTables() const { std::lock_guard lck(dht_mtx); - dht_->dumpTables(); + activeDht()->dumpTables(); } InfoHash DhtRunner::getId() const { - if (!dht_) + if (!activeDht()) return {}; - return dht_->getId(); + return activeDht()->getId(); } InfoHash DhtRunner::getNodeId() const { - if (!dht_) + if (!activeDht()) return {}; - return dht_->getNodeId(); + return activeDht()->getNodeId(); } @@ -211,19 +232,30 @@ DhtRunner::exportValues() const { void DhtRunner::setLoggers(LogMethod error, LogMethod warn, LogMethod debug) { std::lock_guard lck(dht_mtx); - dht_->setLoggers(std::forward(error), std::forward(warn), std::forward(debug)); + if (dht_) + dht_->setLoggers(std::forward(error), std::forward(warn), std::forward(debug)); +#if OPENDHT_PROXY_CLIENT + if (dht_via_proxy_) + dht_via_proxy_->setLoggers(std::forward(error), std::forward(warn), std::forward(debug)); +#endif } void DhtRunner::setLogFilter(const InfoHash& f) { std::lock_guard lck(dht_mtx); - dht_->setLogFilter(f); + activeDht()->setLogFilter(f); + if (dht_) + dht_->setLogFilter(f); +#if OPENDHT_PROXY_CLIENT + if (dht_via_proxy_) + dht_via_proxy_->setLogFilter(f); +#endif } void DhtRunner::registerType(const ValueType& type) { std::lock_guard lck(dht_mtx); - dht_->registerType(type); + activeDht()->registerType(type); } void @@ -236,7 +268,7 @@ unsigned DhtRunner::getNodesStats(sa_family_t af, unsigned *good_return, unsigned *dubious_return, unsigned *cached_return, unsigned *incoming_return) const { std::lock_guard lck(dht_mtx); - const auto stats = dht_->getNodesStats(af); + const auto stats = activeDht()->getNodesStats(af); if (good_return) *good_return = stats.good_nodes; if (dubious_return) @@ -252,76 +284,82 @@ NodeStats DhtRunner::getNodesStats(sa_family_t af) const { std::lock_guard lck(dht_mtx); - return dht_->getNodesStats(af); + return activeDht()->getNodesStats(af); } std::vector DhtRunner::getNodeMessageStats(bool in) const { std::lock_guard lck(dht_mtx); - return dht_->getNodeMessageStats(in); + return activeDht()->getNodeMessageStats(in); } std::string DhtRunner::getStorageLog() const { std::lock_guard lck(dht_mtx); - return dht_->getStorageLog(); + return activeDht()->getStorageLog(); } std::string DhtRunner::getStorageLog(const InfoHash& f) const { std::lock_guard lck(dht_mtx); - return dht_->getStorageLog(f); + return activeDht()->getStorageLog(f); } std::string DhtRunner::getRoutingTablesLog(sa_family_t af) const { std::lock_guard lck(dht_mtx); - return dht_->getRoutingTablesLog(af); + return activeDht()->getRoutingTablesLog(af); } std::string DhtRunner::getSearchesLog(sa_family_t af) const { std::lock_guard lck(dht_mtx); - return dht_->getSearchesLog(af); + return activeDht()->getSearchesLog(af); } std::string DhtRunner::getSearchLog(const InfoHash& f, sa_family_t af) const { std::lock_guard lck(dht_mtx); - return dht_->getSearchLog(f, af); + return activeDht()->getSearchLog(f, af); } std::vector DhtRunner::getPublicAddress(sa_family_t af) { std::lock_guard lck(dht_mtx); - return dht_->getPublicAddress(af); + return activeDht()->getPublicAddress(af); } std::vector DhtRunner::getPublicAddressStr(sa_family_t af) { auto addrs = getPublicAddress(af); std::vector ret(addrs.size()); - std::transform(addrs.begin(), addrs.end(), ret.begin(), dht::printAddr); + std::transform(addrs.begin(), addrs.end(), ret.begin(), [](const SockAddr& a) { return a.toString(); }); return ret; } void DhtRunner::registerCertificate(std::shared_ptr cert) { std::lock_guard lck(dht_mtx); - dht_->registerCertificate(cert); + activeDht()->registerCertificate(cert); } void DhtRunner::setLocalCertificateStore(CertificateStoreQuery&& query_method) { std::lock_guard lck(dht_mtx); - dht_->setLocalCertificateStore(std::forward(query_method)); +#if OPENDHT_PROXY_CLIENT + if (dht_via_proxy_) + dht_via_proxy_->setLocalCertificateStore(std::forward(query_method)); +#endif + if (dht_) + dht_->setLocalCertificateStore(std::forward(query_method)); } time_point DhtRunner::loop_() { - if (!dht_) + auto dht = activeDht(); + if (not dht) return {}; decltype(pending_ops) ops {}; @@ -332,7 +370,7 @@ DhtRunner::loop_() std::move(pending_ops) : std::move(pending_ops_prio); } while (not ops.empty()) { - ops.front()(*dht_); + ops.front()(*dht); ops.pop(); } @@ -347,15 +385,15 @@ DhtRunner::loop_() for (const auto& pck : received) { auto& buf = pck.first; auto& from = pck.second; - wakeup = dht_->periodic(buf.data(), buf.size()-1, from); + wakeup = dht->periodic(buf.data(), buf.size()-1, from); } received.clear(); } else { - wakeup = dht_->periodic(nullptr, 0, nullptr, 0); + wakeup = dht->periodic(nullptr, 0, nullptr, 0); } - NodeStatus nstatus4 = dht_->getStatus(AF_INET); - NodeStatus nstatus6 = dht_->getStatus(AF_INET6); + NodeStatus nstatus4 = dht->getStatus(AF_INET); + NodeStatus nstatus6 = dht->getStatus(AF_INET6); if (nstatus4 != status4 || nstatus6 != status6) { status4 = nstatus4; status6 = nstatus6; @@ -375,50 +413,52 @@ DhtRunner::loop_() return wakeup; } -void -DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::Config config) -{ - dht_.reset(); - int s4 = -1, - s6 = -1; +int bindSocket(const SockAddr& addr, SockAddr& bound) +{ + bool is_ipv6 = addr.getFamily() == AF_INET6; + int sock = socket(is_ipv6 ? PF_INET6 : PF_INET, SOCK_DGRAM, 0); + if (sock < 0) + throw DhtException(std::string("Can't open socket: ") + strerror(sock)); + int set = 1; +#ifdef SO_NOSIGPIPE + setsockopt(sock, SOL_SOCKET, SO_NOSIGPIPE, (const char*)&set, sizeof(set)); +#endif + if (is_ipv6) + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&set, sizeof(set)); + int rc = bind(sock, addr.get(), addr.getLength()); + if(rc < 0) + throw DhtException("Can't bind socket on " + addr.toString() + " " + strerror(rc)); + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + getsockname(sock, (sockaddr*)&ss, &ss_len); + bound = {ss, ss_len}; + return sock; +} + +void +DhtRunner::startNetwork(const SockAddr sin4, const SockAddr sin6) +{ + running_network = false; + if (rcv_thread.joinable()) + rcv_thread.join(); + s4 = -1; + s6 = -1; bound4 = {}; - if (sin4) { - s4 = socket(PF_INET, SOCK_DGRAM, 0); - if(s4 >= 0) { - int rc = bind(s4, (sockaddr*)sin4, sizeof(sockaddr_in)); - if(rc < 0) - throw DhtException("Can't bind IPv4 socket on " + dht::print_addr((sockaddr*)sin4, sizeof(sockaddr_in))); - bound4.second = sizeof(bound4.first); - getsockname(s4, (sockaddr*)&bound4.first, &bound4.second); - } - } + if (sin4) + s4 = bindSocket(sin4, bound4); #if 1 bound6 = {}; - if (sin6) { - s6 = socket(PF_INET6, SOCK_DGRAM, 0); - if(s6 >= 0) { - int val = 1; - int rc = setsockopt(s6, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&val, sizeof(val)); - if(rc < 0) - throw DhtException("Can't set IPV6_V6ONLY"); - - rc = bind(s6, (sockaddr*)sin6, sizeof(sockaddr_in6)); - if(rc < 0) - throw DhtException("Can't bind IPv6 socket on " + dht::print_addr((sockaddr*)sin6, sizeof(sockaddr_in6))); - bound6.second = sizeof(bound6.first); - getsockname(s6, (sockaddr*)&bound6.first, &bound6.second); - } - } + if (sin6) + s6 = bindSocket(sin6, bound6); #endif - dht_ = std::unique_ptr(new SecureDht {s4, s6, config}); - - rcv_thread = std::thread([this,s4,s6]() { + running_network = true; + rcv_thread = std::thread([this]() { try { - while (true) { + while (running_network) { struct timeval tv {/*.tv_sec = */0, /*.tv_usec = */250000}; fd_set readfds; @@ -436,24 +476,24 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C } } - if(!running) + if (not running_network) break; if(rc > 0) { std::array buf; - SockAddr from; - from.second = sizeof(from.first); + sockaddr_storage from; + socklen_t from_len = sizeof(from); if(s4 >= 0 && FD_ISSET(s4, &readfds)) - rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second); + rc = recvfrom(s4, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len); else if(s6 >= 0 && FD_ISSET(s6, &readfds)) - rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (struct sockaddr*)&from.first, &from.second); + rc = recvfrom(s6, (char*)buf.data(), buf.size(), 0, (sockaddr*)&from, &from_len); else break; if (rc > 0) { { std::lock_guard lck(sock_mtx); - rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, from); + rcv.emplace_back(Blob {buf.begin(), buf.begin()+rc+1}, SockAddr(from, from_len)); } cv.notify_all(); } @@ -466,6 +506,10 @@ DhtRunner::doRun(const sockaddr_in* sin4, const sockaddr_in6* sin6, SecureDht::C close(s4); if (s6 >= 0) close(s6); + s4 = -1; + s6 = -1; + bound4 = {}; + bound6 = {}; }); } @@ -486,7 +530,8 @@ DhtRunner::get(const std::string& key, GetCallback vcb, DoneCallbackSimple dcb, { get(InfoHash::get(key), vcb, dcb, f, w); } -void DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Query q) { +void +DhtRunner::query(const InfoHash& hash, QueryCallback cb, DoneCallback done_cb, Query q) { { std::lock_guard lck(storage_mtx); pending_ops.emplace([=](SecureDht& dht) mutable { @@ -503,7 +548,28 @@ DhtRunner::listen(InfoHash hash, GetCallback vcb, Value::Filter f, Where w) { std::lock_guard lck(storage_mtx); pending_ops.emplace([=](SecureDht& dht) mutable { - ret_token->set_value(dht.listen(hash, vcb, std::move(f), std::move(w))); + auto tokenbGlobal = listener_token_++; + Listener listener {}; + listener.hash = hash; + listener.f = std::move(f); + listener.w = std::move(w); + listener.gcb = [hash,vcb,tokenbGlobal,this](const std::vector>& vals){ + if (not vcb(vals)) { +#if OPENDHT_PROXY_CLIENT + cancelListen(hash, tokenbGlobal); +#endif + return false; + } + return true; + }; +#if OPENDHT_PROXY_CLIENT + if (use_proxy) + listener.tokenProxyDht = dht.listen(hash, listener.gcb, listener.f, listener.w); + else +#endif + listener.tokenClassicDht = dht.listen(hash, listener.gcb, listener.f, listener.w); + listeners_.emplace(tokenbGlobal, std::move(listener)); + ret_token->set_value(tokenbGlobal); }); } cv.notify_all(); @@ -519,12 +585,22 @@ DhtRunner::listen(const std::string& key, GetCallback vcb, Value::Filter f, Wher void DhtRunner::cancelListen(InfoHash h, size_t token) { - { - std::lock_guard lck(storage_mtx); - pending_ops.emplace([=](SecureDht& dht) { - dht.cancelListen(h, token); - }); - } + std::lock_guard lck(storage_mtx); +#if OPENDHT_PROXY_CLIENT + pending_ops.emplace([=](SecureDht&) { + auto it = listeners_.find(token); + if (it == listeners_.end()) return; + if (it->second.tokenClassicDht) + dht_->cancelListen(h, it->second.tokenClassicDht); + if (it->second.tokenProxyDht and dht_via_proxy_) + dht_via_proxy_->cancelListen(h, it->second.tokenProxyDht); + listeners_.erase(it); + }); +#else + pending_ops.emplace([=](SecureDht& dht) { + dht.cancelListen(h, token); + }); +#endif // OPENDHT_PROXY_CLIENT cv.notify_all(); } @@ -660,7 +736,7 @@ DhtRunner::tryBootstrapContinuously() for (auto it = nodes.rbegin(); it != nodes.rend(); it++) { ++ping_count; try { - bootstrap(getAddrInfo(it->first, it->second), [&](bool) { + bootstrap(SockAddr::resolve(it->first, it->second), [&](bool) { if (not running) return; { @@ -691,31 +767,6 @@ DhtRunner::tryBootstrapContinuously() }); } -std::vector> -DhtRunner::getAddrInfo(const std::string& host, const std::string& service) -{ - std::vector> ips {}; - if (host.empty()) - return ips; - - addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_socktype = SOCK_DGRAM; - addrinfo* info = nullptr; - int rc = getaddrinfo(host.c_str(), service.c_str(), &hints, &info); - if(rc != 0) - throw std::invalid_argument(std::string("Error: `") + host + ":" + service + "`: " + gai_strerror(rc)); - - addrinfo* infop = info; - while (infop) { - ips.emplace_back(sockaddr_storage(), infop->ai_addrlen); - std::copy_n((uint8_t*)infop->ai_addr, infop->ai_addrlen, (uint8_t*)&ips.back().first); - infop = infop->ai_next; - } - freeaddrinfo(info); - return ips; -} - void DhtRunner::bootstrap(const std::string& host, const std::string& service) { @@ -733,13 +784,13 @@ DhtRunner::clearBootstrap() } void -DhtRunner::bootstrap(const std::vector>& nodes, DoneCallbackSimple&& cb) +DhtRunner::bootstrap(const std::vector& nodes, DoneCallbackSimple&& cb) { std::lock_guard lck(storage_mtx); pending_ops_prio.emplace([=](SecureDht& dht) mutable { auto rem = cb ? std::make_shared>(nodes.size(), false) : nullptr; - for (auto& node : nodes) - dht.pingNode((sockaddr*)&node.first, node.second, cb ? [rem,cb](bool ok) { + for (const auto& node : nodes) + dht.pingNode(node.get(), node.getLength(), cb ? [rem,cb](bool ok) { auto& r = *rem; r.first--; r.second |= ok; @@ -755,7 +806,7 @@ DhtRunner::bootstrap(const SockAddr& addr, DoneCallbackSimple&& cb) { std::lock_guard lck(storage_mtx); pending_ops_prio.emplace([addr,cb](SecureDht& dht) mutable { - dht.pingNode((const sockaddr*)&addr.first, addr.second, std::move(cb)); + dht.pingNode(addr.get(), addr.getLength(), std::move(cb)); }); cv.notify_all(); } @@ -796,4 +847,119 @@ DhtRunner::findCertificate(InfoHash hash, std::functionshutdown({}); + } + if (proxify) { + // Init the proxy client + auto dht_via_proxy = std::unique_ptr( + new DhtProxyClient([this]{ + if (config_.threaded) { + { + std::lock_guard lck(storage_mtx); + pending_ops_prio.emplace([=](SecureDht&) mutable {}); + } + cv.notify_all(); + } + }, config_.proxy_server, config_.push_node_id) + ); + dht_via_proxy_ = std::unique_ptr(new SecureDht(std::move(dht_via_proxy), config_.dht_config)); +#if OPENDHT_PUSH_NOTIFICATIONS + if (not pushToken_.empty()) + dht_via_proxy_->setPushNotificationToken(pushToken_); +#endif + // add current listeners + for (auto& l: listeners_) + l.second.tokenProxyDht = dht_via_proxy_->listen(l.second.hash, l.second.gcb, l.second.f, l.second.w); + // and use it + use_proxy = proxify; + } else { + use_proxy = proxify; + std::lock_guard lck(storage_mtx); + if (not listeners_.empty()) { + pending_ops.emplace([this](SecureDht& /*dht*/) mutable { + if (not dht_) + return; + for (auto& l : listeners_) { + if (not l.second.tokenClassicDht) { + l.second.tokenClassicDht = dht_->listen(l.second.hash, l.second.gcb, l.second.f, l.second.w); + } + } + }); + } + } +#else + if (proxify) + std::cerr << "DHT proxy requested but OpenDHT built without proxy support." << std::endl; +#endif +} + + +/** + * Updates the push notification device token + */ +void +DhtRunner::setPushNotificationToken(const std::string& token) { +#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS + pushToken_ = token; + if (dht_via_proxy_) + dht_via_proxy_->setPushNotificationToken(token); +#endif +} + +void +DhtRunner::forwardAllMessages(bool forward) +{ +#if OPENDHT_PROXY_SERVER +#if OPENDHT_PROXY_CLIENT + if (dht_via_proxy_) + dht_via_proxy_->forwardAllMessages(forward); +#endif // OPENDHT_PROXY_CLIENT + if (dht_) + dht_->forwardAllMessages(forward); +#endif // OPENDHT_PROXY_SERVER +} + +void +DhtRunner::pushNotificationReceived(const std::map& data) const +{ +#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS + if (dht_via_proxy_) + dht_via_proxy_->pushNotificationReceived(data); +#endif +} + +void +DhtRunner::resubscribe(unsigned token) +{ +#if OPENDHT_PROXY_CLIENT && OPENDHT_PUSH_NOTIFICATIONS + if (dht_via_proxy_) + dht_via_proxy_->resubscribe(token); +#endif +} + } diff --git a/src/infohash.cpp b/src/infohash.cpp index 7631a9d5..9e9f9a61 100644 --- a/src/infohash.cpp +++ b/src/infohash.cpp @@ -17,11 +17,6 @@ */ #include "infohash.h" -#include "rng.h" - -extern "C" { -#include -} #include #include @@ -29,95 +24,32 @@ extern "C" { namespace dht { -InfoHash::InfoHash(const std::string& hex) { - if (hex.size() < 2*HASH_LEN) { - fill(0); - return; - } - const auto p = (const char*)hex.data(); - for (size_t i = 0; i < HASH_LEN; i++) { - unsigned res = 0; - sscanf(p + 2*i, "%02x", &res); - (*this)[i] = res; - } +const HexMap hex_map = {}; + +void +NodeExport::msgpack_unpack(msgpack::object o) +{ + if (o.type != msgpack::type::MAP) + throw msgpack::type_error(); + if (o.via.map.size < 2) + throw msgpack::type_error(); + if (o.via.map.ptr[0].key.as() != "id") + throw msgpack::type_error(); + if (o.via.map.ptr[1].key.as() != "addr") + throw msgpack::type_error(); + const auto& addr = o.via.map.ptr[1].val; + if (addr.type != msgpack::type::BIN) + throw msgpack::type_error(); + if (addr.via.bin.size > sizeof(sockaddr_storage)) + throw msgpack::type_error(); + id.msgpack_unpack(o.via.map.ptr[0].val); + sslen = addr.via.bin.size; + std::copy_n(addr.via.bin.ptr, addr.via.bin.size, (char*)&ss); } -InfoHash -InfoHash::get(const uint8_t* data, size_t data_len) +std::ostream& operator<< (std::ostream& s, const NodeExport& h) { - InfoHash h; - size_t s = h.size(); - const gnutls_datum_t gnudata = {(uint8_t*)data, (unsigned)data_len}; - const gnutls_digest_algorithm_t algo = (HASH_LEN == 64) ? GNUTLS_DIG_SHA512 : ( - (HASH_LEN == 32) ? GNUTLS_DIG_SHA256 : ( - (HASH_LEN == 20) ? GNUTLS_DIG_SHA1 : - GNUTLS_DIG_NULL )); - static_assert(algo != GNUTLS_DIG_NULL, "Can't find hash function to use."); - int rc = gnutls_fingerprint(algo, &gnudata, h.data(), &s); - if (rc == 0 && s == HASH_LEN) - return h; - throw std::runtime_error(std::string("Error hashing: ") + gnutls_strerror(rc)); -} - -InfoHash -InfoHash::getRandom() -{ - InfoHash h; - crypto::random_device rdev; - std::uniform_int_distribution rand_int; - auto a = reinterpret_cast(h.data()); - auto b = reinterpret_cast(h.data() + h.size()); - std::generate(a, b, std::bind(rand_int, std::ref(rdev))); - return h; -} - -struct HexMap : public std::array, 256> { - HexMap() { - for (size_t i=0; i> 4) & 0x0F]; - e[1] = hex_digits[i & 0x0F]; - } - } -private: - static constexpr const char* hex_digits = "0123456789abcdef"; -}; - -const char* -InfoHash::to_c_str() const -{ - static const HexMap map; - thread_local std::array buf; - for (size_t i=0; i> (std::istream& s, InfoHash& h) -{ - std::array dat; - s.exceptions(std::istream::eofbit | std::istream::failbit); - s.read(&(*dat.begin()), dat.size()); - for (size_t i = 0; i < HASH_LEN; i++) { - unsigned res = 0; - sscanf(dat.data() + 2*i, "%02x", &res); - h[i] = res; - } + msgpack::pack(s, h); return s; } diff --git a/src/net.h b/src/net.h new file mode 100644 index 00000000..0aa9dda9 --- /dev/null +++ b/src/net.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +namespace dht { +namespace net { + +enum class MessageType { + Error = 0, + Reply, + Ping, + FindNode, + GetValues, + AnnounceValue, + Refresh, + Listen, + ValueData, + ValueUpdate +}; + +} /* namespace net */ +} /* dht */ diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 56b821f2..200b046d 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -22,6 +22,7 @@ #include "request.h" #include "default_types.h" #include "log_enable.h" +#include "parsed_message.h" #include @@ -52,16 +53,7 @@ constexpr std::chrono::seconds NetworkEngine::RX_MAX_PACKET_TIME; constexpr std::chrono::seconds NetworkEngine::RX_TIMEOUT; const std::string NetworkEngine::my_v {"RNG1"}; -const constexpr uint16_t TransId::INVALID; -std::mt19937 NetworkEngine::rd_device {dht::crypto::random_device{}()}; - -const TransPrefix TransPrefix::PING = {"pn"}; -const TransPrefix TransPrefix::FIND_NODE = {"fn"}; -const TransPrefix TransPrefix::GET_VALUES = {"gt"}; -const TransPrefix TransPrefix::ANNOUNCE_VALUES = {"pt"}; -const TransPrefix TransPrefix::REFRESH = {"rf"}; -const TransPrefix TransPrefix::LISTEN = {"lt"}; -constexpr long unsigned NetworkEngine::MAX_REQUESTS_PER_SEC; +constexpr size_t NetworkEngine::MAX_REQUESTS_PER_SEC; static const uint8_t v4prefix[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0, 0, 0, 0 @@ -95,62 +87,23 @@ set_nonblocking(int fd, int nonblocking) #endif -enum class MessageType { - Error = 0, - Reply, - Ping, - FindNode, - GetValues, - AnnounceValue, - Refresh, - Listen, - ValueData + +/* Transaction-ids are 4-bytes long, with the first two bytes identifying + * the kind of request, and the remaining two a sequence number in + * host order. + */ +struct TransId final : public std::array { + TransId() { std::fill(begin(), end(), 0); } + TransId(const std::array& o) { std::copy(o.begin(), o.end(), begin()); } + TransId(uint32_t id) { + *reinterpret_cast(data()) = htonl(id); + } + + uint32_t toInt() const { + return ntohl(*reinterpret_cast(&(*this)[0])); + } }; -struct ParsedMessage { - MessageType type; - /* Node ID of the sender */ - InfoHash id; - /* Network id */ - NetId network {0}; - /* hash for which values are requested */ - InfoHash info_hash; - /* target id around which to find nodes */ - InfoHash target; - /* transaction id */ - TransId tid; - /* tid for packets going through request socket */ - TransId socket_id; - /* security token */ - Blob token; - /* the value id (announce confirmation) */ - Value::Id value_id; - /* time when value was first created */ - time_point created { time_point::max() }; - /* IPv4 nodes in response to a 'find' request */ - Blob nodes4_raw, nodes6_raw; - std::vector> nodes4, nodes6; - /* values to store or retreive request */ - std::vector> values; - /* index for fields values */ - std::vector> fields; - /** When part of the message header: {index -> (total size, {})} - * When part of partial value data: {index -> (offset, part_data)} */ - std::map> value_parts; - /* query describing a filter to apply on values. */ - Query query; - /* states if ipv4 or ipv6 request */ - want_t want; - /* error code in case of error */ - uint16_t error_code; - /* reported address by the distant node */ - std::string ua; - SockAddr addr; - void msgpack_unpack(msgpack::object o); - - bool append(const ParsedMessage& block); - bool complete(); -}; struct NetworkEngine::PartialMessage { SockAddr from; @@ -169,12 +122,14 @@ serializeValues(const std::vector>& st) return svals; } -NetworkEngine::RequestAnswer::RequestAnswer(ParsedMessage&& msg) +RequestAnswer::RequestAnswer(ParsedMessage&& msg) : ntoken(std::move(msg.token)), values(std::move(msg.values)), fields(std::move(msg.fields)), nodes4(std::move(msg.nodes4)), nodes6(std::move(msg.nodes6)) {} -NetworkEngine::NetworkEngine(Logger& log, Scheduler& scheduler) : myid(zeroes), DHT_LOG(log), scheduler(scheduler) {} -NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& log, Scheduler& scheduler, +NetworkEngine::NetworkEngine(Logger& log, Scheduler& scheduler, const int& s, const int& s6) + : myid(zeroes), DHT_LOG(log), scheduler(scheduler), dht_socket(s), dht_socket6(s6) +{} +NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, const int& s, const int& s6, Logger& log, Scheduler& scheduler, decltype(NetworkEngine::onError) onError, decltype(NetworkEngine::onNewNode) onNewNode, decltype(NetworkEngine::onReportedAddr) onReportedAddr, @@ -196,7 +151,6 @@ NetworkEngine::NetworkEngine(InfoHash& myid, NetId net, int s, int s6, Logger& l if (!set_nonblocking(dht_socket6, 1)) throw DhtException("Can't set socket to non-blocking mode"); } - transaction_id = std::uniform_int_distribution{1}(rd_device); } NetworkEngine::~NetworkEngine() { @@ -204,14 +158,14 @@ NetworkEngine::~NetworkEngine() { } void -NetworkEngine::tellListener(Sp node, uint32_t socket_id, const InfoHash& hash, want_t want, +NetworkEngine::tellListener(Sp node, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken, std::vector>&& nodes, std::vector>&& nodes6, std::vector>&& values, const Query& query) { auto nnodes = bufferNodes(node->getFamily(), hash, want, nodes, nodes6); try { - sendNodesValues(node->addr, TransId((char*)&socket_id, 4), nnodes.first, nnodes.second, values, query, ntoken); + sendNodesValues(node->getAddr(), socket_id, nnodes.first, nnodes.second, values, query, ntoken); } catch (const std::overflow_error& e) { DHT_LOG.e("Can't send value: buffer not large enough !"); } @@ -232,40 +186,9 @@ NetworkEngine::isRunning(sa_family_t af) const } } -Sp -NetworkEngine::openSocket(const Sp& node, TransPrefix tp, SocketCb&& cb) -{ - auto tid = TransId {tp, getNewTid()}; - auto s = opened_sockets.emplace(tid, std::make_shared(node, tid, cb)); - if (not s.second) - DHT_LOG.e(node->id, "[node %s] socket (tid: %d) already opened!", node->id.toString().c_str(), tid.toInt()); - return s.first->second; -} - -void -NetworkEngine::closeSocket(Sp socket) -{ - if (socket) - opened_sockets.erase(socket->id); -} - -void -NetworkEngine::cancelRequest(Sp& req) -{ - if (req) { - req->cancel(); - closeSocket(req->socket); - requests.erase(req->tid); - } -} - void NetworkEngine::clear() -{ - for (auto& req : requests) - req.second->setExpired(); - requests.clear(); -} +{} void NetworkEngine::connectivityChanged(sa_family_t af) @@ -277,32 +200,32 @@ void NetworkEngine::requestStep(Sp sreq) { auto& req = *sreq; - if (not req.pending()) { - if (req.cancelled()) - requests.erase(req.tid); + if (not req.pending()) return; - } auto now = scheduler.time(); auto& node = *req.node; if (req.isExpired(now)) { DHT_LOG.e(node.id, "[node %s] expired !", node.toString().c_str()); node.setExpired(); - requests.erase(req.tid); + if (not node.id) + requests.erase(req.tid); return; } else if (req.attempt_count == 1) { req.on_expired(req, false); } auto err = send((char*)req.msg.data(), req.msg.size(), - (node.reply_time >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM, - node.addr); + (node.getReplyTime() >= now - UDP_REPLY_TIME) ? 0 : MSG_CONFIRM, + node.getAddr()); if (err == ENETUNREACH || err == EHOSTUNREACH || - err == EAFNOSUPPORT) + err == EAFNOSUPPORT || + err == EPIPE) { node.setExpired(); - requests.erase(req.tid); + if (not node.id) + requests.erase(req.tid); } else { if (err != EAGAIN) { ++req.attempt_count; @@ -321,18 +244,13 @@ NetworkEngine::requestStep(Sp sreq) * be made before the request expires. */ void -NetworkEngine::sendRequest(Sp& request) +NetworkEngine::sendRequest(const Sp& request) { + auto& node = request->node; + if (not node->id) + requests.emplace(request->tid, request); request->start = scheduler.time(); - auto e = requests.emplace(request->tid, request); - if (not e.second) { - // Should not happen ! - // Try to handle this scenario as well as we can - e.first->second->setExpired(); - e.first->second = request; - DHT_LOG.e(request->node->id, "Request already existed (tid: %d)!", request->tid.getTid()); - } - request->node->requested(request); + node->requested(request); requestStep(request); } @@ -343,15 +261,15 @@ NetworkEngine::rateLimit(const SockAddr& addr) { const auto& now = scheduler.time(); - // occasional IP limiter maintenance - std::bernoulli_distribution rand_trial(1./128.); - if (rand_trial(rd_device)) { + // occasional IP limiter maintenance (a few times every second at max rate) + if (limiter_maintenance++ == MAX_REQUESTS_PER_SEC/8) { for (auto it = address_rate_limiter.begin(); it != address_rate_limiter.end();) { if (it->second.maintain(now) == 0) address_rate_limiter.erase(it++); else ++it; } + limiter_maintenance = 0; } auto it = address_rate_limiter.emplace(addr, IpLimiter{}); @@ -372,7 +290,7 @@ NetworkEngine::isMartian(const SockAddr& addr) ((address[0] & 0xE0) == 0xE0); } case AF_INET6: { - if (addr.second < sizeof(sockaddr_in6)) + if (addr.getLength() < sizeof(sockaddr_in6)) return true; const auto& sin6 = addr.getIPv6(); const uint8_t* address = (const uint8_t*)&sin6.sin6_addr; @@ -392,15 +310,7 @@ void NetworkEngine::blacklistNode(const Sp& n) { n->setExpired(); - for (auto rit = requests.begin(); rit != requests.end();) { - if (rit->second->node == n) { - //rit->second->cancel(); - requests.erase(rit++); - } else { - ++rit; - } - } - blacklist.emplace(n->addr); + blacklist.emplace(n->getAddr()); } bool @@ -466,7 +376,7 @@ NetworkEngine::processMessage(const uint8_t *buf, size_t buflen, const SockAddr& return; } - if (msg->id == myid || msg->id == zeroes) { + if (msg->id == myid or not msg->id) { DHT_LOG.d("Received message from self"); return; } @@ -501,50 +411,53 @@ void NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& from) { const auto& now = scheduler.time(); + auto node = cache.getNode(msg->id, from, now, true, msg->is_client); + + if (msg->type == MessageType::ValueUpdate) { + auto rsocket = node->getSocket(msg->tid); + if (not rsocket) + throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find socket", msg->id}; + node->received(now, {}); + onNewNode(node, 2); + deserializeNodes(*msg, from); + rsocket->on_receive(node, std::move(*msg)); + } + else if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { + auto rsocket = node->getSocket(msg->tid); + auto req = node->getRequest(msg->tid); - if (msg->type == MessageType::Error or msg->type == MessageType::Reply) { /* either response for a request or data for an opened socket */ - auto req_it = requests.find(msg->tid); - auto rsocket_it = opened_sockets.end(); - if (req_it == requests.end()) - rsocket_it = opened_sockets.find(msg->tid); - if (req_it == requests.end() and rsocket_it == opened_sockets.end()) - throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; - - auto req = req_it != requests.end() ? req_it->second : nullptr; - auto rsocket = rsocket_it != opened_sockets.end() ? rsocket_it->second : nullptr; - - auto& node = req ? req->node : rsocket->node; - if (node->id != msg->id) { - if (node->id == zeroes) // received reply to a message sent when we didn't know the node ID. - node = cache.getNode(msg->id, from, now, true); - else { - // received reply from unexpected node + if (not req and not rsocket) { + auto req_it = requests.find(msg->tid); + if (req_it != requests.end() and not req_it->second->node->id) { + req = req_it->second; + req->node = node; + requests.erase(req_it); + } else { node->received(now, req); - onNewNode(node, 2); - DHT_LOG.w(node->id, "[node %s] message received from unexpected node", node->toString().c_str()); - return; + if (not node->isClient()) + onNewNode(node, 1); + throw DhtProtocolException {DhtProtocolException::UNKNOWN_TID, "Can't find transaction", msg->id}; } - } else - node->update(from); + } node->received(now, req); - onNewNode(node, 2); + if (not node->isClient()) + onNewNode(node, 2); onReportedAddr(msg->id, msg->addr); if (req and (req->cancelled() or req->expired() or req->completed())) { DHT_LOG.w(node->id, "[node %s] response to expired, cancelled or completed request", node->toString().c_str()); - requests.erase(req_it); return; } switch (msg->type) { case MessageType::Error: { - if (msg->id != zeroes and ( - (msg->error_code == DhtProtocolException::NOT_FOUND and msg->tid.matches(TransPrefix::REFRESH)) or - (msg->error_code == DhtProtocolException::UNAUTHORIZED and (msg->tid.matches(TransPrefix::ANNOUNCE_VALUES) - or msg->tid.matches(TransPrefix::LISTEN))))) + if (msg->id and req and ( + (msg->error_code == DhtProtocolException::NOT_FOUND and req->getType() == MessageType::Refresh) or + (msg->error_code == DhtProtocolException::UNAUTHORIZED and (req->getType() == MessageType::AnnounceValue + or req->getType() == MessageType::Listen)))) { req->last_try = time_point::min(); req->reply_time = time_point::min(); @@ -556,28 +469,26 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro break; } case MessageType::Reply: - if (req) { /* request reply */ - if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen) - req->node->authSuccess(); + if (req) { /* request reply */ + if (msg->type == MessageType::AnnounceValue or msg->type == MessageType::Listen) + req->node->authSuccess(); + req->reply_time = scheduler.time(); - // erase before calling callback to make sure iterator is still valid - requests.erase(req_it); - req->reply_time = scheduler.time(); - - deserializeNodes(*msg, from); - req->setDone(std::move(*msg)); - break; - } else { /* request socket data */ - deserializeNodes(*msg, from); - rsocket->on_receive(rsocket->node, std::move(*msg)); - } + deserializeNodes(*msg, from); + req->setDone(std::move(*msg)); + break; + } else { /* request socket data */ + deserializeNodes(*msg, from); + rsocket->on_receive(node, std::move(*msg)); + } + break; default: break; } } else { - auto node = cache.getNode(msg->id, from, now, true); node->received(now, {}); - onNewNode(node, 1); + if (not node->isClient()) + onNewNode(node, 1); try { switch (msg->type) { case MessageType::Ping: @@ -624,9 +535,7 @@ NetworkEngine::process(std::unique_ptr&& msg, const SockAddr& fro case MessageType::Listen: { DHT_LOG.d(msg->info_hash, node->id, "[node %s] got 'listen' request for %s", node->toString().c_str(), msg->info_hash.toString().c_str()); ++in_stats.listen; - /* TODO: backward compatibility check to remove in a few versions (see ::sendListen doc) */ - auto socket_id = msg->socket_id.getTid() ? msg->socket_id : TransId { TransPrefix::GET_VALUES, msg->tid.getTid() }; - RequestAnswer answer = onListen(node, msg->info_hash, msg->token, socket_id.toInt(), std::move(msg->query)); + RequestAnswer answer = onListen(node, msg->info_hash, msg->token, msg->socket_id, std::move(msg->query)); sendListenConfirmation(from, msg->tid); break; } @@ -651,10 +560,10 @@ packToken(msgpack::packer& pk, const Blob& token) void insertAddr(msgpack::packer& pk, const SockAddr& addr) { - size_t addr_len = std::min(addr.second, + size_t addr_len = std::min(addr.getLength(), (addr.getFamily() == AF_INET) ? sizeof(in_addr) : sizeof(in6_addr)); - void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&((sockaddr_in*)&addr.first)->sin_addr - : (void*)&((sockaddr_in6*)&addr.first)->sin6_addr; + void* addr_ptr = (addr.getFamily() == AF_INET) ? (void*)&addr.getIPv4().sin_addr + : (void*)&addr.getIPv6().sin6_addr; pk.pack("sa"); pk.pack_bin(addr_len); pk.pack_bin_body((char*)addr_ptr, addr_len); @@ -663,7 +572,7 @@ insertAddr(msgpack::packer& pk, const SockAddr& addr) int NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr) { - if (addr.second == 0) + if (not addr) return EFAULT; int s; @@ -676,9 +585,15 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr if (s < 0) return EAFNOSUPPORT; - if (sendto(s, buf, len, flags, (const sockaddr*)&addr.first, addr.second) == -1) { +#ifdef MSG_NOSIGNAL + flags &= MSG_NOSIGNAL; +#endif + if (sendto(s, buf, len, flags, addr.get(), addr.getLength()) == -1) { int err = errno; DHT_LOG.e("Can't send message to %s: %s", addr.toString().c_str(), strerror(err)); + if (err == EPIPE) { + throw SocketException(EPIPE); + } return err; } return 0; @@ -686,7 +601,7 @@ NetworkEngine::send(const char *buf, size_t len, int flags, const SockAddr& addr Sp NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::PING, getNewTid()}; + TransId tid (node->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -703,7 +618,7 @@ NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& o pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, node, + auto req = std::make_shared(MessageType::Ping, tid.toInt(), node, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&&) { DHT_LOG.d(req_status.node->id, "[node %s] got pong !", req_status.node->toString().c_str()); @@ -723,7 +638,7 @@ NetworkEngine::sendPing(Sp node, RequestCb&& on_done, RequestExpiredCb&& o } void -NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { +NetworkEngine::sendPong(const SockAddr& addr, Tid tid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -732,8 +647,9 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { pk.pack(std::string("id")); pk.pack(myid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -746,7 +662,7 @@ NetworkEngine::sendPong(const SockAddr& addr, TransId tid) { Sp NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::FIND_NODE, getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -770,7 +686,7 @@ NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::FindNode, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) { @@ -792,7 +708,7 @@ NetworkEngine::sendFindNode(Sp n, const InfoHash& target, want_t want, Sp NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& query, want_t want, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::GET_VALUES, getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -819,7 +735,7 @@ NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::GetValues, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) { @@ -839,20 +755,18 @@ NetworkEngine::sendGetValues(Sp n, const InfoHash& info_hash, const Query& SockAddr deserializeIPv4(const uint8_t* ni) { SockAddr addr; - auto& sin = addr.getIPv4(); - std::fill_n((uint8_t*)&sin, sizeof(sockaddr_in), 0); addr.setFamily(AF_INET); - memcpy(&sin.sin_addr, ni, 4); - memcpy(&sin.sin_port, ni + 4, 2); + auto& sin = addr.getIPv4(); + std::memcpy(&sin.sin_addr, ni, 4); + std::memcpy(&sin.sin_port, ni + 4, 2); return addr; } SockAddr deserializeIPv6(const uint8_t* ni) { SockAddr addr; - auto& sin6 = addr.getIPv6(); - std::fill_n((uint8_t*)&sin6, sizeof(sockaddr_in6), 0); addr.setFamily(AF_INET6); - memcpy(&sin6.sin6_addr, ni, 16); - memcpy(&sin6.sin6_port, ni + 16, 2); + auto& sin6 = addr.getIPv6(); + std::memcpy(&sin6.sin6_addr, ni, 16); + std::memcpy(&sin6.sin6_port, ni + 16, 2); return addr; } @@ -860,41 +774,40 @@ void NetworkEngine::deserializeNodes(ParsedMessage& msg, const SockAddr& from) { if (msg.nodes4_raw.size() % NODE4_INFO_BUF_LEN != 0 || msg.nodes6_raw.size() % NODE6_INFO_BUF_LEN != 0) { throw DhtProtocolException {DhtProtocolException::WRONG_NODE_INFO_BUF_LEN}; - } else { - // deserialize nodes - const auto& now = scheduler.time(); - for (unsigned i = 0; i < msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i++) { - uint8_t *ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN; - const InfoHash& ni_id = *reinterpret_cast(ni); - if (ni_id == myid) - continue; - SockAddr addr = deserializeIPv4(ni + ni_id.size()); - if (addr.isLoopback() and from.getFamily() == AF_INET) { - auto port = addr.getPort(); - addr = from; - addr.setPort(port); - } - if (isMartian(addr) || isNodeBlacklisted(addr)) - continue; - msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, false)); - onNewNode(msg.nodes4.back(), 0); + } + // deserialize nodes + const auto& now = scheduler.time(); + for (unsigned i = 0, n = msg.nodes4_raw.size() / NODE4_INFO_BUF_LEN; i < n; i++) { + const uint8_t* ni = msg.nodes4_raw.data() + i * NODE4_INFO_BUF_LEN; + const auto& ni_id = *reinterpret_cast(ni); + if (ni_id == myid) + continue; + SockAddr addr = deserializeIPv4(ni + ni_id.size()); + if (addr.isLoopback() and from.getFamily() == AF_INET) { + auto port = addr.getPort(); + addr = from; + addr.setPort(port); } - for (unsigned i = 0; i < msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i++) { - uint8_t *ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN; - const InfoHash& ni_id = *reinterpret_cast(ni); - if (ni_id == myid) - continue; - SockAddr addr = deserializeIPv6(ni + ni_id.size()); - if (addr.isLoopback() and from.getFamily() == AF_INET6) { - auto port = addr.getPort(); - addr = from; - addr.setPort(port); - } - if (isMartian(addr) || isNodeBlacklisted(addr)) - continue; - msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, false)); - onNewNode(msg.nodes6.back(), 0); + if (isMartian(addr) || isNodeBlacklisted(addr)) + continue; + msg.nodes4.emplace_back(cache.getNode(ni_id, addr, now, false)); + onNewNode(msg.nodes4.back(), 0); + } + for (unsigned i = 0, n = msg.nodes6_raw.size() / NODE6_INFO_BUF_LEN; i < n; i++) { + const uint8_t* ni = msg.nodes6_raw.data() + i * NODE6_INFO_BUF_LEN; + const auto& ni_id = *reinterpret_cast(ni); + if (ni_id == myid) + continue; + SockAddr addr = deserializeIPv6(ni + ni_id.size()); + if (addr.isLoopback() and from.getFamily() == AF_INET6) { + auto port = addr.getPort(); + addr = from; + addr.setPort(port); } + if (isMartian(addr) || isNodeBlacklisted(addr)) + continue; + msg.nodes6.emplace_back(cache.getNode(ni_id, addr, now, false)); + onNewNode(msg.nodes6.back(), 0); } } @@ -923,7 +836,7 @@ NetworkEngine::packValueHeader(msgpack::sbuffer& buffer, const std::vector& svals, const SockAddr& addr) +NetworkEngine::sendValueParts(const TransId& tid, const std::vector& svals, const SockAddr& addr) { msgpack::sbuffer buffer; unsigned i=0; @@ -953,7 +866,7 @@ NetworkEngine::sendValueParts(TransId tid, const std::vector& svals, const } void -NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& nodes, const Blob& nodes6, +NetworkEngine::sendNodesValues(const SockAddr& addr, Tid tid, const Blob& nodes, const Blob& nodes6, const std::vector>& st, const Query& query, const Blob& token) { msgpack::sbuffer buffer; @@ -994,8 +907,9 @@ NetworkEngine::sendNodesValues(const SockAddr& addr, TransId tid, const Blob& no } } - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1022,21 +936,21 @@ NetworkEngine::bufferNodes(sa_family_t af, const InfoHash& id, std::vectorsin_addr, sizeof(in_addr)); - memcpy(dest + HASH_LEN + sizeof(in_addr), &sin->sin_port, sizeof(in_port_t)); + memcpy(dest + HASH_LEN, &sin.sin_addr, sizeof(in_addr)); + memcpy(dest + HASH_LEN + sizeof(in_addr), &sin.sin_port, sizeof(in_port_t)); } } else if (af == AF_INET6) { bnodes.resize(NODE6_INFO_BUF_LEN * nnode); for (size_t i=0; isin6_addr, sizeof(in6_addr)); - memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6->sin6_port, sizeof(in_port_t)); + memcpy(dest + HASH_LEN, &sin6.sin6_addr, sizeof(in6_addr)); + memcpy(dest + HASH_LEN + sizeof(in6_addr), &sin6.sin6_port, sizeof(in_port_t)); } } return bnodes; @@ -1070,25 +984,21 @@ NetworkEngine::sendListen(Sp n, RequestExpiredCb&& on_expired, SocketCb&& socket_cb) { - Sp socket; - auto tid = TransId { TransPrefix::LISTEN, previous ? previous->tid.getTid() : getNewTid() }; + Tid socket; + TransId tid (n->getNewTid()); if (previous and previous->node == n) { - socket = previous->socket; + socket = previous->getSocket(); } else { if (previous) DHT_LOG.e(hash, "[node %s] trying refresh listen contract with wrong node", previous->node->toString().c_str()); - /* TODO: Manually creating a socket for "listen" backward compatibility. - * As soon as this is not an issue anymore, switch to using - * ::createSocket function! */ - auto sid = TransId {TransPrefix::GET_VALUES, tid.getTid()}; - socket = std::make_shared(n, sid, socket_cb); - opened_sockets.emplace(sid, socket); + socket = n->openSocket(std::move(socket_cb)); } if (not socket) { DHT_LOG.e(hash, "[node %s] unable to get a valid socket for listen. Aborting listen", n->toString().c_str()); return {}; } + TransId sid(socket); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); @@ -1099,8 +1009,8 @@ NetworkEngine::sendListen(Sp n, pk.pack(std::string("id")); pk.pack(myid); pk.pack(std::string("h")); pk.pack(hash); pk.pack(std::string("token")); packToken(pk, token); - pk.pack(std::string("sid")); pk.pack_bin(socket->id.size()); - pk.pack_bin_body((const char*)socket->id.data(), socket->id.size()); + pk.pack(std::string("sid")); pk.pack_bin(sid.size()); + pk.pack_bin_body((const char*)sid.data(), sid.size()); if (has_query) { pk.pack(std::string("q")); pk.pack(query); } @@ -1114,7 +1024,7 @@ NetworkEngine::sendListen(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::Listen, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (on_done) @@ -1132,7 +1042,7 @@ NetworkEngine::sendListen(Sp n, } void -NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) { +NetworkEngine::sendListenConfirmation(const SockAddr& addr, Tid tid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -1141,8 +1051,9 @@ NetworkEngine::sendListenConfirmation(const SockAddr& addr, TransId tid) { pk.pack(std::string("id")); pk.pack(myid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t (tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1161,7 +1072,7 @@ NetworkEngine::sendAnnounceValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::ANNOUNCE_VALUES, getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1185,7 +1096,7 @@ NetworkEngine::sendAnnounceValue(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::AnnounceValue, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (msg.value_id == Value::INVALID_ID) { @@ -1206,7 +1117,7 @@ NetworkEngine::sendAnnounceValue(Sp n, ); sendRequest(req); if (not v.empty()) - sendValueParts(tid, v, n->addr); + sendValueParts(tid, v, n->getAddr()); ++out_stats.put; return req; } @@ -1219,7 +1130,7 @@ NetworkEngine::sendRefreshValue(Sp n, RequestCb&& on_done, RequestExpiredCb&& on_expired) { - auto tid = TransId {TransPrefix::REFRESH, getNewTid()}; + TransId tid (n->getNewTid()); msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(5+(network?1:0)); @@ -1239,7 +1150,7 @@ NetworkEngine::sendRefreshValue(Sp n, pk.pack(std::string("n")); pk.pack(network); } - auto req = std::make_shared(tid, n, + auto req = std::make_shared(MessageType::Refresh, tid.toInt(), n, Blob(buffer.data(), buffer.data() + buffer.size()), [=](const Request& req_status, ParsedMessage&& msg) { /* on done */ if (msg.value_id == Value::INVALID_ID) { @@ -1264,7 +1175,7 @@ NetworkEngine::sendRefreshValue(Sp n, } void -NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id vid) { +NetworkEngine::sendValueAnnounced(const SockAddr& addr, Tid tid, Value::Id vid) { msgpack::sbuffer buffer; msgpack::packer pk(&buffer); pk.pack_map(4+(network?1:0)); @@ -1274,8 +1185,9 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id v pk.pack(std::string("vid")); pk.pack(vid); insertAddr(pk, addr); - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t(tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("r")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1287,7 +1199,7 @@ NetworkEngine::sendValueAnnounced(const SockAddr& addr, TransId tid, Value::Id v void NetworkEngine::sendError(const SockAddr& addr, - TransId tid, + Tid tid, uint16_t code, const std::string& message, bool include_id) @@ -1305,8 +1217,9 @@ NetworkEngine::sendError(const SockAddr& addr, pk.pack(std::string("id")); pk.pack(myid); } - pk.pack(std::string("t")); pk.pack_bin(tid.size()); - pk.pack_bin_body((const char*)tid.data(), tid.size()); + TransId t(tid); + pk.pack(std::string("t")); pk.pack_bin(t.size()); + pk.pack_bin_body((const char*)t.data(), t.size()); pk.pack(std::string("y")); pk.pack(std::string("e")); pk.pack(std::string("v")); pk.pack(my_v); if (network) { @@ -1317,188 +1230,7 @@ NetworkEngine::sendError(const SockAddr& addr, } void -ParsedMessage::msgpack_unpack(msgpack::object msg) -{ - auto y = findMapValue(msg, "y"); - auto r = findMapValue(msg, "r"); - auto e = findMapValue(msg, "e"); - auto v = findMapValue(msg, "p"); - - if (auto t = findMapValue(msg, "t")) - tid = {t->as>()}; - - if (auto rv = findMapValue(msg, "v")) - ua = rv->as(); - - if (auto netid = findMapValue(msg, "n")) - network = netid->as(); - - std::string q; - if (auto rq = findMapValue(msg, "q")) { - if (rq->type != msgpack::type::STR) - throw msgpack::type_error(); - q = rq->as(); - } - - if (e) - type = MessageType::Error; - else if (r) - type = MessageType::Reply; - else if (v) - type = MessageType::ValueData; - else if (y and y->as() != "q") - throw msgpack::type_error(); - else if (q == "ping") - type = MessageType::Ping; - else if (q == "find") - type = MessageType::FindNode; - else if (q == "get") - type = MessageType::GetValues; - else if (q == "listen") - type = MessageType::Listen; - else if (q == "put") - type = MessageType::AnnounceValue; - else if (q == "refresh") - type = MessageType::Refresh; - else - throw msgpack::type_error(); - - if (type == MessageType::ValueData) { - if (v->type != msgpack::type::MAP) - throw msgpack::type_error(); - for (size_t i = 0; i < v->via.map.size; ++i) { - auto& vdat = v->via.map.ptr[i]; - auto o = findMapValue(vdat.val, "o"); - auto d = findMapValue(vdat.val, "d"); - if (not o or not d) - continue; - value_parts.emplace(vdat.key.as(), std::pair(o->as(), unpackBlob(*d))); - } - return; - } - - auto a = findMapValue(msg, "a"); - if (!a && !r && !e) - throw msgpack::type_error(); - auto& req = a ? *a : (r ? *r : *e); - - if (e) { - if (e->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - error_code = e->via.array.ptr[0].as(); - } - - if (auto t = findMapValue(req, "sid")) - socket_id = {t->as>()}; - - if (auto rid = findMapValue(req, "id")) - id = {*rid}; - - if (auto rh = findMapValue(req, "h")) - info_hash = {*rh}; - - if (auto rtarget = findMapValue(req, "target")) - target = {*rtarget}; - - if (auto rquery = findMapValue(req, "q")) - query.msgpack_unpack(*rquery); - - if (auto otoken = findMapValue(req, "token")) - token = unpackBlob(*otoken); - - if (auto vid = findMapValue(req, "vid")) - value_id = vid->as(); - - if (auto rnodes4 = findMapValue(req, "n4")) - nodes4_raw = unpackBlob(*rnodes4); - - if (auto rnodes6 = findMapValue(req, "n6")) - nodes6_raw = unpackBlob(*rnodes6); - - if (auto sa = findMapValue(req, "sa")) { - if (sa->type != msgpack::type::BIN) - throw msgpack::type_error(); - auto l = sa->via.bin.size; - if (l == sizeof(in_addr)) { - auto a = (sockaddr_in*)&addr.first; - std::fill_n((uint8_t*)a, sizeof(sockaddr_in), 0); - a->sin_family = AF_INET; - a->sin_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin_addr); - addr.second = sizeof(sockaddr_in); - } else if (l == sizeof(in6_addr)) { - auto a = (sockaddr_in6*)&addr.first; - std::fill_n((uint8_t*)a, sizeof(sockaddr_in6), 0); - a->sin6_family = AF_INET6; - a->sin6_port = 0; - std::copy_n(sa->via.bin.ptr, l, (char*)&a->sin6_addr); - addr.second = sizeof(sockaddr_in6); - } - } else - addr.second = 0; - - if (auto rcreated = findMapValue(req, "c")) - created = from_time_t(rcreated->as()); - - if (auto rvalues = findMapValue(req, "values")) { - if (rvalues->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - for (size_t i = 0; i < rvalues->via.array.size; i++) { - auto& packed_v = rvalues->via.array.ptr[i]; - if (packed_v.type == msgpack::type::POSITIVE_INTEGER) { - // Skip oversize values with a small margin for header overhead - if (packed_v.via.u64 > MAX_VALUE_SIZE + 32) - continue; - value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{})); - } else { - try { - values.emplace_back(std::make_shared(rvalues->via.array.ptr[i])); - } catch (const std::exception& e) { - //DHT_LOG_WARN("Error reading value: %s", e.what()); - } - } - } - } else if (auto raw_fields = findMapValue(req, "fields")) { - if (auto rfields = findMapValue(*raw_fields, "f")) { - auto vfields = rfields->as>(); - if (auto rvalues = findMapValue(*raw_fields, "v")) { - if (rvalues->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - size_t val_num = rvalues->via.array.size / vfields.size(); - for (size_t i = 0; i < val_num; ++i) { - try { - auto v = std::make_shared(); - v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size()); - fields.emplace_back(std::move(v)); - } catch (const std::exception& e) { } - } - } - } else { - throw msgpack::type_error(); - } - } - - if (auto w = findMapValue(req, "w")) { - if (w->type != msgpack::type::ARRAY) - throw msgpack::type_error(); - want = 0; - for (unsigned i=0; ivia.array.size; i++) { - auto& val = w->via.array.ptr[i]; - try { - auto w = val.as(); - if (w == AF_INET) - want |= WANT4; - else if(w == AF_INET6) - want |= WANT6; - } catch (const std::exception& e) {}; - } - } else { - want = -1; - } -} - -void -NetworkEngine::maintainRxBuffer(const TransId& tid) +NetworkEngine::maintainRxBuffer(Tid tid) { auto msg = partial_messages.find(tid); if (msg != partial_messages.end()) { @@ -1511,44 +1243,6 @@ NetworkEngine::maintainRxBuffer(const TransId& tid) } } -bool -ParsedMessage::append(const ParsedMessage& block) -{ - bool ret(false); - for (const auto& ve : block.value_parts) { - auto part_val = value_parts.find(ve.first); - if (part_val == value_parts.end() - || part_val->second.second.size() >= part_val->second.first) - continue; - // TODO: handle out-of-order packets - if (ve.second.first != part_val->second.second.size()) { - //std::cout << "skipping out-of-order packet" << std::endl; - continue; - } - ret = true; - part_val->second.second.insert(part_val->second.second.end(), - ve.second.second.begin(), - ve.second.second.end()); - } - return ret; -} - -bool -ParsedMessage::complete() -{ - for (auto& e : value_parts) { - //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl; - if (e.second.first > e.second.second.size()) - return false; - } - for (auto& e : value_parts) { - msgpack::unpacked msg; - msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size()); - values.emplace_back(std::make_shared(msg.get())); - } - return true; -} - } /* namespace net */ } /* namespace dht */ diff --git a/src/node.cpp b/src/node.cpp index 1c482af2..1f9c45e3 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -20,6 +20,7 @@ #include "node.h" #include "request.h" +#include "rng.h" #include @@ -29,6 +30,14 @@ constexpr std::chrono::minutes Node::NODE_EXPIRE_TIME; constexpr std::chrono::minutes Node::NODE_GOOD_TIME; constexpr std::chrono::seconds Node::MAX_RESPONSE_TIME; + +Node::Node(const InfoHash& id, const SockAddr& addr, bool client) +: id(id), addr(addr), is_client(client), sockets_() +{ + crypto::random_device rd; + transaction_id = std::uniform_int_distribution{1}(rd); +} + /* This is our definition of a known-good node. */ bool Node::isGood(time_point now) const @@ -41,11 +50,9 @@ Node::isGood(time_point now) const bool Node::isPendingMessage() const { - for (auto w : requests_) { - if (auto r = w.lock()) { - if (r->pending()) - return true; - } + for (auto r : requests_) { + if (r.second->pending()) + return true; } return false; } @@ -54,11 +61,9 @@ size_t Node::getPendingMessageCount() const { size_t count {0}; - for (auto w : requests_) { - if (auto r = w.lock()) { - if (r->pending()) - count++; - } + for (auto r : requests_) { + if (r.second->pending()) + count++; } return count; } @@ -71,27 +76,44 @@ Node::update(const SockAddr& new_addr) /** To be called when a message was sent to the node */ void -Node::requested(std::shared_ptr& req) +Node::requested(const Sp& req) { - requests_.emplace_back(req); + auto e = requests_.emplace(req->getTid(), req); + if (not e.second and req != e.first->second) { + // Should not happen ! + // Try to handle this scenario as well as we can + e.first->second->setExpired(); + e.first->second = req; + } } /** To be called when a message was received from the node. Req should be true if the message was an aswer to a request we made*/ void -Node::received(time_point now, std::shared_ptr req) +Node::received(time_point now, const Sp& req) { time = now; expired_ = false; if (req) { reply_time = now; - for (auto it = requests_.begin(); it != requests_.end();) { - auto r = it->lock(); - if (not r or r == req) - it = requests_.erase(it); - else - ++it; - } + requests_.erase(req->getTid()); + } +} + +Sp +Node::getRequest(Tid tid) +{ + auto it = requests_.find(tid); + return it != requests_.end() ? it->second : nullptr; +} + +void +Node::cancelRequest(const Sp& req) +{ + if (req) { + req->cancel(); + closeSocket(req->closeSocket()); + requests_.erase(req->getTid()); } } @@ -99,13 +121,41 @@ void Node::setExpired() { expired_ = true; - for (auto w : requests_) { - if (auto r = w.lock()) - r->setExpired(); + for (auto r : requests_) { + r.second->setExpired(); } requests_.clear(); + sockets_.clear(); } +Tid +Node::openSocket(SocketCb&& cb) +{ + if (++transaction_id == 0) + transaction_id = 1; + + auto sock = std::make_shared(std::move(cb)); + auto s = sockets_.emplace(transaction_id, std::move(sock)); + if (not s.second) + s.first->second = std::move(sock); + return transaction_id; +} + +Sp +Node::getSocket(Tid id) +{ + auto it = sockets_.find(id); + return it == sockets_.end() ? nullptr : it->second; +} + +void +Node::closeSocket(Tid id) +{ + if (id) { + sockets_.erase(id); + //DHT_LOG.w("Closing socket (tid: %d), %lu remaining", socket->id, sockets_.size()); + } +} std::string Node::toString() const diff --git a/src/node_cache.cpp b/src/node_cache.cpp index 4bc51215..926768da 100644 --- a/src/node_cache.cpp +++ b/src/node_cache.cpp @@ -20,44 +20,53 @@ namespace dht { -std::shared_ptr +NodeCache::~NodeCache() +{ + cache_4.setExpired(); + cache_6.setExpired(); +} + +Sp NodeCache::getNode(const InfoHash& id, sa_family_t family) { - return (family == AF_INET ? cache_4 : cache_6).getNode(id); + return cache(family).getNode(id); } -std::shared_ptr -NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm) { - if (id == zeroes) +Sp +NodeCache::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { + if (not id) return std::make_shared(id, addr); - return (addr.getFamily() == AF_INET ? cache_4 : cache_6).getNode(id, addr, now, confirm); + return cache(addr.getFamily()).getNode(id, addr, now, confirm, client); } -std::vector> -NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) { - const auto& c = (sa_f == AF_INET ? cache_4 : cache_6); - auto it_p = c.lower_bound(id), - it_n = it_p; +std::vector> +NodeCache::getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) const +{ + const auto& c = cache(sa_f); - std::vector> nodes; + std::vector> nodes; nodes.reserve(std::min(c.size(), count)); NodeMap::const_iterator it; + auto dec_it = [&c](NodeMap::const_iterator& it) { + auto ret = it; + it = (it == c.cbegin()) ? c.cend() : std::prev(it); + return ret; + }; - if (it_p != c.begin()) /* Create 2 separate iterator if we could */ - --it_p; + auto it_p = c.lower_bound(id), + it_n = it_p; + if (not c.empty()) + dec_it(it_p); /* Create 2 separate iterator if we could */ - while (nodes.size() < count and (it_n != c.end() or it_p != c.end())) { + while (nodes.size() < count and (it_n != c.cend() or it_p != c.cend())) { /* If one of the iterator is at the end, then take the other one If they are both in middle of somewhere comapre both and take the closest to the id. */ - if (it_p == c.end()) it = it_n++; - else if (it_n == c.end()) it = it_p--; - else it = id.xorCmp(it_p->first, it_n->first) < 0 ? it_p-- : it_n++; - - if (it == c.begin()) - it_p = c.end(); + if (it_p == c.cend()) it = it_n++; + else if (it_n == c.cend()) it = dec_it(it_p); + else it = id.xorCmp(it_p->first, it_n->first) < 0 ? dec_it(it_p) : it_n++; if (auto n = it->second.lock()) - if ( not n->isExpired() ) + if ( not n->isExpired() and not n->isClient() ) nodes.emplace_back(std::move(n)); } @@ -71,11 +80,11 @@ NodeCache::clearBadNodes(sa_family_t family) clearBadNodes(AF_INET); clearBadNodes(AF_INET6); } else { - (family == AF_INET ? cache_4 : cache_6).clearBadNodes(); + cache(family).clearBadNodes(); } } -std::shared_ptr +Sp NodeCache::NodeMap::getNode(const InfoHash& id) { auto wn = find(id); @@ -87,15 +96,15 @@ NodeCache::NodeMap::getNode(const InfoHash& id) return {}; } -std::shared_ptr -NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm) +Sp +NodeCache::NodeMap::getNode(const InfoHash& id, const SockAddr& addr, time_point now, bool confirm, bool client) { auto it = emplace(id, std::weak_ptr{}); auto node = it.first->second.lock(); if (not node) { - node = std::make_shared(id, addr); + node = std::make_shared(id, addr, client); it.first->second = node; - } else if (confirm || node->time < now - Node::NODE_EXPIRE_TIME) { + } else if (confirm or node->isOld(now)) { node->update(addr); } return node; @@ -113,4 +122,12 @@ NodeCache::NodeMap::clearBadNodes() { } } +void +NodeCache::NodeMap::setExpired() { + for (auto& wn : *this) + if (auto n = wn.second.lock()) + n->setExpired(); + clear(); +} + } diff --git a/src/parsed_message.h b/src/parsed_message.h new file mode 100644 index 00000000..c4ac56cf --- /dev/null +++ b/src/parsed_message.h @@ -0,0 +1,307 @@ +/* + * Copyright (C) 2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + #pragma once + +#include "infohash.h" +#include "sockaddr.h" +#include "net.h" + +#include + +namespace dht { +namespace net { + +Tid unpackTid(const msgpack::object& o) { + switch (o.type) { + case msgpack::type::POSITIVE_INTEGER: + return o.as(); + default: + return ntohl(*reinterpret_cast(o.as>().data())); + } +} + +struct ParsedMessage { + MessageType type; + /* Node ID of the sender */ + InfoHash id; + /* Network id */ + NetId network {0}; + /** Is a client node */ + bool is_client {false}; + /* hash for which values are requested */ + InfoHash info_hash; + /* target id around which to find nodes */ + InfoHash target; + /* transaction id */ + Tid tid; + /* tid for packets going through request socket */ + Tid socket_id; + /* security token */ + Blob token; + /* the value id (announce confirmation) */ + Value::Id value_id; + /* time when value was first created */ + time_point created { time_point::max() }; + /* IPv4 nodes in response to a 'find' request */ + Blob nodes4_raw, nodes6_raw; + std::vector> nodes4, nodes6; + /* values to store or retreive request */ + std::vector> values; + /* index for fields values */ + std::vector> fields; + /** When part of the message header: {index -> (total size, {})} + * When part of partial value data: {index -> (offset, part_data)} */ + std::map> value_parts; + /* query describing a filter to apply on values. */ + Query query; + /* states if ipv4 or ipv6 request */ + want_t want; + /* error code in case of error */ + uint16_t error_code; + /* reported address by the distant node */ + std::string ua; + SockAddr addr; + void msgpack_unpack(msgpack::object o); + + bool append(const ParsedMessage& block); + bool complete(); +}; + +bool +ParsedMessage::append(const ParsedMessage& block) +{ + bool ret(false); + for (const auto& ve : block.value_parts) { + auto part_val = value_parts.find(ve.first); + if (part_val == value_parts.end() + || part_val->second.second.size() >= part_val->second.first) + continue; + // TODO: handle out-of-order packets + if (ve.second.first != part_val->second.second.size()) { + //std::cout << "skipping out-of-order packet" << std::endl; + continue; + } + ret = true; + part_val->second.second.insert(part_val->second.second.end(), + ve.second.second.begin(), + ve.second.second.end()); + } + return ret; +} + +bool +ParsedMessage::complete() +{ + for (auto& e : value_parts) { + //std::cout << "part " << e.first << ": " << e.second.second.size() << "/" << e.second.first << std::endl; + if (e.second.first > e.second.second.size()) + return false; + } + for (auto& e : value_parts) { + msgpack::unpacked msg; + msgpack::unpack(msg, (const char*)e.second.second.data(), e.second.second.size()); + values.emplace_back(std::make_shared(msg.get())); + } + return true; +} + +void +ParsedMessage::msgpack_unpack(msgpack::object msg) +{ + auto y = findMapValue(msg, "y"); + auto r = findMapValue(msg, "r"); + auto u = findMapValue(msg, "u"); + auto e = findMapValue(msg, "e"); + auto v = findMapValue(msg, "p"); + + if (auto t = findMapValue(msg, "t")) + tid = unpackTid(*t); + + if (auto rv = findMapValue(msg, "v")) + ua = rv->as(); + + if (auto netid = findMapValue(msg, "n")) + network = netid->as(); + + if (auto is_client_v = findMapValue(msg, "s")) + is_client = is_client_v->as(); + + std::string q; + if (auto rq = findMapValue(msg, "q")) { + if (rq->type != msgpack::type::STR) + throw msgpack::type_error(); + q = rq->as(); + } + + if (e) + type = MessageType::Error; + else if (r) + type = MessageType::Reply; + else if (v) + type = MessageType::ValueData; + else if (u) + type = MessageType::ValueUpdate; + else if (y and y->as() != "q") + throw msgpack::type_error(); + else if (q == "ping") + type = MessageType::Ping; + else if (q == "find") + type = MessageType::FindNode; + else if (q == "get") + type = MessageType::GetValues; + else if (q == "listen") + type = MessageType::Listen; + else if (q == "put") + type = MessageType::AnnounceValue; + else if (q == "refresh") + type = MessageType::Refresh; + else + throw msgpack::type_error(); + + if (type == MessageType::ValueData) { + if (v->type != msgpack::type::MAP) + throw msgpack::type_error(); + for (size_t i = 0; i < v->via.map.size; ++i) { + auto& vdat = v->via.map.ptr[i]; + auto o = findMapValue(vdat.val, "o"); + auto d = findMapValue(vdat.val, "d"); + if (not o or not d) + continue; + value_parts.emplace(vdat.key.as(), std::pair(o->as(), unpackBlob(*d))); + } + return; + } + + auto a = findMapValue(msg, "a"); + if (!a && !r && !e && !u) + throw msgpack::type_error(); + auto& req = a ? *a : (r ? *r : (u ? *u : *e)); + + if (e) { + if (e->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + error_code = e->via.array.ptr[0].as(); + } + + if (auto t = findMapValue(req, "sid")) + socket_id = unpackTid(*t); + + if (auto rid = findMapValue(req, "id")) + id = {*rid}; + + if (auto rh = findMapValue(req, "h")) + info_hash = {*rh}; + + if (auto rtarget = findMapValue(req, "target")) + target = {*rtarget}; + + if (auto rquery = findMapValue(req, "q")) + query.msgpack_unpack(*rquery); + + if (auto otoken = findMapValue(req, "token")) + token = unpackBlob(*otoken); + + if (auto vid = findMapValue(req, "vid")) + value_id = vid->as(); + + if (auto rnodes4 = findMapValue(req, "n4")) + nodes4_raw = unpackBlob(*rnodes4); + + if (auto rnodes6 = findMapValue(req, "n6")) + nodes6_raw = unpackBlob(*rnodes6); + + if (auto sa = findMapValue(req, "sa")) { + if (sa->type != msgpack::type::BIN) + throw msgpack::type_error(); + auto l = sa->via.bin.size; + if (l == sizeof(in_addr)) { + addr.setFamily(AF_INET); + auto& a = addr.getIPv4(); + a.sin_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin_addr); + } else if (l == sizeof(in6_addr)) { + addr.setFamily(AF_INET6); + auto& a = addr.getIPv6(); + a.sin6_port = 0; + std::copy_n(sa->via.bin.ptr, l, (char*)&a.sin6_addr); + } + } else + addr = {}; + + if (auto rcreated = findMapValue(req, "c")) + created = from_time_t(rcreated->as()); + + if (auto rvalues = findMapValue(req, "values")) { + if (rvalues->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + for (size_t i = 0; i < rvalues->via.array.size; i++) { + auto& packed_v = rvalues->via.array.ptr[i]; + if (packed_v.type == msgpack::type::POSITIVE_INTEGER) { + // Skip oversize values with a small margin for header overhead + if (packed_v.via.u64 > MAX_VALUE_SIZE + 32) + continue; + value_parts.emplace(i, std::make_pair(packed_v.via.u64, Blob{})); + } else { + try { + values.emplace_back(std::make_shared(rvalues->via.array.ptr[i])); + } catch (const std::exception& e) { + //DHT_LOG_WARN("Error reading value: %s", e.what()); + } + } + } + } else if (auto raw_fields = findMapValue(req, "fields")) { + if (auto rfields = findMapValue(*raw_fields, "f")) { + auto vfields = rfields->as>(); + if (auto rvalues = findMapValue(*raw_fields, "v")) { + if (rvalues->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + size_t val_num = rvalues->via.array.size / vfields.size(); + for (size_t i = 0; i < val_num; ++i) { + try { + auto v = std::make_shared(); + v->msgpack_unpack_fields(vfields, *rvalues, i*vfields.size()); + fields.emplace_back(std::move(v)); + } catch (const std::exception& e) { } + } + } + } else { + throw msgpack::type_error(); + } + } + + if (auto w = findMapValue(req, "w")) { + if (w->type != msgpack::type::ARRAY) + throw msgpack::type_error(); + want = 0; + for (unsigned i=0; ivia.array.size; i++) { + auto& val = w->via.array.ptr[i]; + try { + auto w = val.as(); + if (w == AF_INET) + want |= WANT4; + else if(w == AF_INET6) + want |= WANT6; + } catch (const std::exception& e) {}; + } + } else { + want = -1; + } +} + +} /* namespace net */ +} /* namespace dht */ diff --git a/src/request.h b/src/request.h index 0174e3b0..3c04cd34 100644 --- a/src/request.h +++ b/src/request.h @@ -20,34 +20,15 @@ #pragma once #include "net.h" +#include "value.h" namespace dht { +struct Node; namespace net { class NetworkEngine; struct ParsedMessage; -/*! - * @class Socket - * @brief Open route to a node for continous incoming packets. - * @details - * A socket lets a remote node send us continuous packets treated using a - * given callback. This is intended to provide an easy management of - * specific updates nodes can send. For e.g, this is used in the case of the - * "listen" operation for treating updates a node has for a given storage. - */ -struct Socket { - Socket() {} - Socket(std::shared_ptr node, - TransId id, - std::function&, ParsedMessage&&)> on_receive) : - node(node), id(id), on_receive(on_receive) {} - - std::shared_ptr node; - TransId id; - std::function&, ParsedMessage&&)> on_receive {}; -}; - /*! * @class Request * @brief An atomic request destined to a node. @@ -59,7 +40,7 @@ struct Socket { struct Request { friend class dht::net::NetworkEngine; - std::shared_ptr node {}; /* the node to whom the request is destined. */ + Sp node {}; /* the node to whom the request is destined. */ time_point reply_time {time_point::min()}; /* time when we received the response to the request. */ enum class State @@ -78,13 +59,19 @@ struct Request { State getState() const { return state_; } Request(State state = State::PENDING) : state_(state) {} - Request(TransId tid, - std::shared_ptr node, + Request(MessageType type, Tid tid, + Sp node, Blob&& msg, std::function on_done, std::function on_expired, - std::shared_ptr socket = {}) : - node(node), on_done(on_done), on_expired(on_expired), tid(tid), msg(std::move(msg)), socket(socket) { } + Tid socket = 0) : + node(node), tid(tid), type(type), on_done(on_done), on_expired(on_expired), msg(std::move(msg)), socket(socket) { } + + Tid getTid() const { return tid; } + MessageType getType() const { return type; } + + Tid getSocket() { return socket; } + Tid closeSocket() { auto ret = socket; socket = 0; return ret; } void setExpired() { if (pending()) { @@ -121,6 +108,8 @@ private: msg.clear(); } + const Tid tid {0}; /* the request id. */ + const MessageType type {}; State state_ {State::PENDING}; unsigned attempt_count {0}; /* number of attempt to process the request. */ @@ -130,9 +119,8 @@ private: std::function on_done {}; std::function on_expired {}; - const TransId tid; /* the request id. */ Blob msg {}; /* the serialized message. */ - std::shared_ptr socket; /* the socket used for further reponses. */ + Tid socket; /* the socket used for further reponses. */ }; } /* namespace net */ diff --git a/src/routing_table.cpp b/src/routing_table.cpp index 495846b2..e9a5bb15 100644 --- a/src/routing_table.cpp +++ b/src/routing_table.cpp @@ -1,4 +1,24 @@ +/* + * Copyright (C) 2014-2017 Savoir-faire Linux Inc. + * Author(s) : Adrien Béraud + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + #include "routing_table.h" + +#include "network_engine.h" #include "rng.h" #include @@ -35,6 +55,15 @@ Bucket::randomNode() return nodes.back(); } +void Bucket::sendCachedPing(net::NetworkEngine& ne) +{ + if (not cached) + return; + //DHT_LOG.d(b.cached->id, "[node %s] sending ping to cached node", cached->toString().c_str()); + ne.sendPing(cached, nullptr, nullptr); + cached = {}; +} + InfoHash RoutingTable::randomId(const RoutingTable::const_iterator& it) const { @@ -47,7 +76,7 @@ RoutingTable::randomId(const RoutingTable::const_iterator& it) const int b = bit/8; InfoHash id_return; - std::copy_n(it->first.begin(), b, id_return.begin()); + std::copy_n(it->first.cbegin(), b, id_return.begin()); id_return[b] = it->first[b] & (0xFF00 >> (bit % 8)); id_return[b] |= rand_byte(rd) >> (bit % 8); for (unsigned i = b + 1; i < HASH_LEN; i++) @@ -172,4 +201,66 @@ RoutingTable::split(const RoutingTable::iterator& b) return true; } +bool +RoutingTable::onNewNode(const Sp& node, int confirm, const time_point& now, const InfoHash& myid, net::NetworkEngine& ne) { + auto b = findBucket(node->id); + if (b == end()) return false; + + if (confirm == 2) + b->time = now; + + for (auto& n : b->nodes) { + if (n == node) + return false; + } + + bool mybucket = contains(b, myid); + if (mybucket) { + grow_time = now; + //scheduler.edit(nextNodesConfirmation, now); + } + + if (b->nodes.size() >= TARGET_NODES) { + /* Try to get rid of an expired node. */ + for (auto& n : b->nodes) + if (n->isExpired()) { + n = node; + return true; + } + /* Bucket full. Ping a dubious node */ + bool dubious = false; + for (auto& n : b->nodes) { + /* Pick the first dubious node that we haven't pinged in the + last 9 seconds. This gives nodes the time to reply, but + tends to concentrate on the same nodes, so that we get rid + of bad nodes fast. */ + if (not n->isGood(now)) { + dubious = true; + if (not n->isPendingMessage()) { + //DHT_LOG.d(n->id, "[node %s] sending ping to dubious node", n->toString().c_str()); + ne.sendPing(n, nullptr, nullptr); + break; + } + } + } + + if ((mybucket || (is_client and depth(b) < 6)) && (!dubious || size() == 1)) { + //DHT_LOG.d("Splitting from depth %u", depth(b)); + b->sendCachedPing(ne); + split(b); + return onNewNode(node, confirm, now, myid, ne); + } + + /* No space for this node. Cache it away for later. */ + if (confirm or not b->cached) + b->cached = node; + } else { + /* Create a new node. */ + b->nodes.emplace_front(node); + } + return true; +} + + + } diff --git a/src/search.h b/src/search.h index 81a3fe94..e5aeee82 100644 --- a/src/search.h +++ b/src/search.h @@ -76,7 +76,19 @@ struct Dht::SearchNode { node is a new candidate for inclusion. */ SearchNode() : node() {} + SearchNode(const SearchNode&) = delete; + SearchNode(SearchNode&&) = default; + SearchNode& operator=(const SearchNode&) = delete; + SearchNode& operator=(SearchNode&&) = default; + SearchNode(const Sp& node) : node(node) {} + ~SearchNode() { + if (node) { + cancelGet(); + cancelListen(); + cancelAnnounce(); + } + } /** * Can we use this node to listen/announce now ? @@ -173,6 +185,15 @@ struct Dht::SearchNode { } } + void cancelGet() { + for (const auto& status : getStatus) { + if (status.second->pending()) { + node->cancelRequest(status.second); + } + } + getStatus.clear(); + } + /** * Tells if a request in the status map is expired. * @@ -180,12 +201,12 @@ struct Dht::SearchNode { * * @return true if there exists an expired request, else false. */ - bool expired(const SyncStatus& status) const { + /*static bool expired(const SyncStatus& status) const { return std::find_if(status.begin(), status.end(), [](const SyncStatus::value_type& r){ return r.second and r.second->expired(); }) != status.end(); - } + }*/ /** * Tells if a request in the status map is pending. @@ -194,19 +215,30 @@ struct Dht::SearchNode { * * @return true if there exists an expired request, else false. */ - bool pending(const SyncStatus& status) const { + static bool pending(const SyncStatus& status) { return std::find_if(status.begin(), status.end(), [](const SyncStatus::value_type& r){ return r.second and r.second->pending(); }) != status.end(); } + bool pendingGet() const { return pending(getStatus); } + bool isAnnounced(Value::Id vid) const { auto ack = acked.find(vid); if (ack == acked.end() or not ack->second.first) return false; return ack->second.first->completed(); } + void cancelAnnounce() { + for (const auto& status : acked) { + const auto& req = status.second.first; + if (req and req->pending()) { + node->cancelRequest(req); + } + } + acked.clear(); + } bool isListening(time_point now) const { auto ls = listenStatus.begin(); @@ -229,6 +261,18 @@ struct Dht::SearchNode { return false; return listen_status->second->reply_time + LISTEN_EXPIRE_TIME > now; } + void cancelListen() { + for (const auto& status : listenStatus) + node->cancelRequest(status.second); + listenStatus.clear(); + } + void cancelListen(const Sp& query) { + auto it = listenStatus.find(query); + if (it != listenStatus.end()) { + node->cancelRequest(it->second); + listenStatus.erase(it); + } + } /** * Assumng the node is synced, should a "put" request be sent to this node now ? @@ -319,7 +363,7 @@ struct Dht::Search { unsigned currentlySolicitedNodeCount() const { unsigned count = 0; for (const auto& n : nodes) - if (not n.isBad() and n.pending(n.getStatus)) + if (not n.isBad() and n.pendingGet()) count++; return count; } @@ -382,6 +426,21 @@ struct Dht::Search { bool isAnnounced(Value::Id id) const; bool isListening(time_point now) const; + void cancelListen(size_t token) { + Sp query; + const auto& ll = listeners.find(token); + if (ll != listeners.cend()) { + query = ll->second.query; + listeners.erase(ll); + } + for (auto& sn : nodes) { + if (listeners.empty()) + sn.cancelListen(); + else if (query) + sn.cancelListen(query); + } + } + /** * @return The number of non-good search nodes. */ @@ -430,10 +489,10 @@ struct Dht::Search { * * @return true if a node has been removed, else false. */ - bool removeExpiredNode(time_point now) { + bool removeExpiredNode(const time_point& now) { for (auto e = nodes.cend(); e != nodes.cbegin();) { const Node& n = *(--e)->node; - if (n.isExpired() and n.time + Node::NODE_EXPIRE_TIME < now) { + if (n.isRemovable(now)) { //std::cout << "Removing expired node " << n.id << " from IPv" << (af==AF_INET?'4':'6') << " search " << id << std::endl; nodes.erase(e); return true; @@ -585,7 +644,7 @@ Dht::Search::insertNode(const Sp& snode, time_point now, const Blob& token step_time = time_point::min(); } n = nodes.insert(n, SearchNode(snode)); - node.time = now; + node.setTime(now); new_search_node = true; if (node.isExpired()) { if (not expired) @@ -672,7 +731,7 @@ Dht::Search::getUpdateTime(time_point now) const for (const auto& sn : nodes) { if (sn.node->isExpired() or (sn.candidate and t >= TARGET_NODES)) continue; - auto pending = sn.pending(sn.getStatus); + auto pending = sn.pendingGet(); if (sn.last_get_reply < std::max(now - Node::NODE_EXPIRE_TIME, last_get) or pending) { // not isSynced if (not pending and solicited_nodes < MAX_REQUESTED_SEARCH_NODES) diff --git a/src/securedht.cpp b/src/securedht.cpp index d2c36008..7dd8dc86 100644 --- a/src/securedht.cpp +++ b/src/securedht.cpp @@ -1,7 +1,8 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. - * Author : Adrien Béraud + * Authors: Adrien Béraud * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,20 +33,10 @@ extern "C" { namespace dht { -Config& getConfig(SecureDht::Config& conf) +SecureDht::SecureDht(std::unique_ptr dht, SecureDht::Config conf) +: dht_(std::move(dht)), key_(conf.id.first), certificate_(conf.id.second) { - auto& c = conf.node_config; - if (c.node_id == InfoHash() and conf.id.second) - c.node_id = InfoHash::get("node:"+conf.id.second->getId().toString()); - return c; -} - -SecureDht::SecureDht(int s, int s6, SecureDht::Config conf) -: Dht(s, s6, getConfig(conf)), key_(conf.id.first), certificate_(conf.id.second) -{ - if (s < 0 && s6 < 0) - return; - + if (!dht_) return; for (const auto& type : DEFAULT_TYPES) registerType(type); @@ -59,7 +50,7 @@ SecureDht::SecureDht(int s, int s6, SecureDht::Config conf) if (key_ and certId != key_->getPublicKey().getId()) throw DhtException("SecureDht: provided certificate doesn't match private key."); - Dht::put(certId, Value { + dht_->put(certId, Value { CERTIFICATE_TYPE, *certificate_, 1 @@ -76,7 +67,7 @@ SecureDht::~SecureDht() ValueType SecureDht::secureType(ValueType&& type) { - type.storePolicy = [this,type](InfoHash id, Sp& v, InfoHash nid, const sockaddr* a, socklen_t al) { + type.storePolicy = [this,type](InfoHash id, Sp& v, const InfoHash& nid, const SockAddr& a) { if (v->isSigned()) { if (!v->owner or !v->owner->checkSignature(v->getToSign(), v->signature)) { DHT_LOG.WARN("Signature verification failed"); @@ -85,11 +76,11 @@ SecureDht::secureType(ValueType&& type) else DHT_LOG.WARN("Signature verification succeeded"); } - return type.storePolicy(id, v, nid, a, al); + return type.storePolicy(id, v, nid, a); }; - type.editPolicy = [this,type](InfoHash id, const Sp& o, Sp& n, InfoHash nid, const sockaddr* a, socklen_t al) { + type.editPolicy = [this,type](InfoHash id, const Sp& o, Sp& n, const InfoHash& nid, const SockAddr& a) { if (!o->isSigned()) - return type.editPolicy(id, o, n, nid, a, al); + return type.editPolicy(id, o, n, nid, a); if (o->owner != n->owner) { DHT_LOG.WARN("Edition forbidden: owner changed."); return false; @@ -190,7 +181,7 @@ SecureDht::findCertificate(const InfoHash& node, std::function(false); - Dht::get(node, [cb,node,found,this](const std::vector>& vals) { + dht_->get(node, [cb,node,found,this](const std::vector>& vals) { if (*found) return false; for (const auto& v : vals) { @@ -238,8 +229,13 @@ SecureDht::getCallbackFilter(GetCallback cb, Value::Filter&& filter) for (const auto& v : values) { // Decrypt encrypted values if (v->isEncrypted()) { - if (not key_) + if (not key_) { +#if OPENDHT_PROXY_SERVER + if (forward_all_) // We are currently a proxy, send messages to clients. + tmpvals.push_back(v); +#endif continue; + } try { Value decrypted_val (decrypt(*v)); if (decrypted_val.recipient == getId()) { @@ -277,13 +273,13 @@ SecureDht::getCallbackFilter(GetCallback cb, Value::Filter&& filter) void SecureDht::get(const InfoHash& id, GetCallback cb, DoneCallback donecb, Value::Filter&& f, Where&& w) { - Dht::get(id, getCallbackFilter(cb, std::forward(f)), donecb, {}, std::forward(w)); + dht_->get(id, getCallbackFilter(cb, std::forward(f)), donecb, {}, std::forward(w)); } size_t -SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter&& f, Where&& w) +SecureDht::listen(const InfoHash& id, GetCallback cb, Value::Filter f, Where w) { - return Dht::listen(id, getCallbackFilter(cb, std::forward(f)), {}, std::forward(w)); + return dht_->listen(id, getCallbackFilter(cb, std::forward(f)), {}, std::forward(w)); } void @@ -295,7 +291,7 @@ SecureDht::putSigned(const InfoHash& hash, Sp val, DoneCallback callback, } // Check if we are already announcing a value - auto p = getPut(hash, val->id); + auto p = dht_->getPut(hash, val->id); if (p && val->seq <= p->seq) { DHT_LOG.DEBUG("Found previous value being announced."); val->seq = p->seq + 1; @@ -317,7 +313,7 @@ SecureDht::putSigned(const InfoHash& hash, Sp val, DoneCallback callback, }, [hash,val,this,callback,permanent] (bool /* ok */) { sign(*val); - put(hash, val, callback, time_point::max(), permanent); + dht_->put(hash, val, callback, time_point::max(), permanent); }, Value::IdFilter(val->id) ); @@ -334,7 +330,7 @@ SecureDht::putEncrypted(const InfoHash& hash, const InfoHash& to, Sp val, } DHT_LOG.WARN("Encrypting data for PK: %s", pk->getId().toString().c_str()); try { - put(hash, encrypt(*val, *pk), callback, time_point::max(), permanent); + dht_->put(hash, encrypt(*val, *pk), callback, time_point::max(), permanent); } catch (const std::exception& e) { DHT_LOG.ERR("Error putting encrypted data: %s", e.what()); if (callback) diff --git a/src/utils.cpp b/src/utils.cpp index afc09f04..c9a7abfd 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -27,6 +27,33 @@ namespace dht { +static constexpr std::array MAPPED_IPV4_PREFIX {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}}; + +std::vector +SockAddr::resolve(const std::string& host, const std::string& service) +{ + std::vector ips {}; + if (host.empty()) + return ips; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_socktype = SOCK_DGRAM; + addrinfo* info = nullptr; + int rc = getaddrinfo(host.c_str(), service.empty() ? nullptr : service.c_str(), &hints, &info); + if(rc != 0) + throw std::invalid_argument(std::string("Error: `") + host + ":" + service + "`: " + gai_strerror(rc)); + + addrinfo* infop = info; + while (infop) { + ips.emplace_back(infop->ai_addr, infop->ai_addrlen); + infop = infop->ai_next; + } + freeaddrinfo(info); + return ips; +} + + std::string print_addr(const sockaddr* sa, socklen_t slen) { @@ -38,7 +65,7 @@ print_addr(const sockaddr* sa, socklen_t slen) out << "[" << hbuf << "]"; else out << hbuf; - if (strcmp(sbuf, "0")) + if (std::strcmp(sbuf, "0")) out << ":" << sbuf; } else out << "[invalid address]"; @@ -51,11 +78,6 @@ print_addr(const sockaddr_storage& ss, socklen_t sslen) return print_addr((const sockaddr*)&ss, sslen); } -std::string -printAddr(const SockAddr& addr) { - return print_addr((const sockaddr*)&addr.first, addr.second); -} - bool SockAddr::isUnspecified() const { @@ -74,7 +96,8 @@ SockAddr::isLoopback() const { switch (getFamily()) { case AF_INET: { - uint8_t b1 = (uint8_t)(getIPv4().sin_addr.s_addr >> 24); + auto addr_host = ntohl(getIPv4().sin_addr.s_addr); + uint8_t b1 = (uint8_t)(addr_host >> 24); return b1 == 127; } case AF_INET6: @@ -91,10 +114,11 @@ SockAddr::isPrivate() const return true; } switch (getFamily()) { - case AF_INET: + case AF_INET: { + auto addr_host = ntohl(getIPv4().sin_addr.s_addr); uint8_t b1, b2; - b1 = (uint8_t)(getIPv4().sin_addr.s_addr >> 24); - b2 = (uint8_t)((getIPv4().sin_addr.s_addr >> 16) & 0x0ff); + b1 = (uint8_t)(addr_host >> 24); + b2 = (uint8_t)((addr_host >> 16) & 0x0ff); // 10.x.y.z if (b1 == 10) return true; @@ -105,6 +129,7 @@ SockAddr::isPrivate() const if ((b1 == 192) && (b2 == 168)) return true; return false; + } case AF_INET6: { const uint8_t* addr6 = reinterpret_cast(&getIPv6().sin6_addr); if (addr6[0] == 0xfc) @@ -116,6 +141,30 @@ SockAddr::isPrivate() const } } +bool +SockAddr::isMappedIPv4() const +{ + if (getFamily() != AF_INET6) + return false; + const uint8_t* addr6 = reinterpret_cast(&getIPv6().sin6_addr); + return std::equal(MAPPED_IPV4_PREFIX.begin(), MAPPED_IPV4_PREFIX.end(), addr6); +} + +SockAddr +SockAddr::getMappedIPv4() const +{ + if (not isMappedIPv4()) + return *this; + SockAddr ret; + ret.setFamily(AF_INET); + ret.setPort(getPort()); + auto addr6 = reinterpret_cast(&getIPv6().sin6_addr); + auto addr4 = reinterpret_cast(&ret.getIPv4().sin_addr); + addr6 += MAPPED_IPV4_PREFIX.size(); + std::copy_n(addr6, sizeof(in_addr), addr4); + return ret; +} + bool operator==(const SockAddr& a, const SockAddr& b) { return a.equals(b); } diff --git a/src/value.cpp b/src/value.cpp index 18456390..fddbcb16 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -22,6 +22,11 @@ #include "default_types.h" #include "securedht.h" // print certificate ID +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT +#include "base64.h" +#endif //OPENDHT_PROXY_SERVER + + namespace dht { const std::string Query::QUERY_PARSE_ERROR {"Error parsing query."}; @@ -35,12 +40,13 @@ Value::Filter bindFilterRaw(FilterRaw raw_filter, void* user_data) { std::ostream& operator<< (std::ostream& s, const Value& v) { + auto flags(s.flags()); s << "Value[id:" << std::hex << v.id << std::dec << " "; if (v.isEncrypted()) s << "encrypted "; else if (v.isSigned()) { s << "signed (v" << v.seq << ") "; - if (v.recipient != InfoHash()) + if (v.recipient) s << "decrypted "; } if (not v.isEncrypted()) { @@ -63,13 +69,14 @@ std::ostream& operator<< (std::ostream& s, const Value& v) } } s << "]"; + s.flags(flags); return s; } const ValueType ValueType::USER_DATA = {0, "User Data"}; bool -ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, InfoHash, const sockaddr*, socklen_t) +ValueType::DEFAULT_STORE_POLICY(InfoHash, std::shared_ptr& v, const InfoHash&, const SockAddr&) { return v->size() <= MAX_VALUE_SIZE; } @@ -165,6 +172,71 @@ Value::msgpack_unpack_body(const msgpack::object& o) } } +#if OPENDHT_PROXY_SERVER || OPENDHT_PROXY_CLIENT +Value::Value(Json::Value& json) +{ + try { + if (json.isMember("id")) + id = ValueType::Id(json["id"].asInt()); + } catch (...) { } + if (json.isMember("cypher")) { + auto cypherStr = json["cypher"].asString(); + cypherStr = base64_decode(cypherStr); + cypher = std::vector(cypherStr.begin(), cypherStr.end()); + } + if (json.isMember("sig")) { + auto sigStr = json["sig"].asString(); + sigStr = base64_decode(sigStr); + signature = std::vector(sigStr.begin(), sigStr.end()); + } + if (json.isMember("seq")) + seq = json["seq"].asInt(); + if (json.isMember("owner")) { + auto ownerStr = json["owner"].asString(); + auto ownerBlob = std::vector(ownerStr.begin(), ownerStr.end()); + owner = std::make_shared(ownerBlob); + } + if (json.isMember("to")) { + auto toStr = json["to"].asString(); + recipient = InfoHash(toStr); + } + if (json.isMember("type")) + type = json["type"].asInt(); + if (json.isMember("data")){ + auto dataStr = json["data"].asString(); + dataStr = base64_decode(dataStr); + data = std::vector(dataStr.begin(), dataStr.end()); + } + if (json.isMember("utype")) + user_type = json["utype"].asString(); +} + +Json::Value +Value::toJson() const +{ + Json::Value val; + val["id"] = std::to_string(id); + if (isEncrypted()) { + val["cypher"] = base64_encode(cypher); + } else { + if (isSigned()) + val["sig"] = base64_encode(signature); + bool has_owner = owner && *owner; + if (has_owner) { // isSigned + val["seq"] = seq; + val["owner"] = owner->toString(); + if (recipient != InfoHash()) + val["to"] = recipient.toString(); + } + val["type"] = type; + val["data"] = base64_encode(data); + if (not user_type.empty()) + val["utype"] = user_type; + } + return val; +} +#endif //OPENDHT_PROXY_SERVER + bool FieldValue::operator==(const FieldValue& vfd) const { @@ -257,9 +329,12 @@ std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) { os << "Index["; for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) { switch (v->first) { - case Value::Field::Id: + case Value::Field::Id: { + auto flags(os.flags()); os << "Id:" << std::hex << v->second.getInt(); + os.flags(flags); break; + } case Value::Field::ValueType: os << "ValueType:" << v->second.getInt(); break; @@ -360,10 +435,10 @@ Where::Where(const std::string& q_str) { std::string s {}; std::istringstream convert {value_str}; convert >> v; - if (convert.failbit + if (not convert and value_str.size() > 1 - and value_str[0] == '\"' - and value_str[value_str.size()-1] == '\"') + and value_str[0] == '"' + and value_str[value_str.size()-1] == '"') s = value_str.substr(1, value_str.size()-2); else s = value_str; @@ -490,4 +565,3 @@ std::ostream& operator<<(std::ostream& s, const dht::Where& where) { } - diff --git a/tools/dhtchat.cpp b/tools/dhtchat.cpp index 8e48723f..8e5654e0 100644 --- a/tools/dhtchat.cpp +++ b/tools/dhtchat.cpp @@ -45,7 +45,7 @@ void print_node_info(const DhtRunner& dht, const dht_params&) { void print_usage() { std::cout << "Usage: dhtchat [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; std::cout << "dhtchat, a simple OpenDHT command line chat client." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } int @@ -67,6 +67,13 @@ main(int argc, char **argv) if (not params.bootstrap.first.empty()) dht.bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str()); +#if OPENDHT_PROXY_CLIENT + if (!params.proxyclient.empty()) { + dht.setProxyServer(params.proxyclient); + dht.enableProxy(true); + } +#endif //OPENDHT_PROXY_CLIENT + print_node_info(dht, params); std::cout << " type 'c {hash}' to join a channel" << std::endl << std::endl; @@ -88,8 +95,6 @@ main(int argc, char **argv) if (line.empty()) continue; - static constexpr dht::InfoHash INVALID_ID {}; - std::istringstream iss(line); std::string op, idstr; iss >> op; @@ -99,7 +104,7 @@ main(int argc, char **argv) else if (op == "c") { iss >> idstr; room = InfoHash(idstr); - if (room == INVALID_ID) { + if (not room) { room = InfoHash::get(idstr); std::cout << "Joining h(" << idstr << ") = " << room << std::endl; } diff --git a/tools/dhtnode.cpp b/tools/dhtnode.cpp index 8aaf75f3..5370e92c 100644 --- a/tools/dhtnode.cpp +++ b/tools/dhtnode.cpp @@ -1,8 +1,9 @@ /* * Copyright (C) 2014-2017 Savoir-faire Linux Inc. * - * Author: Adrien Béraud - * Simon Désaulniers + * Authors: Adrien Béraud + * Simon Désaulniers + * Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,9 +30,9 @@ extern "C" { using namespace dht; void print_usage() { - std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; + std::cout << "Usage: dhtnode [-v [-l logfile]] [-i] [-d] [-n network_id] [-p local_port] [-b bootstrap_host[:port]] [--proxyserver local_port]" << std::endl << std::endl; std::cout << "dhtnode, a simple OpenDHT command line node runner." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } void print_id_req() { @@ -55,15 +56,42 @@ void print_help() { << " ll Print basic information and stats about the current node." << std::endl << " ls [key] Print basic information about current search(es)." << std::endl << " ld [key] Print basic information about currenty stored values on this node (or key)." << std::endl - << " lr Print the full current routing table of this node" << std::endl; + << " lr Print the full current routing table of this node." << std::endl; + +#if OPENDHT_PROXY_SERVER + std::cout << std::endl << "Operations with the proxy:" << std::endl +#if OPENDHT_PUSH_NOTIFICATIONS + << " pst [port] Start the proxy interface on port." << std::endl +#else + << " pst [port] Start the proxy interface on port." << std::endl +#endif // OPENDHT_PUSH_NOTIFICATIONS + << " psp [port] Stop the proxy interface on port." << std::endl; +#endif //OPENDHT_PROXY_SERVER + +#if OPENDHT_PROXY_CLIENT + std::cout << std::endl << "Operations with the proxy:" << std::endl +#if OPENDHT_PUSH_NOTIFICATIONS + << " stt [server_address] Start the proxy client." << std::endl +#else + << " stt [server_address] Start the proxy client." << std::endl +#endif // OPENDHT_PUSH_NOTIFICATIONS +#if OPENDHT_PUSH_NOTIFICATIONS + << " rs [token] Resubscribe to opendht." << std::endl + << " rp [push_notification] Inject a push notification in Opendht." << std::endl +#endif // OPENDHT_PUSH_NOTIFICATIONS + << " stp Stop the proxy client." << std::endl; +#endif //OPENDHT_PROXY_CLIENT std::cout << std::endl << "Operations on the DHT:" << std::endl - << " b Ping potential node at given IP address/port." << std::endl + << " b Ping potential node at given IP address/port." << std::endl << " g Get values at ." << std::endl << " l Listen for value changes at ." << std::endl + << " cl Cancel listen for and ." << std::endl << " p Put string value at ." << std::endl + << " pp Put string value at (persistent version)." << std::endl << " s Put string value at , signed with our generated private key." << std::endl << " e Put string value at , encrypted for with its public key (if found)." << std::endl; + std::cout << std::endl << "Indexation operations on the DHT:" << std::endl << " il [exact match] Lookup the index named with the key ." << std::endl << " Set [exact match] to 'false' for inexact match lookup." << std::endl @@ -71,7 +99,11 @@ void print_help() { << std::endl; } -void cmd_loop(std::shared_ptr& dht, dht_params& params) +void cmd_loop(std::shared_ptr& dht, dht_params& params +#if OPENDHT_PROXY_SERVER + , std::map>& proxies +#endif +) { print_node_info(dht, params); std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl << std::endl; @@ -91,7 +123,7 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) break; std::istringstream iss(line); - std::string op, idstr, value, index, keystr; + std::string op, idstr, value, index, keystr, pushServer, deviceKey; iss >> op; if (op == "x" || op == "exit" || op == "quit") { @@ -115,20 +147,18 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) } else if (op == "ld") { iss >> idstr; InfoHash filter(idstr); - if (filter == InfoHash{}) - std::cout << dht->getStorageLog() << std::endl; - else + if (filter) std::cout << dht->getStorageLog(filter) << std::endl; + else + std::cout << dht->getStorageLog() << std::endl; continue; } else if (op == "ls") { iss >> idstr; InfoHash filter(idstr); - if (filter == InfoHash{}) { - std::cout << "Searches:" << std::endl; - std::cout << dht->getSearchesLog() << std::endl; - } else { + if (filter) std::cout << dht->getSearchLog(filter) << std::endl; - } + else + std::cout << dht->getSearchesLog() << std::endl; continue; } else if (op == "la") { std::cout << "Reported public addresses:" << std::endl; @@ -161,16 +191,72 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) dht->setLogFilter(filter); continue; } +#if OPENDHT_PROXY_SERVER + else if (op == "pst") { +#if OPENDHT_PUSH_NOTIFICATIONS + iss >> idstr >> pushServer; +#else + iss >> idstr; +#endif // OPENDHT_PUSH_NOTIFICATIONS + try { + unsigned int port = std::stoi(idstr); +#if OPENDHT_PUSH_NOTIFICATIONS + proxies.emplace(port, new DhtProxyServer(dht, port, pushServer)); +#else + proxies.emplace(port, new DhtProxyServer(dht, port)); +#endif // OPENDHT_PUSH_NOTIFICATIONS + } catch (...) { } + continue; + } else if (op == "psp") { + iss >> idstr; + try { + auto it = proxies.find(std::stoi(idstr)); + if (it != proxies.end()) + proxies.erase(it); + } catch (...) { } + continue; + } +#endif //OPENDHT_PROXY_SERVER +#if OPENDHT_PROXY_CLIENT + else if (op == "stt") { + /*iss >> idstr >> deviceKey; +#if OPENDHT_PUSH_NOTIFICATIONS + if (not deviceKey.empty()) + dht->setPushNotificationToken(deviceKey); +#endif // OPENDHT_PUSH_NOTIFICATIONS + dht->setProxyServer(idstr);*/ + dht->enableProxy(true); + continue; + } else if (op == "stp") { + dht->enableProxy(false); + continue; + } +#if OPENDHT_PUSH_NOTIFICATIONS + else if (op == "rp") { + iss >> value; + dht->pushNotificationReceived({{"token", value}}); + continue; + } else if (op == "re") { + iss >> value; + try { + unsigned token = std::stoul(value); + dht->resubscribe(token); + } catch (...) { } + continue; + } +#endif // OPENDHT_PUSH_NOTIFICATIONS +#endif //OPENDHT_PROXY_CLIENT if (op.empty()) continue; - static const std::set VALID_OPS {"g", "l", "il", "ii", "p", "s", "e", "a"}; + static const std::set VALID_OPS {"g", "l", "cl", "il", "ii", "p", "pp", "cpp", "s", "e", "a", "q"}; if (VALID_OPS.find(op) == VALID_OPS.cend()) { std::cout << "Unknown command: " << op << std::endl; std::cout << " (type 'h' or 'help' for a list of possible commands)" << std::endl; continue; } + dht::InfoHash id; if (op == "il" or op == "ii") { // Pht syntax @@ -197,12 +283,17 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) else { // Dht syntax iss >> idstr; - InfoHash h {idstr}; - if (not isInfoHash(h)) - continue; + id = dht::InfoHash(idstr); + if (not id) { + if (idstr.empty()) { + std::cerr << "Syntax error: invalid InfoHash." << std::endl; + continue; + } + id = InfoHash::get(idstr); + std::cout << "Using h(" << idstr << ") = " << id << std::endl; + } } - dht::InfoHash id {idstr}; // Dht auto start = std::chrono::high_resolution_clock::now(); if (op == "g") { @@ -236,11 +327,18 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) else if (op == "l") { std::string rem; std::getline(iss, rem); - dht->listen(id, [](std::shared_ptr value) { + auto token = dht->listen(id, [](std::shared_ptr value) { std::cout << "Listen: found value:" << std::endl; std::cout << "\t" << *value << std::endl; return true; }, {}, dht::Where {std::move(rem)}); + auto t = token.get(); + std::cout << "Listening, token: " << t << std::endl; + } + else if (op == "cl") { + std::string rem; + iss >> rem; + dht->cancelListen(id, std::stoul(rem)); } else if (op == "p") { std::string v; @@ -253,6 +351,25 @@ void cmd_loop(std::shared_ptr& dht, dht_params& params) std::cout << "Put: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s)" << std::endl; }); } + else if (op == "pp") { + std::string v; + iss >> v; + auto value = std::make_shared( + dht::ValueType::USER_DATA.id, + std::vector {v.begin(), v.end()} + ); + dht->put(id, value, [start,value](bool ok) { + auto end = std::chrono::high_resolution_clock::now(); + auto flags(std::cout.flags()); + std::cout << "Put: " << (ok ? "success" : "failure") << " (took " << print_dt(end-start) << "s). Value ID: " << std::hex << value->id << std::endl; + std::cout.flags(flags); + }, time_point::max(), true); + } + else if (op == "cpp") { + std::string rem; + iss >> rem; + dht->cancelPut(id, std::stoul(rem, nullptr, 16)); + } else if (op == "s") { if (not params.generate_identity) { print_id_req(); @@ -357,6 +474,7 @@ main(int argc, char **argv) print_usage(); return 0; } + if (params.daemonize) { daemonize(); } else if (params.service) { @@ -365,11 +483,20 @@ main(int argc, char **argv) dht::crypto::Identity crt {}; if (params.generate_identity) { - auto ca_tmp = dht::crypto::generateIdentity("DHT Node CA"); - crt = dht::crypto::generateIdentity("DHT Node", ca_tmp); + auto ca_tmp = dht::crypto::generateEcIdentity("DHT Node CA"); + crt = dht::crypto::generateEcIdentity("DHT Node", ca_tmp); } - dht->run(params.port, crt, true, params.network); + dht::DhtRunner::Config config; + config.dht_config.node_config.network = params.network; + config.dht_config.id = crt; + config.threaded = true; + config.proxy_server = params.proxyclient; + config.push_node_id = "dhtnode"; + if (not params.proxyclient.empty()) + dht->setPushNotificationToken(params.devicekey); + + dht->run(params.port, config); if (params.log) { if (params.syslog or (params.daemonize and params.logfile.empty())) @@ -385,12 +512,27 @@ main(int argc, char **argv) dht->bootstrap(params.bootstrap.first.c_str(), params.bootstrap.second.c_str()); } - if (params.daemonize or params.service) { - while (runner.wait()); - } else { - cmd_loop(dht, params); +#if OPENDHT_PROXY_SERVER + std::map> proxies; +#endif + if (params.proxyserver != 0) { +#if OPENDHT_PROXY_SERVER + proxies.emplace(params.proxyserver, new DhtProxyServer(dht, params.proxyserver, params.pushserver)); +#else + std::cerr << "DHT proxy server requested but OpenDHT built without proxy server support." << std::endl; + exit(EXIT_FAILURE); +#endif } + if (params.daemonize or params.service) + while (runner.wait()); + else + cmd_loop(dht, params +#if OPENDHT_PROXY_SERVER + , proxies +#endif + ); + } catch(const std::exception&e) { std::cerr << std::endl << e.what() << std::endl; } diff --git a/tools/dhtscanner.cpp b/tools/dhtscanner.cpp index 137378c0..46baf597 100644 --- a/tools/dhtscanner.cpp +++ b/tools/dhtscanner.cpp @@ -32,7 +32,7 @@ using namespace dht; void print_usage() { std::cout << "Usage: dhtscanner [-n network_id] [-p local_port] [-b bootstrap_host[:port]]" << std::endl << std::endl; std::cout << "dhtscanner, a simple OpenDHT command line utility generating scan result the network." << std::endl; - std::cout << "Report bugs to: http://opendht.net" << std::endl; + std::cout << "Report bugs to: https://opendht.net" << std::endl; } struct snode_compare { diff --git a/tools/proxy_node.html b/tools/proxy_node.html new file mode 100644 index 00000000..735f0ed7 --- /dev/null +++ b/tools/proxy_node.html @@ -0,0 +1,186 @@ + + + + + + OpenDHT tester + + + + + + + +
+
+
+
+

OpenDHT tester

+
+
+
+
+
+ + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + +
+
+
+
+
+
+ + + + +
+
+
+
+
+
+
+ + diff --git a/tools/tools_common.h b/tools/tools_common.h index 933cc7e2..791a8f11 100644 --- a/tools/tools_common.h +++ b/tools/tools_common.h @@ -2,6 +2,7 @@ * Copyright (C) 2014-2017 Savoir-faire Linux Inc. * * Author: Adrien Béraud + * Author: Sébastien Blin * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -98,10 +99,8 @@ dht::indexation::Pht::Key createPhtKey(std::map pht_ke return pht_key; } -bool isInfoHash(dht::InfoHash& h) { - static constexpr dht::InfoHash INVALID_ID {}; - - if (h == INVALID_ID) { +bool isInfoHash(const dht::InfoHash& h) { + if (not h) { std::cout << "Syntax error: invalid InfoHash." << std::endl; return false; } @@ -121,6 +120,10 @@ struct dht_params { bool daemonize {false}; bool service {false}; std::pair bootstrap {}; + in_port_t proxyserver {0}; + std::string proxyclient {}; + std::string pushserver {}; + std::string devicekey {}; }; static const constexpr struct option long_options[] = { @@ -134,6 +137,10 @@ static const constexpr struct option long_options[] = { {"service", no_argument , nullptr, 's'}, {"logfile", required_argument, nullptr, 'l'}, {"syslog", no_argument , nullptr, 'L'}, + {"proxyserver",required_argument, nullptr, 'S'}, + {"proxyclient",required_argument, nullptr, 'C'}, + {"pushserver", required_argument, nullptr, 'P'}, + {"devicekey", required_argument, nullptr, 'D'}, {nullptr, 0 , nullptr, 0} }; @@ -151,6 +158,23 @@ parseArgs(int argc, char **argv) { std::cout << "Invalid port: " << port_arg << std::endl; } break; + case 'S': { + int port_arg = atoi(optarg); + if (port_arg >= 0 && port_arg < 0x10000) + params.proxyserver = port_arg; + else + std::cout << "Invalid port: " << port_arg << std::endl; + } + break; + case 'P': + params.pushserver = optarg; + break; + case 'C': + params.proxyclient = optarg; + break; + case 'D': + params.devicekey = optarg; + break; case 'n': params.network = strtoul(optarg, nullptr, 0); break;