search: return if added node is actually removed

Link against libatomic also on architectures that need it for 64-bit
atomic operations.  ARM EABI (armel) and little-endian MIPS (mipsel)
are two such architectures.

.github/workflows/ccpp.yml

@@ -57,7 +57,7 @@ jobs:
 
   build-macos:
     name: macOS/Clang build
-    runs-on: macos-11
+    runs-on: macos-12
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies

CMakeLists.txt

@@ -15,7 +15,7 @@ include(cmake/CheckAtomic.cmake)
 include(CTest)
 
 set (opendht_VERSION_MAJOR 3)
-set (opendht_VERSION_MINOR 0.0)
+set (opendht_VERSION_MINOR 1.4)
 set (opendht_VERSION ${opendht_VERSION_MAJOR}.${opendht_VERSION_MINOR})
 set (PACKAGE_VERSION ${opendht_VERSION})
 set (VERSION "${opendht_VERSION}")
@@ -47,7 +47,9 @@ set (CMAKE_CXX_STANDARD 17)
 set (CMAKE_CXX_STANDARD_REQUIRED on)
 
 # Dependencies
-if (NOT HAVE_CXX_ATOMICS_WITHOUT_LIB)
+if (NOT HAVE_CXX_ATOMICS_WITHOUT_LIB
+        # For ARM EABI (armel), little-endian MIPS (mipsel), etc.
+        OR NOT HAVE_CXX_ATOMICS64_WITHOUT_LIB)
     link_libraries (atomic)
 endif ()
 

configure.ac

@@ -1,7 +1,7 @@
 dnl define macros
 m4_define([opendht_major_version], 3)
-m4_define([opendht_minor_version], 0)
-m4_define([opendht_patch_version], 0)
+m4_define([opendht_minor_version], 1)
+m4_define([opendht_patch_version], 4)
 m4_define([opendht_version],
 		  [opendht_major_version.opendht_minor_version.opendht_patch_version])
 

include/opendht/crypto.h

@@ -772,6 +772,7 @@ OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name, const Identi
 OPENDHT_PUBLIC Identity generateEcIdentity(const std::string& name = "dhtnode", const Identity& ca = {});
 
 OPENDHT_PUBLIC void saveIdentity(const Identity& id, const std::string& path, const std::string& privkey_password = {});
+OPENDHT_PUBLIC Identity loadIdentity(const std::string &path,const std::string &privkey_password = {});
 
 /**
  * Performs SHA512, SHA256 or SHA1, depending on hash_length.
@@ -792,7 +793,7 @@ OPENDHT_PUBLIC void hash(const uint8_t* data, size_t data_length, uint8_t* hash,
  * that can be transmitted in clear, and will be generated if
  * not provided (32 bytes).
  */
-OPENDHT_PUBLIC Blob stretchKey(const std::string& password, Blob& salt, size_t key_length = 512/8);
+OPENDHT_PUBLIC Blob stretchKey(std::string_view password, Blob& salt, size_t key_length = 512/8);
 
 /**
  * AES-GCM encryption. Key must be 128, 192 or 256 bits long (16, 24 or 32 bytes).
@@ -801,15 +802,55 @@ OPENDHT_PUBLIC Blob aesEncrypt(const uint8_t* data, size_t data_length, const Bl
 OPENDHT_PUBLIC inline Blob aesEncrypt(const Blob& data, const Blob& key) {
     return aesEncrypt(data.data(), data.size(), key);
 }
-OPENDHT_PUBLIC Blob aesEncrypt(const Blob& data, const std::string& password);
+/**
+ * AES-GCM encryption with argon2 key derivation.
+ * This function uses `stretchKey` to generate an AES key from the password and a random salt.
+ * The result is a bundle including the salt that can be decrypted with `aesDecrypt(data, password)`.
+ * If needed, the salt or encrypted data can be individually extracted from the bundle with `aesGetSalt` and `aesGetEncrypted`.
+ */
+OPENDHT_PUBLIC Blob aesEncrypt(const Blob& data, std::string_view password);
 
 /**
  * AES-GCM decryption.
  */
 OPENDHT_PUBLIC Blob aesDecrypt(const uint8_t* data, size_t data_length, const Blob& key);
 OPENDHT_PUBLIC inline Blob aesDecrypt(const Blob& data, const Blob& key) { return aesDecrypt(data.data(), data.size(), key); }
-OPENDHT_PUBLIC Blob aesDecrypt(const uint8_t* data, size_t data_length, const std::string& password);
-OPENDHT_PUBLIC inline Blob aesDecrypt(const Blob& data, const std::string& password) { return aesDecrypt(data.data(), data.size(), password); }
+OPENDHT_PUBLIC inline Blob aesDecrypt(std::string_view data, const Blob& key) { return aesDecrypt((uint8_t*)data.data(), data.size(), key); }
+
+OPENDHT_PUBLIC Blob aesDecrypt(const uint8_t* data, size_t data_length, std::string_view password);
+OPENDHT_PUBLIC inline Blob aesDecrypt(const Blob& data, std::string_view password) { return aesDecrypt(data.data(), data.size(), password); }
+OPENDHT_PUBLIC inline Blob aesDecrypt(std::string_view data, std::string_view password) { return aesDecrypt((uint8_t*)data.data(), data.size(), password); }
+
+/**
+ * Get raw AES key from password and salt stored with the encrypted data.
+ */
+OPENDHT_PUBLIC Blob aesGetKey(const uint8_t* data, size_t data_length, std::string_view password);
+OPENDHT_PUBLIC Blob inline aesGetKey(const Blob& data, std::string_view password) {
+    return aesGetKey(data.data(), data.size(), password);
+}
+/** Get the salt part of data password-encrypted with `aesEncrypt(data, password)` */
+OPENDHT_PUBLIC Blob aesGetSalt(const uint8_t* data, size_t data_length);
+OPENDHT_PUBLIC Blob inline aesGetSalt(const Blob& data) {
+    return aesGetSalt(data.data(), data.size());
+}
+/** Get the encrypted data part of data password-encrypted with `aesEncrypt(data, password)` */
+OPENDHT_PUBLIC std::string_view aesGetEncrypted(const uint8_t* data, size_t data_length);
+OPENDHT_PUBLIC std::string_view inline aesGetEncrypted(const Blob& data) {
+    return aesGetEncrypted(data.data(), data.size());
+}
+
+/** Build an encrypted bundle that can be decrypted with aesDecrypt(data, password).
+ *  @param encryptedData: result of `aesEncrypt(data, key)` or `aesGetEncrypted`
+ *  @param salt: should match the encryption key and password so that `stretchKey(password, salk) == key`.
+ *  Can be obtained from an existing bundle with `aesGetSalt`.
+ **/
+OPENDHT_PUBLIC Blob aesBuildEncrypted(const uint8_t* encryptedData, size_t data_length, const Blob& salt);
+OPENDHT_PUBLIC Blob inline aesBuildEncrypted(const Blob& encryptedData, const Blob& salt) {
+    return aesBuildEncrypted(encryptedData.data(), encryptedData.size(), salt);
+}
+OPENDHT_PUBLIC Blob inline aesBuildEncrypted(std::string_view encryptedData, const Blob& salt) {
+    return aesBuildEncrypted((const uint8_t*)encryptedData.data(), encryptedData.size(), salt);
+}
 
 }
 }

meson.build

@@ -1,5 +1,5 @@
 project('opendht', 'c', 'cpp',
-    version: '3.0.0',
+    version: '3.1.4',
     default_options: [
         'cpp_std=c++17',
         'warning_level=3'

python/opendht.pyx

@@ -462,6 +462,26 @@ cdef class Identity(object):
             k._key = self._id.first
             return k
 
+def aesEncrypt(bytes data, str password) -> bytes :
+    cdef size_t d_len = len(data)
+    cdef cpp.uint8_t* d_ptr = <cpp.uint8_t*>data
+    cdef cpp.Blob indat
+    indat.assign(d_ptr, <cpp.uint8_t*>(d_ptr + d_len))
+    cdef cpp.Blob encrypted = cpp.aesEncrypt(indat, password.encode())
+    cdef char* encrypted_c_str = <char *>encrypted.data()
+    cdef Py_ssize_t length = encrypted.size()
+    return encrypted_c_str[:length]
+
+def aesDecrypt(bytes data, str password) -> bytes :
+    cdef size_t d_len = len(data)
+    cdef cpp.uint8_t* d_ptr = <cpp.uint8_t*>data
+    cdef cpp.Blob indat
+    indat.assign(d_ptr, <cpp.uint8_t*>(d_ptr + d_len))
+    cdef cpp.Blob decrypted = cpp.aesDecrypt(indat, password.encode())
+    cdef char* decrypted_c_str = <char *>decrypted.data()
+    cdef Py_ssize_t length = decrypted.size()
+    return decrypted_c_str[:length]
+
 cdef class DhtConfig(object):
     cdef cpp.DhtRunnerConfig _config
     def __init__(self):
@@ -609,6 +629,7 @@ cdef class DhtRunner(_WithID):
                 while pending > 0:
                     lock.wait()
             return res
+
     def put(self, InfoHash key, Value val, done_cb=None, permanent=False):
         """Publish a new value on the DHT at key.
 
@@ -637,6 +658,72 @@ cdef class DhtRunner(_WithID):
                     lock.wait()
             return ok
 
+    def putSigned(self, InfoHash key, Value val, done_cb=None, permanent=False):
+        if done_cb:
+            cb_obj = {'done':done_cb}
+            ref.Py_INCREF(cb_obj)
+            self.thisptr.get().putSigned(key._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj), permanent)
+        else:
+            lock = threading.Condition()
+            pending = 0
+            ok = False
+            def tmp_done(ok_ret, nodes):
+                nonlocal pending, ok, lock
+                with lock:
+                    ok = ok_ret
+                    pending -= 1
+                    lock.notify()
+            with lock:
+                pending += 1
+                self.putSigned(key, val, done_cb=tmp_done)
+                while pending > 0:
+                    lock.wait()
+            return ok
+
+    def putEncrypted(self, InfoHash key, InfoHash to, Value val, done_cb=None, bool permanent=False):
+        if done_cb:
+            cb_obj = {'done':done_cb}
+            ref.Py_INCREF(cb_obj)
+            self.thisptr.get().putEncrypted(key._infohash, to._infohash, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj), permanent)
+        else:
+            lock = threading.Condition()
+            pending = 0
+            ok = False
+            def tmp_done(ok_ret, nodes):
+                nonlocal pending, ok, lock
+                with lock:
+                    ok = ok_ret
+                    pending -= 1
+                    lock.notify()
+            with lock:
+                pending += 1
+                self.putEncrypted(key, to, val, done_cb=tmp_done, permanent=permanent)
+                while pending > 0:
+                    lock.wait()
+            return ok
+
+    def putEncrypted(self, InfoHash key, PublicKey to, Value val, done_cb=None, bool permanent=False):
+        if done_cb:
+            cb_obj = {'done':done_cb}
+            ref.Py_INCREF(cb_obj)
+            self.thisptr.get().putEncrypted(key._infohash, to._key, val._value, cpp.bindDoneCb(done_callback, <void*>cb_obj), permanent)
+        else:
+            lock = threading.Condition()
+            pending = 0
+            ok = False
+            def tmp_done(ok_ret, nodes):
+                nonlocal pending, ok, lock
+                with lock:
+                    ok = ok_ret
+                    pending -= 1
+                    lock.notify()
+            with lock:
+                pending += 1
+                self.putEncrypted(key, to, val, done_cb=tmp_done, permanent=permanent)
+                while pending > 0:
+                    lock.wait()
+            return ok
+
     def cancelPut(self, InfoHash key, Value val):
         self.thisptr.get().cancelPut(key._infohash, val._value)
 

python/opendht_cpp.pxd

@@ -94,6 +94,8 @@ ctypedef vector[uint8_t] Blob
 cdef extern from "opendht/crypto.h" namespace "dht::crypto":
     ctypedef pair[shared_ptr[PrivateKey], shared_ptr[Certificate]] Identity
     cdef Identity generateIdentity(string name, Identity ca, unsigned bits)
+    cdef Blob aesEncrypt(Blob data, string password) except +
+    cdef Blob aesDecrypt(Blob encrypted, string password) except +
 
     cdef cppclass PrivateKey:
         PrivateKey()
@@ -267,6 +269,9 @@ cdef extern from "opendht/dhtrunner.h" namespace "dht":
         string getSearchesLog(sa_family_t af) const
         void get(InfoHash key, GetCallback get_cb, DoneCallback done_cb, nullptr_t f, Where w)
         void put(InfoHash key, shared_ptr[Value] val, DoneCallback done_cb, time_point created, bool permanent)
+        void putSigned(InfoHash key, shared_ptr[Value] val, DoneCallback done_cb, bool permanent) 
+        void putEncrypted(InfoHash key, InfoHash to, shared_ptr[Value] val, DoneCallback done_cb, bool permanent)
+        void putEncrypted(InfoHash key, shared_ptr[PublicKey] to, shared_ptr[Value] val, DoneCallback done_cb, bool permanent)
         void cancelPut(InfoHash key, shared_ptr[Value] val)
         ListenToken listen(InfoHash key, ValueCallback get_cb)
         void cancelListen(InfoHash key, SharedListenToken token)

src/crypto.cpp

@@ -103,13 +103,11 @@ Blob aesEncrypt(const uint8_t* data, size_t data_length, const Blob& key)
     return ret;
 }
 
-Blob aesEncrypt(const Blob& data, const std::string& password)
+Blob aesEncrypt(const Blob& data, std::string_view password)
 {
     Blob salt;
     Blob key = stretchKey(password, salt, 256 / 8);
-    Blob encrypted = aesEncrypt(data, key);
-    encrypted.insert(encrypted.begin(), salt.begin(), salt.end());
-    return encrypted;
+    return aesBuildEncrypted(aesEncrypt(data, key), salt);
 }
 
 Blob aesDecrypt(const uint8_t* data, size_t data_length, const Blob& key)
@@ -152,16 +150,44 @@ Blob aesDecrypt(const uint8_t* data, size_t data_length, const Blob& key)
     return ret;
 }
 
-Blob aesDecrypt(const uint8_t* data, size_t data_len, const std::string& password)
+Blob aesDecrypt(const uint8_t* data, size_t data_length, std::string_view password)
 {
-    if (data_len <= PASSWORD_SALT_LENGTH)
-        throw DecryptError("Wrong data size");
-    Blob salt {data, data+PASSWORD_SALT_LENGTH};
-    Blob key = stretchKey(password, salt, 256/8);
-    return aesDecrypt(data+PASSWORD_SALT_LENGTH, data_len - PASSWORD_SALT_LENGTH, key);
+    return aesDecrypt(
+        aesGetEncrypted(data, data_length),
+        aesGetKey(data, data_length, password)
+    );
 }
 
-Blob stretchKey(const std::string& password, Blob& salt, size_t key_length)
+Blob aesGetSalt(const uint8_t* data, size_t data_length)
+{
+    if (data_length <= PASSWORD_SALT_LENGTH)
+        throw DecryptError("Wrong data size");
+    return Blob {data, data+PASSWORD_SALT_LENGTH};
+}
+
+std::string_view aesGetEncrypted(const uint8_t* data, size_t data_length)
+{
+    if (data_length <= PASSWORD_SALT_LENGTH)
+        throw DecryptError("Wrong data size");
+    return std::string_view((const char*)(data+PASSWORD_SALT_LENGTH), data_length - PASSWORD_SALT_LENGTH);
+}
+
+Blob aesBuildEncrypted(const uint8_t* data, size_t data_length, const Blob& salt)
+{
+    Blob ret;
+    ret.reserve(data_length + salt.size());
+    ret.insert(ret.end(), salt.begin(), salt.end());
+    ret.insert(ret.end(), data, data + data_length);
+    return ret;
+}
+
+Blob aesGetKey(const uint8_t* data, size_t data_length, std::string_view password)
+{
+    Blob salt = aesGetSalt(data, data_length);
+    return stretchKey(password, salt, 256/8);
+}
+
+Blob stretchKey(std::string_view password, Blob& salt, size_t key_length)
 {
     if (salt.empty()) {
         salt.resize(PASSWORD_SALT_LENGTH);
@@ -1132,6 +1158,26 @@ saveIdentity(const Identity& id, const std::string& path, const std::string& pri
     }
 }
 
+Identity 
+loadIdentity(const std::string &path,const std::string &privkey_password)
+{
+    std::ifstream pkStream(path + ".pem", std::ios::in | std::ios::binary);
+    std::vector<uint8_t> pkContent((std::istreambuf_iterator<char>(pkStream)),
+                                    std::istreambuf_iterator<char>());
+    auto key = std::make_shared<PrivateKey>(pkContent, privkey_password);
+    pkStream.close();
+    // Create a certificate
+    gnutls_x509_crt_t gnuCert;
+    if (gnutls_x509_crt_init(&gnuCert) != GNUTLS_E_SUCCESS)
+        throw std::runtime_error("Failed to initialize gnutls certificate struct");
+    gnutls_datum_t crtContent;
+    // Read the certificate file
+    gnutls_load_file((path + ".crt").c_str(), &crtContent);
+    gnutls_x509_crt_import(gnuCert, &crtContent, GNUTLS_X509_FMT_PEM);
+    auto cert = std::make_shared<Certificate>(gnuCert);
+    return {std::move(key), std::move(cert)};
+}
+
 void
 setValidityPeriod(gnutls_x509_crt_t cert, int64_t validity)
 {

src/search.h

@@ -854,12 +854,17 @@ Dht::Search::insertNode(const Sp<Node>& snode, time_point now, const Blob& token
             expired = false;
         }
 
-        while (nodes.size() - bad >  SEARCH_NODES) {
+        while (nodes.size() - bad > SEARCH_NODES) {
+            bool removingNode = nodes.back()->node == snode;
             if (not expired and nodes.back()->isBad())
                 bad--;
             nodes.pop_back();
+            if (removingNode)
+                return false;
         }
     }
+    if (n == nodes.end() or not *n)
+        return false;
     if (not token.empty()) {
         (*n)->candidate = false;
         (*n)->last_get_reply = now;

tests/cryptotester.cpp

@@ -173,6 +173,72 @@ void CryptoTester::testOcsp() {
     CPPUNIT_ASSERT(ocspRequest.second == req.getNonce());
 }
 
+void CryptoTester::testAesEncryption() {
+    auto password = "this is a password 123414!@#%@#$?" + std::to_string(rand());
+
+    std::vector<uint8_t> data1 {5, 10};
+    std::vector<uint8_t> data2(128 * 1024 + 13, 10);
+
+    auto encrypted1 = dht::crypto::aesEncrypt(data1, password);
+    auto encrypted2 = dht::crypto::aesEncrypt(data2, password);
+
+    auto decrypted1 = dht::crypto::aesDecrypt(encrypted1, password);
+    auto decrypted2 = dht::crypto::aesDecrypt(encrypted2, password);
+
+    CPPUNIT_ASSERT(data1 != encrypted1);
+    CPPUNIT_ASSERT(data2 != encrypted2);
+    CPPUNIT_ASSERT(data1 == decrypted1);
+    CPPUNIT_ASSERT(data2 == decrypted2);
+
+    auto key1 = dht::crypto::aesGetKey(encrypted1, password);
+    auto key2 = dht::crypto::aesGetKey(encrypted2, password);
+    auto encrypted1_data = dht::crypto::aesGetEncrypted(encrypted1);
+    auto encrypted2_data = dht::crypto::aesGetEncrypted(encrypted2);
+
+    CPPUNIT_ASSERT(key1 != key2);
+
+    decrypted1 = dht::crypto::aesDecrypt(encrypted1_data, key1);
+    decrypted2 = dht::crypto::aesDecrypt(encrypted2_data, key2);
+
+    CPPUNIT_ASSERT(data1 == decrypted1);
+    CPPUNIT_ASSERT(data2 == decrypted2);
+
+    auto salt1 = dht::crypto::aesGetSalt(encrypted1);
+    auto salt2 = dht::crypto::aesGetSalt(encrypted2);
+
+    CPPUNIT_ASSERT(salt1 != salt2);
+
+    auto encrypted11 = dht::crypto::aesBuildEncrypted(encrypted1_data, salt1);
+    auto encrypted22 = dht::crypto::aesBuildEncrypted(encrypted2_data, salt2);
+
+    CPPUNIT_ASSERT(encrypted11 == encrypted1);
+    CPPUNIT_ASSERT(encrypted22 == encrypted2);
+
+    auto key12 = dht::crypto::stretchKey(password, salt1, 256/8);
+    auto key22 = dht::crypto::stretchKey(password, salt2, 256/8);
+
+    CPPUNIT_ASSERT(key1 == key12);
+    CPPUNIT_ASSERT(key2 == key22);
+
+    decrypted1 = dht::crypto::aesDecrypt(encrypted1_data, key12);
+    decrypted2 = dht::crypto::aesDecrypt(encrypted2_data, key22);
+
+    CPPUNIT_ASSERT(data1 == decrypted1);
+    CPPUNIT_ASSERT(data2 == decrypted2);
+
+    auto encrypted12_data = dht::crypto::aesEncrypt(data1, key12);
+    auto encrypted22_data = dht::crypto::aesEncrypt(data2, key22);
+
+    encrypted11 = dht::crypto::aesBuildEncrypted(encrypted12_data, salt1);
+    encrypted22 = dht::crypto::aesBuildEncrypted(encrypted22_data, salt2);
+
+    decrypted1 = dht::crypto::aesDecrypt(encrypted11, password);
+    decrypted2 = dht::crypto::aesDecrypt(encrypted22, password);
+
+    CPPUNIT_ASSERT(data1 == decrypted1);
+    CPPUNIT_ASSERT(data2 == decrypted2);
+}
+
 void
 CryptoTester::tearDown() {
 

tests/cryptotester.h

@@ -33,6 +33,7 @@ class CryptoTester : public CppUnit::TestFixture {
     CPPUNIT_TEST(testCertificateRequest);
     CPPUNIT_TEST(testCertificateSerialNumber);
     CPPUNIT_TEST(testOcsp);
+    CPPUNIT_TEST(testAesEncryption);
     CPPUNIT_TEST_SUITE_END();
 
  public:
@@ -64,6 +65,10 @@ class CryptoTester : public CppUnit::TestFixture {
      * Test OCSP
      */
     void testOcsp();
+    /**
+     * Test key streching and aes encryption/decryption
+     */
+    void testAesEncryption();
 };
 
 }  // namespace test