feat: add forwarding and announce request handling

This is the "server-side" part of the new friend finding system,
allowing DHT nodes to store small amounts of data and permit searching
for it. A forwarding (proxying) mechanism allows this to be used by TCP
clients, and deals with non-transitivity in the network.

Author: zugz

.github/scripts/flags-clang.sh

@@ -50,8 +50,8 @@ add_flag -Wno-unused-function
 add_flag -Wno-used-but-marked-unused
 # We use variable length arrays a lot.
 add_flag -Wno-vla
-# Disable warning about Doxygen retval tag
-add_flag -fcomment-block-commands=retval
+# Disable warnings about unknown Doxygen commands
+add_flag -Wno-documentation-unknown-command
 
 # Disable specific warning flags for C++.
 

CMakeLists.txt

@@ -197,6 +197,8 @@ set(toxcore_PKGCONFIG_REQUIRES)
 set(toxcore_SOURCES
   third_party/cmp/cmp.c
   third_party/cmp/cmp.h
+  toxcore/announce.c
+  toxcore/announce.h
   toxcore/bin_pack.c
   toxcore/bin_pack.h
   toxcore/bin_unpack.c
@@ -230,6 +232,8 @@ set(toxcore_SOURCES
   toxcore/events/friend_status_message.c
   toxcore/events/friend_typing.c
   toxcore/events/self_connection_status.c
+  toxcore/forwarding.c
+  toxcore/forwarding.h
   toxcore/friend_connection.c
   toxcore/friend_connection.h
   toxcore/friend_requests.c

auto_tests/BUILD.bazel

@@ -48,8 +48,10 @@ flaky_tests = {
         "//c-toxcore/toxcore:TCP_common",
         "//c-toxcore/toxcore:TCP_connection",
         "//c-toxcore/toxcore:TCP_server",
+        "//c-toxcore/toxcore:announce",
         "//c-toxcore/toxcore:ccompat",
         "//c-toxcore/toxcore:crypto_core",
+        "//c-toxcore/toxcore:forwarding",
         "//c-toxcore/toxcore:friend_connection",
         "//c-toxcore/toxcore:logger",
         "//c-toxcore/toxcore:mono_time",

auto_tests/CMakeLists.txt

@@ -19,6 +19,7 @@ function(auto_test target)
 endfunction()
 
 auto_test(TCP)
+auto_test(announce)
 auto_test(conference)
 auto_test(conference_double_invite)
 auto_test(conference_invite_merge)
@@ -31,6 +32,7 @@ auto_test(dht_getnodes_api)
 auto_test(encryptsave)
 auto_test(file_transfer)
 auto_test(file_saving)
+auto_test(forwarding)
 auto_test(friend_connection)
 auto_test(friend_request)
 auto_test(friend_request_spam)

auto_tests/Makefile.inc

@@ -5,6 +5,7 @@ libauto_test_support_la_SOURCES = ../auto_tests/auto_test_support.c ../auto_test
 libauto_test_support_la_LIBADD = libmisc_tools.la libtoxcore.la
 
 TESTS = \
+        announce_test \
 	conference_double_invite_test \
 	conference_invite_merge_test \
 	conference_peer_nick_test \
@@ -13,6 +14,7 @@ TESTS = \
 	conference_two_test \
 	crypto_test \
 	file_transfer_test \
+	forwarding_test \
 	friend_connection_test \
 	friend_request_test \
 	invalid_tcp_proxy_test \
@@ -68,6 +70,10 @@ endif
 
 check_PROGRAMS = $(TESTS)
 
+announce_test_SOURCES = ../auto_tests/announce_test.c
+announce_test_CFLAGS = $(AUTOTEST_CFLAGS)
+announce_test_LDADD = $(AUTOTEST_LDADD)
+
 conference_double_invite_test_SOURCES = ../auto_tests/conference_double_invite_test.c
 conference_double_invite_test_CFLAGS = $(AUTOTEST_CFLAGS)
 conference_double_invite_test_LDADD = $(AUTOTEST_LDADD)
@@ -108,6 +114,10 @@ file_transfer_test_SOURCES = ../auto_tests/file_transfer_test.c
 file_transfer_test_CFLAGS = $(AUTOTEST_CFLAGS)
 file_transfer_test_LDADD = $(AUTOTEST_LDADD)
 
+forwarding_test_SOURCES = ../auto_tests/forwarding_test.c
+forwarding_test_CFLAGS = $(AUTOTEST_CFLAGS)
+forwarding_test_LDADD = $(AUTOTEST_LDADD)
+
 friend_connection_test_SOURCES = ../auto_tests/friend_connection_test.c
 friend_connection_test_CFLAGS = $(AUTOTEST_CFLAGS)
 friend_connection_test_LDADD = $(AUTOTEST_LDADD)

auto_tests/TCP_test.c

@@ -56,7 +56,7 @@ static void test_basic(void)
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
     const Network *ns = system_network();
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create a TCP relay server.");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS,
                   "Failed to bind a TCP relay server to all %d attempted ports.", NUM_PORTS);
@@ -306,7 +306,7 @@ static void test_some(void)
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create TCP relay server");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS, "Failed to bind to all ports.");
 
@@ -497,7 +497,7 @@ static void test_client(void)
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
     const Network *ns = system_network();
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create a TCP relay server.");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS, "Failed to bind the relay server to all ports.");
 
@@ -704,7 +704,7 @@ static void test_tcp_connection(void)
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(pk_equal(tcp_server_public_key(tcp_s), self_public_key), "Wrong public key");
 
     TCP_Proxy_Info proxy_info;
@@ -815,7 +815,7 @@ static void test_tcp_connection2(void)
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(pk_equal(tcp_server_public_key(tcp_s), self_public_key), "Wrong public key");
 
     TCP_Proxy_Info proxy_info;

auto_tests/announce_test.c

@@ -0,0 +1,123 @@
+#include <stdint.h>
+#include <string.h>
+
+#include "../toxcore/announce.h"
+#include "../toxcore/tox.h"
+#include "../testing/misc_tools.h"
+#include "../toxcore/mono_time.h"
+#include "../toxcore/forwarding.h"
+#include "../toxcore/net_crypto.h"
+#include "../toxcore/util.h"
+#include "auto_test_support.h"
+#include "check_compat.h"
+
+static void test_bucketnum(void)
+{
+    const Random *rng = system_random();
+    ck_assert(rng != nullptr);
+    uint8_t key1[CRYPTO_PUBLIC_KEY_SIZE], key2[CRYPTO_PUBLIC_KEY_SIZE];
+    random_bytes(rng, key1, sizeof(key1));
+    memcpy(key2, key1, CRYPTO_PUBLIC_KEY_SIZE);
+
+    ck_assert_msg(get_bucketnum(key1, key2) == 0, "Bad bucketnum");
+
+    key2[4] ^= 0x09;
+    key2[5] ^= 0xc5;
+
+    ck_assert_msg(get_bucketnum(key1, key2) == 7, "Bad bucketnum");
+
+    key2[4] ^= 0x09;
+
+    ck_assert_msg(get_bucketnum(key1, key2) == 17, "Bad bucketnum");
+
+    key2[5] ^= 0xc5;
+    key2[31] ^= 0x09;
+
+    ck_assert_msg(get_bucketnum(key1, key2) == 4, "Bad bucketnum");
+}
+
+typedef struct Announce_Test_Data {
+    uint8_t data[MAX_ANNOUNCEMENT_SIZE];
+    uint16_t length;
+    bool passed;
+} Announce_Test_Data;
+
+static void test_announce_data(void *object, const uint8_t *data, uint16_t length)
+{
+    Announce_Test_Data *test_data = (Announce_Test_Data *) object;
+    test_data->passed = test_data->length == length && memcmp(test_data->data, data, length) == 0;
+}
+
+static void test_store_data(void)
+{
+    const Random *rng = system_random();
+    ck_assert(rng != nullptr);
+    const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    Logger *log = logger_new();
+    ck_assert(log != nullptr);
+    logger_callback_log(log, (logger_cb *)print_debug_log, nullptr, nullptr);
+    Mono_Time *mono_time = mono_time_new();
+    Networking_Core *net = new_networking_no_udp(log, ns);
+    DHT *dht = new_dht(log, rng, ns, mono_time, net, true, true);
+    Forwarding *forwarding = new_forwarding(log, rng, mono_time, dht);
+    Announcements *announce = new_announcements(log, rng, mono_time, forwarding);
+    ck_assert(announce != nullptr);
+
+    /* Just to prevent CI from complaining that set_synch_offset is unused: */
+    set_synch_offset(announce, 0);
+
+    Announce_Test_Data test_data;
+    random_bytes(rng, test_data.data, sizeof(test_data.data));
+    test_data.length = sizeof(test_data.data);
+
+    uint8_t key[CRYPTO_PUBLIC_KEY_SIZE];
+    random_bytes(rng, key, sizeof(key));
+
+    ck_assert_msg(!on_stored(announce, key, nullptr, nullptr), "Unstored announcement exists");
+
+    ck_assert_msg(store_data(announce, key, test_data.data, sizeof(test_data.data),
+                             MAX_MAX_ANNOUNCEMENT_TIMEOUT), "Failed to store announcement");
+
+    ck_assert_msg(on_stored(announce, key, test_announce_data, &test_data), "Failed to get stored announcement");
+
+    ck_assert_msg(test_data.passed, "Bad stored announcement data");
+
+    const uint8_t *const base = dht_get_self_public_key(dht);
+    ck_assert_msg(store_data(announce, base, test_data.data, sizeof(test_data.data), 1), "failed to store base");
+
+    uint8_t test_keys[ANNOUNCE_BUCKET_SIZE + 1][CRYPTO_PUBLIC_KEY_SIZE];
+
+    for (uint8_t i = 0; i < ANNOUNCE_BUCKET_SIZE + 1; ++i) {
+        memcpy(test_keys[i], base, CRYPTO_PUBLIC_KEY_SIZE);
+        test_keys[i][i] ^= 1;
+        ck_assert_msg(store_data(announce, test_keys[i], test_data.data, sizeof(test_data.data), 1),
+                      "Failed to store announcement %d", i);
+    }
+
+    ck_assert_msg(on_stored(announce, base, nullptr, nullptr), "base was evicted");
+    ck_assert_msg(!on_stored(announce, test_keys[0], nullptr, nullptr), "furthest was not evicted");
+    ck_assert_msg(!store_data(announce, test_keys[0], nullptr, 0, 1), "furthest evicted closer");
+
+    kill_announcements(announce);
+    kill_forwarding(forwarding);
+    kill_dht(dht);
+    kill_networking(net);
+    mono_time_free(mono_time);
+    logger_kill(log);
+}
+
+static void basic_announce_tests(void)
+{
+    test_bucketnum();
+    test_store_data();
+}
+
+
+int main(void)
+{
+    setvbuf(stdout, nullptr, _IONBF, 0);
+
+    basic_announce_tests();
+    return 0;
+}