refactor: Add mem module to allow tests to override allocators.

This will allow us to do more interesting things with memory allocation
within toxcore, and allow fuzzers to explore various allocation failure
paths.

Author: iphydf

auto_tests/announce_test.c

@@ -54,14 +54,17 @@ static void test_store_data(void)
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
     ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
+
     Logger *log = logger_new();
     ck_assert(log != nullptr);
     logger_callback_log(log, print_debug_logger, nullptr, nullptr);
     Mono_Time *mono_time = mono_time_new(nullptr, nullptr);
     Networking_Core *net = new_networking_no_udp(log, ns);
-    DHT *dht = new_dht(log, rng, ns, mono_time, net, true, true);
+    DHT *dht = new_dht(log, mem, rng, ns, mono_time, net, true, true);
     Forwarding *forwarding = new_forwarding(log, rng, mono_time, dht);
-    Announcements *announce = new_announcements(log, rng, mono_time, forwarding);
+    Announcements *announce = new_announcements(log, mem, rng, mono_time, forwarding);
     ck_assert(announce != nullptr);
 
     /* Just to prevent CI from complaining that set_synch_offset is unused: */

auto_tests/forwarding_test.c

@@ -116,6 +116,8 @@ static Forwarding_Subtox *new_forwarding_subtox(bool no_udp, uint32_t *index, ui
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
     ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     if (no_udp) {
         subtox->net = new_networking_no_udp(subtox->log, ns);
@@ -124,15 +126,15 @@ static Forwarding_Subtox *new_forwarding_subtox(bool no_udp, uint32_t *index, ui
         subtox->net = new_networking_ex(subtox->log, ns, &ip, port, port, nullptr);
     }
 
-    subtox->dht = new_dht(subtox->log, rng, ns, subtox->mono_time, subtox->net, true, true);
+    subtox->dht = new_dht(subtox->log, mem, rng, ns, subtox->mono_time, subtox->net, true, true);
 
     const TCP_Proxy_Info inf = {{{{0}}}};
     subtox->c = new_net_crypto(subtox->log, rng, ns, subtox->mono_time, subtox->dht, &inf);
 
     subtox->forwarding = new_forwarding(subtox->log, rng, subtox->mono_time, subtox->dht);
     ck_assert(subtox->forwarding != nullptr);
 
-    subtox->announce = new_announcements(subtox->log, rng, subtox->mono_time, subtox->forwarding);
+    subtox->announce = new_announcements(subtox->log, mem, rng, subtox->mono_time, subtox->forwarding);
     ck_assert(subtox->announce != nullptr);
 
     return subtox;

auto_tests/onion_test.c

@@ -224,20 +224,23 @@ static void test_basic(void)
 {
     uint32_t index[] = { 1, 2, 3 };
     const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
+    const Random *rng = system_random();
+    ck_assert(rng != nullptr);
 
     Logger *log1 = logger_new();
     logger_callback_log(log1, print_debug_logger, nullptr, &index[0]);
     Logger *log2 = logger_new();
     logger_callback_log(log2, print_debug_logger, nullptr, &index[1]);
 
-    const Random *rng = system_random();
-    ck_assert(rng != nullptr);
     Mono_Time *mono_time1 = mono_time_new(nullptr, nullptr);
     Mono_Time *mono_time2 = mono_time_new(nullptr, nullptr);
 
     IP ip = get_loopback();
-    Onion *onion1 = new_onion(log1, mono_time1, rng, new_dht(log1, rng, ns, mono_time1, new_networking(log1, ns, &ip, 36567), true, false));
-    Onion *onion2 = new_onion(log2, mono_time2, rng, new_dht(log2, rng, ns, mono_time2, new_networking(log2, ns, &ip, 36568), true, false));
+    Onion *onion1 = new_onion(log1, mono_time1, rng, new_dht(log1, mem, rng, ns, mono_time1, new_networking(log1, ns, &ip, 36567), true, false));
+    Onion *onion2 = new_onion(log2, mono_time2, rng, new_dht(log2, mem, rng, ns, mono_time2, new_networking(log2, ns, &ip, 36568), true, false));
     ck_assert_msg((onion1 != nullptr) && (onion2 != nullptr), "Onion failed initializing.");
     networking_registerhandler(onion2->net, NET_PACKET_ANNOUNCE_REQUEST, &handle_test_1, onion2);
 
@@ -333,7 +336,7 @@ static void test_basic(void)
 
     Mono_Time *mono_time3 = mono_time_new(nullptr, nullptr);
 
-    Onion *onion3 = new_onion(log3, mono_time3, rng, new_dht(log3, rng, ns, mono_time3, new_networking(log3, ns, &ip, 36569), true, false));
+    Onion *onion3 = new_onion(log3, mono_time3, rng, new_dht(log3, mem, rng, ns, mono_time3, new_networking(log3, ns, &ip, 36569), true, false));
     ck_assert_msg((onion3 != nullptr), "Onion failed initializing.");
 
     random_nonce(rng, nonce);
@@ -400,7 +403,7 @@ typedef struct {
     Onion_Client *onion_c;
 } Onions;
 
-static Onions *new_onions(const Random *rng, uint16_t port, uint32_t *index)
+static Onions *new_onions(const Memory *mem, const Random *rng, uint16_t port, uint32_t *index)
 {
     IP ip = get_loopback();
     ip.ip.v6.uint8[15] = 1;
@@ -437,7 +440,7 @@ static Onions *new_onions(const Random *rng, uint16_t port, uint32_t *index)
         return nullptr;
     }
 
-    DHT *dht = new_dht(on->log, rng, ns, on->mono_time, net, true, false);
+    DHT *dht = new_dht(on->log, mem, rng, ns, on->mono_time, net, true, false);
 
     if (!dht) {
         kill_networking(net);
@@ -576,10 +579,12 @@ static void test_announce(void)
     Onions *onions[NUM_ONIONS];
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     for (i = 0; i < NUM_ONIONS; ++i) {
         index[i] = i + 1;
-        onions[i] = new_onions(rng, i + 36655, &index[i]);
+        onions[i] = new_onions(mem, rng, i + 36655, &index[i]);
         ck_assert_msg(onions[i] != nullptr, "Failed to create onions. %u", i);
     }
 

other/DHT_bootstrap.c

@@ -148,7 +148,8 @@ int main(int argc, char *argv[])
     const uint16_t start_port = PORT;
     const uint16_t end_port = start_port + (TOX_PORTRANGE_TO - TOX_PORTRANGE_FROM);
     const Network *ns = system_network();
-    DHT *dht = new_dht(logger, rng, ns, mono_time, new_networking_ex(logger, ns, &ip, start_port, end_port, nullptr), true, true);
+    const Memory *mem = system_memory();
+    DHT *dht = new_dht(logger, mem, rng, ns, mono_time, new_networking_ex(logger, ns, &ip, start_port, end_port, nullptr), true, true);
     Onion *onion = new_onion(logger, mono_time, rng, dht);
     Forwarding *forwarding = new_forwarding(logger, rng, mono_time, dht);
     GC_Announces_List *gc_announces_list = new_gca_list();

other/bootstrap_daemon/src/tox-bootstrapd.c

@@ -322,8 +322,9 @@ int main(int argc, char *argv[])
 
     mono_time_update(mono_time);
 
+    const Memory *mem = system_memory();
     const Random *rng = system_random();
-    DHT *const dht = new_dht(logger, rng, ns, mono_time, net, true, enable_lan_discovery);
+    DHT *const dht = new_dht(logger, mem, rng, ns, mono_time, net, true, enable_lan_discovery);
 
     if (dht == nullptr) {
         log_write(LOG_LEVEL_ERROR, "Couldn't initialize Tox DHT instance. Exiting.\n");
@@ -350,7 +351,7 @@ int main(int argc, char *argv[])
         return 1;
     }
 
-    Announcements *announce = new_announcements(logger, rng, mono_time, forwarding);
+    Announcements *announce = new_announcements(logger, mem, rng, mono_time, forwarding);
 
     if (announce == nullptr) {
         log_write(LOG_LEVEL_ERROR, "Couldn't initialize DHT announcements. Exiting.\n");

testing/Messenger_test.c

@@ -102,7 +102,7 @@ int main(int argc, char *argv[])
     Messenger_Options options = {0};
     options.ipv6enabled = ipv6enabled;
     Messenger_Error err;
-    m = new_messenger(mono_time, system_random(), system_network(), &options, &err);
+    m = new_messenger(mono_time, system_memory(), system_random(), system_network(), &options, &err);
 
     if (!m) {
         fprintf(stderr, "Failed to allocate messenger datastructure: %d\n", err);

testing/fuzzing/BUILD.bazel

@@ -28,6 +28,7 @@ cc_library(
 
 cc_fuzz_test(
     name = "bootstrap_fuzz_test",
+    #size = "small",
     srcs = ["bootstrap_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:bootstrap_fuzzer"],
@@ -42,6 +43,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "e2e_fuzz_test",
+    #size = "small",
     srcs = ["e2e_fuzz_test.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:e2e_fuzz_test"],
@@ -57,6 +59,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "toxsave_fuzz_test",
+    #size = "small",
     srcs = ["toxsave_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:toxsave_fuzzer"],
@@ -89,6 +92,7 @@ fuzzing_binary(
 
 cc_fuzz_test(
     name = "protodump_reduce",
+    #size = "small",
     srcs = ["protodump_reduce.cc"],
     copts = ["-UNDEBUG"],
     deps = [

toxcore/BUILD.bazel

@@ -15,6 +15,14 @@ cc_library(
     visibility = ["//c-toxcore:__subpackages__"],
 )
 
+cc_library(
+    name = "mem",
+    srcs = ["mem.c"],
+    hdrs = ["mem.h"],
+    visibility = ["//c-toxcore:__subpackages__"],
+    deps = [":attributes"],
+)
+
 cc_library(
     name = "ccompat",
     srcs = ["ccompat.c"],
@@ -315,6 +323,7 @@ cc_library(
         ":ccompat",
         ":crypto_core",
         ":logger",
+        ":mem",
         ":mono_time",
         ":network",
         ":ping_array",
@@ -766,6 +775,7 @@ cc_library(
         ":group",
         ":group_moderation",
         ":logger",
+        ":mem",
         ":mono_time",
         ":network",
         "//c-toxcore/toxencryptsave:defines",

toxcore/DHT.c

@@ -91,6 +91,7 @@ struct DHT {
     const Logger *log;
     const Network *ns;
     Mono_Time *mono_time;
+    const Memory *mem;
     const Random *rng;
     Networking_Core *net;
 
@@ -414,12 +415,13 @@ int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_
     }
 }
 
-int dht_create_packet(const Random *rng, const uint8_t public_key[CRYPTO_PUBLIC_KEY_SIZE],
+int dht_create_packet(const Memory *mem, const Random *rng,
+                      const uint8_t public_key[CRYPTO_PUBLIC_KEY_SIZE],
                       const uint8_t *shared_key, const uint8_t type,
                       const uint8_t *plain, size_t plain_length,
                       uint8_t *packet, size_t length)
 {
-    uint8_t *encrypted = (uint8_t *)malloc(plain_length + CRYPTO_MAC_SIZE);
+    uint8_t *encrypted = (uint8_t *)mem_malloc(mem, plain_length + CRYPTO_MAC_SIZE);
     uint8_t nonce[CRYPTO_NONCE_SIZE];
 
     if (encrypted == nullptr) {
@@ -431,12 +433,12 @@ int dht_create_packet(const Random *rng, const uint8_t public_key[CRYPTO_PUBLIC_
     const int encrypted_length = encrypt_data_symmetric(shared_key, nonce, plain, plain_length, encrypted);
 
     if (encrypted_length == -1) {
-        free(encrypted);
+        mem_free(mem, encrypted);
         return -1;
     }
 
     if (length < 1 + CRYPTO_PUBLIC_KEY_SIZE + CRYPTO_NONCE_SIZE + encrypted_length) {
-        free(encrypted);
+        mem_free(mem, encrypted);
         return -1;
     }
 
@@ -445,7 +447,7 @@ int dht_create_packet(const Random *rng, const uint8_t public_key[CRYPTO_PUBLIC_
     memcpy(packet + 1 + CRYPTO_PUBLIC_KEY_SIZE, nonce, CRYPTO_NONCE_SIZE);
     memcpy(packet + 1 + CRYPTO_PUBLIC_KEY_SIZE + CRYPTO_NONCE_SIZE, encrypted, encrypted_length);
 
-    free(encrypted);
+    mem_free(mem, encrypted);
     return 1 + CRYPTO_PUBLIC_KEY_SIZE + CRYPTO_NONCE_SIZE + encrypted_length;
 }
 
@@ -937,7 +939,8 @@ static bool send_announce_ping(DHT *dht, const uint8_t *public_key, const IP_Por
 
     uint8_t request[1 + CRYPTO_PUBLIC_KEY_SIZE + CRYPTO_NONCE_SIZE + sizeof(plain) + CRYPTO_MAC_SIZE];
 
-    if (dht_create_packet(dht->rng, dht->self_public_key, shared_key, NET_PACKET_DATA_SEARCH_REQUEST,
+    if (dht_create_packet(dht->mem, dht->rng,
+                          dht->self_public_key, shared_key, NET_PACKET_DATA_SEARCH_REQUEST,
                           plain, sizeof(plain), request, sizeof(request)) != sizeof(request)) {
         return false;
     }
@@ -1392,7 +1395,7 @@ bool dht_getnodes(DHT *dht, const IP_Port *ip_port, const uint8_t *public_key, c
 
     const uint8_t *shared_key = dht_get_shared_key_sent(dht, public_key);
 
-    const int len = dht_create_packet(dht->rng,
+    const int len = dht_create_packet(dht->mem, dht->rng,
                                       dht->self_public_key, shared_key, NET_PACKET_GET_NODES,
                                       plain, sizeof(plain), data, sizeof(data));
 
@@ -1442,7 +1445,7 @@ static int sendnodes_ipv6(const DHT *dht, const IP_Port *ip_port, const uint8_t
     const uint32_t crypto_size = 1 + CRYPTO_PUBLIC_KEY_SIZE + CRYPTO_NONCE_SIZE + CRYPTO_MAC_SIZE;
     VLA(uint8_t, data, 1 + nodes_length + length + crypto_size);
 
-    const int len = dht_create_packet(dht->rng,
+    const int len = dht_create_packet(dht->mem, dht->rng,
                                       dht->self_public_key, shared_encryption_key, NET_PACKET_SEND_NODES_IPV6,
                                       plain, 1 + nodes_length + length, data, SIZEOF_VLA(data));
 
@@ -2606,7 +2609,8 @@ static int handle_LANdiscovery(void *object, const IP_Port *source, const uint8_
 
 /*----------------------------------------------------------------------------------*/
 
-DHT *new_dht(const Logger *log, const Random *rng, const Network *ns, Mono_Time *mono_time, Networking_Core *net,
+DHT *new_dht(const Logger *log, const Memory *mem, const Random *rng, const Network *ns,
+             Mono_Time *mono_time, Networking_Core *net,
              bool hole_punching_enabled, bool lan_discovery_enabled)
 {
     if (net == nullptr) {
@@ -2625,6 +2629,7 @@ DHT *new_dht(const Logger *log, const Random *rng, const Network *ns, Mono_Time
     dht->log = log;
     dht->net = net;
     dht->rng = rng;
+    dht->mem = mem;
 
     dht->hole_punching_enabled = hole_punching_enabled;
     dht->lan_discovery_enabled = lan_discovery_enabled;

toxcore/DHT.h

@@ -14,6 +14,7 @@
 #include "attributes.h"
 #include "crypto_core.h"
 #include "logger.h"
+#include "mem.h"
 #include "mono_time.h"
 #include "network.h"
 #include "ping_array.h"
@@ -219,7 +220,7 @@ int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_
  * @retval -1 on failure.
  */
 non_null()
-int dht_create_packet(const Random *rng,
+int dht_create_packet(const Memory *mem, const Random *rng,
                       const uint8_t public_key[CRYPTO_PUBLIC_KEY_SIZE],
                       const uint8_t *shared_key, const uint8_t type,
                       const uint8_t *plain, size_t plain_length,
@@ -494,8 +495,8 @@ int dht_load(DHT *dht, const uint8_t *data, uint32_t length);
 
 /** Initialize DHT. */
 non_null()
-DHT *new_dht(const Logger *log, const Random *rng, const Network *ns, Mono_Time *mono_time, Networking_Core *net,
-             bool hole_punching_enabled, bool lan_discovery_enabled);
+DHT *new_dht(const Logger *log, const Memory *mem, const Random *rng, const Network *ns,
+             Mono_Time *mono_time, Networking_Core *net, bool hole_punching_enabled, bool lan_discovery_enabled);
 
 nullable(1)
 void kill_dht(DHT *dht);