diff --git a/staticmemory/memory-bucket-optimizer/README.md b/staticmemory/memory-bucket-optimizer/README.md new file mode 100644 index 000000000..d528c803a --- /dev/null +++ b/staticmemory/memory-bucket-optimizer/README.md @@ -0,0 +1,68 @@ +# Memory Bucket Optimizer for wolfSSL + +** Note that the staticmemory build of wolfSSL does not use less memory ** +The staticmemory feature ends up using a bit more memory and is a simple sectioning up of a static buffer used dynamically instead of malloc/free. wolfSSL has the option for users to define custom XMALLOC/XFREE if wanting to use a different allocater. + + +The exact optimized configuration is a difficult problem (think traveling salesman problem), but this optimizer gives a good starting point. It uses a simple algorithm in that it fill the first half of bucket sizes with largest allocations and the second half based on max concurrent uses. + +## Directory Structure + +``` +memory-bucket-optimizer/ +├── optimizer/ # Source code for the optimizer +├── tester/ # Source code for testing configuration +└── README.md # This file +``` + +## Prerequisites + +- wolfSSL (built with `CPPFLAGS="-DWOLFSSL_DEBUG_MEMORY -DWOLFSSL_DEBUG_MEMORY_PRINT` and `--enable-staticmemory`) +- GCC compiler +- gnuplot (for visualization) + +## Building + +```bash +make +``` + +## Usage + +### Basic Usage + +1. Build wolfSSL with memory logging enabled: + +```bash +cd ~/wolfssl +./configure CPPFLAGS="-DWOLFSSL_DEBUG_MEMORY -DWOLFSSL_DEBUG_MEMORY_PRINT" --enable-staticmemory +make +sudo make install +``` + +2. Run the application linked to wolfssl: + +```bash +./wolfcrypt/test/testwolfcrypt &> testwolfcrypt.log +``` + +This will run the application with memory log output. + +3. Run the log through the optimizer + +```bash +cd optimizer +make +./memory_bucket_optimizer testwolfcrypt.log +``` + +4. Build and run tester (optional) + +``` +cd ~/wolfssl +./configure CPPFLAGS="-DWOLFSSL_NO_MALLOC -DWOLFSSL_DEBUG_MEMORY -DWOLFSSL_DEBUG_MEMORY_PRINT" --enable-staticmemory +make && sudo make install +cd tester && make +./memory_bucket_tester ../testwolfcrypt.log --buckets "289,832,1056,1072,1152,1616,1632,3160,4240" --dist "2,1,2,1,1,1,1,19,1" --buffer-size 74298 +``` + diff --git a/staticmemory/memory-bucket-optimizer/optimizer/Makefile b/staticmemory/memory-bucket-optimizer/optimizer/Makefile new file mode 100644 index 000000000..0a087d1e0 --- /dev/null +++ b/staticmemory/memory-bucket-optimizer/optimizer/Makefile @@ -0,0 +1,32 @@ +# Makefile for memory_bucket_optimizer +# +# Copyright (C) 2006-2025 wolfSSL Inc. +# +# This file is part of wolfSSL. +# +# wolfSSL is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# wolfSSL is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + +CC = gcc +LDFLAGS = -lwolfssl + +all: memory_bucket_optimizer + +memory_bucket_optimizer: memory_bucket_optimizer.c + $(CC) $(CFLAGS) -o memory_bucket_optimizer memory_bucket_optimizer.c $(LDFLAGS) + +clean: + rm -f memory_bucket_optimizer + +.PHONY: all clean diff --git a/staticmemory/memory-bucket-optimizer/optimizer/memory_bucket_optimizer.c b/staticmemory/memory-bucket-optimizer/optimizer/memory_bucket_optimizer.c new file mode 100644 index 000000000..657ade1b1 --- /dev/null +++ b/staticmemory/memory-bucket-optimizer/optimizer/memory_bucket_optimizer.c @@ -0,0 +1,716 @@ +/* memory_bucket_optimizer.c + * + * Copyright (C) 2025 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#include +#include +#include +#include +#include /* Required for INT_MAX */ + +#include +#include + +#define MAX_LINE_LENGTH 1024 +#define MAX_UNIQUE_BUCKETS 9 /* Maximum number of unique bucket sizes to create */ + +/* Configuration notes: + * - MAX_UNIQUE_BUCKETS: Limits the total number of unique bucket sizes + * This helps control memory overhead and bucket management complexity + * Default: 9 buckets (can be adjusted based on memory constraints) + */ + +/* Linked list node for allocation events */ +typedef struct AllocationEventNode { + int size; + int timestamp; /* Simple counter for allocation order */ + int active; /* 1 if allocated, 0 if freed */ + struct AllocationEventNode* next; +} AllocationEventNode; +AllocationEventNode* event_head = NULL; + +/* Linked list node for unique allocation sizes */ +typedef struct AllocSizeNode { + int size; + int count; + int concurrent; + int max_concurrent; /* Maximum number of concurrent allocations of this size */ + struct AllocSizeNode* next; /* next in list of sizes sorted by size */ + struct AllocSizeNode* nextFreq; /* sorted by count size descending */ +} AllocSizeNode; + +/* Linked list for allocation events */ +typedef struct { + AllocationEventNode* head; + AllocationEventNode* tail; + int count; +} AllocationEventList; + +/* Linked list for unique allocation sizes */ +typedef struct { + AllocSizeNode* head; + int count; +} AllocSizeList; + +/* Helper functions for linked lists */ +AllocationEventNode* create_allocation_event_node(int size, int timestamp, + int active) +{ + AllocationEventNode* node; + + node = (AllocationEventNode*)malloc(sizeof(AllocationEventNode)); + if (node) { + node->size = size; + node->timestamp = timestamp; + node->active = active; + node->next = NULL; + } + return node; +} + +void add_allocation_event(AllocationEventNode** list, int size, int timestamp, + int active) +{ + AllocationEventNode* node; + + node = create_allocation_event_node(size, timestamp, active); + if (node) { + if (*list == NULL) { + event_head = node; + *list = node; + } else { + (*list)->next = node; + *list = node; /* Update the list pointer to point to the new node */ + } + } +} + +AllocSizeNode* create_alloc_size_node(int size) +{ + AllocSizeNode* node; + + node = (AllocSizeNode*)malloc(sizeof(AllocSizeNode)); + if (node) { + node->size = size; + node->count = 0; + node->concurrent = 0; + node->max_concurrent = 0; + node->next = NULL; + node->nextFreq = NULL; + } + return node; +} + +AllocSizeNode* find_or_create_alloc_size(AllocSizeNode** list, int size) +{ + AllocSizeNode* current = *list; + AllocSizeNode* previous = NULL; + AllocSizeNode* node = NULL; + + /* Look for existing size */ + while (current) { + if (current->size == size) { + return current; + } + current = current->next; + } + + /* Create new size node */ + node = create_alloc_size_node(size); + if (node) { + /* insert node into list ordered from largest size first to smallest */ + current = *list; + if (current == NULL) { + *list = node; + } + else { + while (current != NULL) { + if (current->size < size) { + node->next = current; + if (previous != NULL) { + previous->next = node; + } + else { + *list = node; + } + break; + } + previous = current; + current = current->next; + } + /* If we reached the end of the list, append the node */ + if (current == NULL) { + previous->next = node; + } + } + } + return node; +} + +void free_allocation_event_list(AllocationEventNode* list) +{ + AllocationEventNode* current = list; + while (current) { + AllocationEventNode* next = current->next; + free(current); + current = next; + } +} + +void free_alloc_size_list(AllocSizeNode* list) +{ + AllocSizeNode* current = list; + while (current) { + AllocSizeNode* next = current->next; + free(current); + current = next; + } +} + +/* Function to calculate memory padding size per bucket */ +int calculate_padding_size() +{ + return wolfSSL_MemoryPaddingSz(); +} + +/* Function to calculate total memory overhead */ +int calculate_total_overhead(int num_buckets) +{ + /* Total overhead includes: + * - WOLFSSL_HEAP structure + * - WOLFSSL_HEAP_HINT structure + * - Alignment padding + * Note: Padding is already included in bucket sizes + */ + int total_overhead = sizeof(WOLFSSL_HEAP) + + sizeof(WOLFSSL_HEAP_HINT) + + (WOLFSSL_STATIC_ALIGN - 1); + total_overhead += num_buckets * wolfSSL_MemoryPaddingSz(); + return total_overhead; +} + +/* Function to parse memory allocation logs with concurrent usage tracking */ +int parse_memory_logs(const char* filename, AllocationEventNode** events, + int* peak_heap_usage, int* buckets) +{ + int current_heap_usage = 0; + char line[MAX_LINE_LENGTH]; + int timestamp = 0; + FILE* file; + + file = fopen(filename, "r"); + if (!file) { + printf("Error: Could not open file %s\n", filename); + return -1; + } + + *peak_heap_usage = 0; /* Initialize peak heap usage */ + + while (fgets(line, sizeof(line), file)) { + /* Look for lines containing "Alloc:" or "Free:" */ + char* alloc_pos = strstr(line, "Alloc:"); + char* free_pos = strstr(line, "Free:"); + int size; + + if (alloc_pos) { + /* Handle multiple formats: + * Format 1: Alloc: 0x55fde046b490 -> 4 (11) at wolfTLSv1_3_client_method_ex:src/tls.c:16714 + * Format 2: [HEAP 0x1010e2110] Alloc: 0x101108a40 -> 1024 at simple_mem_test:18561 + * Format 3: (Using global heap hint 0x1010e2110) [HEAP 0x0] Alloc: 0x101107440 -> 1584 at _sp_exptmod_nct:14231 + */ + if (sscanf(alloc_pos, "Alloc: %*s -> %d", &size) == 1) { + /* Here we begin the bucket list, as a simple tracking of + * largest allocs encountered. */ + int i; + for (i = 0; i < MAX_UNIQUE_BUCKETS; i++) { + if (size > buckets[i]) { + buckets[i] = size; + break; + } + } + current_heap_usage += size; + if (current_heap_usage > *peak_heap_usage) { + *peak_heap_usage = current_heap_usage; + } + add_allocation_event(events, size, timestamp++, 1); + } + } + else if (free_pos) { + /* Handle multiple formats: + * Format 1: Free: 0x55fde046b490 -> 4 at wolfTLSv1_3_client_method_ex:src/tls.c:16714 + * Format 2: [HEAP 0x1010e2110] Free: 0x101108a40 -> 1024 at simple_mem_test:18576 + * Format 3: (Using global heap hint 0x1010e2110) [HEAP 0x0] Free: 0x101107440 -> 1584 at _sp_exptmod_nct:14462 + */ + if (sscanf(free_pos, "Free: %*s -> %d", &size) == 1) { + current_heap_usage -= size; + if (current_heap_usage < 0) { + current_heap_usage = 0; + } + add_allocation_event(events, size, timestamp++, 0); + } + } + } + + fclose(file); + return 0; +} + + +/* This goes through all the events and finds unique allocations and the max + * concurent use of each unique allocation */ +static void find_max_concurent_allocations(AllocSizeNode** alloc_sizes) +{ + AllocationEventNode* current = event_head; + while (current != NULL) { + if (current->active) { + AllocSizeNode* alloc_size = + find_or_create_alloc_size(alloc_sizes, current->size); + alloc_size->concurrent++; + alloc_size->count++; + if (alloc_size->max_concurrent < alloc_size->concurrent) { + alloc_size->max_concurrent = alloc_size->concurrent; + } + } + else { + AllocSizeNode* alloc_size = + find_or_create_alloc_size(alloc_sizes, current->size); + alloc_size->concurrent--; + } + current = current->next; + } +} + + +/* This function makes sure that for every alloc there is a bucket avilable */ +static void set_distributions(int* buckets, int* dist, int num_buckets) +{ + AllocationEventNode* current = event_head; + int max_concurrent_use[num_buckets]; + int current_use[num_buckets]; + int i; + + /* Initialize arrays to zero */ + memset(max_concurrent_use, 0, sizeof(max_concurrent_use)); + memset(current_use, 0, sizeof(current_use)); + + while (current != NULL) { + /* find bucket this would go in */ + for (i = 0; i < num_buckets; i++) { + if (current->size <= (buckets[i] - wolfSSL_MemoryPaddingSz())) { + break; + } + } + + /* Only process if we found a valid bucket */ + if (i < num_buckets) { + if (current->active) { + current_use[i] += 1; + if (current_use[i] > max_concurrent_use[i]) { + max_concurrent_use[i] = current_use[i]; + } + } + else { + current_use[i] -= 1; + } + } else { + printf("ERROR: allocation size %d is larger than all bucket sizes!\n", + current->size); + printf("This indicates a bug in the bucket optimization algorithm.\n"); + printf("Largest bucket size: %d, allocation size: %d\n", + num_buckets > 0 ? buckets[num_buckets-1] : 0, current->size); + exit(1); + } + current = current->next; + } + + for (i = 0; i < num_buckets; i++) { + dist[i] = max_concurrent_use[i]; + } +} + +static void sort_alloc_by_frequency(AllocSizeNode* alloc_sizes, + AllocSizeNode** sorted) +{ + AllocSizeNode* max; + AllocSizeNode* current; + AllocSizeNode* tail; + int current_count = 0; + int current_upper_bound = INT_MAX; + + *sorted = NULL; /* Initialize to NULL */ + + do { + max = NULL; + current = alloc_sizes; /* Reset current to beginning of list */ + while (current != NULL) { + if (current->count > current_count && + current->size < current_upper_bound) { + current_count = current->count; + max = current; + } + current = current->next; + } + + if (max == NULL) { + break; /* No more nodes to process */ + } + + current_upper_bound = max->size; + if (*sorted == NULL) { + *sorted = max; + tail = max; + } + else { + tail->nextFreq = max; + tail = max; + } + tail->nextFreq = NULL; + } while (max != NULL); +} + +/* returns what the bucket size would be */ +static int get_bucket_size(int size) +{ + int padding; + + padding = size % WOLFSSL_STATIC_ALIGN; + if (padding > 0) { + padding = WOLFSSL_STATIC_ALIGN - padding; + } + return size + padding + wolfSSL_MemoryPaddingSz(); +} + +/* Function to optimize bucket sizes */ +/* + * Optimization heuristic: + * - Always include the largest allocation size + * - For other sizes, only create a new bucket if the waste from using + * existing buckets is >= padding size per bucket + * - This reduces bucket management overhead when waste is minimal + * - Limited to MAX_UNIQUE_BUCKETS total unique bucket sizes + */ +void optimize_buckets(AllocSizeNode* alloc_sizes, + AllocSizeNode* alloc_sizes_by_freq, int num_sizes, int* buckets, int* dist, + int* num_buckets) +{ + int i, j; + AllocSizeNode* current; + + /* Initialize bucket count */ + *num_buckets = 0; + + /* Always include the largest allocation sizes (with padding) */ + current = alloc_sizes; + for (i = 0; i < MAX_UNIQUE_BUCKETS/2 && current != NULL; i++) { + buckets[*num_buckets] = get_bucket_size(current->size); + dist[*num_buckets] = current->max_concurrent; + (*num_buckets)++; + current = current->next; + } + + /* Fill out the other half based on max concurent use */ + for (i = *num_buckets; i < MAX_UNIQUE_BUCKETS; i++) { + int max_concurrent = 0; + AllocSizeNode* max = NULL; + + current = alloc_sizes; + while (current != NULL) { + if (current->max_concurrent > max_concurrent) { + /* Skip if already included */ + int already_included = 0; + for (j = 0; j < *num_buckets; j++) { + if (buckets[j] == get_bucket_size(current->size)) { + already_included = 1; + break; + } + } + if (!already_included) { + max_concurrent = current->max_concurrent; + max = current; + } + } + current = current->next; + } + if (max != NULL) { + buckets[*num_buckets] = get_bucket_size(max->size); + dist[*num_buckets] = max->max_concurrent; + *num_buckets += 1; + } + else { + break; + } + } + + /* Sort buckets by size (ascending) */ + for (i = 0; i < *num_buckets - 1; i++) { + for (j = 0; j < *num_buckets - i - 1; j++) { + if (buckets[j] > buckets[j + 1]) { + /* Swap bucket sizes */ + int temp = buckets[j]; + buckets[j] = buckets[j + 1]; + buckets[j + 1] = temp; + + /* Swap distribution values */ + temp = dist[j]; + dist[j] = dist[j + 1]; + dist[j + 1] = temp; + } + } + } + set_distributions(buckets, dist, *num_buckets); + + /* Print optimization summary */ + printf("Optimization Summary:\n"); + printf("Padding size per bucket: %d bytes\n", calculate_padding_size()); + printf("Maximum unique buckets allowed: %d\n", MAX_UNIQUE_BUCKETS); + printf("Total buckets created: %d\n", *num_buckets); + if (*num_buckets >= MAX_UNIQUE_BUCKETS) { + printf("Note: Reached maximum bucket limit (%d). Some allocations may use larger buckets.\n", MAX_UNIQUE_BUCKETS); + } + printf("Note: Allocations with waste < padding size use existing buckets to reduce overhead\n"); + printf("Note: Bucket limit helps balance memory efficiency vs. management overhead\n\n"); +} + +/* Function to calculate memory efficiency metrics */ +void calculate_memory_efficiency(AllocSizeNode* alloc_sizes, int num_sizes, + int* buckets, int* dist, int num_buckets) +{ + AllocSizeNode* current = alloc_sizes; + int i, j; + float total_waste = 0.0; + int total_allocations = 0; + int allocations_handled = 0; + int padding_size = calculate_padding_size(); + + printf("Memory Efficiency Analysis:\n"); + printf("Note: Allocations with waste < %d bytes (padding size) use existing buckets\n", padding_size); + printf("Size Count Concurrent Bucket Waste Coverage\n"); + printf("---- ----- ---------- ------ ----- --------\n"); + + for (i = 0; i < num_sizes && current != NULL; i++) { + int size = current->size; + int count = current->count; + total_allocations += count; + + /* Find the smallest bucket that can fit this allocation */ + int best_bucket = -1; + int min_waste = INT_MAX; + + for (j = 0; j < num_buckets; j++) { + /* Bucket sizes now include padding, so we need to subtract it for comparison */ + int bucket_data_size = buckets[j] - calculate_padding_size(); + if (bucket_data_size >= size) { + int waste = bucket_data_size - size; + if (waste < min_waste) { + min_waste = waste; + best_bucket = j; + } + } + } + + if (best_bucket >= 0) { + allocations_handled += count; + total_waste += (float)min_waste * count; + printf("%-7d %-7d %-10d %-7d %-7d %s\n", + size, count, current->max_concurrent, buckets[best_bucket], + min_waste, "✓"); + } else { + printf("%-7d %-7d %-10d %-7s %-7s %s\n", + size, count, current->max_concurrent, "N/A", "N/A", "✗"); + } + current = current->next; + } + + printf("\nEfficiency Summary:\n"); + printf("Total allocations: %d\n", total_allocations); + printf("Allocations handled: %d (%.1f%%)\n", + allocations_handled, + (float)allocations_handled * 100 / total_allocations); + printf("Total memory waste: %.2f bytes\n", total_waste); + printf("Average waste per allocation: %.2f bytes\n", + total_waste / total_allocations); + + /* Calculate total memory needed for buckets */ + int total_bucket_memory = 0; + int total_num_buckets = 0; + for (i = 0; i < num_buckets; i++) { + total_bucket_memory += buckets[i] * dist[i]; + total_num_buckets += dist[i]; + } + + /* Calculate total overhead */ + int total_overhead = calculate_total_overhead(total_num_buckets); + int total_memory_needed = total_bucket_memory + total_overhead; + + printf("Total bucket memory: %d bytes\n", total_bucket_memory); + printf("Memory overhead: %d bytes\n", total_overhead); + printf(" - Padding per bucket: %d bytes (included in bucket sizes)\n", + calculate_padding_size()); + printf(" - Heap structures: %ld bytes\n", sizeof(WOLFSSL_HEAP) + + sizeof(WOLFSSL_HEAP_HINT)); + printf(" - Alignment: %d bytes\n", WOLFSSL_STATIC_ALIGN - 1); + printf("Total memory needed: %d bytes\n", total_memory_needed); + + /* Calculate efficiency based on actual data vs total memory */ + float data_memory = 0; + current = alloc_sizes; + for (i = 0; i < num_sizes && current != NULL; i++) { + data_memory += current->size * current->count; + current = current->next; + } + printf("Data memory: %.0f bytes\n", data_memory); +} + +/* Function to provide buffer size recommendations */ +void print_buffer_recommendations(int* buckets, int* dist, int num_buckets) +{ + int total_bucket_memory = 0, total_overhead = 0, total_memory_needed, i; + + for (i = 0; i < num_buckets; i++) { + total_bucket_memory += buckets[i] * dist[i]; + total_overhead += dist[i] * wolfSSL_MemoryPaddingSz(); + } + + total_overhead += sizeof(WOLFSSL_HEAP_HINT) + sizeof(WOLFSSL_HEAP) + + WOLFSSL_STATIC_ALIGN; + total_memory_needed = total_bucket_memory + total_overhead; + + printf("\nBuffer Size Recommendations:\n"); + printf("============================\n"); + printf("Minimum buffer size needed: %d bytes\n", total_memory_needed); + + printf("\nUsage in wolfSSL application:\n"); + printf("============================\n"); + printf("// Allocate buffer\n"); + printf("byte staticBuffer[%d];\n", total_memory_needed); + printf("\n// Load static memory\n"); + printf("WOLFSSL_HEAP_HINT* heapHint = NULL;\n"); + printf("if (wc_LoadStaticMemory_ex(&heapHint, %d, bucket_sizes, bucket_dist,\n", + num_buckets); + printf(" staticBuffer, %d, 0, 0) != 0) {\n", total_memory_needed); + printf(" // Handle error\n"); + printf("}\n"); + printf("\n// Use in wolfSSL context\n"); + printf("wolfSSL_CTX_load_static_memory(&method, NULL, staticBuffer,\n"); + printf(" %d, 0, 1);\n", total_memory_needed); +} + +int main(int argc, char** argv) +{ + int i; + int buckets[MAX_UNIQUE_BUCKETS]; + int dist[MAX_UNIQUE_BUCKETS]; + int num_sizes = 0; + int peak_heap_usage = 0; + int num_buckets = 0; + AllocationEventNode* events = NULL; + AllocSizeNode* alloc_sizes = NULL; + AllocSizeNode* alloc_sizes_by_freq = NULL; + AllocSizeNode* current; + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + return 1; + } + + /* Initialize buckets array to 0 */ + memset(buckets, 0, sizeof(buckets)); + + /* Parse memory allocation logs */ + if (parse_memory_logs(argv[1], &events, &peak_heap_usage, buckets) != 0) { + return 1; + } + + + find_max_concurent_allocations(&alloc_sizes); + sort_alloc_by_frequency(alloc_sizes, &alloc_sizes_by_freq); + + current = alloc_sizes; + while (current!= NULL) { + num_sizes++; + current = current->next; + } + + printf("Found %d unique allocation sizes\n", num_sizes); + printf("Peak heap usage: %d bytes (maximum concurrent memory usage)\n\n", + peak_heap_usage); + + /* Print allocation sizes, frequencies, and concurrent usage */ + printf("Allocation Sizes, Frequencies, and Concurrent Usage:\n"); + printf("Size Count Max Concurrent\n"); + printf("---- ----- --------------\n"); + current = alloc_sizes; + while (current != NULL) { + printf("%-7d %-7d %d\n", current->size, current->count, + current->max_concurrent); + current = current->next; + } + printf("\n"); + + /* Optimize bucket sizes */ + optimize_buckets(alloc_sizes, alloc_sizes_by_freq, num_sizes, buckets, dist, + &num_buckets); + + /* Print optimized bucket sizes and distribution */ + printf("Optimized Bucket Sizes and Distribution:\n"); + printf("Data Size + Padding = Bucket Size Dist\n"); + printf("----------------------------------------\n"); + + for (i = 0; i < num_buckets; i++) { + int data_size = buckets[i] - calculate_padding_size(); + printf("%-7d + %-7d = %-7d %d\n", + data_size, calculate_padding_size(), buckets[i], dist[i]); + } + printf("\n"); + + /* Print WOLFMEM_BUCKETS and WOLFMEM_DIST macros */ + printf("WOLFMEM_BUCKETS and WOLFMEM_DIST Macros:\n"); + printf("#define WOLFMEM_BUCKETS "); + for (i = 0; i < num_buckets; i++) { + printf("%d", buckets[i]); + if (i < num_buckets - 1) { + printf(","); + } + } + printf("\n"); + + printf("#define WOLFMEM_DIST "); + for (i = 0; i < num_buckets; i++) { + printf("%d", dist[i]); + if (i < num_buckets - 1) { + printf(","); + } + } + printf("\n"); + + /* Calculate and print memory efficiency metrics */ + calculate_memory_efficiency(alloc_sizes, num_sizes, buckets, dist, + num_buckets); + + /* Print buffer size recommendations */ + print_buffer_recommendations(buckets, dist, num_buckets); + + /* Clean up events list */ + free_allocation_event_list(events); + free_alloc_size_list(alloc_sizes); + /* alloc_sizes_by_freq is the same nodes as alloc_sizes */ + + return 0; +} diff --git a/staticmemory/memory-bucket-optimizer/tester/Makefile b/staticmemory/memory-bucket-optimizer/tester/Makefile new file mode 100644 index 000000000..4faf41202 --- /dev/null +++ b/staticmemory/memory-bucket-optimizer/tester/Makefile @@ -0,0 +1,37 @@ +# Makefile for memory bucket tester +# +# Copyright (C) 2025 wolfSSL Inc. +# +# This file is part of wolfSSL. +# +# wolfSSL is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# wolfSSL is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + +CC = gcc +CFLAGS = -Wall -Wextra -O2 -g +LIBS = -lwolfssl + +TARGET = memory_bucket_tester +SOURCE = memory_bucket_tester.c + +.PHONY: all clean test + +all: $(TARGET) + +$(TARGET): $(SOURCE) + $(CC) $(CFLAGS) $(INCLUDES) -o $@ $< $(LIBS) + +clean: + rm -f $(TARGET) + diff --git a/staticmemory/memory-bucket-optimizer/tester/memory_bucket_tester.c b/staticmemory/memory-bucket-optimizer/tester/memory_bucket_tester.c new file mode 100644 index 000000000..9552b8b32 --- /dev/null +++ b/staticmemory/memory-bucket-optimizer/tester/memory_bucket_tester.c @@ -0,0 +1,383 @@ +/* memory_bucket_tester.c + * + * Copyright (C) 2025 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#include +#include +#include +#include + +#include +#include + +/* Memory overhead constants - these should match wolfSSL's actual values */ +#ifndef WOLFSSL_STATIC_ALIGN + #define WOLFSSL_STATIC_ALIGN 8 +#endif + +#ifndef MAX_ALLOCATIONS + #define MAX_ALLOCATIONS 10000 +#endif + +#define MAX_LINE_LENGTH 1024 +#define MAX_BUCKETS 16 + +typedef struct { + int size; + int is_alloc; /* 1 for alloc, 0 for free */ + void* ptr; /* For tracking allocated pointers */ +} AllocationEvent; + +typedef struct { + int size; + int count; +} BucketConfig; + +/* Function to parse bucket configuration from command line */ +int parse_bucket_config(int argc, char** argv, int start_idx, + BucketConfig* buckets, int* num_buckets, + int* total_buffer_size) +{ + int i; + + if (start_idx >= argc) { + printf("Error: No bucket configuration provided\n"); + return -1; + } + + *num_buckets = 0; + *total_buffer_size = 0; + i = start_idx; + + while (i < argc && *num_buckets < MAX_BUCKETS) { + if (strcmp(argv[i], "--buckets") == 0) { + i++; + if (i < argc) { + char* bucket_str = argv[i]; + char* token = strtok(bucket_str, ","); + + while (token != NULL && *num_buckets < MAX_BUCKETS) { + buckets[*num_buckets].size = atoi(token); + buckets[*num_buckets].count = 1; /* Default count */ + (*num_buckets)++; + token = strtok(NULL, ","); + } + i++; + } + } else if (strcmp(argv[i], "--dist") == 0) { + i++; + if (i < argc) { + char* dist_str = argv[i]; + char* token = strtok(dist_str, ","); + int dist_idx = 0; + + while (token != NULL && dist_idx < *num_buckets) { + buckets[dist_idx].count = atoi(token); + dist_idx++; + token = strtok(NULL, ","); + } + i++; + } + } else if (strcmp(argv[i], "--buffer-size") == 0) { + i++; + if (i < argc) { + *total_buffer_size = atoi(argv[i]); + i++; + } + } else { + i++; + } + } + + return 0; +} + +/* Function to parse memory allocation logs */ +int parse_memory_logs(const char* filename, AllocationEvent* events, + int* num_events) +{ + char line[MAX_LINE_LENGTH]; + FILE* file; + + file = fopen(filename, "r"); + if (!file) { + printf("Error: Could not open file %s\n", filename); + return -1; + } + + *num_events = 0; + + while (fgets(line, sizeof(line), file) && *num_events < MAX_ALLOCATIONS) { + /* Look for lines containing "Alloc:" or "Free:" */ + char* alloc_pos = strstr(line, "Alloc:"); + char* free_pos = strstr(line, "Free:"); + + if (alloc_pos) { + int size; + /* Handle multiple formats: + * Format 1: Alloc: 0x55fde046b490 -> 4 (11) at wolfTLSv1_3_client_method_ex:src/tls.c:16714 + * Format 2: [HEAP 0x1010e2110] Alloc: 0x101108a40 -> 1024 at simple_mem_test:18561 + * Format 3: (Using global heap hint 0x1010e2110) [HEAP 0x0] Alloc: 0x101107440 -> 1584 at _sp_exptmod_nct:14231 + */ + if (sscanf(alloc_pos, "Alloc: %*s -> %d", &size) == 1) { + events[*num_events].size = size; + events[*num_events].is_alloc = 1; + events[*num_events].ptr = NULL; + (*num_events)++; + } + } else if (free_pos) { + int size; + /* Handle multiple formats: + * Format 1: Free: 0x55fde046b490 -> 4 at wolfTLSv1_3_client_method_ex:src/tls.c:16714 + * Format 2: [HEAP 0x1010e2110] Free: 0x101108a40 -> 1024 at simple_mem_test:18576 + * Format 3: (Using global heap hint 0x1010e2110) [HEAP 0x0] Free: 0x101107440 -> 1584 at _sp_exptmod_nct:14462 + */ + if (sscanf(free_pos, "Free: %*s -> %d", &size) == 1) { + events[*num_events].size = size; + events[*num_events].is_alloc = 0; + events[*num_events].ptr = NULL; + (*num_events)++; + } + } + } + + if (*num_events >= MAX_ALLOCATIONS) { + printf("Error: Too many allocation events (over %d)\n", MAX_ALLOCATIONS); + return -1; + } + + fclose(file); + return 0; +} + + +#ifdef WOLFSSL_NO_MALLOC +/* Function to replay allocation sequence */ +int replay_allocation_sequence(AllocationEvent* events, int num_events, + WOLFSSL_HEAP_HINT* heap_hint ) +{ + int i; + int success_count = 0; + int failure_count = 0; + + for (i = 0; i < num_events; i++) { + if (events[i].is_alloc) { + /* Try to allocate memory */ + void* ptr = XMALLOC(events[i].size, heap_hint, + DYNAMIC_TYPE_TMP_BUFFER); + if (ptr == NULL) { + printf("FAILURE: malloc failed for size %d at event %d\n", + events[i].size, i); + failure_count++; + return -1; /* Exit on first failure */ + } else { + events[i].ptr = ptr; + success_count++; + printf("SUCCESS: Allocated %d bytes at event %d\n", + events[i].size, i); + } + } else { + /* Find the corresponding allocation to free */ + int found = 0; + for (int j = i - 1; j >= 0; j--) { + if (events[j].is_alloc && events[j].size == events[i].size && + events[j].ptr != NULL) { + XFREE(events[j].ptr, heap_hint, DYNAMIC_TYPE_TMP_BUFFER); + events[j].ptr = NULL; + events[i].ptr = NULL; + found = 1; + printf("SUCCESS: Freed %d bytes at event %d\n", + events[i].size, i); + break; + } + } + if (!found) { + printf("WARNING: No matching allocation found for free of size" + " %d at event %d\n", events[i].size, i); + } + } + } + + printf("\nReplay Summary:\n"); + printf("Total events: %d\n", num_events); + printf("Successful allocations: %d\n", success_count); + printf("Failed allocations: %d\n", failure_count); + + if (failure_count > 0) { + printf("TEST FAILED: Some allocations failed\n"); + return -1; + } else { + printf("TEST PASSED: All allocations succeeded\n"); + return 0; + } +} +#endif + +/* Function to calculate required buffer size */ +int calculate_required_buffer_size(BucketConfig* buckets, int num_buckets) +{ + int total_size = 0; + int i; + + for (i = 0; i < num_buckets; i++) { + total_size += buckets[i].size * buckets[i].count; + } + + /* Add overhead for wolfSSL heap structures */ + total_size += 64 + 32 + (WOLFSSL_STATIC_ALIGN - 1); + + return total_size; +} + +void print_usage(const char* program_name) +{ + printf("Usage: %s --buckets \",,...\" --dist \",,...\" --buffer-size \n", program_name); + printf("\n"); + printf("Arguments:\n"); + printf(" Path to the memory allocation log file\n"); + printf(" --buckets Bucket sizes (comma-separated in quotes)\n"); + printf(" --dist Distribution counts for each bucket (comma-separated in quotes)\n"); + printf(" --buffer-size Total buffer size to use (in bytes)\n"); + printf("\n"); + printf("Examples:\n"); + printf(" %s test.log --buckets \"1024,256,128\" --dist \"2,4,8\" --buffer-size 8192\n", program_name); + printf(" %s test.log --buckets \"1584,1024,256,128,32\" --dist \"2,2,4,2,1\" --buffer-size 16384\n", program_name); + printf(" %s test.log --buckets \"1024,256,128\" --dist \"2,2,4,2,1,3,1,16,1\" --buffer-size 4096\n", program_name); + printf("\n"); + printf("The tester will:\n"); + printf(" 1. Parse the allocation log file\n"); + printf(" 2. Replay the exact same allocation sequence\n"); + printf(" 3. Use the provided bucket configuration and buffer size\n"); + printf(" 4. Fail if any XMALLOC fails\n"); +} + +int main(int argc, char** argv) +{ + const char* log_file; + BucketConfig buckets[MAX_BUCKETS]; + AllocationEvent events[MAX_ALLOCATIONS]; + unsigned int bucket_sizes[MAX_BUCKETS]; + unsigned int bucket_dist[MAX_BUCKETS]; + int i, ret, buffer_size; + int num_buckets = 0, total_buffer_size = 0, num_events = 0; + byte* static_buffer; + WOLFSSL_HEAP_HINT* heap_hint = NULL; + + if (argc < 6) { + print_usage(argv[0]); + return 1; + } + log_file = argv[1]; + + /* Parse bucket configuration */ + if (parse_bucket_config(argc, argv, 2, buckets, &num_buckets, + &total_buffer_size) != 0) { + print_usage(argv[0]); + return 1; + } + + if (num_buckets == 0) { + printf("Error: No buckets configured\n"); + print_usage(argv[0]); + return 1; + } + + if (total_buffer_size <= 0) { + printf("Error: Invalid buffer size (%d). Must be greater than 0.\n", + total_buffer_size); + print_usage(argv[0]); + return 1; + } + + if (parse_memory_logs(log_file, events, &num_events) != 0) { + return 1; + } + + printf("Parsed %d allocation events from %s\n", num_events, log_file); + + /* Use provided buffer size */ + buffer_size = total_buffer_size; + printf("Using provided buffer size: %d bytes\n\n", buffer_size); + + /* Convert bucket config to wolfSSL format */ + for (i = 0; i < num_buckets; i++) { + bucket_sizes[i] = (unsigned int)buckets[i].size; + bucket_dist[i] = (unsigned int)buckets[i].count; + } + + /* Allocate static buffer */ + static_buffer = (byte*)malloc(buffer_size); + if (!static_buffer) { + printf("Error: Failed to allocate static buffer\n"); + return 1; + } + + /* Initialize static memory */ + ret = wc_LoadStaticMemory_ex(&heap_hint, num_buckets, bucket_sizes, + bucket_dist, static_buffer, buffer_size, 0, 0); + if (ret != 0) { + printf("Error: Failed to load static memory (ret=%d)\n", ret); + free(static_buffer); + return 1; + } + + /* Display heap hint information if available */ + if (heap_hint != NULL) { + printf("Heap hint initialized successfully\n"); + printf("Heap hint address: %p\n", (void*)heap_hint); + + /* Try to access the heap structure if available */ + if (heap_hint->memory != NULL) { + WOLFSSL_MEM_STATS mem_stats; + /* Extract bucket information from heap structure */ + WOLFSSL_HEAP* heap = (WOLFSSL_HEAP*)heap_hint->memory; + + wolfSSL_GetMemStats(heap, &mem_stats); + + /* Print actual buckets created in the heap */ + printf("Actual buckets created in heap:\n"); + for (i = 0; i < WOLFMEM_MAX_BUCKETS; i++) { + printf(" Bucket %d: size=%d, count=%d\n", i, + mem_stats.blockSz[i], mem_stats.avaBlock[i]); + } + } else { + printf("Heap structure not accessible\n"); + } + + printf("Static memory system ready for allocation testing\n"); + } else { + printf("Warning: Heap hint is NULL\n"); + } + printf("=====================================\n\n"); + fflush(stdout); + +#ifdef WOLFSSL_NO_MALLOC + /* Replay allocation sequence */ + ret = replay_allocation_sequence(events, num_events, heap_hint); +#else + printf("ERROR: WOLFSSL_NO_MALLOC is not defined\n"); + ret = -1; +#endif + + /* Cleanup */ + free(static_buffer); + + return ret; +} \ No newline at end of file