acceptance WIP

Author: pavel-kirienko

libudpard/udpard.c

@@ -733,8 +733,8 @@ typedef struct
     meta_t          meta;
 } rx_frame_t;
 
-/// We require that the fragment tree does not contain fully-contained or equal-range fragments.
-/// One implication is that no two fragments can have the same offset.
+/// We require that the fragment tree does not contain fully-contained or equal-range fragments. This implies that no
+/// two fragments have the same offset, and that fragments ordered by offset also order by their ends.
 static int32_t rx_cavl_compare_fragment_offset(const void* const user, const udpard_tree_t* const node)
 {
     const size_t u = *(const size_t*)user;
@@ -743,12 +743,69 @@ static int32_t rx_cavl_compare_fragment_offset(const void* const user, const udp
     if (u > v) { return +1; }
     return 0; // clang-format on
 }
+static int32_t rx_cavl_compare_fragment_end(const void* const user, const udpard_tree_t* const node)
+{
+    const size_t                   u = *(const size_t*)user;
+    const udpard_fragment_t* const f = (const udpard_fragment_t*)node;
+    const size_t                   v = f->offset + f->view.size; // clang-format off
+    if (u < v) { return -1; }
+    if (u > v) { return +1; }
+    return 0; // clang-format on
+}
 
-/// Find the first fragment where offset >= left in log time. Returns NULL if no such fragment exists.
-/// This is intended for overlap removal and gap detection when deciding if a new fragment is needed.
-static udpard_fragment_t* rx_fragment_tree_lower_bound(udpard_tree_t* const root, const size_t left)
+/// Checks the tree for fragments that can be removed if a new one spanning [left, right) is to be inserted.
+/// Returns the first fragment that can be removed, or NULL if none can be removed.
+/// Invoke repeatedly until NULL is returned to remove all redundant fragments.
+/// The complexity is logarithmic in the number of fragments in the tree.
+static udpard_fragment_t* rx_fragment_tree_first_redundant(udpard_tree_t* const root,
+                                                           const size_t         left,
+                                                           const size_t         right)
 {
-    return (udpard_fragment_t*)cavl2_lower_bound(root, &left, &rx_cavl_compare_fragment_offset);
+    // The most simple case to check first is to find the fragments that fall fully within the new one.
+    // This may somewhat simplify the reasoning about the remaining checks below.
+    udpard_fragment_t* frag = (udpard_fragment_t*)cavl2_lower_bound(root, &left, &rx_cavl_compare_fragment_offset);
+    if ((frag != NULL) && ((frag->offset + frag->view.size) <= right)) {
+        return frag; // We may land here multiple times if the new fragment is large.
+    }
+    // Now we are certain that no fully-contained fragments exist. But the addition of a larger fragment that
+    // joins adjacent fragments together into a larger contiguous block may render smaller fragments that overlap
+    // with its edges redundant. To check for that, we create a new virtual fragment that represents the new fragment
+    // together with those that join it on either end, if any, and then look for fragments fully contained within.
+    // Example:
+    //                |---B---|
+    //             |--X--|
+    //          |--A--|
+    // The addition of fragment A or B will render X redundant, even though it is not contained within any fragment.
+    // This algorithm will detect that and mark X for removal.
+    //
+    // To find the left neighbor, we need to find the fragment crossing the left boundary whose offset is the smallest.
+    // To do that, we simply need to find the fragment with the smallest right boundary that is on the right of our
+    // left boundary. This works because by construction we guarantee that our tree has no fully-contained fragments,
+    // implying that ordering by left is also ordering by right.
+    size_t v_left = left;
+    frag          = (udpard_fragment_t*)cavl2_lower_bound(root, &left, &rx_cavl_compare_fragment_end);
+    if (frag != NULL) {
+        v_left = smaller(v_left, frag->offset); // Ignore fragments ending before our left boundary
+    }
+    // The right neighbor is found by analogy: find the fragment with the largest left boundary that is on the left
+    // of our right boundary. This guarantees that the new virtual right boundary will max out to the right.
+    size_t v_right = right;
+    frag           = (udpard_fragment_t*)cavl2_predecessor(root, &right, &rx_cavl_compare_fragment_offset);
+    if (frag != NULL) {
+        v_right = larger(v_right, frag->offset + frag->view.size); // Ignore fragments starting after our right boundary
+    }
+    UDPARD_ASSERT((v_left <= left) && (right <= v_right));
+    // Adjust the boundaries to avoid removing the neighbors themselves.
+    v_left++;
+    if (v_right > 0) {
+        v_right--;
+    }
+    // Now we repeat the process as above but with the expanded virtual fragment (v_left, v_right-1).
+    frag = (udpard_fragment_t*)cavl2_lower_bound(root, &v_left, &rx_cavl_compare_fragment_offset);
+    if ((frag != NULL) && ((frag->offset + frag->view.size) <= v_right)) {
+        return frag;
+    }
+    return NULL; // Nothing left to remove.
 }
 
 /// True if the fragment tree does not have a contiguous payload coverage in [left, right).
@@ -825,7 +882,7 @@ typedef enum
     rx_fragment_tree_oom,
 } rx_fragment_tree_update_result_t;
 
-/// Takes ownership of the frame payload. Returns true if the transfer is fully reassembled in the tree.
+/// Takes ownership of the frame payload; either a new fragment is inserted or the payload is freed.
 static rx_fragment_tree_update_result_t rx_fragment_tree_update(udpard_tree_t** const       root,
                                                                 const udpard_mem_resource_t fragment_memory,
                                                                 const rx_frame_base_t       frame,
@@ -837,6 +894,61 @@ static rx_fragment_tree_update_result_t rx_fragment_tree_update(udpard_tree_t**
     // First we need to check if the new fragment is needed at all. It is needed in two cases:
     // 1) It covers new ground in the [0, extent) range.
     // 2) It overlaps smaller fragments that are already in the tree, which can be replaced.
+    {
+        const size_t       left = frame.offset;
+        udpard_fragment_t* frag = (udpard_fragment_t*)cavl2_predecessor(*root, &left, &rx_cavl_compare_fragment_offset);
+        if ((frag != NULL) && ((frag->offset + frag->view.size) >= (frame.offset + frame.payload.size))) {
+            mem_free_payload(payload_deleter, frame.origin); // New fragment is fully contained within an existing one.
+            return rx_fragment_tree_not_done;
+        }
+    }
+    udpard_fragment_t* victim =
+      rx_fragment_tree_first_redundant(*root, frame.offset, frame.offset + frame.payload.size);
+    const bool need =
+      (victim != NULL) || rx_fragment_is_needed(*root, frame.offset, frame.payload.size, transfer_payload_size, extent);
+    if (!need) {
+        mem_free_payload(payload_deleter, frame.origin);
+        return rx_fragment_tree_not_done; // Cannot make use of this fragment.
+    }
+
+    // Ensure we can allocate the fragment header for the new frame before pruning the tree to avoid data loss.
+    udpard_fragment_t* frag = mem_alloc(fragment_memory, sizeof(udpard_fragment_t));
+    if (frag == NULL) {
+        mem_free_payload(payload_deleter, frame.origin);
+        return rx_fragment_tree_oom; // Cannot allocate fragment header. Maybe we will succeed later.
+    }
+    mem_zero(sizeof(*frag), frag);
+    frag->view.data       = frame.payload.data;
+    frag->view.size       = frame.payload.size;
+    frag->origin.data     = frame.origin.data;
+    frag->origin.size     = frame.origin.size;
+    frag->offset          = frame.offset;
+    frag->payload_deleter = payload_deleter;
+
+    // Remove all redundant fragments before inserting the new one.
+    while (victim != NULL) {
+        cavl2_remove(root, &victim->index_offset);
+        mem_free_payload(victim->payload_deleter, victim->origin);
+        mem_free(fragment_memory, sizeof(udpard_fragment_t), victim);
+        victim = rx_fragment_tree_first_redundant(*root, frame.offset, frame.offset + frame.payload.size);
+    }
+
+    // Insert the new fragment.
+    udpard_tree_t* const res = cavl2_find_or_insert(root, //
+                                                    &frag->offset,
+                                                    &rx_cavl_compare_fragment_offset,
+                                                    &frag->index_offset,
+                                                    &cavl2_trivial_factory);
+    UDPARD_ASSERT(res == &frag->index_offset);
+    (void)res;
+
+    // Update the covered prefix.
+    *covered_prefix_io = rx_fragment_tree_update_covered_prefix(*root, //
+                                                                *covered_prefix_io,
+                                                                frame.offset,
+                                                                frame.payload.size);
+    return (*covered_prefix_io >= smaller(extent, transfer_payload_size)) ? rx_fragment_tree_done
+                                                                          : rx_fragment_tree_not_done;
 }
 
 typedef enum