CDRIVER-5601 more robust bson append (#1648)

Co-authored-by: Kevin Albertson <[email protected]>
Co-authored-by: Ezra Chung <[email protected]>

Author: 3 people

src/libbson/src/bson/bson.c

@@ -324,7 +324,18 @@ _bson_append_va (bson_t *bson,              /* IN */
 
    buf = _bson_data (bson) + bson->len - 1;
 
+   /* Track running sum of bytes written in a uint64_t to detect possible overflow of `n_bytes`. */
+   uint64_t n_bytes_sum = 0;
    do {
+      // Size of any individual data being appended should not exceed the total byte limit.
+      if (BSON_UNLIKELY (bson_cmp_less_uu (n_bytes, data_len))) {
+         return false;
+      }
+      // Total size of data being appended should not exceed the total byte limit.
+      if (BSON_UNLIKELY (bson_cmp_greater_uu (n_bytes_sum, n_bytes - data_len))) {
+         return false;
+      }
+      n_bytes_sum += data_len;
       n_pairs--;
       /* data may be NULL if data_len is 0. memcpy is not safe to call with
        * NULL. */