Fix CVE-2025-6857 (#5799)

Add additional checks for v1 B-tree corruption

Author: jhendersonHDF

release_docs/CHANGELOG.md

@@ -454,6 +454,12 @@ HDF5 release, platforms tested, and known problems in this release.
 # 🪲  Bug Fixes
 
 ## Library
+- Fixed security issue CVE-2025-6857
+
+   An HDF5 file had a corrupted v1 B-tree that would result in a stack overflow when performing a lookup on it. This has been fixed with additional integrity checks.
+
+   Fixes GitHub issue #5575
+
 - Check for overflow in decoded heap block addresses
 
    Currently, we do not check for overflow when decoding addresses from the heap, which can cause overflow problems. We've added a check in H5HL__fl_deserialize to ensure no overflow can occur.

src/H5B.c

@@ -140,6 +140,8 @@ typedef struct H5B_ins_ud_t {
 /********************/
 /* Local Prototypes */
 /********************/
+static herr_t    H5B_find_helper(H5F_t *f, const H5B_class_t *type, haddr_t addr, int exp_level, bool *found,
+                                 void *udata);
 static H5B_ins_t H5B__insert_helper(H5F_t *f, H5B_ins_ud_t *bt_ud, const H5B_class_t *type, uint8_t *lt_key,
                                     bool *lt_key_changed, uint8_t *md_key, void *udata, uint8_t *rt_key,
                                     bool *rt_key_changed, H5B_ins_ud_t *split_bt_ud /*out*/);
@@ -255,26 +257,67 @@ H5B_create(H5F_t *f, const H5B_class_t *type, void *udata, haddr_t *addr_p /*out
 } /* end H5B_create() */
 
 /*-------------------------------------------------------------------------
- * Function:	H5B_find
+ * Function:    H5B_find
  *
- * Purpose:	Locate the specified information in a B-tree and return
- *		that information by filling in fields of the caller-supplied
- *		UDATA pointer depending on the type of leaf node
- *		requested.  The UDATA can point to additional data passed
- *		to the key comparison function.
+ * Purpose:     Locate the specified information in a B-tree and return
+ *              that information by filling in fields of the
+ *              caller-supplied UDATA pointer depending on the type of leaf
+ *              node requested.  The UDATA can point to additional data
+ *              passed to the key comparison function.
  *
- * Note:	This function does not follow the left/right sibling
- *		pointers since it assumes that all nodes can be reached
- *		from the parent node.
+ * Note:        This function does not follow the left/right sibling
+ *              pointers since it assumes that all nodes can be reached
+ *              from the parent node.
  *
- * Return:	Non-negative (true/false) on success (if found, values returned
- *              through the UDATA argument). Negative on failure (if not found,
- *              UDATA is undefined).
+ * Return:      Non-negative (true/false) on success (if found, values
+ *              returned through the UDATA argument). Negative on failure
+ *              (if not found, UDATA is undefined).
  *
  *-------------------------------------------------------------------------
  */
 herr_t
 H5B_find(H5F_t *f, const H5B_class_t *type, haddr_t addr, bool *found, void *udata)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /*
+     * Check arguments.
+     */
+    assert(f);
+    assert(type);
+    assert(type->decode);
+    assert(type->cmp3);
+    assert(type->found);
+    assert(H5_addr_defined(addr));
+
+    if ((ret_value = H5B_find_helper(f, type, addr, H5B_UNKNOWN_NODELEVEL, found, udata)) < 0)
+        HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "can't lookup key");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5B_find() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5B_find_helper
+ *
+ * Purpose:     Recursive helper routine for H5B_find used to track node
+ *              levels and attempt to detect B-tree corruption during
+ *              lookups.
+ *
+ * Note:        This function does not follow the left/right sibling
+ *              pointers since it assumes that all nodes can be reached
+ *              from the parent node.
+ *
+ * Return:      Non-negative on success (if found, values returned through
+ *              the UDATA argument). Negative on failure (if not found,
+ *              UDATA is undefined).
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5B_find_helper(H5F_t *f, const H5B_class_t *type, haddr_t addr, int exp_level, bool *found, void *udata)
 {
     H5B_t         *bt = NULL;
     H5UC_t        *rc_shared;           /* Ref-counted shared info */
@@ -309,7 +352,7 @@ H5B_find(H5F_t *f, const H5B_class_t *type, haddr_t addr, bool *found, void *uda
     cache_udata.f         = f;
     cache_udata.type      = type;
     cache_udata.rc_shared = rc_shared;
-    cache_udata.exp_level = H5B_UNKNOWN_NODELEVEL;
+    cache_udata.exp_level = exp_level;
     if (NULL == (bt = (H5B_t *)H5AC_protect(f, H5AC_BT, addr, &cache_udata, H5AC__READ_ONLY_FLAG)))
         HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree node");
 
@@ -333,7 +376,17 @@ H5B_find(H5F_t *f, const H5B_class_t *type, haddr_t addr, bool *found, void *uda
         assert(idx < bt->nchildren);
 
         if (bt->level > 0) {
-            if ((ret_value = H5B_find(f, type, bt->child[idx], found, udata)) < 0)
+            /* Sanity check to catch the case where the current node points to
+             * itself and the current node was loaded with an expected node level
+             * of H5B_UNKNOWN_NODELEVEL, thus bypassing the expected node level
+             * check during deserialization and in the future if the node was
+             * cached.
+             */
+            if (bt->child[idx] == addr)
+                HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, FAIL, "cyclic B-tree detected");
+
+            if ((ret_value = H5B_find_helper(f, type, bt->child[idx], (int)(bt->level - 1), found, udata)) <
+                0)
                 HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "can't lookup key in subtree");
         } /* end if */
         else {
@@ -347,7 +400,7 @@ H5B_find(H5F_t *f, const H5B_class_t *type, haddr_t addr, bool *found, void *uda
         HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release node");
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5B_find() */
+} /* end H5B_find_helper() */
 
 /*-------------------------------------------------------------------------
  * Function:	H5B__split