salad: fix crash on insertion into bps tree with OOM conditions

On insertion into bps tree we first reserve blocks for the case we
need to insert new leaf during rebalancing. Next there can be a free
block on last extent in matras but no more free extents in tuple arena.
So `bps_tree_reserve_blocks()` is able to allocate a block but then
`bps_tree_garbage_push()` is called that touches a block. As there no
free extents touch is failed which is unexpected and causes a crash.

We don't need to touch the block on this path as block is newly allocated
and is not shared with any read view. We don't need to touch block on
other paths too (disposing a leaf or a inner node) because we move data
from this node before so it is already should be touched.

So just don't touch the block in `bps_tree_garbage_push()`.

The issue is introduced in the commit 51c56d9 ("salad: reserve
extents for matras_touch on BPS tree operations") and unlocked in the
commit 32b0678 ("memtx: reserve extents in the RTREE index itself")
where we removed crutches where we heuristically reserve memory for
index operations.

Closes tarantool#11326

NO_DOC=bugfix

Author: nshy

changelogs/unreleased/gh-11326-fix-crash-on-insertion-on-oom.md

@@ -0,0 +1,3 @@
+## bugfix/memtx
+
+* Fixed a crash on OOM on insertion in tree index of memtx engine (gh-11326).

src/lib/salad/bps_tree.h

@@ -3156,7 +3156,6 @@ static inline void
 bps_tree_garbage_push(struct bps_tree_common *tree, struct bps_block *block,
 		      bps_tree_block_id_t id)
 {
-	assert(block); (void) block;
 	bps_tree_block_id_t next_leaf_id = (bps_tree_block_id_t)(-1);
 	bps_tree_block_id_t prev_leaf_id = (bps_tree_block_id_t)(-1);
 	if (block->type == BPS_TREE_BT_LEAF) {
@@ -3165,8 +3164,7 @@ bps_tree_garbage_push(struct bps_tree_common *tree, struct bps_block *block,
 		prev_leaf_id = leaf->prev_id;
 	}
 
-	struct bps_garbage *garbage = (struct bps_garbage *)
-		bps_tree_touch_block(tree, id);
+	struct bps_garbage *garbage = (struct bps_garbage *)block;
 	garbage->header.type = BPS_TREE_BT_GARBAGE;
 	garbage->next_id = tree->garbage_head_id;
 	garbage->next_leaf_id = next_leaf_id;

test/unit/bps_tree.cc

@@ -174,11 +174,14 @@ compare(type_t a, type_t b)
 	return a < b ? -1 : a > b ? 1 : 0;
 }
 static int extents_count = 0;
+static bool extent_alloc_failure = false;
 
 static void *
 extent_alloc(struct matras_allocator *allocator)
 {
 	(void)allocator;
+	if (extent_alloc_failure)
+		return NULL;
 	++extents_count;
 	return xmalloc(BPS_TREE_EXTENT_SIZE);
 }
@@ -920,10 +923,44 @@ insert_successor_test()
 	ok(true, "successor test");
 }
 
+static void
+gh_11326_oom_on_insertion_test()
+{
+	plan(1);
+	header();
+
+	test tree;
+	test_view view;
+	type_t replaced;
+	struct test_iterator iterator;
+
+	test_create(&tree, 0, &allocator, NULL);
+	test_insert(&tree, 0, &replaced, NULL);
+	test_view_create(&view, &tree);
+
+	extent_alloc_failure = true;
+	fail_unless(test_insert(&tree, 1, &replaced, NULL) != 0);
+	debug_check(&tree);
+	fail_unless(test_size(&tree) == 1);
+	iterator = test_first(&tree);
+	type_t *v = test_iterator_get_elem(&tree, &iterator);
+	fail_unless(v != NULL && *v == 0);
+	fail_unless(test_iterator_next(&tree, &iterator) == false);
+	extent_alloc_failure = false;
+
+	test_view_destroy(&view);
+	test_destroy(&tree);
+
+	ok(true, "gh-11326: OOM on insertion test");
+
+	footer();
+	check_plan();
+}
+
 int
 main(void)
 {
-	plan(12);
+	plan(13);
 	header();
 
 	matras_allocator_create(&allocator, BPS_TREE_EXTENT_SIZE,
@@ -944,6 +981,11 @@ main(void)
 
 	matras_allocator_destroy(&allocator);
 
+	matras_allocator_create(&allocator, BPS_TREE_EXTENT_SIZE,
+				extent_alloc, extent_free);
+	gh_11326_oom_on_insertion_test();
+	matras_allocator_destroy(&allocator);
+
 	footer();
 	return check_plan();
 }