Bypass WAL in bottom-up build mode (couchbase#11)

Author: greensky00

src/btree.cc

@@ -1705,6 +1705,7 @@ btree_result btree_init_and_load(
         struct btree_meta *meta,
         uint64_t num_keys,
         btree_load_get_next_kv* next_kv,
+        btree_load_write_done* write_done,
         void* aux)
 {
     void *addr;
@@ -1830,6 +1831,7 @@ btree_result btree_init_and_load(
             } else {
                 if (cur_node[jj]->nentry >= max_num_entries_non_root) {
                     // Otherwise: full, should allocate a new one.
+                    write_done(btree->blk_handle, node_bids[jj], aux);
                     cur_node[jj] = NULL;
                     node_bids[jj] = 0;
                 }

src/btree.h

@@ -186,6 +186,7 @@ btree_result btree_init(
         bnode_flag_t flag, struct btree_meta *meta);
 
 typedef int btree_load_get_next_kv(void** key_out, void** value_out, void* aux);
+typedef void btree_load_write_done(void* blk_handle, bid_t bid, void* aux);
 btree_result btree_init_and_load(
         struct btree *btree,
         void *blk_handle,
@@ -198,6 +199,7 @@ btree_result btree_init_and_load(
         struct btree_meta *meta,
         uint64_t num_keys,
         btree_load_get_next_kv* next_kv,
+        btree_load_write_done* write_done,
         void* aux);
 
 btree_result btree_iterator_init(struct btree *btree, struct btree_iterator *it, void *initial_key);

src/btreeblock.cc

@@ -945,6 +945,38 @@ fdb_status btreeblk_operation_end(void *voidhandle)
     return status;
 }
 
+void btreeblk_write_done(void* voidhandle, bid_t bid) {
+    // flush and write the block for given BID in the allocation list.
+    struct btreeblk_handle *handle = (struct btreeblk_handle *)voidhandle;
+
+    // write and free items in allocation list
+    struct list_elem *e = list_begin(&handle->alc_list);
+    while (e) {
+        struct btreeblk_block *block = _get_entry(e, struct btreeblk_block, le);
+        if (block->bid != bid) {
+            e = list_next(e);
+            continue;
+        }
+        int writable = filemgr_is_writable(handle->file, block->bid);
+        if (writable) {
+            fdb_status status = _btreeblk_write_dirty_block(handle, block);
+            if (status != FDB_RESULT_SUCCESS) {
+                return;
+            }
+        } else {
+            fdb_log(nullptr, FDB_LOG_FATAL,
+                    FDB_RESULT_WRITE_FAIL,
+                    "b+tree node write fail, BID %zu, file %s",
+                    block->bid,
+                    handle->file->filename);
+            return;
+        }
+
+        e = list_remove(&handle->alc_list, &block->le);
+        _btreeblk_free_dirty_block(handle, block);
+    }
+}
+
 void btreeblk_discard_blocks(struct btreeblk_handle *handle)
 {
     // discard all writable blocks in the read list

src/btreeblock.h

@@ -96,6 +96,7 @@ void btreeblk_reset_subblock_info(struct btreeblk_handle *handle);
 void btreeblk_free(struct btreeblk_handle *handle);
 void btreeblk_discard_blocks(struct btreeblk_handle *handle);
 fdb_status btreeblk_end(struct btreeblk_handle *handle);
+void btreeblk_write_done(void* voidhandle, bid_t bid);
 
 #ifdef __cplusplus
 }

src/forestdb.cc

@@ -2284,6 +2284,14 @@ fdb_status _fdb_open(fdb_kvs_handle *handle,
         }
     }
 
+    if (config->bottom_up_index_build) {
+        handle->bottom_up_build_entries = (struct list*)malloc(sizeof(struct list));
+        list_init(handle->bottom_up_build_entries);
+        handle->num_bottom_up_build_entries = 0;
+    } else {
+        handle->bottom_up_build_entries = NULL;
+    }
+
     return status;
 }
 
@@ -4250,29 +4258,47 @@ fdb_status fdb_set(fdb_kvs_handle *handle, fdb_doc *doc)
     if (!txn) {
         txn = &file->global_txn;
     }
-    if (handle->kvs) {
-        // multi KV instance mode
-        fdb_doc kv_ins_doc = *doc;
-        kv_ins_doc.key = _doc.key;
-        kv_ins_doc.keylen = _doc.length.keylen;
-        if (!immediate_remove) {
-            wal_insert(txn, file, &cmp_info, &kv_ins_doc, offset,
-                       WAL_INS_WRITER);
-        } else {
-            wal_immediate_remove(txn, file, &cmp_info, &kv_ins_doc, offset,
-                                 WAL_INS_WRITER);
-        }
+
+    if (handle->config.bottom_up_index_build) {
+        // Bottom-up build mode, bypass WAL.
+        struct bottom_up_build_entry* bub_entry =
+            (struct bottom_up_build_entry*)
+            malloc(sizeof(struct bottom_up_build_entry));
+        bub_entry->keylen = _doc.length.keylen;
+        bub_entry->key = (void*)malloc(_doc.length.keylen);
+        memcpy(bub_entry->key, _doc.key, _doc.length.keylen);
+        bub_entry->seqnum = doc->seqnum;
+        bub_entry->offset = doc->offset;
+
+        fdb_kvs_handle* root_handle = handle->fhandle->root;
+        list_push_back(root_handle->bottom_up_build_entries, &bub_entry->le);
+        root_handle->num_bottom_up_build_entries++;
+
     } else {
-        if (!immediate_remove) {
-            wal_insert(txn, file, &cmp_info, doc, offset, WAL_INS_WRITER);
+        if (handle->kvs) {
+            // multi KV instance mode
+            fdb_doc kv_ins_doc = *doc;
+            kv_ins_doc.key = _doc.key;
+            kv_ins_doc.keylen = _doc.length.keylen;
+            if (!immediate_remove) {
+                wal_insert(txn, file, &cmp_info, &kv_ins_doc, offset,
+                           WAL_INS_WRITER);
+            } else {
+                wal_immediate_remove(txn, file, &cmp_info, &kv_ins_doc, offset,
+                                     WAL_INS_WRITER);
+            }
         } else {
-            wal_immediate_remove(txn, file, &cmp_info, doc, offset,
-                                 WAL_INS_WRITER);
+            if (!immediate_remove) {
+                wal_insert(txn, file, &cmp_info, doc, offset, WAL_INS_WRITER);
+            } else {
+                wal_immediate_remove(txn, file, &cmp_info, doc, offset,
+                                     WAL_INS_WRITER);
+            }
         }
-    }
 
-    if (wal_get_dirty_status(file)== FDB_WAL_CLEAN) {
-        wal_set_dirty_status(file, FDB_WAL_DIRTY);
+        if (wal_get_dirty_status(file)== FDB_WAL_CLEAN) {
+            wal_set_dirty_status(file, FDB_WAL_DIRTY);
+        }
     }
 
     if (handle->config.auto_commit &&
@@ -4648,11 +4674,11 @@ fdb_status fdb_commit_non_durable(fdb_file_handle *fhandle,
 
 void* _fdb_bottom_up_index_build_next(void* cur_entry, void* aux) {
     if (!cur_entry) {
-        struct avl_tree* tree = (struct avl_tree*)aux;
-        return avl_first(tree);
+        struct list* ll = (struct list*)aux;
+        return list_begin(ll);
     }
-    struct avl_node* an = (struct avl_node*)cur_entry;
-    return avl_next(an);
+    struct list_elem* le = (struct list_elem*)cur_entry;
+    return list_next(le);
 }
 
 void _fdb_bottom_up_index_build_get(void* cur_entry,
@@ -4661,15 +4687,13 @@ void _fdb_bottom_up_index_build_get(void* cur_entry,
                                     void** value_out,
                                     void* aux)
 {
-    struct wal_item_header* header =
-        _get_entry(cur_entry, struct wal_item_header, avl_key);
-    *key_out = header->key;
-    *keylen_out = header->keylen;
+    struct bottom_up_build_entry* entry =
+        _get_entry(cur_entry, struct bottom_up_build_entry, le);
+    *key_out = entry->key;
+    *keylen_out = entry->keylen;
 
     thread_local uint64_t enc_bid;
-    struct list_elem* le = list_begin(&header->items);
-    struct wal_item* first_item = _get_entry(le, struct wal_item, list_elem);
-    enc_bid = _endian_encode(first_item->offset);
+    enc_bid = _endian_encode(entry->offset);
     *value_out = &enc_bid;
 }
 
@@ -4683,27 +4707,23 @@ void _fdb_bottom_up_index_build_get_seq(void* cur_entry,
                                         void** value_out,
                                         void* aux)
 {
-    struct wal_item* elem =
-        _get_entry(cur_entry, struct wal_item, avl_seq);
+    struct bottom_up_build_entry* entry =
+        _get_entry(cur_entry, struct bottom_up_build_entry, le);
     thread_local uint64_t enc_seq;
-    enc_seq = _endian_encode(elem->seqnum);
+    enc_seq = _endian_encode(entry->seqnum);
     *key_out = &enc_seq;
     *keylen_out = sizeof(enc_seq);
 
     thread_local uint64_t enc_bid;
-    enc_bid = _endian_encode(elem->offset);
+    enc_bid = _endian_encode(entry->offset);
     *value_out = &enc_bid;
 }
 
 fdb_status _fdb_bottom_up_index_build(fdb_kvs_handle *handle)
 {
     fdb_status fs = FDB_RESULT_SUCCESS;
 
-    uint64_t num_entries = handle->file->wal->num_flushable;
-    struct avl_tree* avl_key = &handle->file->wal->key_shards[0]._map;
-    struct avl_tree* avl_seq =
-        handle->file->wal->seq_shards
-        ? &handle->file->wal->seq_shards[0]._map : NULL;
+    uint64_t num_entries = handle->num_bottom_up_build_entries;
 
     // Build key-index first (tree is sorted by key).
     struct hbtrie new_key_trie;
@@ -4717,7 +4737,7 @@ fdb_status _fdb_bottom_up_index_build(fdb_kvs_handle *handle)
                          _fdb_bottom_up_index_build_next,
                          _fdb_bottom_up_index_build_get,
                          _fdb_bottom_up_index_btreeblk_end,
-                         avl_key);
+                         handle->bottom_up_build_entries);
     handle->trie->root_bid = new_key_trie.root_bid;
 
     // Build seq-index next.
@@ -4733,7 +4753,7 @@ fdb_status _fdb_bottom_up_index_build(fdb_kvs_handle *handle)
                              _fdb_bottom_up_index_build_next,
                              _fdb_bottom_up_index_build_get_seq,
                              _fdb_bottom_up_index_btreeblk_end,
-                             avl_seq);
+                             handle->bottom_up_build_entries);
         handle->seqtrie->root_bid = new_seq_trie.root_bid;
     } else if (handle->seqtree) {
         // Not supported yet.
@@ -4842,33 +4862,35 @@ fdb_status _fdb_commit(fdb_kvs_handle *handle,
         //    (in this case, flush the rest of entries)
         // 3. user forces to manually flush wal
 
-        struct filemgr_dirty_update_node *prev_node = NULL, *new_node = NULL;
+        if (handle->config.bottom_up_index_build) {
+            _fdb_bottom_up_index_build(handle);
 
-        _fdb_dirty_update_ready(handle, &prev_node, &new_node,
-                                &dirty_idtree_root, &dirty_seqtree_root, false);
+        } else {
+            struct filemgr_dirty_update_node *prev_node = NULL, *new_node = NULL;
 
-        wr = wal_flush(handle->file, (void *)handle,
-                       _fdb_wal_flush_func, _fdb_wal_get_old_offset,
-                       _fdb_wal_flush_seq_purge, _fdb_wal_flush_kvs_delta_stats,
-                       &flush_items);
+            _fdb_dirty_update_ready(handle, &prev_node, &new_node,
+                                    &dirty_idtree_root, &dirty_seqtree_root, false);
 
-        if (handle->config.bottom_up_index_build) {
-            _fdb_bottom_up_index_build(handle);
-        }
+            wr = wal_flush(handle->file, (void *)handle,
+                           _fdb_wal_flush_func, _fdb_wal_get_old_offset,
+                           _fdb_wal_flush_seq_purge, _fdb_wal_flush_kvs_delta_stats,
+                           &flush_items);
 
-        if (wr != FDB_RESULT_SUCCESS) {
-            btreeblk_clear_dirty_update(handle->bhandle);
-            filemgr_dirty_update_close_node(handle->file, prev_node);
-            filemgr_dirty_update_remove_node(handle->file, new_node);
-            filemgr_mutex_unlock(handle->file);
-            atomic_cas_uint8_t(&handle->handle_busy, 1, 0);
-            return wr;
-        }
-        wal_set_dirty_status(handle->file, FDB_WAL_CLEAN);
-        wal_flushed = true;
 
-        _fdb_dirty_update_finalize(handle, prev_node, new_node,
-                                   &dirty_idtree_root, &dirty_seqtree_root, true);
+            if (wr != FDB_RESULT_SUCCESS) {
+                btreeblk_clear_dirty_update(handle->bhandle);
+                filemgr_dirty_update_close_node(handle->file, prev_node);
+                filemgr_dirty_update_remove_node(handle->file, new_node);
+                filemgr_mutex_unlock(handle->file);
+                atomic_cas_uint8_t(&handle->handle_busy, 1, 0);
+                return wr;
+            }
+            wal_set_dirty_status(handle->file, FDB_WAL_CLEAN);
+            wal_flushed = true;
+
+            _fdb_dirty_update_finalize(handle, prev_node, new_node,
+                                       &dirty_idtree_root, &dirty_seqtree_root, true);
+        }
     }
 
     // Note: Appending KVS header must be done after flushing WAL
@@ -8182,6 +8204,21 @@ fdb_status _fdb_close_root(fdb_kvs_handle *handle)
     return fs;
 }
 
+void _destroy_bottom_up_build_entries(fdb_kvs_handle* handle) {
+    if (!handle->bottom_up_build_entries) {
+        return;
+    }
+    struct list_elem* le = list_begin(handle->bottom_up_build_entries);
+    while (le) {
+        struct bottom_up_build_entry* entry =
+            _get_entry(le, struct bottom_up_build_entry, le);
+        le = list_next(le);
+        free(entry->key);
+        free(entry);
+    }
+    free(handle->bottom_up_build_entries);
+}
+
 fdb_status _fdb_close(fdb_kvs_handle *handle)
 {
     fdb_status fs;
@@ -8235,6 +8272,8 @@ fdb_status _fdb_close(fdb_kvs_handle *handle)
         handle->filename = NULL;
     }
 
+    _destroy_bottom_up_build_entries(handle);
+
     return fs;
 }
 

src/hbtrie.cc

@@ -19,6 +19,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "btreeblock.h"
 #include "option.h"
 #include "forestdb_endian.h"
 #include "hbtrie.h"
@@ -2495,6 +2496,10 @@ int _building_btree_next_kv(void** key_out, void** val_out, void* aux) {
     return 0;
 }
 
+void _building_btree_write_done(void* voidhandle, bid_t bid, void* aux) {
+    btreeblk_write_done(voidhandle, bid);
+}
+
 bid_t _hbtrie_load_recursive(struct hbtrie *trie,
                              int cur_chunk_idx,
                              int cp_start_chunk_idx,
@@ -2699,6 +2704,7 @@ bid_t _hbtrie_load_recursive(struct hbtrie *trie,
                              &meta,
                              num_entries,
                              _building_btree_next_kv,
+                             _building_btree_write_done,
                              &params );
         do_btreeblk_end(trie->btreeblk_handle);
         ret_bid = btree.root_bid;

src/internal_types.h

@@ -307,6 +307,7 @@ struct _fdb_kvs_handle {
         dirty_updates = kv_handle.dirty_updates;
         node = kv_handle.node;
         num_iterators = kv_handle.num_iterators;
+        bottom_up_build_entries = kv_handle.bottom_up_build_entries;
         return *this;
     }
 
@@ -432,6 +433,24 @@ struct _fdb_kvs_handle {
      * Number of active iterator instances created from this handle
      */
     uint32_t num_iterators;
+    /**
+     * Used when `bottom_up_index_build` is `true`.
+     * Instead of inserting into the WAL, it keeps <key, seqnum, offset> tuples
+     * to build the index.
+     */
+    struct list* bottom_up_build_entries;
+    /**
+     * The number of entries in `bottom_up_build_entries`.
+     */
+    uint64_t num_bottom_up_build_entries;
+};
+
+struct bottom_up_build_entry {
+    struct list_elem le;
+    void* key;
+    size_t keylen;
+    uint64_t offset;
+    uint64_t seqnum;
 };
 
 struct hbtrie_iterator;