Import wiredtiger: 15e567db3a1adbc9b520a479a9d99bd448a1bd9c from branch mongodb-master

ref: 13392cc987..15e567db3a
for: 7.2.0-rc0

WT-10716 MVP implementation for automatic prefetching

Author: ruby-oujo

src/third_party/wiredtiger/bench/wtperf/track.c

@@ -111,15 +111,13 @@ uint64_t
 sum_scan_ops(WTPERF *wtperf)
 {
     CONFIG_OPTS *opts;
-    uint64_t total;
 
     opts = wtperf->opts;
 
-    if (opts->scan_interval > 0)
-        total = wtperf->scanthreads->scan.ops;
-    else
-        total = 0;
-    return (total);
+    if (opts->scan_interval == 0)
+        return (0);
+
+    return (wtperf->scanthreads->scan.ops);
 }
 
 /*

src/third_party/wiredtiger/dist/api_data.py

@@ -1218,6 +1218,10 @@ def __ge__(self, other):
         permit sharing between processes (will automatically start an RPC server for primary
         processes and use RPC for secondary processes). <b>Not yet supported in WiredTiger</b>''',
         type='boolean'),
+    Config('prefetch', 'false', r'''
+        Enable automatic detection of scans by applications, and attempt to pre-fetch future
+        content into the cache''',
+        type='boolean'),
     Config('readonly', 'false', r'''
         open connection in read-only mode. The database must exist. All methods that may
         modify a database are disabled. See @ref readonly for more information''',

src/third_party/wiredtiger/dist/filelist

@@ -32,6 +32,7 @@ src/btree/bt_import.c
 src/btree/bt_misc.c
 src/btree/bt_ovfl.c
 src/btree/bt_page.c
+src/btree/bt_prefetch.c
 src/btree/bt_random.c
 src/btree/bt_read.c
 src/btree/bt_ret.c
@@ -80,6 +81,7 @@ src/conn/conn_dhandle.c
 src/conn/conn_handle.c
 src/conn/conn_log.c
 src/conn/conn_open.c
+src/conn/conn_prefetch.c
 src/conn/conn_reconfig.c
 src/conn/conn_stat.c
 src/conn/conn_sweep.c
@@ -199,6 +201,7 @@ src/schema/schema_worker.c
 src/session/session_api.c
 src/session/session_compact.c
 src/session/session_dhandle.c
+src/session/session_prefetch.c
 src/rollback_to_stable/rts_api.c
 src/rollback_to_stable/rts_btree.c
 src/rollback_to_stable/rts_btree_walk.c

src/third_party/wiredtiger/dist/s_string.ok

@@ -282,6 +282,7 @@ PID
 POS
 POSIX
 PPC
+PREFETCH
 PRId
 PRIu
 PRNG
@@ -1083,6 +1084,7 @@ pre
 pread
 prealloc
 precomp
+prefetch
 preload
 prepend
 prepended

src/third_party/wiredtiger/dist/stat_data.py

@@ -236,6 +236,13 @@ def __init__(self, name, desc, flags=''):
     BlockCacheStat('block_cache_misses', 'number of misses'),
     BlockCacheStat('block_cache_not_evicted_overhead', 'number of blocks not evicted due to overhead'),
 
+    BlockCacheStat('block_prefetch_attempts', 'pre-fetch triggered by page read'),
+    BlockCacheStat('block_prefetch_disk_one', 'pre-fetch not triggered after single disk read'),
+    BlockCacheStat('block_prefetch_pages_queued', 'pre-fetch pages queued'),
+    BlockCacheStat('block_prefetch_pages_read', 'pre-fetch pages read in background'),
+    BlockCacheStat('block_prefetch_skipped', 'pre-fetch not triggered by page read'),
+    BlockCacheStat('block_prefetch_pages_fail', 'pre-fetch page not on disk when reading'),
+
     ##########################################
     # Block manager statistics
     ##########################################
@@ -270,6 +277,7 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_eviction_app', 'pages evicted by application threads'),
     CacheStat('cache_eviction_app_dirty', 'modified pages evicted by application threads'),
     CacheStat('cache_eviction_clear_ordinary', 'pages removed from the ordinary queue to be queued for urgent eviction'),
+    CacheStat('cache_eviction_consider_prefetch', 'pages considered for eviction that were brought in by pre-fetch', 'no_clear,no_scale'),
     CacheStat('cache_eviction_empty_score', 'eviction empty score', 'no_clear,no_scale'),
     CacheStat('cache_eviction_fail', 'pages selected for eviction unable to be evicted'),
     CacheStat('cache_eviction_fail_active_children_on_an_internal_page', 'pages selected for eviction unable to be evicted because of active children on an internal page'),
@@ -940,6 +948,7 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_hs_write_squash', 'history store table writes requiring squashed modifies'),
     CacheStat('cache_inmem_split', 'in-memory page splits'),
     CacheStat('cache_inmem_splittable', 'in-memory page passed criteria to be split'),
+    CacheStat('cache_pages_prefetch', 'pages requested from the cache due to pre-fetch'),
     CacheStat('cache_pages_requested', 'pages requested from the cache'),
     CacheStat('cache_read', 'pages read into cache'),
     CacheStat('cache_read_deleted', 'pages read into cache after truncate'),

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "13392cc987b86ff0d131ea20255f019f54f42521"
+    "commit": "15e567db3a1adbc9b520a479a9d99bd448a1bd9c"
 }

src/third_party/wiredtiger/src/block_cache/block_io.c

@@ -70,6 +70,7 @@ __wt_blkcache_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr,
     blkcache_found = found = false;
     skip_cache_put = (blkcache->type == WT_BLKCACHE_UNCONFIGURED);
 
+    WT_ASSERT_ALWAYS(session, session->dhandle != NULL, "The block cache requires a dhandle");
     /*
      * If anticipating a compressed or encrypted block, start with a scratch buffer and convert into
      * the caller's buffer. Else, start with the caller's buffer.

src/third_party/wiredtiger/src/btree/bt_prefetch.c

@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2014-present MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ *	All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __wt_btree_prefetch --
+ *     Pre-load a set of pages into the cache. This session holds a hazard pointer on the ref passed
+ *     in, so there must be a valid page and a valid parent page (though that parent could change if
+ *     a split happens).
+ */
+int
+__wt_btree_prefetch(WT_SESSION_IMPL *session, WT_REF *ref)
+{
+    WT_CONNECTION_IMPL *conn;
+    WT_DECL_RET;
+    WT_REF *next_ref;
+    uint64_t block_preload;
+
+    conn = S2C(session);
+    block_preload = 0;
+
+    /*
+     * FIXME-WT-11759 Consider whether we should have these asserts here or swallow up the errors
+     * instead.
+     */
+    WT_ASSERT_ALWAYS(session, F_ISSET(ref, WT_REF_FLAG_LEAF),
+      "Pre-fetch starts with a leaf page and reviews the parent");
+
+    WT_ASSERT_ALWAYS(session, __wt_session_gen(session, WT_GEN_SPLIT) != 0,
+      "Pre-fetch requires a split generation to traverse internal page(s)");
+
+    session->prefetch_prev_ref = ref;
+    /* Load and decompress a set of pages into the block cache. */
+    WT_INTL_FOREACH_BEGIN (session, ref->home, next_ref) {
+        /* Don't let the pre-fetch queue get overwhelmed. */
+        if (conn->prefetch_queue_count > WT_MAX_PREFETCH_QUEUE ||
+          block_preload > WT_PREFETCH_QUEUE_PER_TRIGGER)
+            break;
+
+        /*
+         * Skip queuing pages that are already in cache or are internal. They aren't the pages we
+         * are looking for. This pretty much assumes that all children of an internal page remain in
+         * cache during the scan. If a previous pre-fetch of this internal page read a page in, then
+         * that page was evicted and now a future page wants to be pre-fetched, this algorithm needs
+         * a tweak. It would need to remember which child was last queued and start again from
+         * there, rather than this approximation which assumes recently pre-fetched pages are still
+         * in cache.
+         */
+        if (next_ref->state == WT_REF_DISK && F_ISSET(next_ref, WT_REF_FLAG_LEAF)) {
+            ret = __wt_conn_prefetch_queue_push(session, next_ref);
+            if (ret == 0)
+                ++block_preload;
+            else if (ret != EBUSY)
+                WT_RET(ret);
+        }
+    }
+    WT_INTL_FOREACH_END;
+
+    WT_STAT_CONN_INCRV(session, block_prefetch_pages_queued, block_preload);
+    return (0);
+}
+
+/*
+ * __wt_prefetch_page_in --
+ *     Does the heavy lifting of reading a page into the cache. Immediately releases the page since
+ *     reading it in is the useful side effect here. Must be called while holding a dhandle.
+ */
+int
+__wt_prefetch_page_in(WT_SESSION_IMPL *session, WT_PREFETCH *pf)
+{
+    WT_ADDR_COPY addr;
+
+    if (pf->ref->home != pf->first_home)
+        __wt_verbose(
+          session, WT_VERB_PREFETCH, "The home changed while queued for pre-fetch %s", "");
+
+    /*
+     * FIXME-WT-11759 Consider whether we should have these asserts here or swallow up the errors
+     * instead.
+     */
+    WT_ASSERT_ALWAYS(session, pf->dhandle != NULL, "Pre-fetch needs to save a valid dhandle");
+    WT_ASSERT_ALWAYS(
+      session, !F_ISSET(pf->ref, WT_REF_FLAG_INTERNAL), "Pre-fetch should only see leaf pages");
+
+    if (pf->ref->state != WT_REF_DISK) {
+        WT_STAT_CONN_INCR(session, block_prefetch_pages_fail);
+        return (0);
+    }
+
+    WT_STAT_CONN_INCR(session, block_prefetch_pages_read);
+
+    if (__wt_ref_addr_copy(session, pf->ref, &addr)) {
+        WT_RET(__wt_page_in(session, pf->ref, WT_READ_PREFETCH));
+        WT_RET(__wt_page_release(session, pf->ref, 0));
+    } else
+        return (WT_ERROR);
+
+    return (0);
+}

src/third_party/wiredtiger/src/btree/bt_read.c

@@ -194,6 +194,8 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags)
     page_flags = WT_DATA_IN_ITEM(&tmp) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED;
     if (LF_ISSET(WT_READ_IGNORE_CACHE_SIZE))
         FLD_SET(page_flags, WT_PAGE_EVICT_NO_PROGRESS);
+    if (LF_ISSET(WT_READ_PREFETCH))
+        FLD_SET(page_flags, WT_PAGE_PREFETCH);
     WT_ERR(__wt_page_inmem(session, ref, tmp.data, page_flags, &notused, &prepare));
     tmp.mem = NULL;
     if (prepare)
@@ -282,6 +284,9 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
     if (!LF_ISSET(WT_READ_CACHE))
         WT_STAT_CONN_DATA_INCR(session, cache_pages_requested);
 
+    if (LF_ISSET(WT_READ_PREFETCH))
+        WT_STAT_CONN_INCR(session, cache_pages_prefetch);
+
     /*
      * If configured, free stashed memory more aggressively to encourage finding bugs in generation
      * tracking code.
@@ -321,11 +326,12 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
              * If configured to not trash the cache, leave the page generation unset, we'll set it
              * before returning to the oldest read generation, so the page is forcibly evicted as
              * soon as possible. We don't do that set here because we don't want to evict the page
-             * before we "acquire" it.
+             * before we "acquire" it. Also avoid queuing a pre-fetch page for forced eviction
+             * before it has a chance of being used. Otherwise the work we've just done is wasted.
              */
             wont_need = LF_ISSET(WT_READ_WONT_NEED) ||
               F_ISSET(session, WT_SESSION_READ_WONT_NEED) ||
-              F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_NOKEEP);
+              (!LF_ISSET(WT_READ_PREFETCH) && F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_NOKEEP));
             continue;
         case WT_REF_LOCKED:
             if (LF_ISSET(WT_READ_NO_WAIT))
@@ -434,6 +440,25 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
             }
 
 skip_evict:
+            page = ref->page;
+            /*
+             * Keep track of whether a session is reading leaf pages into the cache. This allows for
+             * the session to decide whether pre-fetch would be helpful. It might not work if a
+             * session has multiple cursors on different tables open, since the operations on
+             * different tables get in the way of the heuristic. That isn't super likely - this is
+             * to catch traversals through a btree, not complex multi-table user transactions.
+             */
+            if (!LF_ISSET(WT_READ_PREFETCH) && F_ISSET(ref, WT_REF_FLAG_LEAF)) {
+                /*
+                 * If the page was read by this retrieval or was pulled into the cache via the
+                 * pre-fetch mechanism, count that as a page read directly from disk.
+                 */
+                if (F_ISSET_ATOMIC_16(page, WT_PAGE_PREFETCH) ||
+                  page->read_gen == WT_READGEN_NOTSET)
+                    ++session->prefetch_disk_read_count;
+                else
+                    session->prefetch_disk_read_count = 0;
+            }
             /*
              * If we read the page and are configured to not trash the cache, and no other thread
              * has already used the page, set the read generation so the page is evicted soon.
@@ -442,7 +467,6 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags
              * generation and the page isn't already flagged for forced eviction, update the page
              * read generation.
              */
-            page = ref->page;
             if (page->read_gen == WT_READGEN_NOTSET) {
                 if (wont_need)
                     page->read_gen = WT_READGEN_WONT_NEED;

src/third_party/wiredtiger/src/btree/bt_walk.c

@@ -402,6 +402,10 @@ __tree_walk_internal(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp
                 couple = NULL;
                 *refp = ref;
                 WT_ASSERT(session, ref != ref_orig);
+
+                if (__wt_session_prefetch_check(session, ref))
+                    WT_ERR(__wt_btree_prefetch(session, ref));
+
                 goto done;
             }
 
@@ -470,6 +474,9 @@ __tree_walk_internal(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp
                 /* Success, so "couple" has been released. */
                 couple = NULL;
 
+                if (__wt_session_prefetch_check(session, ref))
+                    WT_ERR(__wt_btree_prefetch(session, ref));
+
                 /* Return leaf pages to our caller. */
                 if (F_ISSET(ref, WT_REF_FLAG_LEAF)) {
                     *refp = ref;