Import wiredtiger: 850b579d4b526de7f296cb617999ccc20adb4aff from branch mongodb-4.6

ref: 4cacb06e31..850b579d4b
for: 4.5.1

WT-5693       Enable test_wt4105_large_doc_small_upd
WT-6390       Extend compact02 timeout from 8 => 10 minutes
WT-6569       Squash the prepared updates into a single update before writing it to data store
WT-6578       Prevent reconciliation from looking past the on-disk value
WT-6602       Allow operation timeout ms to be passed to commit and rollback

Author: lukech

src/third_party/wiredtiger/dist/api_data.py

@@ -1450,11 +1450,11 @@ def __ge__(self, other):
     Config('name', '', r'''
         name of the transaction for tracing and debugging'''),
     Config('operation_timeout_ms', '0', r'''
-        when non-zero, a requested limit on the number of elapsed real time milliseconds taken
-        to complete database operations in this transaction.  Time is measured from the start
-        of each WiredTiger API call.  There is no guarantee any operation will not take longer
-        than this amount of time. If WiredTiger notices the limit has been exceeded, an operation
-        may return a WT_ROLLBACK error. Default is to have no limit''',
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
         min=1),
     Config('priority', 0, r'''
         priority of the transaction for resolving conflicts.
@@ -1501,6 +1501,13 @@ def __ge__(self, other):
         current transaction.  The value must also not be older than the
         current stable timestamp.  See
         @ref transaction_timestamps'''),
+    Config('operation_timeout_ms', '0', r'''
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
+        min=1),
     Config('sync', '', r'''
         override whether to sync log records when the transaction commits,
         inherited from ::wiredtiger_open \c transaction_sync.
@@ -1542,7 +1549,15 @@ def __ge__(self, other):
         for a transaction. See @ref transaction_timestamps'''),
 ]),
 
-'WT_SESSION.rollback_transaction' : Method([]),
+'WT_SESSION.rollback_transaction' : Method([
+    Config('operation_timeout_ms', '0', r'''
+        when non-zero, a requested limit on the time taken to complete operations in this
+        transaction. Time is measured in real time milliseconds from the start of each WiredTiger
+        API call. There is no guarantee any operation will not take longer than this amount of time.
+        If WiredTiger notices the limit has been exceeded, an operation may return a WT_ROLLBACK
+        error. Default is to have no limit''',
+        min=1),
+]),
 
 'WT_SESSION.checkpoint' : Method([
     Config('drop', '', r'''

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.6",
-    "commit": "4cacb06e31b67cd17e0ecdcc366c3e9466e5ac3a"
+    "commit": "850b579d4b526de7f296cb617999ccc20adb4aff"
 }

src/third_party/wiredtiger/src/btree/bt_page.c

@@ -633,6 +633,20 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
             tombstone->prepare_state = WT_PREPARE_INPROGRESS;
             F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
             F_SET(upd, WT_UPDATE_RESTORED_FROM_DS);
+
+            /*
+             * Mark the update also as in-progress if the update and tombstone are from same
+             * transaction by comparing both the transaction and timestamps as the transaction
+             * information gets lost after restart.
+             */
+            if (unpack.tw.start_ts == unpack.tw.stop_ts &&
+              unpack.tw.durable_start_ts == unpack.tw.durable_stop_ts &&
+              unpack.tw.start_txn == unpack.tw.stop_txn) {
+                upd->durable_ts = WT_TS_NONE;
+                upd->prepare_state = WT_PREPARE_INPROGRESS;
+                F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
+            }
+
             tombstone->next = upd;
         } else {
             upd->durable_ts = WT_TS_NONE;

src/third_party/wiredtiger/src/config/config_def.c

@@ -215,6 +215,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_checkpoint[] = {
 static const WT_CONFIG_CHECK confchk_WT_SESSION_commit_transaction[] = {
   {"commit_timestamp", "string", NULL, NULL, NULL, 0},
   {"durable_timestamp", "string", NULL, NULL, NULL, 0},
+  {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0},
   {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -345,6 +346,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = {
     NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
+static const WT_CONFIG_CHECK confchk_WT_SESSION_rollback_transaction[] = {
+  {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
+
 static const WT_CONFIG_CHECK confchk_WT_SESSION_salvage[] = {
   {"force", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -887,7 +891,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     confchk_WT_SESSION_checkpoint, 5},
   {"WT_SESSION.close", "", NULL, 0},
   {"WT_SESSION.commit_transaction",
-    "commit_timestamp=,durable_timestamp=,sync=", confchk_WT_SESSION_commit_transaction, 3},
+    "commit_timestamp=,durable_timestamp=,operation_timeout_ms=0,"
+    "sync=",
+    confchk_WT_SESSION_commit_transaction, 4},
   {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1},
   {"WT_SESSION.create",
     "access_pattern_hint=none,allocation_size=4KB,app_metadata=,"
@@ -939,7 +945,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "isolation=read-committed",
     confchk_WT_SESSION_reconfigure, 3},
   {"WT_SESSION.rename", "", NULL, 0}, {"WT_SESSION.reset", "", NULL, 0},
-  {"WT_SESSION.rollback_transaction", "", NULL, 0},
+  {"WT_SESSION.rollback_transaction", "operation_timeout_ms=0",
+    confchk_WT_SESSION_rollback_transaction, 1},
   {"WT_SESSION.salvage", "force=false", confchk_WT_SESSION_salvage, 1},
   {"WT_SESSION.strerror", "", NULL, 0},
   {"WT_SESSION.timestamp_transaction",

src/third_party/wiredtiger/src/history/hs.c

@@ -1160,7 +1160,7 @@ __wt_hs_cursor_search_near(WT_SESSION_IMPL *session, WT_CURSOR *cursor, int *exa
  */
 int
 __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_format, uint64_t recno,
-  WT_UPDATE_VALUE *upd_value, bool allow_prepare, WT_ITEM *on_disk_buf)
+  WT_UPDATE_VALUE *upd_value, bool allow_prepare, WT_ITEM *on_disk_buf, WT_TIME_WINDOW *on_disk_tw)
 {
     WT_CURSOR *hs_cursor;
     WT_CURSOR_BTREE *hs_cbt;
@@ -1341,6 +1341,21 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma
                 break;
             }
 
+            /*
+             * If we find a history store record that either corresponds to the on-disk value or is
+             * newer than it then we should use the on-disk value as the base value and apply our
+             * modifies on top of it.
+             */
+            if (on_disk_tw->start_ts < hs_start_ts_tmp ||
+              (on_disk_tw->start_ts == hs_start_ts_tmp &&
+                on_disk_tw->start_txn <= hs_cbt->upd_value->tw.start_txn)) {
+                /* Fallback to the onpage value as the base value. */
+                orig_hs_value_buf = hs_value;
+                hs_value = on_disk_buf;
+                upd_type = WT_UPDATE_STANDARD;
+                break;
+            }
+
             WT_ERR(hs_cursor->get_value(hs_cursor, &hs_stop_durable_ts_tmp, &durable_timestamp_tmp,
               &upd_type_full, hs_value));
             upd_type = (uint8_t)upd_type_full;

src/third_party/wiredtiger/src/include/btmem.h

@@ -1075,12 +1075,13 @@ struct __wt_update {
 
 /* AUTOMATIC FLAG VALUE GENERATION START */
 #define WT_UPDATE_CLEARED_HS 0x01u               /* Update that cleared the history store. */
-#define WT_UPDATE_HS 0x02u                       /* Update has been written to history store. */
-#define WT_UPDATE_OBSOLETE 0x04u                 /* Update that is obsolete. */
-#define WT_UPDATE_PREPARE_RESTORED_FROM_DS 0x08u /* Prepared update restored from data store. */
-#define WT_UPDATE_RESTORED_FAST_TRUNCATE 0x10u   /* Fast truncate instantiation */
-#define WT_UPDATE_RESTORED_FROM_DS 0x20u         /* Update restored from data store. */
-#define WT_UPDATE_RESTORED_FROM_HS 0x40u         /* Update restored from history store. */
+#define WT_UPDATE_DS 0x02u                       /* Update has been written to the data store. */
+#define WT_UPDATE_HS 0x04u                       /* Update has been written to history store. */
+#define WT_UPDATE_OBSOLETE 0x08u                 /* Update that is obsolete. */
+#define WT_UPDATE_PREPARE_RESTORED_FROM_DS 0x10u /* Prepared update restored from data store. */
+#define WT_UPDATE_RESTORED_FAST_TRUNCATE 0x20u   /* Fast truncate instantiation */
+#define WT_UPDATE_RESTORED_FROM_DS 0x40u         /* Update restored from data store. */
+#define WT_UPDATE_RESTORED_FROM_HS 0x80u         /* Update restored from history store. */
                                                  /* AUTOMATIC FLAG VALUE GENERATION STOP */
     uint8_t flags;
 

src/third_party/wiredtiger/src/include/extern.h

@@ -771,8 +771,8 @@ extern int __wt_hs_cursor_search_near(WT_SESSION_IMPL *session, WT_CURSOR *curso
 extern int __wt_hs_delete_key_from_ts(WT_SESSION_IMPL *session, uint32_t btree_id,
   const WT_ITEM *key, wt_timestamp_t ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_format,
-  uint64_t recno, WT_UPDATE_VALUE *upd_value, bool allow_prepare, WT_ITEM *on_disk_buf)
-  WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+  uint64_t recno, WT_UPDATE_VALUE *upd_value, bool allow_prepare, WT_ITEM *on_disk_buf,
+  WT_TIME_WINDOW *on_disk_tw) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_hs_get_btree(WT_SESSION_IMPL *session, WT_BTREE **hs_btreep)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi)

src/third_party/wiredtiger/src/include/txn.i

@@ -993,7 +993,7 @@ retry:
     /* If there's no visible update in the update chain or ondisk, check the history store file. */
     if (F_ISSET(S2C(session), WT_CONN_HS_OPEN) && !F_ISSET(S2BT(session), WT_BTREE_HS))
         WT_RET_NOTFOUND_OK(__wt_hs_find_upd(session, key, cbt->iface.value_format, recno,
-          cbt->upd_value, false, &cbt->upd_value->buf));
+          cbt->upd_value, false, &cbt->upd_value->buf, &tw));
 
     /*
      * Retry if we race with prepared commit or rollback. If we race with prepared rollback, the
@@ -1045,8 +1045,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[])
 
     WT_ASSERT(session, !F_ISSET(txn, WT_TXN_RUNNING));
 
-    if (cfg != NULL)
-        WT_RET(__wt_txn_config(session, cfg));
+    WT_RET(__wt_txn_config(session, cfg));
 
     /* Allocate a snapshot if required. */
     if (txn->isolation == WT_ISO_SNAPSHOT) {

src/third_party/wiredtiger/src/include/wiredtiger.in

@@ -1766,12 +1766,12 @@ struct __wt_session {
 	 * \c "read-committed"\, \c "snapshot"; default empty.}
 	 * @config{name, name of the transaction for tracing and debugging., a string; default
 	 * empty.}
-	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the number of elapsed
-	 * real time milliseconds taken to complete database operations in this transaction.  Time
-	 * is measured from the start of each WiredTiger API call.  There is no guarantee any
-	 * operation will not take longer than this amount of time.  If WiredTiger notices the limit
-	 * has been exceeded\, an operation may return a WT_ROLLBACK error.  Default is to have no
-	 * limit., an integer greater than or equal to 1; default \c 0.}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
 	 * @config{priority, priority of the transaction for resolving conflicts.  Transactions with
 	 * higher values are less likely to abort., an integer between -100 and 100; default \c 0.}
 	 * @config{read_timestamp, read using the specified timestamp.  The supplied value must not
@@ -1820,6 +1820,12 @@ struct __wt_session {
 	 * supplied value must not be older than the commit timestamp set for the current
 	 * transaction.  The value must also not be older than the current stable timestamp.  See
 	 * @ref transaction_timestamps., a string; default empty.}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
 	 * @config{sync, override whether to sync log records when the transaction commits\,
 	 * inherited from ::wiredtiger_open \c transaction_sync.  The \c background setting
 	 * initiates a background synchronization intended to be used with a later call to
@@ -1869,7 +1875,14 @@ struct __wt_session {
 	 * @snippet ex_all.c transaction commit/rollback
 	 *
 	 * @param session the session handle
-	 * @configempty{WT_SESSION.rollback_transaction, see dist/api_data.py}
+	 * @configstart{WT_SESSION.rollback_transaction, see dist/api_data.py}
+	 * @config{operation_timeout_ms, when non-zero\, a requested limit on the time taken to
+	 * complete operations in this transaction.  Time is measured in real time milliseconds from
+	 * the start of each WiredTiger API call.  There is no guarantee any operation will not take
+	 * longer than this amount of time.  If WiredTiger notices the limit has been exceeded\, an
+	 * operation may return a WT_ROLLBACK error.  Default is to have no limit., an integer
+	 * greater than or equal to 1; default \c 0.}
+	 * @configend
 	 * @errors
 	 */
 	int __F(rollback_transaction)(WT_SESSION *session, const char *config);

src/third_party/wiredtiger/src/reconcile/rec_row.c

@@ -590,6 +590,17 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins)
             continue;
         }
 
+        /*
+         * If we've selected an update, it should be flagged as being destined for the data store.
+         *
+         * If not, it's either because we're not doing a history store reconciliation or because the
+         * update is globally visible (in which case, subsequent updates become irrelevant for
+         * reconciliation).
+         */
+        WT_ASSERT(session,
+          F_ISSET(upd, WT_UPDATE_DS) || !F_ISSET(r, WT_REC_HS) ||
+            __wt_txn_tw_start_visible_all(session, &upd_select.tw));
+
         WT_TIME_WINDOW_COPY(&tw, &upd_select.tw);
 
         switch (upd->type) {
@@ -839,6 +850,18 @@ __wt_rec_row_leaf(
                     r->ovfl_items = true;
             }
         } else {
+            /*
+             * If we've selected an update, it should be flagged as being destined for the data
+             * store.
+             *
+             * If not, it's either because we're not doing a history store reconciliation or because
+             * the update is globally visible (in which case, subsequent updates become irrelevant
+             * for reconciliation).
+             */
+            WT_ASSERT(session,
+              F_ISSET(upd, WT_UPDATE_DS) || !F_ISSET(r, WT_REC_HS) ||
+                __wt_txn_tw_start_visible_all(session, &upd_select.tw));
+
             /* The first time we find an overflow record, discard the underlying blocks. */
             if (F_ISSET(vpack, WT_CELL_UNPACK_OVERFLOW) && vpack->raw != WT_CELL_VALUE_OVFL_RM)
                 WT_ERR(__wt_ovfl_remove(session, page, vpack));