Import wiredtiger: 956384325e5ff99567ea7b63b1b87b7bb62c76b8 from branch mongodb-4.4

ref: 7dfd939186..956384325e
for: 4.3.1

WT-4831       Add option to python tests to not fail if stdout is not empty
WT-4935       Add a perf test to find out the wiredtiger_calc_modify overhead
WT-5062       Adjust the record size to consume less size
WT-5063       Return proper error message for cursor modify operation for not supported cursor types

Author: lukech

src/third_party/wiredtiger/bench/wtperf/config.c

@@ -220,6 +220,33 @@ config_threads(WTPERF *wtperf, const char *config, size_t len)
                     goto err;
                 continue;
             }
+            if (STRING_MATCH("modify", k.str, k.len)) {
+                if ((workp->modify = v.val) < 0)
+                    goto err;
+                continue;
+            }
+            if (STRING_MATCH("modify_delta", k.str, k.len)) {
+                if (v.type == WT_CONFIG_ITEM_ID || v.type == WT_CONFIG_ITEM_STRING) {
+                    if (strncmp(v.str, "rand", 4) != 0)
+                        goto err;
+                    /* Special random value */
+                    workp->modify_delta = INT64_MAX;
+                    F_SET(wtperf, CFG_GROW);
+                } else {
+                    workp->modify_delta = v.val;
+                    if (v.val > 0)
+                        F_SET(wtperf, CFG_GROW);
+                    if (v.val < 0)
+                        F_SET(wtperf, CFG_SHRINK);
+                }
+                continue;
+            }
+            if (STRING_MATCH("modify_force_update", k.str, k.len)) {
+                if (v.type != WT_CONFIG_ITEM_BOOL)
+                    goto err;
+                workp->modify_force_update = v.val;
+                continue;
+            }
             if (STRING_MATCH("ops_per_txn", k.str, k.len)) {
                 if ((workp->ops_per_txn = v.val) < 0)
                     goto err;
@@ -302,7 +329,8 @@ config_threads(WTPERF *wtperf, const char *config, size_t len)
         scan = NULL;
         if (ret != 0)
             goto err;
-        if (workp->insert == 0 && workp->read == 0 && workp->update == 0 && workp->truncate == 0)
+        if (workp->insert == 0 && workp->modify == 0 && workp->read == 0 && workp->truncate == 0 &&
+          workp->update == 0)
             goto err;
         /* Why run with truncate if we don't want any truncation. */
         if (workp->truncate != 0 && workp->truncate_pct == 0 && workp->truncate_count == 0)
@@ -312,7 +340,8 @@ config_threads(WTPERF *wtperf, const char *config, size_t len)
         /* Truncate should have its own exclusive thread. */
         if (workp->truncate != 0 && workp->threads > 1)
             goto err;
-        if (workp->truncate != 0 && (workp->insert > 0 || workp->read > 0 || workp->update > 0))
+        if (workp->truncate != 0 &&
+          (workp->insert > 0 || workp->modify > 0 || workp->read > 0 || workp->update > 0))
             goto err;
         wtperf->workers_cnt += (u_int)workp->threads;
     }
@@ -765,11 +794,11 @@ config_sanity(WTPERF *wtperf)
 
     if (wtperf->workload != NULL)
         for (i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) {
-            if (opts->readonly &&
-              (workp->insert != 0 || workp->update != 0 || workp->truncate != 0)) {
+            if (opts->readonly && (workp->insert != 0 || workp->modify != 0 ||
+                                    workp->truncate != 0 || workp->update != 0)) {
                 fprintf(stderr,
-                  "Invalid workload: insert, update or "
-                  "truncate specified with readonly\n");
+                  "Invalid workload: insert, modify, truncate or update specified with "
+                  "readonly\n");
                 return (EINVAL);
             }
             if (workp->insert != 0 && workp->table_index != INT32_MAX) {
@@ -785,6 +814,12 @@ config_sanity(WTPERF *wtperf)
                   workp->table_index, opts->table_count);
                 return (EINVAL);
             }
+            if (workp->modify != 0 && workp->ops_per_txn == 0) {
+                fprintf(stderr,
+                  "Modify operations must be executed with explicit transaction, specify "
+                  "ops_per_txn.");
+                return (EINVAL);
+            }
         }
     return (0);
 }

src/third_party/wiredtiger/bench/wtperf/runners/modify-force-update-large-record-btree.wtperf

@@ -0,0 +1,14 @@
+# wtperf options file: btree with updates. The goal here is to have a workload of
+# large documents that uses a lot of transaction IDs for modify with forced update
+# to ensure that update operation happened instead of modify.
+conn_config="cache_size=20G"
+sess_config="isolation=snapshot"
+table_config="type=file"
+key_sz=40
+value_sz=10000
+icount=500000
+report_interval=5
+random_value=true
+run_time=120
+populate_threads=1
+threads=((count=4,modify=1,ops_per_txn=1,modify_force_update=true))

src/third_party/wiredtiger/bench/wtperf/runners/modify-large-record-btree.wtperf

@@ -0,0 +1,13 @@
+# wtperf options file: btree with updates. The goal here is to have a workload
+# of large documents that uses a lot of transaction IDs for modify operations
+conn_config="cache_size=20G"
+sess_config="isolation=snapshot"
+table_config="type=file"
+key_sz=40
+value_sz=10000
+icount=500000
+report_interval=5
+random_value=true
+run_time=120
+populate_threads=1
+threads=((count=4,modify=1,ops_per_txn=1))

src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh

@@ -39,18 +39,18 @@ rm -f $outfile
 echo "Parsed $# args: test: $wttest runmax: $runmax args: $wtarg" >> $outfile
 
 # Each of these has an entry for each op in ops below.
-avg=(0 0 0 0)
-max=(0 0 0 0)
-min=(0 0 0 0)
-sum=(0 0 0 0)
+avg=(0 0 0 0 0)
+max=(0 0 0 0 0)
+min=(0 0 0 0 0)
+sum=(0 0 0 0 0)
 # Load needs floating point and bc, handle separately.
-loadindex=5
+loadindex=6
 avg[$loadindex]=0
 max[$loadindex]=0
 min[$loadindex]=0
 sum[$loadindex]=0
-ops=(read insert update truncate)
-outp=("Read count:" "Insert count:" "Update count:" "Truncate count:")
+ops=(insert modify read truncate update)
+outp=("Insert count:" "Modify count:" "Read count:" "Truncate count:" "Update count:"  )
 outp[$loadindex]="Load time:"
 
 # getval min/max val cur

src/third_party/wiredtiger/bench/wtperf/track.c

@@ -118,6 +118,11 @@ sum_insert_ops(WTPERF *wtperf)
     return (sum_ops(wtperf, offsetof(WTPERF_THREAD, insert)));
 }
 uint64_t
+sum_modify_ops(WTPERF *wtperf)
+{
+    return (sum_ops(wtperf, offsetof(WTPERF_THREAD, modify)));
+}
+uint64_t
 sum_read_ops(WTPERF *wtperf)
 {
     return (sum_ops(wtperf, offsetof(WTPERF_THREAD, read)));
@@ -185,11 +190,11 @@ latency_op(WTPERF *wtperf, size_t field_offset, uint32_t *avgp, uint32_t *minp,
     }
 }
 void
-latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
+latency_insert(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
 {
     static uint32_t last_avg = 0, last_max = 0, last_min = 0;
 
-    latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp);
+    latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp);
 
     /*
      * If nothing happened, graph the average, minimum and maximum as they were the last time, it
@@ -206,11 +211,32 @@ latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
     }
 }
 void
-latency_insert(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
+latency_modify(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
 {
     static uint32_t last_avg = 0, last_max = 0, last_min = 0;
 
-    latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp);
+    latency_op(wtperf, offsetof(WTPERF_THREAD, modify), avgp, minp, maxp);
+
+    /*
+     * If nothing happened, graph the average, minimum and maximum as they were the last time, it
+     * keeps the graphs from having discontinuities.
+     */
+    if (*minp == 0) {
+        *avgp = last_avg;
+        *minp = last_min;
+        *maxp = last_max;
+    } else {
+        last_avg = *avgp;
+        last_min = *minp;
+        last_max = *maxp;
+    }
+}
+void
+latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp)
+{
+    static uint32_t last_avg = 0, last_max = 0, last_min = 0;
+
+    latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp);
 
     /*
      * If nothing happened, graph the average, minimum and maximum as they were the last time, it
@@ -286,6 +312,11 @@ sum_insert_latency(WTPERF *wtperf, TRACK *total)
     sum_latency(wtperf, offsetof(WTPERF_THREAD, insert), total);
 }
 static void
+sum_modify_latency(WTPERF *wtperf, TRACK *total)
+{
+    sum_latency(wtperf, offsetof(WTPERF_THREAD, modify), total);
+}
+static void
 sum_read_latency(WTPERF *wtperf, TRACK *total)
 {
     sum_latency(wtperf, offsetof(WTPERF_THREAD, read), total);
@@ -344,6 +375,8 @@ latency_print(WTPERF *wtperf)
 
     sum_insert_latency(wtperf, &total);
     latency_print_single(wtperf, &total, "insert");
+    sum_modify_latency(wtperf, &total);
+    latency_print_single(wtperf, &total, "modify");
     sum_read_latency(wtperf, &total);
     latency_print_single(wtperf, &total, "read");
     sum_update_latency(wtperf, &total);