Merge #151474 #151537

151474: storage: bump Pebble format version r=RaduBerinde a=RaduBerinde

#### go.mod: bump Pebble to 258531bdf87e

Changes:

 * [`258531bd`](cockroachdb/pebble@258531bd) blob: add FileFormatV2 which has a properties block
 * [`4be89e71`](cockroachdb/pebble@4be89e71) go.mod: require go1.24
 * [`d3bef264`](cockroachdb/pebble@d3bef264) sstable: remove CompressionOptions
 * [`2d7040f1`](cockroachdb/pebble@2d7040f1) manifest: improve API around TableStats
 * [`7d7cc282`](cockroachdb/pebble@7d7cc282) db: fix bug in scaling compression stats for virtual tables
 * [`c9692798`](cockroachdb/pebble@c9692798) db: add some tests for compression stats

Release note: none.
Epic: none.

#### storage: bump Pebble format version

Start using the latest Pebble format version.

Epic: none
Release note: None

151537: workload: add a contention option to KV r=miraradeva a=miraradeva

This commit adds a new set of flags to the KV workload to simulate long-running writing transactions.

- `--long-running-txn`, when true, enables this option.
- `--long-running-txn-num-writes` specifies the number of writes the  long-running transaction performs.

Even though each long-running write transaction performs multiple writes, for simplicity, it counts for a single write from the perspective of `--read-percent`.

This new option also works with the other write options: `--sfu-writes` and `--sel1-writes`. In those cases, in addition to performing multiple writes, each of the writes is preceded by a `SELECT` and a sleep, configured as before.

Fixes: #151038

Release note: None

Co-authored-by: Radu Berinde <[email protected]>
Co-authored-by: Mira Radeva <[email protected]>

Author: 3 people

docs/generated/settings/settings-for-tenants.txt

@@ -420,4 +420,4 @@ trace.zipkin.collector	string		the address of a Zipkin instance to receive trace
 ui.database_locality_metadata.enabled	boolean	true	if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute	application
 ui.default_timezone	string		the default timezone used to format timestamps in the ui	application
 ui.display_timezone	enumeration	etc/utc	the timezone used to format timestamps in the ui. This setting is deprecatedand will be removed in a future version. Use the 'ui.default_timezone' setting instead. 'ui.default_timezone' takes precedence over this setting. [etc/utc = 0, america/new_york = 1]	application
-version	version	1000025.3-upgrading-to-1000025.4-step-004	set the active cluster version in the format '<major>.<minor>'	application
+version	version	1000025.3-upgrading-to-1000025.4-step-006	set the active cluster version in the format '<major>.<minor>'	application

docs/generated/settings/settings.html

@@ -378,6 +378,6 @@
 <tr><td><div id="setting-ui-database-locality-metadata-enabled" class="anchored"><code>ui.database_locality_metadata.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-default-timezone" class="anchored"><code>ui.default_timezone</code></div></td><td>string</td><td><code></code></td><td>the default timezone used to format timestamps in the ui</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-display-timezone" class="anchored"><code>ui.display_timezone</code></div></td><td>enumeration</td><td><code>etc/utc</code></td><td>the timezone used to format timestamps in the ui. This setting is deprecatedand will be removed in a future version. Use the &#39;ui.default_timezone&#39; setting instead. &#39;ui.default_timezone&#39; takes precedence over this setting. [etc/utc = 0, america/new_york = 1]</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000025.3-upgrading-to-1000025.4-step-004</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000025.3-upgrading-to-1000025.4-step-006</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 </tbody>
 </table>

pkg/cli/testdata/ear-list

@@ -8,13 +8,13 @@ list
 000004.log:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: 31 d3 cd 5a 69 e2 13 64 21 53 57 64
-  counter: 3952287331
+  nonce: 86 a7 78 ad 4b da 62 56 d5 e2 d1 70
+  counter: 798955289
 000005.sst:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: 23 d9 b2 e1 39 b0 87 ed f9 6d 49 20
-  counter: 3481614039
+  nonce: d1 05 79 53 68 35 a0 f1 44 01 22 79
+  counter: 1497766936
 COCKROACHDB_DATA_KEYS_000001_monolith:
   env type: Store, AES128_CTR
   keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
@@ -35,11 +35,11 @@ marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith:
   keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
   nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8
   counter: 467030050
-marker.format-version.000011.024:
+marker.format-version.000013.026:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: c3 6d b2 7b 3f 3e 67 b9 28 b9 81 b1
-  counter: 3050109376
+  nonce: 23 d9 b2 e1 39 b0 87 ed f9 6d 49 20
+  counter: 3481614039
 marker.manifest.000001.MANIFEST-000001:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef

pkg/clusterversion/cockroach_versions.go

@@ -210,6 +210,9 @@ const (
 	// constructing the split write batch.
 	V25_4_WriteInitialTruncStateBeforeSplitApplication
 
+	// V25_4_PebbleFormatV2BlobFiles bumps the pebble format to FormatV2BlobFiles.
+	V25_4_PebbleFormatV2BlobFiles
+
 	// *************************************************
 	// Step (1) Add new versions above this comment.
 	// Do not add new versions to a patch release.
@@ -263,6 +266,7 @@ var versionTable = [numKeys]roachpb.Version{
 	V25_4_Start: {Major: 25, Minor: 3, Internal: 2},
 
 	V25_4_WriteInitialTruncStateBeforeSplitApplication: {Major: 25, Minor: 3, Internal: 4},
+	V25_4_PebbleFormatV2BlobFiles:                      {Major: 25, Minor: 3, Internal: 6},
 
 	// *************************************************
 	// Step (2): Add new versions above this comment.

pkg/storage/pebble.go

@@ -2326,8 +2326,9 @@ func (p *Pebble) CreateCheckpoint(dir string, spans []roachpb.Span) error {
 // named version, it can be assumed all *nodes* have ratcheted to the pebble
 // version associated with it, since they did so during the fence version.
 var pebbleFormatVersionMap = map[clusterversion.Key]pebble.FormatMajorVersion{
-	clusterversion.V25_3: pebble.FormatValueSeparation,
-	clusterversion.V25_2: pebble.FormatTableFormatV6,
+	clusterversion.V25_4_PebbleFormatV2BlobFiles: pebble.FormatV2BlobFiles,
+	clusterversion.V25_3:                         pebble.FormatValueSeparation,
+	clusterversion.V25_2:                         pebble.FormatTableFormatV6,
 }
 
 // MinimumSupportedFormatVersion is the version that provides features that the

pkg/workload/kv/kv.go

@@ -83,6 +83,8 @@ type kv struct {
 	sequential                           bool
 	zipfian                              bool
 	sfuDelay                             time.Duration
+	longRunningTxn                       bool
+	longRunningTxnNumWrites              int
 	splits                               int
 	scatter                              bool
 	secondaryIndex                       bool
@@ -119,18 +121,20 @@ var kvMeta = workload.Meta{
 		g := &kv{}
 		g.flags.FlagSet = pflag.NewFlagSet(`kv`, pflag.ContinueOnError)
 		g.flags.Meta = map[string]workload.FlagMeta{
-			`batch`:             {RuntimeOnly: true},
-			`sfu-wait-delay`:    {RuntimeOnly: true},
-			`sfu-writes`:        {RuntimeOnly: true},
-			`read-percent`:      {RuntimeOnly: true},
-			`span-percent`:      {RuntimeOnly: true},
-			`span-limit`:        {RuntimeOnly: true},
-			`del-percent`:       {RuntimeOnly: true},
-			`splits`:            {RuntimeOnly: true},
-			`scatter`:           {RuntimeOnly: true},
-			`timeout`:           {RuntimeOnly: true},
-			`prepare-read-only`: {RuntimeOnly: true},
-			`sel1-writes`:       {RuntimeOnly: true},
+			`batch`:                       {RuntimeOnly: true},
+			`sfu-wait-delay`:              {RuntimeOnly: true},
+			`sfu-writes`:                  {RuntimeOnly: true},
+			`long-running-txn`:            {RuntimeOnly: true},
+			`long-running-txn-num-writes`: {RuntimeOnly: true},
+			`read-percent`:                {RuntimeOnly: true},
+			`span-percent`:                {RuntimeOnly: true},
+			`span-limit`:                  {RuntimeOnly: true},
+			`del-percent`:                 {RuntimeOnly: true},
+			`splits`:                      {RuntimeOnly: true},
+			`scatter`:                     {RuntimeOnly: true},
+			`timeout`:                     {RuntimeOnly: true},
+			`prepare-read-only`:           {RuntimeOnly: true},
+			`sel1-writes`:                 {RuntimeOnly: true},
 		}
 		g.flags.IntVar(&g.batchSize, `batch`, 1,
 			`Number of blocks to read/insert in a single SQL statement.`)
@@ -187,6 +191,13 @@ var kvMeta = workload.Meta{
 		g.flags.BoolVar(&g.prepareReadOnly, `prepare-read-only`, false, `Prepare and perform only read statements.`)
 		g.flags.BoolVar(&g.writesUseSelect1, `sel1-writes`, false,
 			`Use SELECT 1 as the first statement of transactional writes with a sleep after SELECT 1.`)
+		g.flags.BoolVar(&g.longRunningTxn, `long-running-txn`, false,
+			`Use a long-running transaction for running lock contention scenarios. If run with `+
+				`--sfu-writes or --sel1-writes, it will use those writes in the long-running transaction; `+
+				`otherwise, it will use regular writes. Each long-running write transaction counts for a`+
+				`single write, as measured by --read-percent.`)
+		g.flags.IntVar(&g.longRunningTxnNumWrites, `long-running-txn-num-writes`, 10,
+			`Number of writes in the long-running transaction when using --long-running-txn.`)
 		g.connFlags = workload.NewConnFlags(&g.flags)
 		return g
 	},
@@ -670,23 +681,26 @@ func (o *kvOp) run(ctx context.Context) (retErr error) {
 		o.hists.Get(`span`).Record(elapsed)
 		return err
 	}
-	const argCount = 2
-	writeArgs := make([]interface{}, argCount*o.config.batchSize)
-	var sfuArgs []interface{}
-	if o.config.writesUseSelectForUpdate {
-		sfuArgs = make([]interface{}, o.config.batchSize)
-	}
-	for i := 0; i < o.config.batchSize; i++ {
-		j := i * argCount
-		writeArgs[j+0] = o.t.getKey(o.g.writeKey())
-		if sfuArgs != nil {
-			sfuArgs[i] = writeArgs[j]
+	makeWriteBatchArgs := func() ([]interface{}, []interface{}) {
+		const argCount = 2
+		writeArgs := make([]interface{}, argCount*o.config.batchSize)
+		var sfuArgs []interface{}
+		if o.config.writesUseSelectForUpdate {
+			sfuArgs = make([]interface{}, o.config.batchSize)
+		}
+		for i := 0; i < o.config.batchSize; i++ {
+			j := i * argCount
+			writeArgs[j+0] = o.t.getKey(o.g.writeKey())
+			if sfuArgs != nil {
+				sfuArgs[i] = writeArgs[j]
+			}
+			writeArgs[j+1] = o.config.randBlock(o.g.rand())
 		}
-		writeArgs[j+1] = o.config.randBlock(o.g.rand())
+		return writeArgs, sfuArgs
 	}
 	start := timeutil.Now()
 	var err error
-	if o.config.writesUseSelect1 || o.config.writesUseSelectForUpdate {
+	if o.config.writesUseSelect1 || o.config.writesUseSelectForUpdate || o.config.longRunningTxn {
 		// We could use crdb.ExecuteTx, but we avoid retries in this workload so
 		// that each run call makes 1 attempt, so that rate limiting in workerRun
 		// behaves as expected.
@@ -695,44 +709,52 @@ func (o *kvOp) run(ctx context.Context) (retErr error) {
 		if err != nil {
 			return err
 		}
-
 		defer func() {
 			rollbackErr := tx.Rollback(ctx)
 			if !errors.Is(rollbackErr, pgx.ErrTxClosed) {
 				retErr = errors.CombineErrors(retErr, rollbackErr)
 			}
 		}()
-		if o.config.writesUseSelect1 {
-			rows, err := o.sel1Stmt.QueryTx(ctx, tx)
-			if err != nil {
-				return err
-			}
-			rows.Close()
-			if err = rows.Err(); err != nil {
-				return err
-			}
+		iterations := 1
+		if o.config.longRunningTxn {
+			iterations = o.config.longRunningTxnNumWrites
 		}
-		if o.config.writesUseSelectForUpdate {
-			rows, err := o.sfuStmt.QueryTx(ctx, tx, sfuArgs...)
-			if err != nil {
-				return err
+		for i := 0; i < iterations; i++ {
+			writeArgs, sfuArgs := makeWriteBatchArgs()
+			if o.config.writesUseSelect1 {
+				rows, err := o.sel1Stmt.QueryTx(ctx, tx)
+				if err != nil {
+					return err
+				}
+				rows.Close()
+				if err = rows.Err(); err != nil {
+					return err
+				}
+			}
+			if o.config.writesUseSelectForUpdate {
+				rows, err := o.sfuStmt.QueryTx(ctx, tx, sfuArgs...)
+				if err != nil {
+					return err
+				}
+				rows.Close()
+				if err = rows.Err(); err != nil {
+					// The transaction may have experienced an error in the meantime.
+					return o.tryHandleWriteErr("write-write-err", start, err)
+				}
 			}
-			rows.Close()
-			if err = rows.Err(); err != nil {
-				return err
+			// Simulate a transaction that does other work between the sel1 / SFU and write.
+			time.Sleep(o.config.sfuDelay)
+			if _, err = o.writeStmt.ExecTx(ctx, tx, writeArgs...); err != nil {
+				// Multiple write transactions can contend and encounter
+				// a serialization failure. We swallow such an error.
+				return o.tryHandleWriteErr("write-write-err", start, err)
 			}
 		}
-		// Simulate a transaction that does other work between the sel1 / SFU and write.
-		time.Sleep(o.config.sfuDelay)
-		if _, err = o.writeStmt.ExecTx(ctx, tx, writeArgs...); err != nil {
-			// Multiple write transactions can contend and encounter
-			// a serialization failure. We swallow such an error.
-			return o.tryHandleWriteErr("write-write-err", start, err)
-		}
 		if err = tx.Commit(ctx); err != nil {
 			return o.tryHandleWriteErr("write-commit-err", start, err)
 		}
 	} else {
+		writeArgs, _ := makeWriteBatchArgs()
 		_, err = o.writeStmt.Exec(ctx, writeArgs...)
 	}
 	if err != nil {