drtprod: 300-node TPCC bench

This change adds the YAML automation required for the 300-node scale test for
benchmarking TPCC for publishing results. It contains several cluster
optimisations for the benchmark. It uses server side partitioning. It is tuned
to the optimal parameters for the workload to achieve maximum tpmC.

Epic: None
Release note: None

Author: herkolategan

pkg/cmd/drtprod/configs/drt_scale_300_bench.yaml

@@ -0,0 +1,241 @@
+# Yaml for creating and configuring the drt-scale cluster. This also configures Datadog.
+# Build the drtprod and roachtest binaries (using --cross=linux) before running this script
+environment:
+  ROACHPROD_GCE_DEFAULT_SERVICE_ACCOUNT: [email protected]
+  ROACHPROD_DNS: drt.crdb.io
+  ROACHPROD_GCE_DNS_DOMAIN: drt.crdb.io
+  ROACHPROD_GCE_DNS_ZONE: drt
+  ROACHPROD_GCE_DEFAULT_PROJECT: cockroach-drt
+  CLUSTER: drt-scale-300
+  WORKLOAD_CLUSTER: workload-scale-300
+  CLUSTER_NODES: 300
+  RACKS: 300
+  NODES_PER_ZONE: 100
+  TOTAL_PARTITIONS: 300
+  PARTITION_TYPE: partitions
+  WORKLOAD_NODES: 12
+  VERSION: v25.3.0-beta.3
+  WAREHOUSES: 4000000
+
+dependent_file_locations:
+  - pkg/cmd/drtprod/scripts/setup_datadog_cluster
+  - pkg/cmd/drtprod/scripts/setup_datadog_workload
+  - pkg/cmd/drtprod/scripts/tpcc_init.sh
+  - pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
+  - pkg/cmd/drtprod/scripts/populate_workload_keys.sh
+  - artifacts/roachtest
+  - artifacts/drtprod
+
+targets:
+  # crdb cluster specs
+  - target_name: $CLUSTER
+    steps:
+      - command: create
+        args:
+          - $CLUSTER
+        flags:
+          clouds: gce
+          gce-managed: true
+          gce-enable-multiple-stores: true
+          gce-zones: "us-central1-a:$NODES_PER_ZONE,us-central1-b:$NODES_PER_ZONE,us-central1-c:$NODES_PER_ZONE"
+          nodes: $CLUSTER_NODES
+          gce-machine-type: n2-standard-16
+          local-ssd: false
+          gce-pd-volume-size: 2048
+          gce-pd-volume-type: pd-ssd
+          gce-pd-volume-count: 2
+          os-volume-size: 100
+          username: drt
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20250112"
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $CLUSTER
+          - release
+          - $VERSION
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_cluster"
+      - command: start
+        args:
+          - $CLUSTER
+          - "--binary"
+          - "./cockroach"
+        flags:
+          # add flag to set provisioned throughput on each store according to their cloud provider limits
+          enable-fluent-sink: true
+          store-count: 2
+          args: --wal-failover=among-stores
+          restart: false
+          sql-port: 26257
+          racks: $RACKS
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='256 MB'"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING server.consistency_check.interval = '0s'"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.range_merge.queue_enabled = false"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING rocksdb.min_wal_sync_interval = '500us'"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING admission.kv.enabled = false"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.replication_reports.interval = '0s'"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "ALTER RANGE default CONFIGURE ZONE USING gc.ttlseconds = 600"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING storage.columnar_blocks.enabled = true"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING server.goroutine_dump.num_goroutines_threshold = '10000000'"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING storage.max_sync_duration.fatal.enabled = false"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "ALTER DATABASE system CONFIGURE ZONE USING num_replicas = 5"
+      - command: sql
+        args:
+          - $CLUSTER:1
+          - --
+          - -e
+          - "SET CLUSTER SETTING kv.transaction.write_buffering.enabled = true"
+  # workload cluster specs
+  - target_name: $WORKLOAD_CLUSTER
+    steps:
+      - command: create
+        args:
+          - $WORKLOAD_CLUSTER
+        flags:
+          clouds: gce
+          gce-zones: "us-central1-a"
+          nodes: $WORKLOAD_NODES
+          gce-machine-type: n2-standard-8
+          os-volume-size: 100
+          username: workload
+          lifetime: 8760h
+          gce-image: "ubuntu-2204-jammy-v20250112"
+        on_rollback:
+          - command: destroy
+            args:
+              - $WORKLOAD_CLUSTER
+      - command: sync
+        flags:
+          clouds: gce
+      - command: stage
+        args:
+          - $WORKLOAD_CLUSTER
+          - release
+          - $VERSION
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/roachtest
+          - roachtest-operations
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - artifacts/drtprod
+      - script: "pkg/cmd/drtprod/scripts/setup_datadog_workload"
+  - target_name: post_tasks
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: rm
+        args:
+          - -rf
+          - certs-$CLUSTER
+      - command: fetch-certs
+        args:
+          - $CLUSTER:1
+          - certs-$CLUSTER
+      - command: put
+        args:
+          - $WORKLOAD_CLUSTER
+          - certs-$CLUSTER
+          - certs
+      - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
+        args:
+          - cct_tpcc
+          - false
+        flags:
+          partitions: $TOTAL_PARTITIONS
+          replicate-static-columns: true
+          partition-strategy: leases
+          warehouses: $WAREHOUSES
+          db: cct_tpcc
+      - script: pkg/cmd/drtprod/scripts/populate_workload_keys.sh
+  - target_name: tpcc_run
+    dependent_targets:
+      - $CLUSTER
+      - $WORKLOAD_CLUSTER
+    steps:
+      - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc
+          - false
+        flags:
+          db: cct_tpcc
+          warehouses:  $WAREHOUSES
+          active-warehouses: 333333
+          workers: 333333
+          conns: 1000
+          active-workers: 1000
+          duration: 12h
+          ramp: 5m
+          wait: 0