Implement microbench test target type

Author: michaelsembwever

.build/run-tests.sh

@@ -30,6 +30,9 @@ set -o pipefail
 [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)"
 [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build"
 
+# target types
+TARGET_TYPES="build_dtest_jars stress-test fqltool-test microbench test-burn long-test cqlsh-test simulator-dtest test test-cdc test-compression test-oa test-system-keyspace-directory test-latest jvm-dtest jvm-dtest-upgrade jvm-dtest-novnode jvm-dtest-upgrade-novnode"
+
 # pre-conditions
 command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
 command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
@@ -47,7 +50,7 @@ error() {
 
 print_help() {
   echo "Usage: $0 [-a|-t|-c|-e|-i|-b|-s|-h]"
-  echo "   -a Test target type: test, test-compression, test-cdc, ..."
+  echo "   -a Test target type: ${TARGET_TYPES}"
   echo "   -t Test name regexp to run."
   echo "   -c Chunk to run in the form X/Y: Run chunk X from a total of Y chunks."
   echo "   -b Specify the base git branch for comparison when determining changed tests to"
@@ -65,8 +68,7 @@ print_help() {
 
 
 # legacy argument handling
-case ${1} in
-  "build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+if [[ " ${TARGET_TYPES} " =~ " ${1} " ]]; then
     test_type="-a ${1}"
     if [[ -z ${2} ]]; then
       test_list=""
@@ -78,7 +80,7 @@ case ${1} in
     echo "Using deprecated legacy arguments.  Please update to new parameter format: ${test_type} ${test_list}"
     $0 ${test_type} ${test_list}
     exit $?
-esac
+fi
 
 
 env_vars=""
@@ -88,6 +90,7 @@ detect_changed_tests=true
 while getopts "a:t:c:e:ib:shj:" opt; do
   case $opt in
     a ) test_target="$OPTARG"
+        [[ " ${TARGET_TYPES} " =~ " ${test_target/-repeat/} " ]] || error 1 "Invalid test target type '${test_target}'. Valid types: ${TARGET_TYPES}"
         ;;
     t ) test_name_regexp="$OPTARG"
         ;;

.jenkins/Jenkinsfile

@@ -29,21 +29,24 @@
 //
 // This Jenkinsfile is expected to work on any Jenkins infrastructure.
 // The controller should have 4 cpu, 12GB ram (and be configured to use `-XX:+UseG1GC -Xmx8G`)
-// It is required to have agents providing five labels, each that can provide docker and the following capabilities:
-//  - cassandra-amd64-small  : 1 cpu, 1GB ram
-//  - cassandra-small        : 1 cpu, 1GB ram (alias for above but for any arch)
-//  - cassandra-amd64-medium : 3 cpu, 5GB ram
-//  - cassandra-medium       : 3 cpu, 5GB ram (alias for above but for any arch)
-//  - cassandra-amd64-large  : 7 cpu, 16GB ram
-//  - cassandra-large        : 7 cpu, 16GB ram
+//
+// It is required to have agents providing 6+ labels, each that can provide docker and the following capabilities:
+//  
+//  - cassandra-small + cassandra-${arch}-small       : 1 cpu, 1GB ram (alias for above but for any arch)
+//  - cassandra-medium + cassandra-${arch}-medium     : 3 cpu, 5GB ram
+//  - cassandra-large + cassandra-${arch}-large       : 7 cpu, 16GB ram
+//
+// Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes.
+//
+// Nodes running on spot instances have to have labels with `-spot` suffixes.
+//  For example: cassandra-amd64-large-spot
 //
 // When running builds parameterised to other architectures the corresponding labels are expected.
 //  For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large.
 //
-// The built-in node must has the "controller" label.  There must be more than two agents for each label.
-//
 // Plugins required are:
-//  git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact.
+//  git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact, jmh-report.
+// See .jenkins/k8s/jenkins-deployment.yaml for up to date list of plugins.
 //
 // Any functionality that depends upon ASF Infra ( i.e. the canonical ci-cassandra.a.o )
 //  will be ignored when run on other environments.
@@ -143,6 +146,7 @@ def pipelineProfiles() {
     'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
     'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
     'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
+    'performance': ['microbench'],
     'custom': []
   ]
 }
@@ -204,12 +208,19 @@ def tasks() {
     'dtest-upgrade-novnode': [splits: 128, size: 'large'],
     'dtest-upgrade-large': [splits: 32, size: 'large'],
     'dtest-upgrade-novnode-large': [splits: 32, size: 'large'],
+    'test-system-keyspace-directory': [splits: 16],
+    // performance tests need 'cassandra-*large-dedicated' nodes
+    'microbench': [splits: 1, size: 'large', timeout: 6, benchmark: true],
   ]
   testSteps.each() {
     it.value.put('type', 'test')
     if (!it.value['size']) {
       it.value.put('size', 'medium')
     }
+    if (!it.value['timeout']) {
+      // default 1 hour
+      it.value.put('timeout', 1)
+    }
     if (it.key.startsWith('dtest')) {
       it.value.put('python-dtest', true)
     }
@@ -391,7 +402,7 @@ def test(command, cell) {
             script_vars = "${script_vars} cython=\'${cell.cython}\'"
           }
           script_vars = fetchDTestsSource(command, script_vars)
-          timeout(time: 1, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
+          timeout(time: command.timeout, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
             def timer = System.currentTimeMillis()
             try {
               buildJVMDTestJars(cell, script_vars, logfile)
@@ -425,7 +436,7 @@ def test(command, cell) {
                 find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
                 echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
               """
-            archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz", fingerprint: true
+            archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
             copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
           }
           cleanAgent(cell.step)
@@ -483,8 +494,12 @@ def fetchDockerImages(dockerfiles) {
 }
 
 def getNodeLabel(command, cell) {
-  echo "using node label: cassandra-${cell.arch}-${command.size}"
-  return "cassandra-${cell.arch}-${command.size}"
+  def label = "cassandra-${cell.arch}-${command.size}"
+  if (command.containsKey('benchmark') && command.benchmark) {
+    label = "${label}-dedicated"
+  }
+  echo "using node label: ${label}"
+  return label
 }
 
 def copyToNightlies(sourceFiles, remoteDirectory='') {
@@ -557,7 +572,7 @@ def generateTestReports() {
           unzip -x -d build/test -q output.zip ) ${teeSuffix}
          """
     } else {
-      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
     }
     if (fileExists('build/test/output')) {
       // merge splits for each target's test report, other axes are kept separate
@@ -585,7 +600,10 @@ def generateTestReports() {
 
       dir('build/') {
         archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
-        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
+        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json')
+        if (fileExists('test/jmh-result.json')) {
+          jmhReport('test/jmh-result.json')
+        }
       }
     }
   }

.jenkins/k8s/jenkins-deployment.yaml

@@ -58,6 +58,7 @@ controller:
     - pipeline-rest-api
     - test-stability
     - copyartifact
+    - jmh-plugin
   node-selector:
     cassandra.jenkins.controller: true
   scriptApproval: