Script for indexing micro benchmark results (elastic#128557)

Author: idegtiarenko

.buildkite/hooks/pre-command

@@ -94,6 +94,18 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
   fi
 fi
 
+if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
+  PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/esbench-metics)
+  PERF_METRICS_INDEX="dummy-micro-benchmarks"
+  PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/esbench-metics)
+  PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/esbench-metics)
+
+  export PERF_METRICS_HOST
+  export PERF_METRICS_INDEX
+  export PERF_METRICS_USERNAME
+  export PERF_METRICS_PASSWORD
+fi
+
 # Authenticate to the Docker Hub public read-only registry
 if which docker > /dev/null 2>&1; then
   DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"

.buildkite/pipelines/periodic-micro-benchmarks.yml

@@ -2,6 +2,9 @@ steps:
   - label: periodic-micro-benchmarks
     command: |
       .ci/scripts/run-gradle.sh :benchmarks:run --args 'org.elasticsearch.benchmark._nightly -rf json -rff build/result.json'
+      .buildkite/scripts/index-micro-benchmark-results.sh
+    env:
+      USE_PERF_CREDENTIALS: "true"
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/scripts/index-micro-benchmark-results.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+jq -c '.[]' "benchmarks/build/result.json" | while read -r doc; do
+  doc=$(echo "$doc" | jq --argjson timestamp "$(date +%s000)" '. + {"@timestamp": $timestamp}')
+  echo "Indexing $(echo "$doc" | jq -r '.benchmark')"
+  curl -s -X POST "https://$PERF_METRICS_HOST/$PERF_METRICS_INDEX/_doc" \
+    -u "$PERF_METRICS_USERNAME:$PERF_METRICS_PASSWORD" \
+    -H 'Content-Type: application/json' \
+    -d "$doc"
+done