Merge remote-tracking branch 'origin/main' into kibana-endpoint

Author: pquentin

.buildkite/kibana.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Since we're into the current repo, move to the top-level
+cd ..
+
+echo "--- Clone elasticsearch-js"
+git clone -v -- [email protected]:elastic/elasticsearch-js.git
+pushd elasticsearch-js
+git checkout $BUILDKITE_BRANCH
+popd
+
+echo "--- Clone Kibana"
+git clone -v --reference /usr/local/git-references/git-github-com-elastic-kibana-git -- [email protected]:elastic/kibana.git
+cd kibana
+
+echo "--- Install Node.js and Yarn"
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+set +e # https://github.com/nvm-sh/nvm/issues/3117
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+set -e
+
+nvm install
+nvm use
+npm install --global yarn
+
+echo "--- Install elasticsearch-js"
+pushd ../elasticsearch-js
+npm install
+npm run build
+npm pack
+popd
+yarn add ../elasticsearch-js/elastic-elasticsearch-*.tgz
+
+echo "--- Bootstrap Kibana"
+git --no-pager diff
+yarn kbn bootstrap --allow-root
+
+echo "--- Check types"
+node scripts/type_check.js

.buildkite/kibana.yml

@@ -0,0 +1,9 @@
+agents:
+  memory: "24G"
+  cpu: "4"
+  ephemeralStorage: 15Gi
+
+
+steps:
+  - label: "Run Kibana type checks"
+    command: .buildkite/kibana.sh

.github/validate-pr/index.js

@@ -19,23 +19,37 @@
 
 /* global argv, path, cd */
 
-'use strict'
+import { dirname } from 'path'
+import { fileURLToPath } from 'url'
+import 'zx/globals'
+import assert from 'assert'
+import * as core from '@actions/core'
+import { copyFile } from 'fs/promises'
+import * as github from '@actions/github'
+import specification from '../../output/schema/schema.json' assert { type: 'json' }
+import { run as getReport } from '../../../clients-flight-recorder/scripts/types-validator/index.js'
+import {
+  getNamespace,
+  getName
+} from '../../../clients-flight-recorder/scripts/types-validator/utils.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
 
-require('zx/globals')
-const assert = require('assert')
-const core = require('@actions/core')
-const { copyFile } = require('fs/promises')
-const github = require('@actions/github')
 const octokit = github.getOctokit(argv.token)
-const specification = require('../../output/schema/schema.json')
-const getReport = require('../../../clients-flight-recorder/scripts/types-validator')
-const { getNamespace, getName } = require('../../../clients-flight-recorder/scripts/types-validator/utils')
 
 const privateNames = ['_global']
 const tick = '`'
-const tsValidationPath = path.join(__dirname, '..', '..', '..', 'clients-flight-recorder', 'scripts', 'types-validator')
-
-async function run () {
+const tsValidationPath = path.join(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  'clients-flight-recorder',
+  'scripts',
+  'types-validator'
+)
+
+async function run() {
   await copyFile(
     path.join(__dirname, '..', '..', 'output', 'typescript', 'types.ts'),
     path.join(tsValidationPath, 'types.ts')
@@ -53,14 +67,20 @@ async function run () {
       per_page: 100
     })
     if (data.length > 0) {
-      files.push(...data.map(entry => entry.filename))
+      files.push(
+        ...data
+          .filter((entry) => entry.status !== 'deleted')
+          .map((entry) => entry.filename)
+      )
       page += 1
     } else {
       break
     }
   }
 
-  const specFiles = files.filter(file => file.includes('specification') && !file.includes('compiler/test'))
+  const specFiles = files.filter(
+    (file) => file.includes('specification') && !file.includes('compiler/test')
+  )
   const table = []
 
   cd(tsValidationPath)
@@ -163,7 +183,7 @@ function generateResponse (r) {
   return `${r.passingResponse}/${r.totalResponse}`
 }
 
-run().catch(err => {
+run().catch((err) => {
   core.error(err)
   process.exit(1)
 })

.github/validate-pr/package.json

@@ -2,7 +2,8 @@
   "name": "validate-pr",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "type": "module",
+  "exports": "./index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },

.github/workflows/check-backport-labels.yml

@@ -0,0 +1,118 @@
+name: Check
+
+on:
+  pull_request:
+    branches:
+      - main
+    types: [labeled, unlabeled, opened, reopened, synchronize]
+
+permissions:
+  pull-requests: "read"
+
+jobs:
+  check-backport-label:
+    name: backport label 
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: "Check backport label"
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          json_pr_labels='${{ toJSON(github.event.pull_request.labels) }}'
+          readarray -t pr_labels < <(echo "${json_pr_labels}" | jq -r -c '.[].name')
+
+          json_all_labels="$(gh label list --repo ${{ github.repository }} --json name --search "backport" --limit 1000)"
+          readarray -t all_labels < <(echo "${json_all_labels}" | jq -r -c '.[].name')
+
+          declare -A all_backport_labels=()
+          declare -A all_floating_majors=()
+
+          backport_regex="^backport ([0-9])+\.([0-9]+|x)$"
+
+          echo "::group::Available Labels"
+          echo "skip-backport"
+
+          for label in "${all_labels[@]}"; do
+            if [[ "${label}" =~ ${backport_regex} ]]; then
+              major="${BASH_REMATCH[1]}"
+              minor="${BASH_REMATCH[2]}"
+              all_backport_labels["${label}"]=1
+              echo "${label}"
+
+              if [ "${minor}" = "x" ]; then
+                all_floating_majors["${major}"]=1
+              fi
+            fi
+          done
+
+          echo "::endgroup::"
+
+          has_backport_label=false
+          has_skip_backport_label=false
+          declare -A pr_exact_majors=()
+          declare -A pr_floating_majors=()
+
+          echo "::group::Detected Labels"
+
+          for pr_label in "${pr_labels[@]}"; do
+            if [ "${pr_label}" = "skip-backport" ]; then
+              has_skip_backport_label=true
+              echo "${pr_label}"
+              continue
+            fi
+
+            if [ -z "${all_backport_labels[${pr_label}]}" ]; then
+              continue
+            fi
+
+            has_backport_label=true
+
+            if [[ "${pr_label}" =~ ${backport_regex} ]]; then
+              major="${BASH_REMATCH[1]}"
+              minor="${BASH_REMATCH[2]}"
+              if [ "${minor}" != "x" ]; then
+                pr_exact_majors["${major}"]=1
+              else
+                pr_floating_majors["${major}"]=1
+              fi
+            fi
+
+            echo "${pr_label}"
+          done
+
+          echo "::endgroup::"
+
+          if [ "${has_skip_backport_label}" = true ] && [ "${has_backport_label}" = true ]; then
+            echo "::error::The 'skip-backport' not be used in combination with another backport"\
+                  "label."
+            exit 1
+          fi
+
+          if [ "${has_skip_backport_label}" != true ] && [ "${has_backport_label}" != true ]; then
+            echo "::error::No backport label found. Please add at least one of the"\
+                  "'backport {major}.{minor|x}' labels or use 'skip-backport',"\
+                  "if this PR should not be backported."
+            exit 1
+          fi
+
+          # Validate that a floating backport label exists for each exact backport label major
+          # version.
+
+          has_required_floating_labels=true
+
+          for pr_major in "${!pr_exact_majors[@]}"; do
+            if [ -z "${all_floating_majors[${pr_major}]}" ]; then
+              # There is no floating version branch for the given major version.
+              continue
+            fi
+
+            if [ -z "${pr_floating_majors[${pr_major}]}" ]; then
+              has_required_floating_labels=false
+              echo "::error::Missing floating backport label for '${pr_major}.x'"
+            fi
+          done
+
+          if [ "${has_required_floating_labels}" != true ]; then
+            exit 1
+          fi

.github/workflows/gh-pages-report.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '8.x', '8.16', '8.17', 7.17']
+        branch: ['main', '9.0', '8.x', '8.18', '8.17', '8.16', '7.17']
 
     steps:
       - uses: actions/checkout@v4
@@ -51,7 +51,7 @@ jobs:
           commit-message: 'Update rest-api-spec'
           labels: specification
           delete-branch: true
-          reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,srikanthmanvi,swallez,technige
+          reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,swallez,technige
           branch: automated/rest-api-spec-update-${{ matrix.branch }}
 
       - name: Open an issue if the action fails

.github/workflows/update-rest-api-json.yml

@@ -39,7 +39,7 @@ setup:	## Install dependencies for contrib target
 	@make clean-dep
 	@npm install --prefix compiler
 	@npm install --prefix typescript-generator
-	@npm install @stoplight/spectral-cli
+	@npm install @redocly/cli
 
 clean-dep:	## Clean npm dependencies
 	@rm -rf compiler/node_modules
@@ -57,8 +57,6 @@ filter-for-serverless: ## Generate the serverless version from the compiled sche
 dump-routes: ## Create a new schema with all generics expanded
 	@npm run dump-routes --prefix compiler
 
-contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless ## Pre contribution target
-
 overlay-docs: ## Apply overlays to OpenAPI documents
 	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp1.json"
 	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp2.json"
@@ -70,13 +68,15 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
 lint-docs: ## Lint the OpenAPI documents after overlays
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml
+	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
-lint-docs-errs: ## Lint the OpenAPI documents after overlays and return only errors
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml -D
+lint-docs-stateful: ## Lint only the elasticsearch-openapi.examples.json file
+	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 lint-docs-serverless: ## Lint only the serverless OpenAPI document after overlays
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-serverless-openapi.examples.json --ruleset .spectral.yaml
+	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+
+contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless lint-docs ## Pre contribution target
 
 help:  ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)