Merge branch '8.17' into automated/rest-api-spec-update-8.17

Author: l-trotta

.buildkite/kibana.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Since we're into the current repo, move to the top-level
+cd ..
+
+echo "--- Clone elasticsearch-js"
+git clone -v -- [email protected]:elastic/elasticsearch-js.git
+pushd elasticsearch-js
+git checkout $BUILDKITE_BRANCH
+popd
+
+echo "--- Clone Kibana"
+git clone -v --reference /usr/local/git-references/git-github-com-elastic-kibana-git -- [email protected]:elastic/kibana.git
+cd kibana
+
+echo "--- Install Node.js and Yarn"
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+set +e # https://github.com/nvm-sh/nvm/issues/3117
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+set -e
+
+nvm install
+nvm use
+npm install --global yarn
+
+echo "--- Install elasticsearch-js"
+pushd ../elasticsearch-js
+npm install
+npm run build
+npm pack
+popd
+yarn add ../elasticsearch-js/elastic-elasticsearch-*.tgz
+
+echo "--- Bootstrap Kibana"
+git --no-pager diff
+yarn kbn bootstrap --allow-root
+
+echo "--- Check types"
+node scripts/type_check.js

.buildkite/kibana.yml

@@ -0,0 +1,9 @@
+agents:
+  memory: "24G"
+  cpu: "4"
+  ephemeralStorage: 15Gi
+
+
+steps:
+  - label: "Run Kibana type checks"
+    command: .buildkite/kibana.sh

.github/validate-pr/index.js

@@ -19,23 +19,37 @@
 
 /* global argv, path, cd */
 
-'use strict'
+import { dirname } from 'path'
+import { fileURLToPath } from 'url'
+import 'zx/globals'
+import assert from 'assert'
+import * as core from '@actions/core'
+import { copyFile } from 'fs/promises'
+import * as github from '@actions/github'
+import specification from '../../output/schema/schema.json' assert { type: 'json' }
+import { run as getReport } from '../../../clients-flight-recorder/scripts/types-validator/index.js'
+import {
+  getNamespace,
+  getName
+} from '../../../clients-flight-recorder/scripts/types-validator/utils.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
 
-require('zx/globals')
-const assert = require('assert')
-const core = require('@actions/core')
-const { copyFile } = require('fs/promises')
-const github = require('@actions/github')
 const octokit = github.getOctokit(argv.token)
-const specification = require('../../output/schema/schema.json')
-const getReport = require('../../../clients-flight-recorder/scripts/types-validator')
-const { getNamespace, getName } = require('../../../clients-flight-recorder/scripts/types-validator/utils')
 
 const privateNames = ['_global']
 const tick = '`'
-const tsValidationPath = path.join(__dirname, '..', '..', '..', 'clients-flight-recorder', 'scripts', 'types-validator')
-
-async function run () {
+const tsValidationPath = path.join(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  'clients-flight-recorder',
+  'scripts',
+  'types-validator'
+)
+
+async function run() {
   await copyFile(
     path.join(__dirname, '..', '..', 'output', 'typescript', 'types.ts'),
     path.join(tsValidationPath, 'types.ts')
@@ -53,14 +67,20 @@ async function run () {
       per_page: 100
     })
     if (data.length > 0) {
-      files.push(...data.map(entry => entry.filename))
+      files.push(
+        ...data
+          .filter((entry) => entry.status !== 'deleted')
+          .map((entry) => entry.filename)
+      )
       page += 1
     } else {
       break
     }
   }
 
-  const specFiles = files.filter(file => file.includes('specification') && !file.includes('compiler/test'))
+  const specFiles = files.filter(
+    (file) => file.includes('specification') && !file.includes('compiler/test')
+  )
   const table = []
 
   cd(tsValidationPath)
@@ -163,7 +183,7 @@ function generateResponse (r) {
   return `${r.passingResponse}/${r.totalResponse}`
 }
 
-run().catch(err => {
+run().catch((err) => {
   core.error(err)
   process.exit(1)
 })

.github/validate-pr/package.json

@@ -2,7 +2,8 @@
   "name": "validate-pr",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "type": "module",
+  "exports": "./index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },

Makefile

@@ -39,7 +39,7 @@ setup:	## Install dependencies for contrib target
 	@make clean-dep
 	@npm install --prefix compiler
 	@npm install --prefix typescript-generator
-	@npm install @stoplight/spectral-cli
+	@npm install @redocly/cli
 
 clean-dep:	## Clean npm dependencies
 	@rm -rf compiler/node_modules
@@ -57,19 +57,19 @@ filter-for-serverless: ## Generate the serverless version from the compiled sche
 dump-routes: ## Create a new schema with all generics expanded
 	@npm run dump-routes --prefix compiler
 
-contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless ## Pre contribution target
-
 overlay-docs: ## Apply overlays to OpenAPI documents
 	@npx bump overlay "output/openapi/elasticsearch-openapi.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp1.json"
 	@npx bump overlay "output/openapi/elasticsearch-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp2.json"
 	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-openapi.examples.json
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
 lint-docs: ## Lint the OpenAPI documents after overlays
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml
+	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+
+lint-docs-stateful: ## Lint only the elasticsearch-openapi.examples.json file
+	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
-lint-docs-errs: ## Lint the OpenAPI documents after overlays and return only errors
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml -D
+contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless lint-docs ## Pre contribution target
 
 help:  ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

catalog-info.yaml

@@ -0,0 +1,57 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: elasticsearch-specification
+
+spec:
+  type: library
+  owner: group:devtools-team
+  lifecycle: beta
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-type-checks-pipeline
+  description: Run Kibana type checks against the specification
+
+spec:
+  type: buildkite-pipeline
+  owner: group:devtools-team
+  system: buildkite
+
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: Kibana type checks
+    spec:
+      repository: elastic/elasticsearch-specification
+      pipeline_file: .buildkite/kibana.yml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        devtools-team:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Daily main:
+          branch: 'main'
+          cronline: '0 4 * * *'
+          message: 'Daily run for main branch'
+        Daily 8.x:
+          branch: '8.x'
+          cronline: '0 4 * * *'
+          message: 'Daily run for 8.x branch'
+        Daily 8.17:
+          branch: '8.17'
+          cronline: '0 4 * * *'
+          message: 'Daily run for 8.17 branch'
+        Daily 8.18:
+          branch: '8.18'
+          cronline: '0 4 * * *'
+          message: 'Daily run for 8.18 branch'

compiler/src/model/utils.ts

@@ -668,9 +668,9 @@ export function hoistRequestAnnotations (
       const privileges = [
         'all', 'cancel_task', 'create_snapshot', 'grant_api_key', 'manage', 'manage_api_key', 'manage_ccr',
         'manage_enrich', 'manage_ilm', 'manage_index_templates', 'manage_inference', 'manage_ingest_pipelines', 'manage_logstash_pipelines',
-        'manage_ml', 'manage_oidc', 'manage_own_api_key', 'manage_pipeline', 'manage_rollup', 'manage_saml', 'manage_search_application', 'manage_search_query_rules',
+        'manage_ml', 'manage_oidc', 'manage_own_api_key', 'manage_pipeline', 'manage_rollup', 'manage_saml', 'manage_search_application', 'manage_search_query_rules', 'manage_search_synonyms',
         'manage_security', 'manage_service_account', 'manage_slm', 'manage_token', 'manage_transform', 'manage_user_profile',
-        'manage_watcher', 'monitor', 'monitor_ml', 'monitor_rollup', 'monitor_snapshot', 'monitor_text_structure',
+        'manage_watcher', 'monitor', 'monitor_inference', 'monitor_ml', 'monitor_rollup', 'monitor_snapshot', 'monitor_text_structure',
         'monitor_transform', 'monitor_watcher', 'read_ccr', 'read_ilm', 'read_pipeline', 'read_security', 'read_slm', 'transport_client'
       ]
       const values = parseCommaSeparated(value)

docs/add-new-api.md

@@ -153,3 +153,35 @@ class Response {
   ]
 }
 ```
+
+### Add endpoint request and response examples
+
+Add an `examples` folder and `request` and `xxx_response` subfolders (where `xxx` is the relevant response code).
+
+These examples are for use in the API documentation and must adhere to the [OpenAPI 3.0 Example object specification](https://spec.openapis.org/oas/v3.0.3#example-object). They must have a `value` field that contains the request or response body.
+If there are multiple examples for the endpoint, they must each have a brief `summary` field, which is used as the label for the example. You can also optionaly provide an explanation in a `description` field.
+
+For example:
+
+```yaml
+summary: Sequence query
+# method_request: GET /my-data-stream/_eql/search
+# type: request
+description: >
+  Run `GET /my-data-stream/_eql/search` to search for a sequence of events.
+  The sequence starts with an event with an `event.category` of `file`, a `file.name` of `cmd.exe`, and a `process.pid` other than `2013`.
+  It is followed by an event with an `event.category` of `process` and a `process.executable` that contains the substring `regsvr32`.
+  These events must also share the same `process.pid` value.
+value: |-
+  {
+    "query": """
+      sequence by process.pid
+        [ file where file.name == "cmd.exe" and process.pid != 2013 ]
+        [ process where stringContains(process.executable, "regsvr32") ]
+    """
+  }
+```
+
+NOTE: A good example covers a very common use case or demonstrates a more complex but critical use case.
+It involves realistic data sets ( rather than generic "hello world" values).
+If it requires detailed setup or explanations, however, it is more appropriate for coverage in longer-form narrative documentation.

.spectral.yaml renamed to docs/linters/.spectral.yaml

@@ -0,0 +1,55 @@
+extends:
+  - recommended
+rules:
+# Built-in rules
+ # Descriptions
+  parameter-description: warn
+  tag-description: warn
+  operation-description: off
+ # Document info
+  info-contact: warn
+  info-license: warn
+  no-empty-servers: warn
+ # Examples
+  no-invalid-media-type-examples:
+    severity: off
+    allowAdditionalProperties: false
+  no-invalid-schema-examples:
+    severity: warn
+    allowAdditionalProperties: false
+ # Operations
+  operation-operationId: error
+  operation-operationId-unique: error
+  operation-operationId-url-safe: warn
+  operation-summary: warn
+ # Parameters
+  path-parameters-defined: warn
+ # Paths
+  no-ambiguous-paths: off
+  no-identical-paths: warn
+  path-excludes-patterns:
+    severity: error
+    patterns:
+      - ^\/internal
+ # Responses
+  operation-4xx-response: off
+  operation-2xx-response: off
+ # Schema
+  struct: error
+  spec-components-invalid-map-name: off
+  spec-strict-refs: off
+  security-defined: off
+ # Tags
+  operation-tag-defined: off
+  tags-alphabetical: off
+  operation-singular-tag: off
+# Custom rules
+  rule/operation-summary-length:
+    subject:
+      type: Operation
+      property: summary
+    message: Operation summary must have a minimum of 5 and maximum of 45 characters
+    severity: warn
+    assertions:
+      maxLength: 45
+      minLength: 5