Merge branch 'main' into fix-cluster-stats-resp

Author: pquentin

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "repoOwner": "elastic",
   "repoName": "elasticsearch-specification",
-  "targetBranchChoices": ["9.0", "8.19", "8.18", "8.17"],
+  "targetBranchChoices": ["9.1", "9.0", "8.19", "8.18"],
   "fork": false
 }

.github/validate-pr/index.js

@@ -86,6 +86,9 @@ async function run() {
 
   cd(tsValidationPath)
 
+  // Collect all APIs to validate
+  const apisToValidate = new Set()
+
   for (const file of specFiles) {
     if (file.startsWith('specification/_types')) continue
     if (file.startsWith('specification/_spec_utils')) continue
@@ -102,32 +105,35 @@ async function run() {
         .filter(endpoint => endpoint.name.split('.').filter(s => !privateNames.includes(s))[0] === getApi(file).split('.')[0])
         .map(endpoint => endpoint.name)
       for (const api of apis) {
-        const report = await getReport({
-          api,
-          'generate-report': false,
-          request: true,
-          response: true,
-          ci: false,
-          verbose: false
-        })
-        const [namespace, _method] = api.split('.')
-        // Asked to validate a specific API, so we only store that one
-        reports.set(api, report.get(namespace)[0])
+        apisToValidate.add(api)
       }
     } else {
       const api = getApi(file)
-      const report = await getReport({
-        api,
-        'generate-report': false,
-        request: true,
-        response: true,
-        ci: false,
-        verbose: false
-      })
-
-      const [namespace, _method] = api.split('.')
-      // Asked to validate a specific API, so we only store that one
-      reports.set(api, report.get(namespace)[0])
+      apisToValidate.add(api)
+    }
+  }
+
+  // Call getReport once with all APIs
+  if (apisToValidate.size > 0) {
+    const allApis = Array.from(apisToValidate).join(',')
+    const report = await getReport({
+      api: allApis,
+      'generate-report': false,
+      request: true,
+      response: true,
+      ci: false,
+      verbose: false
+    })
+
+    // Extract individual API reports from the combined result
+    for (const api of apisToValidate) {
+      const namespace = getNamespace(api)
+      if (report.has(namespace)) {
+        const namespaceReport = report.get(namespace).find(r => r.api === getName(api))
+        if (namespaceReport) {
+          reports.set(api, namespaceReport)
+        }
+      }
     }
   }
 
@@ -165,7 +171,6 @@ function getApi (file) {
   return file.split('/').slice(1, 3).filter(s => !privateNames.includes(s)).filter(Boolean).join('.')
 }
 
-
 function findBaselineReport(apiName, baselineValidation) {
   const [namespace, method] = apiName.split('.')
 

.github/workflows/update-rest-api-json.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '9.0', '8.19', '8.18', '8.17']
+        branch: ['main', '9.1', '9.0', '8.19', '8.18']
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/validate-pr.yml

@@ -72,7 +72,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: 'Following you can find the validation results'
+          body-includes: 'Following you can find the validation changes'
 
       - name: Create or update comment
         if: steps.validation.outputs.has_results == 'true'

.gitignore

@@ -65,10 +65,7 @@ output/schema/schema
 # Test suite outputs
 compiler/test/**/output/
 
-# Temporary openAPI files
-output/openapi/elasticsearch-serverless-openapi.tmp*.json
-output/openapi/elasticsearch-serverless-openapi.examples.json
-output/openapi/elasticsearch-openapi.tmp*.json
-output/openapi/elasticsearch-openapi.examples.json
-output/openapi/elasticsearch-serverless-openapi-docs.json
-output/openapi/elasticsearch-openapi-docs.json
+# Temporary openAPI documentation files
+output/openapi/elasticsearch-serverless-openapi-docs*.json
+output/openapi/elasticsearch-openapi-docs*.json
+output/openapi/elasticsearch*.redirects.csv

Makefile

@@ -69,14 +69,14 @@ dump-routes: ## Create a new schema with all generics expanded
 	@npm run dump-routes --prefix compiler
 
 overlay-docs: ## Apply overlays to OpenAPI documents
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi.examples.json
-	@npx bump overlay "output/openapi/elasticsearch-openapi.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-openapi.examples.json
-	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
-	rm output/openapi/elasticsearch-openapi.tmp*.json
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi-docs-final.json
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-openapi-docs-final.json
+	rm output/openapi/elasticsearch-serverless-openapi-docs.tmp*.json
+	rm output/openapi/elasticsearch-openapi-docs.tmp*.json
 
 generate-language-examples:
 	@node docs/examples/generate-language-examples.js
@@ -89,11 +89,11 @@ generate-language-examples-with-java:
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
-lint-docs-stateful: ## Lint only the elasticsearch-openapi.examples.json file
-	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+lint-docs-stateful: ## Lint only the elasticsearch-openapi-docs-final.json file
+	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 lint-docs-serverless: ## Lint only the serverless OpenAPI document after overlays
-	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless lint-docs ## Pre contribution target
 

README.md

@@ -74,6 +74,11 @@ $ make overlay-docs
 # The generated output can be found in ./output/openapi/
 ```
 
+## Specification Viewer
+
+An interactive viewer for the Elasticsearch specification is available
+[here](https://elastic.github.io/specification-viewer/).
+
 ## Contributing
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md)
@@ -234,6 +239,9 @@ make validate api=xpack.info type=request branch=main
 
 # this will validate the xpack.info request and response types against the 8.15 branch
 make validate api=xpack.info branch=8.15
+
+# this will validate the xpack.info and search request and response types against the 8.15 branch
+make validate api=xpack.info,search branch=8.15
 ```
 
 The last command above will install all the dependencies and run, download

compiler-rs/clients_schema/src/transform/expand_generics.rs

@@ -211,8 +211,10 @@ pub fn expand(model: IndexedModel, config: ExpandConfig) -> anyhow::Result<Index
 
         expand_behaviors(&mut resp.behaviors, &mappings, model, ctx)?;
         expand_body(&mut resp.body, &mappings, model, ctx)?;
-
-        // TODO: exceptions
+        
+        for exception in &mut resp.exceptions {
+            expand_body(&mut exception.body, &mappings, model, ctx)?;
+        }
 
         Ok(resp.into())
     }

compiler/run-validations.js

@@ -78,8 +78,11 @@ async function run () {
     process.exit(1)
   }
 
-  if (!apis.includes(options.api)) {
-    spinner.fail(`The api '${options.api}' does not exists, did you mean '${closest(options.api, apis)}'?`)
+  const apiList = options.api.split(',').map(api => api.trim())
+  const invalidApis = apiList.filter(api => !apis.includes(api))
+  if (invalidApis.length > 0) {
+    const suggestions = invalidApis.map(api => `'${api}' (did you mean '${closest(api, apis)}'?)`).join(', ')
+    spinner.fail(`The following APIs do not exist: ${suggestions}`)
     process.exit(1)
   }
   // if the empty string it's because the make target wasn't configured with a type argument

compiler/src/steps/validate-model.ts

@@ -497,6 +497,23 @@ export default async function validateModel (apiModel: model.Model, restSpec: Ma
         // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
         throw new Error(`Unknown kind: ${typeDef.body.kind}`)
     }
+
+    if (typeDef.exceptions != null) {
+      for (const ex of typeDef.exceptions) {
+        switch (ex.body.kind) {
+          case 'properties':
+            validateProperties(ex.body.properties, openGenerics, new Set<string>())
+            break
+          case 'value':
+            validateValueOf(ex.body.value, openGenerics)
+            break
+          case 'no_body':
+            // Nothing to validate
+            break
+        }
+      }
+    }
+
     context.pop()
   }