Merge branch 'main' into update-indeces

Author: kosabogi

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "repoOwner": "elastic",
   "repoName": "elasticsearch-specification",
-  "targetBranchChoices": ["9.0", "8.19", "8.18", "8.17"],
+  "targetBranchChoices": ["9.1", "9.0", "8.19", "8.18"],
   "fork": false
 }

.github/validate-pr/index.js

@@ -86,6 +86,9 @@ async function run() {
 
   cd(tsValidationPath)
 
+  // Collect all APIs to validate
+  const apisToValidate = new Set()
+
   for (const file of specFiles) {
     if (file.startsWith('specification/_types')) continue
     if (file.startsWith('specification/_spec_utils')) continue
@@ -102,32 +105,35 @@ async function run() {
         .filter(endpoint => endpoint.name.split('.').filter(s => !privateNames.includes(s))[0] === getApi(file).split('.')[0])
         .map(endpoint => endpoint.name)
       for (const api of apis) {
-        const report = await getReport({
-          api,
-          'generate-report': false,
-          request: true,
-          response: true,
-          ci: false,
-          verbose: false
-        })
-        const namespace = getNamespace(api)
-        // Asked to validate a specific API, so we only store that one
-        reports.set(api, report.get(namespace)[0])
+        apisToValidate.add(api)
       }
     } else {
       const api = getApi(file)
-      const report = await getReport({
-        api,
-        'generate-report': false,
-        request: true,
-        response: true,
-        ci: false,
-        verbose: false
-      })
+      apisToValidate.add(api)
+    }
+  }
 
+  // Call getReport once with all APIs
+  if (apisToValidate.size > 0) {
+    const allApis = Array.from(apisToValidate).join(',')
+    const report = await getReport({
+      api: allApis,
+      'generate-report': false,
+      request: true,
+      response: true,
+      ci: false,
+      verbose: false
+    })
+
+    // Extract individual API reports from the combined result
+    for (const api of apisToValidate) {
       const namespace = getNamespace(api)
-      // Asked to validate a specific API, so we only store that one
-      reports.set(api, report.get(namespace)[0])
+      if (report.has(namespace)) {
+        const namespaceReport = report.get(namespace).find(r => r.api === getName(api))
+        if (namespaceReport) {
+          reports.set(api, namespaceReport)
+        }
+      }
     }
   }
 

.github/workflows/update-rest-api-json.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '9.0', '8.19', '8.18', '8.17']
+        branch: ['main', '9.1', '9.0', '8.19', '8.18']
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/validate-pr.yml

@@ -72,7 +72,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: 'Following you can find the validation results'
+          body-includes: 'Following you can find the validation changes'
 
       - name: Create or update comment
         if: steps.validation.outputs.has_results == 'true'

.gitignore

@@ -65,10 +65,7 @@ output/schema/schema
 # Test suite outputs
 compiler/test/**/output/
 
-# Temporary openAPI files
-output/openapi/elasticsearch-serverless-openapi.tmp*.json
-output/openapi/elasticsearch-serverless-openapi.examples.json
-output/openapi/elasticsearch-openapi.tmp*.json
-output/openapi/elasticsearch-openapi.examples.json
-output/openapi/elasticsearch-serverless-openapi-docs.json
-output/openapi/elasticsearch-openapi-docs.json
+# Temporary openAPI documentation files
+output/openapi/elasticsearch-serverless-openapi-docs*.json
+output/openapi/elasticsearch-openapi-docs*.json
+output/openapi/elasticsearch*.redirects.csv

CONTRIBUTING.md

@@ -39,7 +39,7 @@ Documents under the docs/ directory will help you to change the specification. T
  * [Fixing a definition, the complete story](docs/validation-example.md)
 
 
-## Send your pull request
+## Send your pull request from a branch
 
 To fix any style issues and generate the outputs (that we store in git), please run:
 
@@ -56,5 +56,4 @@ Then, follow those steps:
   - If the API is unusable without the change -> every supported version
   - If the API is usable, but fix is on the response side -> every supported version
   - If the API is usable, but fix is on the request side -> no backport, unless the API is _partially_ usable and the fix unlocks a missing feature that has no workaround
-
-
+- Send the pull request from a branch, **not a fork**

Makefile

@@ -69,14 +69,14 @@ dump-routes: ## Create a new schema with all generics expanded
 	@npm run dump-routes --prefix compiler
 
 overlay-docs: ## Apply overlays to OpenAPI documents
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi.examples.json
-	@npx bump overlay "output/openapi/elasticsearch-openapi.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-openapi.examples.json
-	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
-	rm output/openapi/elasticsearch-openapi.tmp*.json
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi-docs-final.json
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-openapi-docs-final.json
+	rm output/openapi/elasticsearch-serverless-openapi-docs.tmp*.json
+	rm output/openapi/elasticsearch-openapi-docs.tmp*.json
 
 generate-language-examples:
 	@node docs/examples/generate-language-examples.js
@@ -89,11 +89,11 @@ generate-language-examples-with-java:
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
-lint-docs-stateful: ## Lint only the elasticsearch-openapi.examples.json file
-	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+lint-docs-stateful: ## Lint only the elasticsearch-openapi-docs-final.json file
+	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 lint-docs-serverless: ## Lint only the serverless OpenAPI document after overlays
-	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless lint-docs ## Pre contribution target
 

README.md

@@ -74,6 +74,11 @@ $ make overlay-docs
 # The generated output can be found in ./output/openapi/
 ```
 
+## Specification Viewer
+
+An interactive viewer for the Elasticsearch specification is available
+[here](https://elastic.github.io/specification-viewer/).
+
 ## Contributing
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md)
@@ -234,6 +239,9 @@ make validate api=xpack.info type=request branch=main
 
 # this will validate the xpack.info request and response types against the 8.15 branch
 make validate api=xpack.info branch=8.15
+
+# this will validate the xpack.info and search request and response types against the 8.15 branch
+make validate api=xpack.info,search branch=8.15
 ```
 
 The last command above will install all the dependencies and run, download
@@ -242,7 +250,7 @@ If you need to download the recordings again, run `make validate-no-cache api=xp
 
 Once you see the errors, you can fix the original definition in `/specification`
 and then run the command again until the types validator does not trigger any new error.
-Finally open a pull request with your changes.
+Finally open a pull request with your changes. Please open it from a branch in the repository, and not from a fork.
 
 ## Documentation
 

compiler-rs/clients_schema/src/lib.rs

@@ -1003,7 +1003,6 @@ pub struct UrlTemplate {
 pub struct ModelInfo {
     pub title: String,
     pub license: License,
-    pub version: Option<String>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

compiler-rs/clients_schema/src/transform/expand_generics.rs

@@ -211,8 +211,10 @@ pub fn expand(model: IndexedModel, config: ExpandConfig) -> anyhow::Result<Index
 
         expand_behaviors(&mut resp.behaviors, &mappings, model, ctx)?;
         expand_body(&mut resp.body, &mappings, model, ctx)?;
-
-        // TODO: exceptions
+        
+        for exception in &mut resp.exceptions {
+            expand_body(&mut exception.body, &mappings, model, ctx)?;
+        }
 
         Ok(resp.into())
     }