Merge branch 'main' into fix/logstash-pipeline-settings

Author: kaiwenleee

.buildkite/kibana.sh

@@ -5,15 +5,21 @@ set -euo pipefail
 # Since we're into the current repo, move to the top-level
 cd ..
 
+echo "--- Install dependencies"
+lsb_release -a
+apt-get update -y
+apt-get install -y unzip
+
 echo "--- Clone elasticsearch-js"
 git clone -v -- [email protected]:elastic/elasticsearch-js.git
-pushd elasticsearch-js
-git checkout $BUILDKITE_BRANCH
-popd
+
+echo "--- Clone elastic-client-generator-js"
+git clone -v -- [email protected]:elastic/elastic-client-generator-js.git
+mkdir elastic-client-generator-js/output
 
 echo "--- Clone Kibana"
 git clone -v --reference /usr/local/git-references/git-github-com-elastic-kibana-git -- [email protected]:elastic/kibana.git
-cd kibana
+pushd kibana
 
 echo "--- Install Node.js and Yarn"
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
@@ -25,13 +31,17 @@ set -e
 nvm install
 nvm use
 npm install --global yarn
+popd
 
 echo "--- Install elasticsearch-js"
-pushd ../elasticsearch-js
+pushd elasticsearch-js
 npm install
+node .buildkite/make.mjs --task codegen main
 npm run build
 npm pack
 popd
+
+pushd kibana
 yarn add ../elasticsearch-js/elastic-elasticsearch-*.tgz
 
 echo "--- Bootstrap Kibana"

.buildkite/kibana.yml

@@ -3,7 +3,6 @@ agents:
   cpu: "4"
   ephemeralStorage: 15Gi
 
-
 steps:
   - label: "Run Kibana type checks"
     command: .buildkite/kibana.sh

.github/workflows/update-rest-api-json.yml

@@ -3,7 +3,7 @@ name: Update rest-api-spec
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 4 * * *'  # At 04:00.
+    - cron: '0 4 * * *' # At 04:00.
 
 jobs:
   update-rest-api:
@@ -49,7 +49,7 @@ jobs:
           title: Update rest-api-spec ${{ matrix.branch }}
           body: 'As titled.'
           commit-message: 'Update rest-api-spec'
-          labels: specification
+          labels: specification,skip-backport
           delete-branch: true
           reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,swallez,technige
           branch: automated/rest-api-spec-update-${{ matrix.branch }}
@@ -62,4 +62,3 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           labels: bug
           body: The rest-api-spec action is currently failing, see [here](https://github.com/elastic/elasticsearch-specification/actions/workflows/update-rest-api-json.yml).
-

compiler-rs/clients_schema/src/lib.rs

@@ -482,6 +482,23 @@ impl TypeDefinition {
     }
 }
 
+/// The Example type is used for both requests and responses.
+///
+/// This type definition is taken from the OpenAPI spec
+///     https://spec.openapis.org/oas/v3.1.0#example-object
+/// with the exception of using String as the 'value' type.
+///
+/// The OpenAPI v3 spec also defines the 'Example' type, so
+/// to distinguish them, this type is called SchemaExample.
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SchemaExample {
+    pub summary: Option<String>,
+    pub description: Option<String>,
+    pub value: Option<String>,
+    pub external_value: Option<String>,
+}
+
 /// Common attributes for all type definitions
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -675,6 +692,8 @@ pub struct Request {
 
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub attached_behaviors: Vec<String>,
+
+    pub examples: Option<IndexMap<String, SchemaExample>>
 }
 
 impl WithBaseType for Request {
@@ -703,6 +722,8 @@ pub struct Response {
 
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub exceptions: Vec<ResponseException>,
+
+    pub examples: Option<IndexMap<String, SchemaExample>>
 }
 
 impl WithBaseType for Response {

compiler-rs/clients_schema_to_openapi/package-lock.json

@@ -20,12 +20,15 @@ use std::fmt::Write;
 
 use anyhow::{anyhow, bail};
 use clients_schema::Property;
+use indexmap::IndexMap;
 use indexmap::indexmap;
 use icu_segmenter::SentenceSegmenter;
 use openapiv3::{
     MediaType, Parameter, ParameterData, ParameterSchemaOrContent, PathItem, PathStyle, Paths, QueryStyle, ReferenceOr,
-    RequestBody, Response, Responses, StatusCode,
+    RequestBody, Response, Responses, StatusCode, Example
 };
+use clients_schema::SchemaExample;
+use serde_json::json;
 
 use crate::components::TypesAndComponents;
 
@@ -116,15 +119,42 @@ pub fn add_endpoint(
 
     //---- Prepare request body
 
+    // This function converts the IndexMap<String, SchemaExample> examples of
+    // schema.json to IndexMap<String, ReferenceOr<Example>> which is the format
+    // that OpenAPI expects.
+    fn get_openapi_examples(schema_examples: IndexMap<String, SchemaExample>) -> IndexMap<String, ReferenceOr<Example>> {
+        let mut openapi_examples = indexmap! {};
+        for (name, schema_example) in schema_examples {
+            let openapi_example = Example {
+                value: Some(json!(schema_example.value)),
+                description: schema_example.description.clone(),
+                summary: schema_example.summary.clone(),
+                external_value: None,
+                extensions: Default::default(),
+            };
+            openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));    
+        }
+        return openapi_examples;
+    }
+
+
+    let mut request_examples: IndexMap<String, ReferenceOr<Example>> = indexmap! {};
+    // If this endpoint request has examples in schema.json, convert them to the 
+    // OpenAPI format and add them to the endpoint request in the OpenAPI document.
+    if request.examples.is_some() {
+        request_examples = get_openapi_examples(request.examples.as_ref().unwrap().clone());
+    }
+
     let request_body = tac.convert_request(request)?.map(|schema| {
         let media = MediaType {
             schema: Some(schema),
             example: None,
-            examples: Default::default(),
+            examples: request_examples,
             encoding: Default::default(),
             extensions: Default::default(),
         };
 
+
         let body = RequestBody {
             description: None,
             // FIXME: nd-json requests
@@ -142,17 +172,24 @@ pub fn add_endpoint(
 
     //---- Prepare request responses
 
+
     // FIXME: buggy for responses with no body
     // TODO: handle binary responses
     let response_def = tac.model.get_response(endpoint.response.as_ref().unwrap())?;
+    let mut response_examples: IndexMap<String, ReferenceOr<Example>> = indexmap! {};
+    // If this endpoint response has examples in schema.json, convert them to the 
+    // OpenAPI format and add them to the endpoint response in the OpenAPI document.
+    if response_def.examples.is_some() {
+        response_examples = get_openapi_examples(response_def.examples.as_ref().unwrap().clone());
+    }
     let response = Response {
         description: "".to_string(),
         headers: Default::default(),
         content: indexmap! {
             "application/json".to_string() => MediaType {
                 schema: tac.convert_response(response_def)?,
                 example: None,
-                examples: Default::default(),
+                examples: response_examples,
                 encoding: Default::default(),
                 extensions: Default::default(),
             }

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -31,7 +31,9 @@ try {
   closest = fl.closest
   minimist = require('minimist')
 } catch (err) {
-  console.log('It looks like you didn\'t install the project dependencies, please run \'make setup\'')
+  console.log(
+    "It looks like you didn't install the project dependencies, please run 'make setup'"
+  )
   process.exit(1)
 }
 
@@ -60,7 +62,8 @@ const apis = require('../output/schema/schema.json')
 async function run () {
   const options = minimist(process.argv.slice(2), {
     string: ['api', 'type', 'branch'],
-    boolean: ['cache']
+    boolean: ['cache'],
+    default: { cache: true }
   })
 
   spinner.text = 'Checking requirements'
@@ -93,7 +96,7 @@ async function run () {
 
   const isFlightRecorderCloned = await $`[[ -d ${path.join(__dirname, '..', '..', 'clients-flight-recorder')} ]]`.exitCode === 0
   if (!isFlightRecorderCloned) {
-    spinner.text = 'It looks like you didn\'t cloned the flight recorder, doing that for you'
+    spinner.text = 'It looks like you didn\'t clone the flight recorder, doing that for you'
     await $`git clone https://github.com/elastic/clients-flight-recorder.git ${path.join(__dirname, '..', '..', 'clients-flight-recorder')}`
   } else if (isStale) {
     spinner.text = 'Pulling the latest flight recorder changes'
@@ -105,7 +108,7 @@ async function run () {
   const isCompilerInstalled = await $`[[ -d ${path.join(compilerPath, 'node_modules')} ]]`.exitCode === 0
   const isTsGeneratorInstalled = await $`[[ -d ${path.join(tsGeneratorPath, 'node_modules')} ]]`.exitCode === 0
   if (noCache || !isCompilerInstalled || !isTsGeneratorInstalled) {
-    spinner.text = 'It looks like you didn\'t installed the project dependencies, doing that for you'
+    spinner.text = "It looks like you didn't install the project dependencies, doing that for you"
     await $`npm install --prefix ${compilerPath}`
     await $`npm install --prefix ${tsGeneratorPath}`
   }

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib_bg.wasm

@@ -28,6 +28,7 @@ import validateModel from './steps/validate-model'
 import addContentType from './steps/add-content-type'
 import readDefinitionValidation from './steps/read-definition-validation'
 import addDeprecation from './steps/add-deprecation'
+import ExamplesProcessor from './steps/add-examples'
 
 const nvmrc = readFileSync(join(__dirname, '..', '..', '.nvmrc'), 'utf8')
 const nodejsMajor = process.version.split('.').shift()?.slice(1) ?? ''
@@ -65,6 +66,9 @@ if (outputFolder === '' || outputFolder === undefined) {
 
 const compiler = new Compiler(specsFolder, outputFolder)
 
+const examplesProcessor = new ExamplesProcessor(specsFolder)
+const addExamples = examplesProcessor.addExamples.bind(examplesProcessor)
+
 compiler
   .generateModel()
   .step(addInfo)
@@ -74,6 +78,7 @@ compiler
   .step(validateRestSpec)
   .step(addDescription)
   .step(validateModel)
+  .step(addExamples)
   .write()
   .then(() => {
     console.log('Done')

compiler/run-validations.js

@@ -383,7 +383,18 @@ function compileClassOrInterfaceDeclaration (declaration: ClassDeclaration | Int
             if (property.valueOf.kind === 'instance_of' && property.valueOf.type.name === 'Void') {
               type.body = { kind: 'no_body' }
             } else {
-              type.body = { kind: 'value', value: property.valueOf }
+              const tags = parseJsDocTags((member as PropertySignature).getJsDocs())
+              // TODO: Enable this after adding the missing codegen names.
+              // assert(
+              //     member as Node,
+              //     tags.codegen_name != null,
+              //     'You should configure a body @codegen_name'
+              // )
+              type.body = {
+                kind: 'value',
+                value: property.valueOf,
+                codegenName: tags.codegen_name
+              }
             }
           } else {
             type.body = { kind: 'properties', properties: property.properties }