Merge branch '9.0' into backport-4562-to-9.0

Author: kosabogi

Makefile

@@ -8,7 +8,6 @@ validate-no-cache: ## Validate a given endpoint request or response without loca
 
 generate:	  ## Generate the output spec
 	@echo ">> generating the spec .."
-	@make generate-language-examples
 	@npm run generate-schema --prefix compiler -- --spec ../specification/ --output ../output/
 	@npm run start --prefix typescript-generator
 
@@ -55,7 +54,9 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
 transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --output output/openapi/elasticsearch-openapi-docs.json
+	@make generate-language-examples
+	@make generate
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-openapi-docs.json
 
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
@@ -73,6 +74,10 @@ generate-language-examples:
 	@node docs/examples/generate-language-examples.js
 	@npm run format:fix-examples --prefix compiler
 
+generate-language-examples-with-java:
+	@node docs/examples/generate-language-examples.js java
+	@npm run format:fix-examples --prefix compiler
+
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -35,6 +35,10 @@ pub struct Cli {
     /// output a redirection map when merging multipath endpoints
     #[argh(switch)]
     pub multipath_redirects: bool,
+
+    /// include the x-codeSamples extension with language examples for all endpoints
+    #[argh(switch)]
+    pub include_language_examples: bool,
 }
 
 impl Cli {
@@ -74,6 +78,7 @@ impl From<Cli> for Configuration {
             lift_enum_descriptions: cli.lift_enum_descriptions,
             merge_multipath_endpoints: cli.merge_multipath_endpoints,
             multipath_redirects: cli.multipath_redirects,
+            include_language_examples: cli.include_language_examples,
             namespaces: if cli.namespace.is_empty() {
                 None
             } else {

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -23,8 +23,9 @@ pub mod cli;
 
 use indexmap::IndexMap;
 
-use clients_schema::{Availabilities, Flavor, IndexedModel, Stability, Visibility};
+use clients_schema::{Availabilities, Availability, Flavor, IndexedModel, Stability, Visibility};
 use openapiv3::{Components, OpenAPI};
+use serde_json::Value;
 use clients_schema::transform::ExpandConfig;
 use crate::components::TypesAndComponents;
 
@@ -43,6 +44,9 @@ pub struct Configuration {
 
     /// Should we output a redirect map when merging multipath endpoints?
     pub multipath_redirects: bool,
+
+    /// include the x-codeSamples extension with language examples for all endpoints
+    pub include_language_examples: bool,
 }
 
 pub struct OpenApiConversion {
@@ -51,7 +55,7 @@ pub struct OpenApiConversion {
 }
 
 /// Convert an API model into an OpenAPI v3 schema, optionally filtered for a given flavor
-pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow::Result<OpenApiConversion> {
+pub fn convert_schema(mut schema: IndexedModel, config: Configuration, product_meta: IndexMap<String,String>) -> anyhow::Result<OpenApiConversion> {
     // Expand generics
     schema = clients_schema::transform::expand_generics(schema, ExpandConfig::default())?;
 
@@ -72,7 +76,7 @@ pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow
         schema = clients_schema::transform::filter_availability(schema, filter)?;
     }
 
-    convert_expanded_schema(&schema, &config)
+    convert_expanded_schema(&schema, &config, &product_meta)
 }
 
 /// Convert an API model into an OpenAPI v3 schema. The input model must have all generics expanded, conversion
@@ -81,7 +85,7 @@ pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow
 /// Note: there are ways to represent [generics in JSON Schema], but its unlikely that tooling will understand it.
 ///
 /// [generics in JSON Schema]: https://json-schema.org/blog/posts/dynamicref-and-generics
-pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration) -> anyhow::Result<OpenApiConversion> {
+pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration, product_meta: &IndexMap<String,String>) -> anyhow::Result<OpenApiConversion> {
     let mut openapi = OpenAPI {
         openapi: "3.0.3".into(),
         info: info(model),
@@ -119,7 +123,7 @@ pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration) ->
                 continue;
             }
         }
-        paths::add_endpoint(endpoint, &mut tac, &mut openapi.paths)?;
+        paths::add_endpoint(endpoint, &mut tac, &mut openapi.paths, product_meta)?;
     }
 
     // // Sort maps to ensure output stability
@@ -179,30 +183,49 @@ fn info(model: &IndexedModel) -> openapiv3::Info {
     }
 }
 
-pub fn availability_as_extensions(availabilities: &Option<Availabilities>) -> IndexMap<String, serde_json::Value> {
+pub fn product_meta_as_extensions(namespace: &str, product_meta: &IndexMap<String,String>) -> IndexMap<String, Value> {
     let mut result = IndexMap::new();
+    let mut additional_namespace= "".to_string();
+    if let Some(meta) = product_meta.get(namespace) {
+        additional_namespace = format!(", {meta}");
+    }
+    
+    let product_str = format!("elasticsearch{additional_namespace}");
+    result.insert("x-product-feature".to_string(), Value::String(product_str));
+    result
+}
 
+pub fn availability_as_extensions(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>) -> IndexMap<String, Value> {
+    let mut result = IndexMap::new();
+    convert_availabilities(availabilities, flavor, &mut result);
+    result
+}
+
+pub fn convert_availabilities(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>, result: &mut IndexMap<String, Value>) {
     if let Some(avails) = availabilities {
-        // We may have several availabilities, but since generally exists only on stateful (stack)
-        for (_, availability) in avails {
-            if let Some(stability) = &availability.stability {
-                match stability {
+        if let Some(flav) = flavor {
+            if let Some(availability) = avails.get(flav) {
+                let Availability {since,stability,..} = &availability;
+                let stab = stability.clone().unwrap_or(Stability::Stable);
+                let mut since_str = "".to_string();
+                if let Some(since) = since {
+                    since_str = format!("; Added in {since}");
+                }
+                match stab {
                     Stability::Beta => {
-                        result.insert("x-beta".to_string(), serde_json::Value::Bool(true));
+                        let beta_since = format!("Beta{since_str}");
+                        result.insert("x-state".to_string(), Value::String(beta_since));
                     }
                     Stability::Experimental => {
-                        result.insert("x-state".to_string(), serde_json::Value::String("Technical preview".to_string()));
+                        let exp_since = format!("Technical preview{since_str}");
+                        result.insert("x-state".to_string(), Value::String(exp_since));
                     }
-                    Stability::Stable => {
-                        if let Some(since) = &availability.since {
-                            let stable_since = "Added in ".to_string() + since;
-                            result.insert("x-state".to_string(), serde_json::Value::String(stable_since));
-                        }
+                    Stability::Stable => { 
+                        let stable_since = format!("Generally available{since_str}");
+                        result.insert("x-state".to_string(), Value::String(stable_since));
                     }
                 }
             }
         }
     }
-
-    result
 }

compiler-rs/clients_schema_to_openapi/src/main.rs

@@ -16,7 +16,8 @@
 // under the License.
 
 use std::fs::File;
-use clients_schema::IndexedModel;
+use indexmap::IndexMap;
+use clients_schema::{IndexedModel};
 use tracing::Level;
 use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::FmtSubscriber;
@@ -32,10 +33,12 @@ fn main() -> anyhow::Result<()> {
         .finish();
     tracing::subscriber::set_global_default(subscriber)?;
 
+    let product_meta: IndexMap<String, String> = serde_json::from_reader(File::open("../../specification/_doc_ids/product-meta.json")?)?;
     let schema = IndexedModel::from_reader(File::open(&cli.schema)?)?;
     let output = cli.output.clone();
     let redirect_path = cli.redirect_path(&cli.output);
-    let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into())?;
+    let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into(), product_meta)?;
+    serde_json::to_writer_pretty(File::create(&output)?, &openapi.openapi)?;
     serde_json::to_writer_pretty(File::create(&output)?, &openapi.openapi)?;
 
     if let Some(redirects) = openapi.redirects {

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -28,16 +28,18 @@ use openapiv3::{
     MediaType, Parameter, ParameterData, ParameterSchemaOrContent, PathItem, PathStyle, Paths, QueryStyle, ReferenceOr,
     RequestBody, Response, Responses, StatusCode, Example
 };
+use serde_json::Value;
 use clients_schema::SchemaExample;
-
 use crate::components::TypesAndComponents;
+use crate::convert_availabilities;
 
 /// Add an endpoint to the OpenAPI schema. This will result in the addition of a number of elements to the
 /// openapi schema's `paths` and `components` sections.
 pub fn add_endpoint(
     endpoint: &clients_schema::Endpoint,
     tac: &mut TypesAndComponents,
     out: &mut Paths,
+    product_meta: &IndexMap<String,String>
 ) -> anyhow::Result<()> {
     if endpoint.request.is_none() {
         // tracing::warn!("Endpoint {} is missing a request -- ignored", &endpoint.name);
@@ -61,6 +63,8 @@ pub fn add_endpoint(
     let request = tac.model.get_request(endpoint.request.as_ref().unwrap())?;
 
     fn parameter_data(prop: &Property, in_path: bool, tac: &mut TypesAndComponents) -> anyhow::Result<ParameterData> {
+        let mut extensions: IndexMap<String,Value> = Default::default();
+        convert_availabilities(&prop.availability, &tac.config.flavor, &mut extensions);
         Ok(ParameterData {
             name: prop.name.clone(),
             description: tac.property_description(prop)?,
@@ -341,33 +345,37 @@ pub fn add_endpoint(
         };
 
         // add the x-state extension for availability
-        let mut extensions = crate::availability_as_extensions(&endpoint.availability);
-
-        // add the x-codeSamples extension
-        let mut code_samples = vec![];
-        if let Some(examples) = request.examples.clone() {
-            if let Some((_, example)) = examples.first() {
-                let request_line = example.method_request.clone().unwrap_or(String::from(""));
-                let request_body = example.value.clone().unwrap_or(String::from(""));
-                if !request_line.is_empty() {
-                    code_samples.push(serde_json::json!({
-                        "lang": "Console",
-                        "source": request_line + "\n" + request_body.as_str(),
-                    }));
-                }
-                if let Some(alternatives) = example.alternatives.clone() {
-                    for alternative in alternatives.iter() {
+        let mut extensions = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
+
+        if tac.config.include_language_examples {
+            // add the x-codeSamples extension
+            let mut code_samples = vec![];
+            if let Some(examples) = request.examples.clone() {
+                if let Some((_, example)) = examples.first() {
+                    let request_line = example.method_request.clone().unwrap_or(String::from(""));
+                    let request_body = example.value.clone().unwrap_or(String::from(""));
+                    if !request_line.is_empty() {
                         code_samples.push(serde_json::json!({
-                            "lang": alternative.language,
-                            "source": alternative.code.as_str(),
+                            "lang": "Console",
+                            "source": request_line + "\n" + request_body.as_str(),
                         }));
                     }
+                    if let Some(alternatives) = example.alternatives.clone() {
+                        for alternative in alternatives.iter() {
+                            code_samples.push(serde_json::json!({
+                                "lang": alternative.language,
+                                "source": alternative.code.as_str(),
+                            }));
+                        }
+                    }
                 }
             }
+            if !code_samples.is_empty() {
+                extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
+            }
         }
-        if !code_samples.is_empty() {
-            extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
-        }
+
+        extensions.append(&mut crate::product_meta_as_extensions(namespace, product_meta));
 
         // Create the operation, it will be repeated if we have several methods
         let operation = openapiv3::Operation {
@@ -392,7 +400,7 @@ pub fn add_endpoint(
             deprecated: endpoint.deprecation.is_some(),
             security: None,
             servers: vec![],
-            extensions,
+            extensions
         };
 
 

compiler-rs/clients_schema_to_openapi/src/schemas.rs

@@ -473,7 +473,7 @@ impl<'a> TypesAndComponents<'a> {
         data.external_docs = self.convert_external_docs(prop);
         data.deprecated = prop.deprecation.is_some();
         data.description = self.property_description(prop)?;
-        data.extensions = crate::availability_as_extensions(&prop.availability);
+        data.extensions = crate::availability_as_extensions(&prop.availability, &self.config.flavor);
         // TODO: prop.aliases as extensions
         // TODO: prop.server_default as extension
         // TODO: prop.doc_id as extension (new representation of since and stability)

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib.js

@@ -15,10 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::path::PathBuf;
+use std::path::{PathBuf};
 use argh::FromArgs;
-use clients_schema::IndexedModel;
+use clients_schema::{IndexedModel};
 use wasm_bindgen::prelude::*;
+use clients_schema::indexmap::IndexMap;
 use clients_schema_to_openapi::cli::Cli;
 
 /// Minimal bindings to Node's `fs` module.
@@ -63,8 +64,15 @@ pub fn convert0(cli: Cli, cwd: Option<String>) -> anyhow::Result<()> {
 
     let json = node_fs::read_file_sync_to_string(&input.to_string_lossy(), "utf8");
     let schema = IndexedModel::from_reader(json.as_bytes())?;
+    
+    let product_meta_path = match cwd {
+        Some(ref cwd) => format!("{cwd}/specification/_doc_ids/product-meta.json"),
+        None => "specification/_doc_ids/product-meta.json".to_string(),
+    };
+    let json_product_map = node_fs::read_file_sync_to_string(&product_meta_path, "utf8");
+    let product_meta: IndexMap<String, String> = serde_json::from_str(&json_product_map).expect("Cannot parse product metadata file");
 
-    let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into())?;
+    let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into(), product_meta)?;
 
     let result = serde_json::to_string_pretty(&openapi.openapi)?;
     node_fs::write_file_sync(&output.to_string_lossy(), &result);

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib_bg.wasm

@@ -173,12 +173,13 @@ Add an `examples` folder and `request` and `xxx_response` subfolders (where `xxx
 
 These examples are for use in the API documentation and must adhere to the [OpenAPI 3.0 Example object specification](https://spec.openapis.org/oas/v3.0.3#example-object). They must have a `value` field that contains the request or response body.
 If there are multiple examples for the endpoint, they must each have a brief `summary` field, which is used as the label for the example. You can also optionaly provide an explanation in a `description` field.
+In order to generate curl and console examples automatically, the request examples must also contain a `method_request`.
 
 For example:
 
 ```yaml
 summary: Sequence query
-# method_request: GET /my-data-stream/_eql/search
+method_request: GET /my-data-stream/_eql/search
 # type: request
 description: >
   Run `GET /my-data-stream/_eql/search` to search for a sequence of events.