Merge remote-tracking branch 'origin/main' into feature/mistral-chat-completion-integration

# Conflicts:
#	output/openapi/elasticsearch-openapi.json
#	output/openapi/elasticsearch-serverless-openapi.json

Author: Jan-Kazlouski-elastic

.backportrc.json

@@ -0,0 +1,6 @@
+{
+  "repoOwner": "elastic",
+  "repoName": "elasticsearch-specification",
+  "targetBranchChoices": ["9.0", "8.19", "8.18", "8.17"],
+  "fork": false
+}

compiler-rs/Cargo.lock

@@ -264,7 +264,7 @@ pub struct Availability {
     pub visibility: Option<Visibility>,
 }
 
-/// The availability of an
+/// The availability of an endpoint, field or parameter
 pub type Availabilities = IndexMap<Flavor, Availability>;
 
 pub trait AvailabilityFilter: Fn(&Option<Availabilities>) -> bool {}
@@ -494,6 +494,7 @@ impl TypeDefinition {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaExample {
     pub summary: Option<String>,
+    pub method_request: Option<String>,
     pub description: Option<String>,
     pub value: Option<String>,
     pub external_value: Option<String>,

compiler-rs/clients_schema/src/lib.rs

@@ -10,7 +10,7 @@ clients_schema = {path="../clients_schema"}
 argh = { workspace = true }
 derive_more = { version = "2", features = ["from_str"] }
 serde_json = { workspace = true }
-serde_ignored = { workspace = true }
+itertools = { workspace = true }
 icu_segmenter = { workspace = true }
 openapiv3 = { workspace = true }
 anyhow = { workspace = true }

compiler-rs/clients_schema_to_openapi/Cargo.toml

@@ -23,8 +23,9 @@ pub mod cli;
 
 use indexmap::IndexMap;
 
-use clients_schema::{Availabilities, Flavor, IndexedModel, Stability, Visibility};
+use clients_schema::{Availabilities, Availability, Flavor, IndexedModel, Stability, Visibility};
 use openapiv3::{Components, OpenAPI};
+use serde_json::Value;
 use clients_schema::transform::ExpandConfig;
 use crate::components::TypesAndComponents;
 
@@ -149,30 +150,37 @@ fn info(model: &IndexedModel) -> openapiv3::Info {
     }
 }
 
-pub fn availability_as_extensions(availabilities: &Option<Availabilities>) -> IndexMap<String, serde_json::Value> {
+pub fn availability_as_extensions(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>) -> IndexMap<String, serde_json::Value> {
     let mut result = IndexMap::new();
+    convert_availabilities(availabilities, flavor, &mut result);
+    result
+}
 
+pub fn convert_availabilities(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>, result: &mut IndexMap<String, Value>) {
     if let Some(avails) = availabilities {
-        // We may have several availabilities, but since generally exists only on stateful (stack)
-        for (_, availability) in avails {
-            if let Some(stability) = &availability.stability {
-                match stability {
+        if let Some(flav) = flavor {
+            if let Some(availability) = avails.get(flav) {
+                let Availability {since,stability,..} = &availability;
+                let stab = stability.clone().unwrap_or(Stability::Stable);
+                let mut since_str = "".to_string();
+                if let Some(since) = since {
+                    since_str = format!("; Added in {since}");
+                }
+                match stab {
                     Stability::Beta => {
-                        result.insert("x-beta".to_string(), serde_json::Value::Bool(true));
+                        let beta_since = format!("Beta{since_str}");
+                        result.insert("x-state".to_string(), Value::String(beta_since));
                     }
                     Stability::Experimental => {
-                        result.insert("x-state".to_string(), serde_json::Value::String("Technical preview".to_string()));
+                        let exp_since = format!("Technical preview{since_str}");
+                        result.insert("x-state".to_string(), Value::String(exp_since));
                     }
-                    Stability::Stable => {
-                        if let Some(since) = &availability.since {
-                            let stable_since = "Added in ".to_string() + since;
-                            result.insert("x-state".to_string(), serde_json::Value::String(stable_since));
-                        }
+                    Stability::Stable => { 
+                        let stable_since = format!("Generally available{since_str}");
+                        result.insert("x-state".to_string(), Value::String(stable_since));
                     }
                 }
             }
         }
     }
-
-    result
 }

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -19,24 +19,26 @@ use std::collections::HashMap;
 use std::fmt::Write;
 
 use anyhow::{anyhow, bail};
-use clients_schema::Property;
+use clients_schema::{Privileges, Property};
 use indexmap::IndexMap;
 use indexmap::indexmap;
 use icu_segmenter::SentenceSegmenter;
+use itertools::Itertools;
 use openapiv3::{
     MediaType, Parameter, ParameterData, ParameterSchemaOrContent, PathItem, PathStyle, Paths, QueryStyle, ReferenceOr,
     RequestBody, Response, Responses, StatusCode, Example
 };
+use serde_json::Value;
 use clients_schema::SchemaExample;
-
 use crate::components::TypesAndComponents;
+use crate::convert_availabilities;
 
 /// Add an endpoint to the OpenAPI schema. This will result in the addition of a number of elements to the
 /// openapi schema's `paths` and `components` sections.
 pub fn add_endpoint(
     endpoint: &clients_schema::Endpoint,
     tac: &mut TypesAndComponents,
-    out: &mut Paths,
+    out: &mut Paths
 ) -> anyhow::Result<()> {
     if endpoint.request.is_none() {
         // tracing::warn!("Endpoint {} is missing a request -- ignored", &endpoint.name);
@@ -60,6 +62,8 @@ pub fn add_endpoint(
     let request = tac.model.get_request(endpoint.request.as_ref().unwrap())?;
 
     fn parameter_data(prop: &Property, in_path: bool, tac: &mut TypesAndComponents) -> anyhow::Result<ParameterData> {
+        let mut extensions: IndexMap<String,Value> = Default::default();
+        convert_availabilities(&prop.availability, &tac.config.flavor, &mut extensions);
         Ok(ParameterData {
             name: prop.name.clone(),
             description: tac.property_description(prop)?,
@@ -198,7 +202,7 @@ pub fn add_endpoint(
     // If this endpoint response has examples in schema.json, convert them to the
     // OpenAPI format and add them to the endpoint response in the OpenAPI document.
     let response_examples = if let Some(examples) = &response_def.examples {
-         get_openapi_examples(examples)
+        get_openapi_examples(examples)
     } else {
         IndexMap::new()
     };
@@ -250,6 +254,51 @@ pub fn add_endpoint(
         parameters.append(&mut query_params.clone());
 
         let sum_desc = split_summary_desc(&endpoint.description);
+        
+        let privilege_desc = add_privileges(&endpoint.privileges);
+        
+        let full_desc = match (sum_desc.description, privilege_desc) {
+            (Some(a), Some(b)) => Some(a+ &b),
+            (opt_a, opt_b) => opt_a.or(opt_b)
+        };
+
+        // add the x-state extension for availability
+        let mut extensions = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
+
+        // add the x-codeSamples extension
+        let mut code_samples = vec![];
+        if let Some(examples) = request.examples.clone() {
+            if let Some((_, example)) = examples.first() {
+                let request_line = example.method_request.clone().unwrap_or(String::from(""));
+                let request_body = example.value.clone().unwrap_or(String::from(""));
+                if !request_line.is_empty() {
+                    code_samples.push(serde_json::json!({
+                        "lang": "Console",
+                        "source": request_line + "\n" + request_body.as_str(),
+                    }));
+                }
+            }
+        }
+        if code_samples.is_empty() {
+            // if there are no example requests we look for example responses
+            // this can only happen for examples that do not have a request body
+            if let Some(examples) = response_def.examples.clone() {
+                if let Some((_, example)) = examples.first() {
+                    let request_line = example.method_request.clone().unwrap_or(String::from(""));
+                    if !request_line.is_empty() {
+                        code_samples.push(serde_json::json!({
+                            "lang": "Console",
+                            "source": request_line + "\n",
+                        }));
+                    }
+                }
+            }
+        }
+        if !code_samples.is_empty() {
+            extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
+        }
+        let mut ext_availability = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
+        extensions.append(&mut ext_availability);
 
         // Create the operation, it will be repeated if we have several methods
         let operation = openapiv3::Operation {
@@ -259,7 +308,7 @@ pub fn add_endpoint(
                 vec![namespace.to_string()]
             },
             summary: sum_desc.summary,
-            description: sum_desc.description,
+            description: full_desc,
             external_docs: tac.convert_external_docs(endpoint),
             // external_docs: None, // Need values that differ from client purposes
             operation_id: None, // set in clone_operation below with operation_counter
@@ -274,7 +323,7 @@ pub fn add_endpoint(
             deprecated: endpoint.deprecation.is_some(),
             security: None,
             servers: vec![],
-            extensions: crate::availability_as_extensions(&endpoint.availability),
+            extensions
         };
 
 
@@ -403,6 +452,26 @@ fn split_summary_desc(desc: &str) -> SplitDesc{
     }
 }
 
+fn add_privileges(privileges: &Option<Privileges>) -> Option<String>{
+    if let Some(privs) = privileges {
+        let mut result = "\n ##Required authorization\n".to_string();
+        if !privs.index.is_empty() {
+            result += "* Index privileges: ";
+            result += &privs.index.iter()
+                .map(|a| format!("`{a}`"))
+                .join(",");
+        }
+        if !privs.cluster.is_empty() {
+            result += "* Cluster privileges: ";
+            result += &privs.cluster.iter()
+                .map(|a| format!("`{a}`"))
+                .join(",");
+        }
+        return Some(result)
+    }
+    None
+}
+
 #[derive(PartialEq,Debug)]
 struct SplitDesc {
     summary: Option<String>,

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -469,7 +469,7 @@ impl<'a> TypesAndComponents<'a> {
         data.external_docs = self.convert_external_docs(prop);
         data.deprecated = prop.deprecation.is_some();
         data.description = self.property_description(prop)?;
-        data.extensions = crate::availability_as_extensions(&prop.availability);
+        data.extensions = crate::availability_as_extensions(&prop.availability, &self.config.flavor);
         // TODO: prop.aliases as extensions
         // TODO: prop.server_default as extension
         // TODO: prop.doc_id as extension (new representation of since and stability)

compiler-rs/clients_schema_to_openapi/src/schemas.rs

@@ -271,6 +271,8 @@ export class Example {
   summary?: string
   /** Long description. */
   description?: string
+  /** request method and URL */
+  method_request?: string
   /** Embedded literal example. Mutually exclusive with `external_value` */
   value?: string
   /** A URI that points to the literal example */