Merge branch 'main' into lifecycle-links

Author: lcawl

Makefile

@@ -55,6 +55,9 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --output output/openapi/elasticsearch-openapi.json
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
+transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --output output/openapi/elasticsearch-openapi-docs.json
+
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
 

compiler-rs/Cargo.lock

@@ -264,7 +264,7 @@ pub struct Availability {
     pub visibility: Option<Visibility>,
 }
 
-/// The availability of an
+/// The availability of an endpoint, field or parameter
 pub type Availabilities = IndexMap<Flavor, Availability>;
 
 pub trait AvailabilityFilter: Fn(&Option<Availabilities>) -> bool {}

compiler-rs/clients_schema/src/lib.rs

@@ -10,7 +10,7 @@ clients_schema = {path="../clients_schema"}
 argh = { workspace = true }
 derive_more = { version = "2", features = ["from_str"] }
 serde_json = { workspace = true }
-serde_ignored = { workspace = true }
+itertools = { workspace = true }
 icu_segmenter = { workspace = true }
 openapiv3 = { workspace = true }
 anyhow = { workspace = true }

compiler-rs/clients_schema_to_openapi/Cargo.toml

@@ -20,13 +20,17 @@ pub struct Cli {
     #[argh(option, default = "SchemaFlavor::All")]
     pub flavor: SchemaFlavor,
 
-    /// add enum descriptions to property descriptions [default = true]
-    #[argh(option, default = "true")]
-    pub lift_enum_descriptions: bool,
-
     /// generate only this namespace (can be repeated)
     #[argh(option)]
     pub namespace: Vec<String>,
+
+    /// add enum descriptions to property descriptions [default = true]
+    #[argh(switch)]
+    pub lift_enum_descriptions: bool,
+
+    /// merge endpoints with multiple paths into a single OpenAPI operation [default = false]
+    #[argh(switch)]
+    pub merge_multipath_endpoints: bool,
 }
 
 use derive_more::FromStr;
@@ -42,20 +46,21 @@ pub enum SchemaFlavor {
 }
 
 impl From<Cli> for Configuration {
-    fn from(val: Cli) -> Configuration {
-        let flavor = match val.flavor {
+    fn from(cli: Cli) -> Configuration {
+        let flavor = match cli.flavor {
             SchemaFlavor::All => None,
             SchemaFlavor::Serverless => Some(Flavor::Serverless),
             SchemaFlavor::Stack => Some(Flavor::Stack),
         };
 
         Configuration {
             flavor,
-            lift_enum_descriptions: val.lift_enum_descriptions,
-            namespaces: if val.namespace.is_empty() {
+            lift_enum_descriptions: cli.lift_enum_descriptions,
+            merge_multipath_endpoints: cli.merge_multipath_endpoints,
+            namespaces: if cli.namespace.is_empty() {
                 None
             } else {
-                Some(val.namespace)
+                Some(cli.namespace)
             },
         }
     }

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -23,15 +23,24 @@ pub mod cli;
 
 use indexmap::IndexMap;
 
-use clients_schema::{Availabilities, Flavor, IndexedModel, Stability, Visibility};
+use clients_schema::{Availabilities, Availability, Flavor, IndexedModel, Stability, Visibility};
 use openapiv3::{Components, OpenAPI};
+use serde_json::Value;
 use clients_schema::transform::ExpandConfig;
 use crate::components::TypesAndComponents;
 
 pub struct Configuration {
     pub flavor: Option<Flavor>,
     pub namespaces: Option<Vec<String>>,
+
+    /// If a property value is an enumeration, the description of possible values will be copied in the
+    /// property's description (also works for arrays of enums).
     pub lift_enum_descriptions: bool,
+
+    /// Will output endpoints having multiple paths into a single operation. The operation's path will
+    /// be the longest one (with values for all optional parameters), and the other paths will be added
+    /// at the beginning of the operation's description.
+    pub merge_multipath_endpoints: bool,
 }
 
 /// Convert an API model into an OpenAPI v3 schema, optionally filtered for a given flavor
@@ -149,30 +158,37 @@ fn info(model: &IndexedModel) -> openapiv3::Info {
     }
 }
 
-pub fn availability_as_extensions(availabilities: &Option<Availabilities>) -> IndexMap<String, serde_json::Value> {
+pub fn availability_as_extensions(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>) -> IndexMap<String, serde_json::Value> {
     let mut result = IndexMap::new();
+    convert_availabilities(availabilities, flavor, &mut result);
+    result
+}
 
+pub fn convert_availabilities(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>, result: &mut IndexMap<String, Value>) {
     if let Some(avails) = availabilities {
-        // We may have several availabilities, but since generally exists only on stateful (stack)
-        for (_, availability) in avails {
-            if let Some(stability) = &availability.stability {
-                match stability {
+        if let Some(flav) = flavor {
+            if let Some(availability) = avails.get(flav) {
+                let Availability {since,stability,..} = &availability;
+                let stab = stability.clone().unwrap_or(Stability::Stable);
+                let mut since_str = "".to_string();
+                if let Some(since) = since {
+                    since_str = format!("; Added in {since}");
+                }
+                match stab {
                     Stability::Beta => {
-                        result.insert("x-beta".to_string(), serde_json::Value::Bool(true));
+                        let beta_since = format!("Beta{since_str}");
+                        result.insert("x-state".to_string(), Value::String(beta_since));
                     }
                     Stability::Experimental => {
-                        result.insert("x-state".to_string(), serde_json::Value::String("Technical preview".to_string()));
+                        let exp_since = format!("Technical preview{since_str}");
+                        result.insert("x-state".to_string(), Value::String(exp_since));
                     }
-                    Stability::Stable => {
-                        if let Some(since) = &availability.since {
-                            let stable_since = "Added in ".to_string() + since;
-                            result.insert("x-state".to_string(), serde_json::Value::String(stable_since));
-                        }
+                    Stability::Stable => { 
+                        let stable_since = format!("Generally available{since_str}");
+                        result.insert("x-state".to_string(), Value::String(stable_since));
                     }
                 }
             }
         }
     }
-
-    result
 }

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -19,24 +19,26 @@ use std::collections::HashMap;
 use std::fmt::Write;
 
 use anyhow::{anyhow, bail};
-use clients_schema::Property;
+use clients_schema::{Privileges, Property};
 use indexmap::IndexMap;
 use indexmap::indexmap;
 use icu_segmenter::SentenceSegmenter;
+use itertools::Itertools;
 use openapiv3::{
     MediaType, Parameter, ParameterData, ParameterSchemaOrContent, PathItem, PathStyle, Paths, QueryStyle, ReferenceOr,
     RequestBody, Response, Responses, StatusCode, Example
 };
+use serde_json::Value;
 use clients_schema::SchemaExample;
-
 use crate::components::TypesAndComponents;
+use crate::convert_availabilities;
 
 /// Add an endpoint to the OpenAPI schema. This will result in the addition of a number of elements to the
 /// openapi schema's `paths` and `components` sections.
 pub fn add_endpoint(
     endpoint: &clients_schema::Endpoint,
     tac: &mut TypesAndComponents,
-    out: &mut Paths,
+    out: &mut Paths
 ) -> anyhow::Result<()> {
     if endpoint.request.is_none() {
         // tracing::warn!("Endpoint {} is missing a request -- ignored", &endpoint.name);
@@ -60,6 +62,8 @@ pub fn add_endpoint(
     let request = tac.model.get_request(endpoint.request.as_ref().unwrap())?;
 
     fn parameter_data(prop: &Property, in_path: bool, tac: &mut TypesAndComponents) -> anyhow::Result<ParameterData> {
+        let mut extensions: IndexMap<String,Value> = Default::default();
+        convert_availabilities(&prop.availability, &tac.config.flavor, &mut extensions);
         Ok(ParameterData {
             name: prop.name.clone(),
             description: tac.property_description(prop)?,
@@ -144,14 +148,16 @@ pub fn add_endpoint(
                 // }
             };
 
-            let openapi_example = Example {
-                value: example,
-                description: schema_example.description.clone(),
-                summary: schema_example.summary.clone(),
-                external_value: None,
-                extensions: Default::default(),
-            };
-            openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            if example.is_some() {
+                let openapi_example = Example {
+                    value: example,
+                    description: schema_example.description.clone(),
+                    summary: schema_example.summary.clone(),
+                    external_value: None,
+                    extensions: Default::default(),
+                };
+                openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            }
         }
         openapi_examples
     }
@@ -228,6 +234,67 @@ pub fn add_endpoint(
         // TODO: add error responses
     };
 
+    //---- Merge multipath endpoints if asked for
+    let mut new_endpoint: clients_schema::Endpoint;
+
+    let endpoint = if is_multipath && tac.config.merge_multipath_endpoints {
+        new_endpoint = endpoint.clone();
+        let endpoint = &mut new_endpoint;
+
+        // Sort paths from smallest to longest
+        endpoint.urls.sort_by_key(|x| x.path.len());
+
+        // Keep the longest and its last method so that the operation's path+method are the same as the last one
+        // (avoids the perception that it may have been chosen randomly).
+        let mut longest_path = endpoint.urls.last().unwrap().clone();
+        while longest_path.methods.len() > 1 {
+            longest_path.methods.remove(0);
+        }
+
+        // Replace endpoint urls with the longest path
+        let mut urls = vec![longest_path];
+        std::mem::swap(&mut endpoint.urls, &mut urls);
+
+        let split_desc = split_summary_desc(&endpoint.description);
+
+        // Make sure the description is stays at the top
+        let mut description = match split_desc.summary {
+            Some(summary) => format!("{summary}\n\n"),
+            None => String::new(),
+        };
+
+        // Convert removed paths to descriptions
+        write!(description, "**All methods and paths for this operation:**\n\n")?;
+
+        for url in urls {
+            for method in url.methods {
+                let lower_method = method.to_lowercase();
+                let path = &url.path;
+                write!(
+                    description,
+                    r#"<div>
+                      <span class="operation-verb {lower_method}">{method}</span>
+                      <span class="operation-path">{path}</span>
+                      </div>
+                    "#
+                )?;
+            }
+        }
+
+        if let Some(desc) = &split_desc.description {
+            write!(description, "\n\n{}", desc)?;
+        }
+
+        // Replace description
+        endpoint.description = description;
+
+        // Done
+        endpoint
+    } else {
+        // Not multipath or not asked to merge multipath
+        endpoint
+    };
+
     //---- Build a path for each url + method
     let mut operation_counter = 0;
 
@@ -250,9 +317,16 @@ pub fn add_endpoint(
         parameters.append(&mut query_params.clone());
 
         let sum_desc = split_summary_desc(&endpoint.description);
+        
+        let privilege_desc = add_privileges(&endpoint.privileges);
+        
+        let full_desc = match (sum_desc.description, privilege_desc) {
+            (Some(a), Some(b)) => Some(a+ &b),
+            (opt_a, opt_b) => opt_a.or(opt_b)
+        };
 
         // add the x-state extension for availability
-        let mut extensions = crate::availability_as_extensions(&endpoint.availability);
+        let mut extensions = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
 
         // add the x-codeSamples extension
         let mut code_samples = vec![];
@@ -268,24 +342,11 @@ pub fn add_endpoint(
                 }
             }
         }
-        if code_samples.is_empty() {
-            // if there are no example requests we look for example responses
-            // this can only happen for examples that do not have a request body
-            if let Some(examples) = response_def.examples.clone() {
-                if let Some((_, example)) = examples.first() {
-                    let request_line = example.method_request.clone().unwrap_or(String::from(""));
-                    if !request_line.is_empty() {
-                        code_samples.push(serde_json::json!({
-                            "lang": "Console",
-                            "source": request_line + "\n",
-                        }));
-                    }
-                }
-            }
-        }
         if !code_samples.is_empty() {
             extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
         }
+        let mut ext_availability = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
+        extensions.append(&mut ext_availability);
 
         // Create the operation, it will be repeated if we have several methods
         let operation = openapiv3::Operation {
@@ -295,7 +356,7 @@ pub fn add_endpoint(
                 vec![namespace.to_string()]
             },
             summary: sum_desc.summary,
-            description: sum_desc.description,
+            description: full_desc,
             external_docs: tac.convert_external_docs(endpoint),
             // external_docs: None, // Need values that differ from client purposes
             operation_id: None, // set in clone_operation below with operation_counter
@@ -310,7 +371,7 @@ pub fn add_endpoint(
             deprecated: endpoint.deprecation.is_some(),
             security: None,
             servers: vec![],
-            extensions,
+            extensions
         };
 
 
@@ -439,6 +500,26 @@ fn split_summary_desc(desc: &str) -> SplitDesc{
     }
 }
 
+fn add_privileges(privileges: &Option<Privileges>) -> Option<String>{
+    if let Some(privs) = privileges {
+        let mut result = "\n ##Required authorization\n".to_string();
+        if !privs.index.is_empty() {
+            result += "* Index privileges: ";
+            result += &privs.index.iter()
+                .map(|a| format!("`{a}`"))
+                .join(",");
+        }
+        if !privs.cluster.is_empty() {
+            result += "* Cluster privileges: ";
+            result += &privs.cluster.iter()
+                .map(|a| format!("`{a}`"))
+                .join(",");
+        }
+        return Some(result)
+    }
+    None
+}
+
 #[derive(PartialEq,Debug)]
 struct SplitDesc {
     summary: Option<String>,

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -469,7 +469,7 @@ impl<'a> TypesAndComponents<'a> {
         data.external_docs = self.convert_external_docs(prop);
         data.deprecated = prop.deprecation.is_some();
         data.description = self.property_description(prop)?;
-        data.extensions = crate::availability_as_extensions(&prop.availability);
+        data.extensions = crate::availability_as_extensions(&prop.availability, &self.config.flavor);
         // TODO: prop.aliases as extensions
         // TODO: prop.server_default as extension
         // TODO: prop.doc_id as extension (new representation of since and stability)