Merge branch 'main' into docs-commands

Author: lcawl

.gitignore

@@ -69,4 +69,6 @@ compiler/test/**/output/
 output/openapi/elasticsearch-serverless-openapi-docs.tmp*.json
 output/openapi/elasticsearch-serverless-openapi-docs-final.json
 output/openapi/elasticsearch-openapi-docs.tmp*.json
-output/openapi/elasticsearch-openapi-docs-final.json
+output/openapi/elasticsearch-openapi-docs-final.json
+output/openapi/elasticsearch-serverless-openapi-docs.json
+output/openapi/elasticsearch-openapi-docs.json

Makefile

@@ -56,7 +56,8 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
 transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --output output/openapi/elasticsearch-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --output output/openapi/elasticsearch-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --output output/openapi/elasticsearch-serverless-openapi-docs.json
 
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json

README.md

@@ -60,6 +60,13 @@ Follow the steps to generate the JSON representation, then:
 ```
 # Generate the OpenAPI representation
 $ make transform-to-openapi
+```
+
+To generate the JSON representation that is used for documentation purposes, the commands are different:
+
+```
+# Generate the OpenAPI files
+$ make transform-to-openapi-for-docs
 
 # Apply fixes
 $ make overlay-docs

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -31,6 +31,22 @@ pub struct Cli {
     /// merge endpoints with multiple paths into a single OpenAPI operation [default = false]
     #[argh(switch)]
     pub merge_multipath_endpoints: bool,
+
+    /// output a redirection map when merging multipath endpoints
+    #[argh(switch)]
+    pub multipath_redirects: bool,
+}
+
+impl Cli {
+    pub fn redirect_path(&self, output: &PathBuf) -> Option<String> {
+        if self.multipath_redirects {
+            let path = output.to_string_lossy();
+            let path = path.rsplit_once('.').unwrap().0;
+            Some(format!("{}.redirects.csv", path))
+        } else {
+            None
+        }
+    }
 }
 
 use derive_more::FromStr;
@@ -57,6 +73,7 @@ impl From<Cli> for Configuration {
             flavor,
             lift_enum_descriptions: cli.lift_enum_descriptions,
             merge_multipath_endpoints: cli.merge_multipath_endpoints,
+            multipath_redirects: cli.multipath_redirects,
             namespaces: if cli.namespace.is_empty() {
                 None
             } else {

compiler-rs/clients_schema_to_openapi/src/components.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::collections::BTreeMap;
 use clients_schema::TypeName;
 use openapiv3::{Components, Parameter, ReferenceOr, RequestBody, Response, Schema, StatusCode};
 use crate::Configuration;
@@ -32,11 +33,19 @@ pub struct TypesAndComponents<'a> {
     pub config: &'a Configuration,
     pub model: &'a clients_schema::IndexedModel,
     pub components: &'a mut Components,
+    // Redirections (if paths multipaths endpoints are merged)
+    pub redirects: Option<BTreeMap<String, String>>,
 }
 
 impl<'a> TypesAndComponents<'a> {
     pub fn new(config: &'a Configuration, model: &'a clients_schema::IndexedModel, components: &'a mut Components) -> TypesAndComponents<'a> {
-        TypesAndComponents { config, model, components }
+        let redirects = if config.merge_multipath_endpoints && config.multipath_redirects {
+            Some(BTreeMap::new())
+        } else {
+            None
+        };
+
+        TypesAndComponents { config, model, components, redirects }
     }
 
     pub fn add_request_body(&mut self, endpoint: &str, body: RequestBody) -> ReferenceOr<RequestBody> {

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -41,10 +41,18 @@ pub struct Configuration {
     /// be the longest one (with values for all optional parameters), and the other paths will be added
     /// at the beginning of the operation's description.
     pub merge_multipath_endpoints: bool,
+
+    /// Should we output a redirect map when merging multipath endpoints?
+    pub multipath_redirects: bool,
+}
+
+pub struct OpenApiConversion {
+    pub openapi: OpenAPI,
+    pub redirects: Option<String>,
 }
 
 /// Convert an API model into an OpenAPI v3 schema, optionally filtered for a given flavor
-pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow::Result<OpenAPI> {
+pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow::Result<OpenApiConversion> {
     // Expand generics
     schema = clients_schema::transform::expand_generics(schema, ExpandConfig::default())?;
 
@@ -74,7 +82,7 @@ pub fn convert_schema(mut schema: IndexedModel, config: Configuration) -> anyhow
 /// Note: there are ways to represent [generics in JSON Schema], but its unlikely that tooling will understand it.
 ///
 /// [generics in JSON Schema]: https://json-schema.org/blog/posts/dynamicref-and-generics
-pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration) -> anyhow::Result<OpenAPI> {
+pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration) -> anyhow::Result<OpenApiConversion> {
     let mut openapi = OpenAPI {
         openapi: "3.0.3".into(),
         info: info(model),
@@ -130,7 +138,21 @@ pub fn convert_expanded_schema(model: &IndexedModel, config: &Configuration) ->
     //     comp.security_schemes.sort_keys();
     // }
 
-    Ok(openapi)
+    let redirects = if let Some(redirects) = tac.redirects {
+        use std::fmt::Write;
+        let mut result = String::new();
+        for (source, target) in redirects.iter() {
+            writeln!(&mut result, "{},{}", source, target)?;
+        }
+        Some(result)
+    } else {
+        None
+    };
+
+    Ok(OpenApiConversion {
+        openapi,
+        redirects,
+    })
 }
 
 fn info(model: &IndexedModel) -> openapiv3::Info {
@@ -183,7 +205,7 @@ pub fn convert_availabilities(availabilities: &Option<Availabilities>, flavor: &
                         let exp_since = format!("Technical preview{since_str}");
                         result.insert("x-state".to_string(), Value::String(exp_since));
                     }
-                    Stability::Stable => { 
+                    Stability::Stable => {
                         let stable_since = format!("Generally available{since_str}");
                         result.insert("x-state".to_string(), Value::String(stable_since));
                     }

compiler-rs/clients_schema_to_openapi/src/main.rs

@@ -34,7 +34,14 @@ fn main() -> anyhow::Result<()> {
 
     let schema = IndexedModel::from_reader(File::open(&cli.schema)?)?;
     let output = cli.output.clone();
+    let redirect_path = cli.redirect_path(&cli.output);
     let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into())?;
-    serde_json::to_writer_pretty(File::create(&output)?, &openapi)?;
+    serde_json::to_writer_pretty(File::create(&output)?, &openapi.openapi)?;
+
+    if let Some(redirects) = openapi.redirects {
+        let path = redirect_path.unwrap();
+        std::fs::write(path, &redirects)?;
+    }
+
     Ok(())
 }

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -148,14 +148,16 @@ pub fn add_endpoint(
                 // }
             };
 
-            let openapi_example = Example {
-                value: example,
-                description: schema_example.description.clone(),
-                summary: schema_example.summary.clone(),
-                external_value: None,
-                extensions: Default::default(),
-            };
-            openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            if example.is_some() {
+                let openapi_example = Example {
+                    value: example,
+                    description: schema_example.description.clone(),
+                    summary: schema_example.summary.clone(),
+                    external_value: None,
+                    extensions: Default::default(),
+                };
+                openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            }
         }
         openapi_examples
     }
@@ -233,9 +235,27 @@ pub fn add_endpoint(
     };
 
     //---- Merge multipath endpoints if asked for
+
+    let operation_id: String = endpoint
+        .name
+        .chars()
+        .map(|x| match x {
+            '_' | '.' => '-',
+            _ => x,
+        })
+        .collect();
+
     let mut new_endpoint: clients_schema::Endpoint;
 
     let endpoint = if is_multipath && tac.config.merge_multipath_endpoints {
+
+        // Add redirects for operations that would have been generated otherwise
+        if let Some(ref mut map) = &mut tac.redirects {
+            for i in 1..endpoint.urls.len() {
+                map.insert(format!("{operation_id}-{i}"), operation_id.clone());
+            }
+        }
+
         new_endpoint = endpoint.clone();
         let endpoint = &mut new_endpoint;
 
@@ -315,9 +335,9 @@ pub fn add_endpoint(
         parameters.append(&mut query_params.clone());
 
         let sum_desc = split_summary_desc(&endpoint.description);
-        
+
         let privilege_desc = add_privileges(&endpoint.privileges);
-        
+
         let full_desc = match (sum_desc.description, privilege_desc) {
             (Some(a), Some(b)) => Some(a+ &b),
             (opt_a, opt_b) => opt_a.or(opt_b)
@@ -340,21 +360,6 @@ pub fn add_endpoint(
                 }
             }
         }
-        if code_samples.is_empty() {
-            // if there are no example requests we look for example responses
-            // this can only happen for examples that do not have a request body
-            if let Some(examples) = response_def.examples.clone() {
-                if let Some((_, example)) = examples.first() {
-                    let request_line = example.method_request.clone().unwrap_or(String::from(""));
-                    if !request_line.is_empty() {
-                        code_samples.push(serde_json::json!({
-                            "lang": "Console",
-                            "source": request_line + "\n",
-                        }));
-                    }
-                }
-            }
-        }
         if !code_samples.is_empty() {
             extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
         }
@@ -452,14 +457,7 @@ pub fn add_endpoint(
             };
 
             let mut operation = operation.clone();
-            let mut operation_id: String = endpoint
-                .name
-                .chars()
-                .map(|x| match x {
-                    '_' | '.' => '-',
-                    _ => x,
-                })
-                .collect();
+            let mut operation_id = operation_id.clone();
             if operation_counter != 0 {
                 write!(&mut operation_id, "-{}", operation_counter)?;
             }
@@ -515,18 +513,20 @@ fn split_summary_desc(desc: &str) -> SplitDesc{
 
 fn add_privileges(privileges: &Option<Privileges>) -> Option<String>{
     if let Some(privs) = privileges {
-        let mut result = "\n ##Required authorization\n".to_string();
+        let mut result = "\n\n## Required authorization\n\n".to_string();
         if !privs.index.is_empty() {
             result += "* Index privileges: ";
             result += &privs.index.iter()
                 .map(|a| format!("`{a}`"))
                 .join(",");
+            result += "\n";
         }
         if !privs.cluster.is_empty() {
             result += "* Cluster privileges: ";
             result += &privs.cluster.iter()
                 .map(|a| format!("`{a}`"))
                 .join(",");
+            result += "\n";
         }
         return Some(result)
     }

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib_bg.wasm

@@ -59,14 +59,21 @@ pub fn convert0(cli: Cli, cwd: Option<String>) -> anyhow::Result<()> {
         Some(ref cwd) => PathBuf::from(cwd).join(&cli.output),
         None => cli.output.clone(),
     };
+    let redirect_path = cli.redirect_path(&output);
 
     let json = node_fs::read_file_sync_to_string(&input.to_string_lossy(), "utf8");
     let schema = IndexedModel::from_reader(json.as_bytes())?;
 
     let openapi = clients_schema_to_openapi::convert_schema(schema, cli.into())?;
 
-    let result = serde_json::to_string_pretty(&openapi)?;
+    let result = serde_json::to_string_pretty(&openapi.openapi)?;
     node_fs::write_file_sync(&output.to_string_lossy(), &result);
+
+    if let Some(redirects) = openapi.redirects {
+        let path = redirect_path.unwrap();
+        node_fs::write_file_sync(&path, &redirects);
+    }
+
     Ok(())
 }