Merge remote-tracking branch 'origin/main' into eis-rerank-task-type

Author: pquentin

.github/workflows/update-rest-api-json.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '9.0', '8.19', '8.18', '8.17', '8.16', '7.17']
+        branch: ['main', '9.0', '8.19', '8.18', '8.17']
 
     steps:
       - uses: actions/checkout@v4
@@ -28,8 +28,7 @@ jobs:
       - name: Install deps
         run: |
           npm install --prefix .github/download-artifacts
-          npm install --prefix compiler
-          npm install --prefix typescript-generator
+          make setup
 
       - name: Download artifacts
         run: |
@@ -53,12 +52,3 @@ jobs:
           delete-branch: true
           reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,swallez,technige
           branch: automated/rest-api-spec-update-${{ matrix.branch }}
-
-      - name: Open an issue if the action fails
-        if: ${{ failure() }}
-        uses: nashmaniac/[email protected]
-        with:
-          title: rest-api-spec update failed
-          token: ${{ secrets.GITHUB_TOKEN }}
-          labels: bug
-          body: The rest-api-spec action is currently failing, see [here](https://github.com/elastic/elasticsearch-specification/actions/workflows/update-rest-api-json.yml).

.gitignore

@@ -69,4 +69,6 @@ compiler/test/**/output/
 output/openapi/elasticsearch-serverless-openapi.tmp*.json
 output/openapi/elasticsearch-serverless-openapi.examples.json
 output/openapi/elasticsearch-openapi.tmp*.json
-output/openapi/elasticsearch-openapi.examples.json
+output/openapi/elasticsearch-openapi.examples.json
+output/openapi/elasticsearch-serverless-openapi-docs.json
+output/openapi/elasticsearch-openapi-docs.json

Makefile

@@ -43,6 +43,7 @@ setup:	## Install dependencies for contrib target
 	@npm install --prefix validator
 	@npm install --prefix specification
 	@npm install @redocly/cli
+	@npm install --prefix docs/examples
 
 clean-dep:	## Clean npm dependencies
 	@rm -rf compiler/node_modules
@@ -55,6 +56,12 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --output output/openapi/elasticsearch-openapi.json
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
+transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
+	@make generate-language-examples
+	@make generate
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-serverless-openapi-docs.json
+
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
 
@@ -71,6 +78,10 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
+generate-language-examples:
+	@node docs/examples/generate-language-examples.js
+	@npm run format:fix-examples --prefix compiler
+
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 

README.md

@@ -60,6 +60,13 @@ Follow the steps to generate the JSON representation, then:
 ```
 # Generate the OpenAPI representation
 $ make transform-to-openapi
+```
+
+To generate the JSON representation that is used for documentation purposes, the commands are different:
+
+```
+# Generate the OpenAPI files
+$ make transform-to-openapi-for-docs
 
 # Apply fixes
 $ make overlay-docs

compiler-rs/Cargo.lock

@@ -264,7 +264,7 @@ pub struct Availability {
     pub visibility: Option<Visibility>,
 }
 
-/// The availability of an
+/// The availability of an endpoint, field or parameter
 pub type Availabilities = IndexMap<Flavor, Availability>;
 
 pub trait AvailabilityFilter: Fn(&Option<Availabilities>) -> bool {}
@@ -484,20 +484,28 @@ impl TypeDefinition {
 
 /// The Example type is used for both requests and responses.
 ///
-/// This type definition is taken from the OpenAPI spec
+/// This type definition is based on the OpenAPI spec
 ///     https://spec.openapis.org/oas/v3.1.0#example-object
-/// with the exception of using String as the 'value' type.
+/// with the exception of using String as the 'value' type,
+/// and some custom additions.
 ///
 /// The OpenAPI v3 spec also defines the 'Example' type, so
 /// to distinguish them, this type is called SchemaExample.
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExampleAlternative {
+    pub language: String,
+    pub code: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaExample {
     pub summary: Option<String>,
     pub method_request: Option<String>,
     pub description: Option<String>,
     pub value: Option<String>,
     pub external_value: Option<String>,
+    pub alternatives: Option<Vec<ExampleAlternative>>,
 }
 
 /// Common attributes for all type definitions

compiler-rs/clients_schema/src/lib.rs

@@ -10,7 +10,7 @@ clients_schema = {path="../clients_schema"}
 argh = { workspace = true }
 derive_more = { version = "2", features = ["from_str"] }
 serde_json = { workspace = true }
-serde_ignored = { workspace = true }
+itertools = { workspace = true }
 icu_segmenter = { workspace = true }
 openapiv3 = { workspace = true }
 anyhow = { workspace = true }

compiler-rs/clients_schema_to_openapi/Cargo.toml

@@ -20,13 +20,37 @@ pub struct Cli {
     #[argh(option, default = "SchemaFlavor::All")]
     pub flavor: SchemaFlavor,
 
-    /// add enum descriptions to property descriptions [default = true]
-    #[argh(option, default = "true")]
-    pub lift_enum_descriptions: bool,
-
     /// generate only this namespace (can be repeated)
     #[argh(option)]
     pub namespace: Vec<String>,
+
+    /// add enum descriptions to property descriptions [default = true]
+    #[argh(switch)]
+    pub lift_enum_descriptions: bool,
+
+    /// merge endpoints with multiple paths into a single OpenAPI operation [default = false]
+    #[argh(switch)]
+    pub merge_multipath_endpoints: bool,
+
+    /// output a redirection map when merging multipath endpoints
+    #[argh(switch)]
+    pub multipath_redirects: bool,
+
+    /// include the x-codeSamples extension with language examples for all endpoints
+    #[argh(switch)]
+    pub include_language_examples: bool,
+}
+
+impl Cli {
+    pub fn redirect_path(&self, output: &PathBuf) -> Option<String> {
+        if self.multipath_redirects {
+            let path = output.to_string_lossy();
+            let path = path.rsplit_once('.').unwrap().0;
+            Some(format!("{}.redirects.csv", path))
+        } else {
+            None
+        }
+    }
 }
 
 use derive_more::FromStr;
@@ -42,20 +66,23 @@ pub enum SchemaFlavor {
 }
 
 impl From<Cli> for Configuration {
-    fn from(val: Cli) -> Configuration {
-        let flavor = match val.flavor {
+    fn from(cli: Cli) -> Configuration {
+        let flavor = match cli.flavor {
             SchemaFlavor::All => None,
             SchemaFlavor::Serverless => Some(Flavor::Serverless),
             SchemaFlavor::Stack => Some(Flavor::Stack),
         };
 
         Configuration {
             flavor,
-            lift_enum_descriptions: val.lift_enum_descriptions,
-            namespaces: if val.namespace.is_empty() {
+            lift_enum_descriptions: cli.lift_enum_descriptions,
+            merge_multipath_endpoints: cli.merge_multipath_endpoints,
+            multipath_redirects: cli.multipath_redirects,
+            include_language_examples: cli.include_language_examples,
+            namespaces: if cli.namespace.is_empty() {
                 None
             } else {
-                Some(val.namespace)
+                Some(cli.namespace)
             },
         }
     }

compiler-rs/clients_schema_to_openapi/package-lock.json

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::collections::BTreeMap;
 use clients_schema::TypeName;
 use openapiv3::{Components, Parameter, ReferenceOr, RequestBody, Response, Schema, StatusCode};
 use crate::Configuration;
@@ -32,11 +33,19 @@ pub struct TypesAndComponents<'a> {
     pub config: &'a Configuration,
     pub model: &'a clients_schema::IndexedModel,
     pub components: &'a mut Components,
+    // Redirections (if paths multipaths endpoints are merged)
+    pub redirects: Option<BTreeMap<String, String>>,
 }
 
 impl<'a> TypesAndComponents<'a> {
     pub fn new(config: &'a Configuration, model: &'a clients_schema::IndexedModel, components: &'a mut Components) -> TypesAndComponents<'a> {
-        TypesAndComponents { config, model, components }
+        let redirects = if config.merge_multipath_endpoints && config.multipath_redirects {
+            Some(BTreeMap::new())
+        } else {
+            None
+        };
+
+        TypesAndComponents { config, model, components, redirects }
     }
 
     pub fn add_request_body(&mut self, endpoint: &str, body: RequestBody) -> ReferenceOr<RequestBody> {