[Backport 9.1] Fixing a few missing task types and alphabetizing the PUT request

.github/validate-pr/index.js

@@ -149,20 +149,18 @@ async function run() {
   }
   changedApis.sort((a, b) => a.api.localeCompare(b.api))
 
+  let comment = `Following you can find the validation changes against the target branch for the API${changedApis.length === 1 ? '' : 's'}.\n\n`
   if (changedApis.length > 0) {
-    let comment = `Following you can find the validation changes for the API${changedApis.length === 1 ? '' : 's'} you have modified.\n\n`
     comment += '| API | Status | Request | Response |\n'
     comment += '| --- | --- | --- | --- |\n'
     for (const change of changedApis) {
       comment += buildDiffTableLine(change)
     }
-    comment += `\nYou can validate ${changedApis.length === 1 ? 'this' : 'these'} API${changedApis.length === 1 ? '' : 's'} yourself by using the ${tick}make validate${tick} target.\n`
-
-    core.setOutput('has_results', 'true')
-    core.setOutput('comment_body', comment)
   } else {
-    core.setOutput('has_results', 'false')
+    comment += '**No changes detected**.\n'
   }
+  comment += `\nYou can validate ${changedApis.length === 1 ? 'this' : 'these'} API${changedApis.length === 1 ? '' : 's'} yourself by using the ${tick}make validate${tick} target.\n`
+  core.setOutput('comment_body', comment)
 
   core.info('Done!')
 }
@@ -236,4 +234,4 @@ function generateResponse (r) {
 run().catch((err) => {
   core.error(err)
   process.exit(1)
-})
+})

.github/workflows/validate-pr.yml

@@ -75,7 +75,6 @@ jobs:
           body-includes: 'Following you can find the validation changes'
 
       - name: Create or update comment
-        if: steps.validation.outputs.has_results == 'true'
         uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -65,10 +65,7 @@ output/schema/schema
 # Test suite outputs
 compiler/test/**/output/
 
-# Temporary openAPI files
-output/openapi/elasticsearch-serverless-openapi.tmp*.json
-output/openapi/elasticsearch-serverless-openapi.examples.json
-output/openapi/elasticsearch-openapi.tmp*.json
-output/openapi/elasticsearch-openapi.examples.json
-output/openapi/elasticsearch-serverless-openapi-docs.json
-output/openapi/elasticsearch-openapi-docs.json
+# Temporary openAPI documentation files
+output/openapi/elasticsearch-serverless-openapi-docs*.json
+output/openapi/elasticsearch-openapi-docs*.json
+output/openapi/elasticsearch*.redirects.csv

Makefile

@@ -69,14 +69,14 @@ dump-routes: ## Create a new schema with all generics expanded
 	@npm run dump-routes --prefix compiler
 
 overlay-docs: ## Apply overlays to OpenAPI documents
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi.examples.json
-	@npx bump overlay "output/openapi/elasticsearch-openapi.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp1.json"
-	@npx bump overlay "output/openapi/elasticsearch-openapi.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi.tmp2.json"
-	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-openapi.examples.json
-	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
-	rm output/openapi/elasticsearch-openapi.tmp*.json
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.json" "docs/overlays/elasticsearch-serverless-openapi-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-serverless-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-serverless-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-serverless-openapi-docs-final.json
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.json" "docs/overlays/elasticsearch-openapi-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp1.json"
+	@npx bump overlay "output/openapi/elasticsearch-openapi-docs.tmp1.json" "docs/overlays/elasticsearch-shared-overlays.yaml" > "output/openapi/elasticsearch-openapi-docs.tmp2.json"
+	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi-docs.tmp2.json --ext json -o output/openapi/elasticsearch-openapi-docs-final.json
+	rm output/openapi/elasticsearch-serverless-openapi-docs.tmp*.json
+	rm output/openapi/elasticsearch-openapi-docs.tmp*.json
 
 generate-language-examples:
 	@node docs/examples/generate-language-examples.js
@@ -89,11 +89,11 @@ generate-language-examples-with-java:
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
-lint-docs-stateful: ## Lint only the elasticsearch-openapi.examples.json file
-	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+lint-docs-stateful: ## Lint only the elasticsearch-openapi-docs-final.json file
+	@npx @redocly/cli lint "output/openapi/elasticsearch-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 lint-docs-serverless: ## Lint only the serverless OpenAPI document after overlays
-	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi.examples.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
+	@npx @redocly/cli lint "output/openapi/elasticsearch-serverless-openapi-docs-final.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 
 contrib: | generate license-check spec-format-fix transform-to-openapi filter-for-serverless lint-docs ## Pre contribution target
 

compiler-rs/clients_schema/src/lib.rs

@@ -58,6 +58,7 @@ pub trait Documented {
 pub trait ExternalDocument {
     fn ext_doc_id(&self) -> Option<&str>;
     fn ext_doc_url(&self) -> Option<&str>;
+    fn ext_doc_description(&self) -> Option<&str>;
     fn ext_previous_version_doc_url(&self) -> Option<&str>;
 }
 
@@ -323,6 +324,9 @@ pub struct Property {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_doc_description: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_previous_version_doc_url: Option<String>,
 
@@ -375,6 +379,10 @@ impl ExternalDocument for Property {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_doc_description(&self) -> Option<&str> {
+        self.ext_doc_description.as_deref()
+    }
+
     fn ext_previous_version_doc_url(&self) -> Option<&str> {
         self.ext_previous_version_doc_url.as_deref()
     }
@@ -536,6 +544,9 @@ pub struct BaseType {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_doc_description: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_previous_version_doc_url: Option<String>,
 
@@ -579,6 +590,7 @@ impl BaseType {
             spec_location: None,
             ext_doc_id: None,
             ext_doc_url: None,
+            ext_doc_description: None,
             ext_previous_version_doc_url: None,
         }
     }
@@ -603,6 +615,10 @@ impl ExternalDocument for BaseType {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_doc_description(&self) -> Option<&str> {
+        self.ext_doc_description.as_deref()
+    }
+
     fn ext_previous_version_doc_url(&self) -> Option<&str> {
         self.ext_previous_version_doc_url.as_deref()
     }
@@ -635,6 +651,10 @@ impl<T: WithBaseType> ExternalDocument for T {
         self.base().doc_url()
     }
 
+    fn ext_doc_description(&self) -> Option<&str> {
+        self.base().ext_doc_description()
+    }
+
     fn ext_previous_version_doc_url(&self) -> Option<&str> {
         self.base().ext_previous_version_doc_url()
     }
@@ -915,6 +935,9 @@ pub struct Endpoint {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_doc_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ext_doc_description: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ext_previous_version_doc_url: Option<String>,
 
@@ -968,6 +991,10 @@ impl ExternalDocument for Endpoint {
         self.ext_doc_url.as_deref()
     }
 
+    fn ext_doc_description(&self) -> Option<&str> {
+        self.ext_doc_description.as_deref()
+    }
+
     fn ext_previous_version_doc_url(&self) -> Option<&str> {
         self.ext_previous_version_doc_url.as_deref()
     }
@@ -1003,7 +1030,6 @@ pub struct UrlTemplate {
 pub struct ModelInfo {
     pub title: String,
     pub license: License,
-    pub version: Option<String>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

compiler-rs/clients_schema/src/transform/expand_generics.rs

@@ -211,8 +211,10 @@ pub fn expand(model: IndexedModel, config: ExpandConfig) -> anyhow::Result<Index
 
         expand_behaviors(&mut resp.behaviors, &mappings, model, ctx)?;
         expand_body(&mut resp.body, &mappings, model, ctx)?;
-
-        // TODO: exceptions
+
+        for exception in &mut resp.exceptions {
+            expand_body(&mut exception.body, &mappings, model, ctx)?;
+        }
 
         Ok(resp.into())
     }

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -24,6 +24,10 @@ pub struct Cli {
     #[argh(option)]
     pub namespace: Vec<String>,
 
+    /// the branch to generate [9.0, 8.19, current, etc... - default = current]
+    #[argh(option)]
+    pub branch: Option<String>,
+
     /// add enum descriptions to property descriptions [default = true]
     #[argh(switch)]
     pub lift_enum_descriptions: bool,
@@ -72,9 +76,12 @@ impl From<Cli> for Configuration {
             SchemaFlavor::Serverless => Some(Flavor::Serverless),
             SchemaFlavor::Stack => Some(Flavor::Stack),
         };
+
+        let branch = cli.branch;
 
         Configuration {
             flavor,
+            branch,
             lift_enum_descriptions: cli.lift_enum_descriptions,
             merge_multipath_endpoints: cli.merge_multipath_endpoints,
             multipath_redirects: cli.multipath_redirects,

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -25,13 +25,14 @@ use indexmap::IndexMap;
 
 use clients_schema::{Availabilities, Availability, Flavor, IndexedModel, Stability, Visibility};
 use openapiv3::{Components, OpenAPI};
-use serde_json::Value;
+use serde_json::{Map,Value};
 use clients_schema::transform::ExpandConfig;
 use crate::components::TypesAndComponents;
 
 pub struct Configuration {
     pub flavor: Option<Flavor>,
     pub namespaces: Option<Vec<String>>,
+    pub branch: Option<String>,
 
     /// If a property value is an enumeration, the description of possible values will be copied in the
     /// property's description (also works for arrays of enums).
@@ -189,9 +190,14 @@ pub fn product_meta_as_extensions(namespace: &str, product_meta: &IndexMap<Strin
     if let Some(meta) = product_meta.get(namespace) {
         additional_namespace = format!(", {meta}");
     }
-
-    let product_str = format!("elasticsearch{additional_namespace}");
-    result.insert("x-product-feature".to_string(), Value::String(product_str));
+
+    let product_str = format!("Elasticsearch{additional_namespace}");
+    let mut product_feature_list: Vec<Value> = Vec::new();
+    let mut product_feature: Map<String, Value> = Map::new();
+    product_feature.insert("name".to_string(),Value::String("product_name".to_string()));
+    product_feature.insert("content".to_string(),Value::String(product_str));
+    product_feature_list.push(Value::Object(product_feature));
+    result.insert("x-metaTags".to_string(), Value::Array(product_feature_list));
     result
 }
 

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -74,7 +74,7 @@ pub fn add_endpoint(
             example: None,
             examples: Default::default(),
             explode: None, // Defaults to simple, i.e. comma-separated values for arrays
-            extensions: Default::default(),
+            extensions,
         })
     }
 

compiler-rs/clients_schema_to_openapi/src/schemas.rs

@@ -209,19 +209,14 @@ impl<'a> TypesAndComponents<'a> {
     pub fn convert_external_docs(&self, obj: &impl clients_schema::ExternalDocument) -> Option<ExternalDocumentation> {
         // FIXME: does the model contain resolved doc_id?
         obj.ext_doc_url().map(|url| {
-            let branch: &str = self
-                .model
-                .info
-                .as_ref()
-                .and_then(|i| i.version.as_deref())
-                .unwrap_or("current");
+            let branch = self.config.branch.clone();
             let mut extensions: IndexMap<String,serde_json::Value> = Default::default();
             if let Some(previous_version_doc_url) = obj.ext_previous_version_doc_url() {
                 extensions.insert("x-previousVersionUrl".to_string(), serde_json::json!(previous_version_doc_url));
             }
             ExternalDocumentation {
-                description: None,
-                url: url.trim().replace("{branch}", branch),
+                description: obj.ext_doc_description().map(|desc| { desc.to_string() }),
+                url: url.trim().replace("{branch}", &branch.unwrap_or("current".to_string())),
                 extensions,
             }
         })
@@ -477,9 +472,9 @@ impl<'a> TypesAndComponents<'a> {
         data.external_docs = self.convert_external_docs(prop);
         data.deprecated = prop.deprecation.is_some();
         data.description = self.property_description(prop)?;
+        data.default = prop.server_default.clone().map(|value| { serde_json::json!(value) });
         data.extensions = crate::availability_as_extensions(&prop.availability, &self.config.flavor);
         // TODO: prop.aliases as extensions
-        // TODO: prop.server_default as extension
         // TODO: prop.doc_id as extension (new representation of since and stability)
         // TODO: prop.es_quirk as extension?
         // TODO: prop.codegen_name as extension?

compiler-rs/openapi_to_clients_schema/src/types.rs

@@ -404,6 +404,7 @@ fn generate_interface_def(
             doc_url: None,
             ext_doc_id: None,
             ext_doc_url: None,
+            ext_doc_description: None,
             ext_previous_version_doc_url: None,
             codegen_name: None, // FIXME: extension in workplace search
             description: None,

compiler/src/model/metamodel.ts

@@ -443,6 +443,7 @@ export class Endpoint {
   docId?: string
   extDocId?: string
   extDocUrl?: string
+  extDocDescription?: string
   extPreviousVersionDocUrl?: string
   deprecation?: Deprecation
   availability: Availabilities

compiler/src/model/utils.ts

@@ -703,6 +703,9 @@ export function hoistRequestAnnotations (
       const docUrl = docIds.find(entry => entry[0] === value.trim())
       assert(jsDocs, docUrl != null, `The @ext_doc_id '${value.trim()}' is not present in _doc_ids/table.csv`)
       endpoint.extDocUrl = docUrl[1].replace(/\r/g, '')
+      if (docUrl[3].replace(/\r/g, '') !== '') {
+        endpoint.extDocDescription = docUrl[3].replace(/\r/g, '')
+      }
     } else if (tag === 'availability') {
       // The @availability jsTag is different than most because it allows
       // multiple values within the same docstring, hence needing to parse

compiler/src/steps/add-examples.ts

@@ -54,10 +54,18 @@ export default class ExamplesProcessor {
 class BaseExamplesProcessor {
   model: model.Model
   specsFolder: string
+  static languageExamples: Record<string, model.ExampleAlternative[]> = {}
 
   constructor (model: model.Model, specsFolder: string) {
     this.model = model
     this.specsFolder = specsFolder
+    if (Object.keys(BaseExamplesProcessor.languageExamples).length === 0) {
+      // load the language examples
+      const examplesJson = this.specsFolder + '/../docs/examples/languageExamples.json'
+      if (fs.existsSync(examplesJson)) {
+        BaseExamplesProcessor.languageExamples = JSON.parse(fs.readFileSync(examplesJson, 'utf8'))
+      }
+    }
   }
 
   // Log a 'warning' message.
@@ -139,6 +147,11 @@ class BaseExamplesProcessor {
       const exampleFileContent = fs.readFileSync(filePath, 'utf8')
       const exampleName = path.basename(fileName, path.extname(fileName))
       const example: model.Example = yaml.load(exampleFileContent)
+      // find the language alternative examples and add them
+      const alternativeKey = 'specification/' + filePath.split('/specification/')[1]
+      if (BaseExamplesProcessor.languageExamples[alternativeKey] !== undefined) {
+        example.alternatives = BaseExamplesProcessor.languageExamples[alternativeKey]
+      }
       // Some of the example files set their 'value' as a JSON string,
       // and some files set their 'value' as an object. For consistency,
       // if the value is not a JSON string, convert it to a JSON string.