Merge branch 'main' into mark-role-apis-serverless-available

Author: n1v0lg

.github/CODEOWNERS

@@ -2,9 +2,9 @@
 /specification/ml/** @elastic/ml-core @elastic/ml-ui
 
 # Specification tooling owners
-/.github/                   @elastic/clients-team
-/compiler/                  @elastic/clients-team
-/compiler-rs/               @elastic/clients-team
-/typescript-generator/      @elastic/clients-team
-/specification/_json_spec/  @elastic/clients-team
-/specification/_spec_utils/ @elastic/clients-team
+/.github/                   @elastic/devtools-team
+/compiler/                  @elastic/devtools-team
+/compiler-rs/               @elastic/devtools-team
+/typescript-generator/      @elastic/devtools-team
+/specification/_json_spec/  @elastic/devtools-team
+/specification/_spec_utils/ @elastic/devtools-team

.github/download-artifacts/index.js

@@ -40,23 +40,21 @@ const downloadedSpec = join(esFolder, 'rest-api-spec', 'api')
 const specFolder = join(__dirname, '..', '..', 'specification', '_json_spec')
 
 async function downloadArtifacts (opts) {
-  if (typeof opts.version !== 'string' && typeof opts.branch !== 'string') {
-    throw new Error('Missing version or branch')
+  if (typeof opts.branch !== 'string') {
+    throw new Error('Missing branch')
   }
 
-  core.info('Checking out spec and test')
+  core.info('Resolving artifact URL')
 
-  core.info('Resolving version')
   let resolved
   try {
-    resolved = await resolve(opts.version || fromBranch(opts.branch), opts.hash)
+    resolved = await resolve(opts.branch)
   } catch (err) {
     core.error(err.message)
     process.exit(1)
   }
 
-  opts.version = resolved.version
-  core.info(`Resolved version ${opts.version}`)
+  core.info(`Resolved artifact URL for ${resolved.commit_url}`)
 
   core.info('Cleanup')
   await rm(esFolder)
@@ -96,74 +94,29 @@ async function downloadArtifacts (opts) {
   core.info('Done')
 }
 
-async function resolve (version, hash) {
-  if (version === 'latest') {
-    const response = await fetch('https://artifacts-api.elastic.co/v1/versions')
-    if (!response.ok) {
-      throw new Error(`unexpected response ${response.statusText}`)
-    }
-    const { versions } = await response.json()
-    version = versions.pop()
+async function resolve (branch) {
+  if (branch == 'main') {
+    branch = 'master'
   }
-
-  core.info(`Resolving version ${version}`)
-  const response = await fetch(`https://artifacts-api.elastic.co/v1/versions/${version}`)
+  const url = `https://artifacts-snapshot.elastic.co/elasticsearch/latest/${branch}.json`
+  const response = await fetch(url)
   if (!response.ok) {
-    throw new Error(`unexpected response ${response.statusText}`)
+    throw new Error(`Unexpected response. Invalid version? ${url}: ${response.statusText}`)
   }
-
   const data = await response.json()
-  const esBuilds = data.version.builds
-    .filter(build => build.projects.elasticsearch != null)
-    .map(build => {
-      return {
-        projects: build.projects.elasticsearch,
-        buildId: build.build_id,
-        date: build.start_time,
-        version: build.version
-      }
-    })
-    .sort((a, b) => {
-      const dA = new Date(a.date)
-      const dB = new Date(b.date)
-      if (dA > dB) return -1
-      if (dA < dB) return 1
-      return 0
-    })
-
-  if (hash != null) {
-    const build = esBuilds.find(build => build.projects.commit_hash === hash)
-    if (!build) {
-      throw new Error(`Can't find any build with hash '${hash}'`)
-    }
-    const zipKey = Object.keys(build.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-    return {
-      url: build.projects.packages[zipKey].url,
-      id: build.buildId,
-      hash: build.projects.commit_hash,
-      version: build.version
-    }
-  }
 
-  const lastBuild = esBuilds[0]
-  const zipKey = Object.keys(lastBuild.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-  return {
-    url: lastBuild.projects.packages[zipKey].url,
-    id: lastBuild.buildId,
-    hash: lastBuild.projects.commit_hash,
-    version: lastBuild.version
+  let manifest_url = data.manifest_url
+  const manifestResponse = await fetch(manifest_url)
+  if (!manifestResponse.ok) {
+    throw new Error(`Unexpected manifestResponse. ${manifest_url}: ${manifestResponse.statusText}`)
   }
-}
+  const manifestData = await manifestResponse.json()
+  const elasticsearch = manifestData.projects.elasticsearch
+  const restResourceName = `rest-resources-zip-${manifestData.version}.zip`
 
-function fromBranch (branch) {
-  if (branch === 'main') {
-    return 'latest'
-  } else if (branch === '7.x') {
-    return '7.x-SNAPSHOT'
-  } else if ((branch.startsWith('7.') || branch.startsWith('8.')) && !isNaN(Number(branch.split('.')[1]))) {
-    return `${branch}-SNAPSHOT`
-  } else {
-    throw new Error(`Cannot derive version from branch '${branch}'`)
+  return {
+    url: elasticsearch.packages[restResourceName].url,
+    commit_url: elasticsearch.commit_url,
   }
 }
 
@@ -172,7 +125,7 @@ async function main (options) {
 }
 
 const options = minimist(process.argv.slice(2), {
-  string: ['id', 'version', 'hash', 'branch']
+  string: ['branch']
 })
 main(options).catch(t => {
   core.error(t)

.github/workflows/update-rest-api-json.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Generate output
         run: |
-          SKIP_VERSION_UPDATE=true make contrib
+          make contrib
 
       - name: Debug git status
         run: |

.spectral.yaml

@@ -17,7 +17,7 @@ rules:
   operation-operationId: warn
   operation-operationId-unique: warn
   operation-operationId-valid-in-url: warn
-  operation-tag-defined: warn
+  operation-tag-defined: error
   operation-tags: warn
   # Responses
   operation-success-response: warn
@@ -26,7 +26,7 @@ rules:
   oas2-schema: warn
   # Tags
   openapi-tags: warn
-  openapi-tags-alphabetical: info
+  openapi-tags-alphabetical: false
   # Turn off some built-in rules
   operation-description: false
   operation-singular-tag: false

Makefile

@@ -69,14 +69,14 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	rm output/openapi/elasticsearch-serverless-openapi.tmp*.json
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
-lint-docs: ## Lint the OpenAPI documents
-	@npx @stoplight/spectral-cli lint output/openapi/*.json --ruleset .spectral.yaml
+lint-docs: ## Lint the OpenAPI documents after overlays
+	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml
 
-lint-docs-errs: ## Lint the OpenAPI documents and return only errors
-	@npx @stoplight/spectral-cli lint output/openapi/*.json --ruleset .spectral.yaml -D
+lint-docs-errs: ## Lint the OpenAPI documents after overlays and return only errors
+	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-*.examples.json --ruleset .spectral.yaml -D
 
-lint-docs-serverless: ## Lint only the serverless OpenAPI document
-	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-serverless-openapi.json --ruleset .spectral.yaml
+lint-docs-serverless: ## Lint only the serverless OpenAPI document after overlays
+	@npx @stoplight/spectral-cli lint output/openapi/elasticsearch-serverless-openapi.examples.json --ruleset .spectral.yaml
 
 help:  ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

api-design-guidelines/naming.md

@@ -100,6 +100,8 @@ Care should be given to ensure that:
 - All member APIs of a given namespace are logically related and form a coherent set.
 - Related functionality is not distributed across multiple arbitrary namespaces
 
+NOTE: The endpoint namespaces are used to generate tags in the OpenAPI documents. The tags are ultimately used to group the endpoints in the API documentation. To override the default tag, use `@doc_tag`.
+
 ### Use the global namespace sparingly
 
 The top-level global namespace should be treated with particular care. It is traditionally reserved for search and document endpoints only. A case should be made and a broader discussion carried out before new endpoints unrelated to these functions are added to the global namespace.

compiler-rs/clients_schema/src/lib.rs

@@ -815,6 +815,9 @@ pub struct Endpoint {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub availability: Option<Availabilities>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub doc_tag: Option<String>,
+
     /// If missing, there is not yet a request definition for this endpoint.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub request: Option<TypeName>,

compiler-rs/clients_schema_to_openapi/package-lock.json

@@ -155,18 +155,20 @@ pub fn availability_as_extensions(availabilities: &Option<Availabilities>) -> In
     if let Some(avails) = availabilities {
         // We may have several availabilities, but since generally exists only on stateful (stack)
         for (_, availability) in avails {
-            if let Some(since) = &availability.since {
-                result.insert("x-available-since".to_string(), serde_json::Value::String(since.clone()));
-            }
             if let Some(stability) = &availability.stability {
                 match stability {
                     Stability::Beta => {
                         result.insert("x-beta".to_string(), serde_json::Value::Bool(true));
                     }
                     Stability::Experimental => {
-                        result.insert("x-technical-preview".to_string(), serde_json::Value::Bool(true));
+                        result.insert("x-state".to_string(), serde_json::Value::String("Technical preview".to_string()));
+                    }
+                    Stability::Stable => {
+                        if let Some(since) = &availability.since {
+                            let stable_since = "Added in ".to_string() + since;
+                            result.insert("x-state".to_string(), serde_json::Value::String(stable_since));
+                        }
                     }
-                    _ => {}
                 }
             }
         }

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -196,7 +196,11 @@ pub fn add_endpoint(
 
         // Create the operation, it will be repeated if we have several methods
         let operation = openapiv3::Operation {
-            tags: vec![namespace.to_string()],
+            tags: if let Some(doc_tag) = &endpoint.doc_tag {
+                vec![doc_tag.clone()]
+            } else {
+                vec![namespace.to_string()]
+            },
             summary: sum_desc.summary,
             description: sum_desc.description,
             // external_docs: tac.convert_external_docs(endpoint),