Merge branch 'main' into make-score-required

Author: pquentin

.backportrc.json

@@ -0,0 +1,6 @@
+{
+  "repoOwner": "elastic",
+  "repoName": "elasticsearch-specification",
+  "targetBranchChoices": ["9.1", "9.0", "8.19", "8.18"],
+  "fork": false
+}

.buildkite/kibana.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Since we're into the current repo, move to the top-level
+cd ..
+
+echo "--- Install dependencies"
+lsb_release -a
+apt-get update -y
+apt-get install -y unzip
+
+echo "--- Clone elasticsearch-js"
+git clone -v -- [email protected]:elastic/elasticsearch-js.git
+
+echo "--- Clone elastic-client-generator-js"
+git clone -v -- [email protected]:elastic/elastic-client-generator-js.git
+mkdir elastic-client-generator-js/output
+
+echo "--- Clone Kibana"
+git clone -v --reference /usr/local/git-references/git-github-com-elastic-kibana-git -- [email protected]:elastic/kibana.git
+pushd kibana
+
+echo "--- Install Node.js and Yarn"
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+set +e # https://github.com/nvm-sh/nvm/issues/3117
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+set -e
+
+nvm install
+nvm use
+npm install --global yarn
+popd
+
+echo "--- Install elasticsearch-js"
+pushd elasticsearch-js
+npm install
+node .buildkite/make.mjs --task codegen main
+npm run build
+npm pack
+popd
+
+pushd kibana
+yarn add ../elasticsearch-js/elastic-elasticsearch-*.tgz
+
+echo "--- Bootstrap Kibana"
+git --no-pager diff
+yarn kbn bootstrap --allow-root
+
+echo "--- Check types"
+node scripts/type_check.js

.buildkite/kibana.yml

@@ -0,0 +1,8 @@
+agents:
+  memory: "24G"
+  cpu: "4"
+  ephemeralStorage: 15Gi
+
+steps:
+  - label: "Run Kibana type checks"
+    command: .buildkite/kibana.sh

.gitattributes

@@ -232,13 +232,8 @@ dist/*            binary
 
 ### Linguist Overrides #############################################################################
 
-output/dangling-types/dangling.csv linguist-generated=true
-output/schema/schema.json linguist-generated=true
-output/openapi/elasticsearch-serverless-openapi.json linguist-generated=true
-output/schema/import-type-graph.json linguist-generated=true
-output/schema/import-namespace-graph-compact.json linguist-generated=true
-output/schema/import-namespace-graph-expanded.json linguist-generated=true
-output/schema/validation-errors.json linguist-generated=true
-output/typescript/types.ts linguist-generated=true
+output/** linguist-generated=true
+output/schema/validation-errors.json linguist-generated=false
+compiler-rs/compiler-wasm-lib/pkg/* linguist-generated=true
 
 ####################################################################################################

.github/CODEOWNERS

@@ -2,9 +2,9 @@
 /specification/ml/** @elastic/ml-core @elastic/ml-ui
 
 # Specification tooling owners
-/.github/                   @elastic/clients-team
-/compiler/                  @elastic/clients-team
-/compiler-rs/               @elastic/clients-team
-/typescript-generator/      @elastic/clients-team
-/specification/_json_spec/  @elastic/clients-team
-/specification/_spec_utils/ @elastic/clients-team
+/.github/                   @elastic/devtools-team
+/compiler/                  @elastic/devtools-team
+/compiler-rs/               @elastic/devtools-team
+/typescript-generator/      @elastic/devtools-team
+/specification/_json_spec/  @elastic/devtools-team
+/specification/_spec_utils/ @elastic/devtools-team

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1 @@
-<!--
-
-Hello there!
-
-Thank you for opening a pull request!
-Please make sure to follow the steps below when opening a pr:
-
-- Sign the CLA https://www.elastic.co/contributor-agreement/
-- Tag the relative issue (if any) and give a brief explanation on what your changes are doing
-- If you did a spec change, remember to generate again the outputs, you can do it by running `make contrib`
-- Add the appropriate backport labels. If you need to backport a breaking change (e.g. changing the structure of a type or changing the type/optionality of a field), please follow these rules:
-  - If the API is unusable without the change -> every supported version
-  - If the API is usable, but fix is on the response side -> every supported version
-  - If the API is usable, but fix is on the request side -> no backport, unless the API is _partially_ usable and the fix unlocks a missing feature that has no workaround
-
-Happy coding!
-
--->
+<!-- Hello there! Thank you for opening a pull request. See CONTRIBUTING.md for instructions. -->

.github/download-artifacts/index.js

@@ -17,50 +17,48 @@
  * under the License.
  */
 
-'use strict'
-
-const core = require('@actions/core')
-const { join } = require('path')
-const minimist = require('minimist')
-const stream = require('stream')
-const { promisify } = require('util')
-const { createWriteStream, promises } = require('fs')
-const rimraf = require('rimraf')
-const fetch = require('node-fetch')
-const crossZip = require('cross-zip')
+import core from '@actions/core'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import minimist from 'minimist'
+import stream from 'stream'
+import { promisify } from 'util'
+import { createWriteStream, promises } from 'fs'
+import { rimraf } from 'rimraf'
+import fetch from 'node-fetch'
+import crossZip from 'cross-zip'
 
 const { mkdir, rename, readdir, unlink } = promises
 const pipeline = promisify(stream.pipeline)
 const unzip = promisify(crossZip.unzip)
-const rm = promisify(rimraf)
 
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
 const esFolder = join(__dirname, '..', '..', 'artifacts')
 const zipFolder = join(esFolder, 'artifacts.zip')
 const downloadedSpec = join(esFolder, 'rest-api-spec', 'api')
 const specFolder = join(__dirname, '..', '..', 'specification', '_json_spec')
 
 async function downloadArtifacts (opts) {
-  if (typeof opts.version !== 'string' && typeof opts.branch !== 'string') {
-    throw new Error('Missing version or branch')
+  if (typeof opts.branch !== 'string') {
+    throw new Error('Missing branch')
   }
 
-  core.info('Checking out spec and test')
+  core.info('Resolving artifact URL')
 
-  core.info('Resolving version')
   let resolved
   try {
-    resolved = await resolve(opts.version || fromBranch(opts.branch), opts.hash)
+    resolved = await resolve(opts.branch)
   } catch (err) {
     core.error(err.message)
     process.exit(1)
   }
 
-  opts.version = resolved.version
-  core.info(`Resolved version ${opts.version}`)
+  core.info(`Resolved artifact URL for ${resolved.commit_url}`)
 
   core.info('Cleanup')
-  await rm(esFolder)
-  await rm(specFolder)
+  await rimraf(esFolder)
+  await rimraf(specFolder)
   await mkdir(esFolder, { recursive: true })
   await mkdir(specFolder, { recursive: true })
 
@@ -76,7 +74,7 @@ async function downloadArtifacts (opts) {
   await unzip(zipFolder, esFolder)
 
   core.info('Cleanup')
-  await rm(zipFolder)
+  await rimraf(zipFolder)
 
   core.info('Moving files')
   const files = await readdir(downloadedSpec)
@@ -96,74 +94,29 @@ async function downloadArtifacts (opts) {
   core.info('Done')
 }
 
-async function resolve (version, hash) {
-  if (version === 'latest') {
-    const response = await fetch('https://artifacts-api.elastic.co/v1/versions')
-    if (!response.ok) {
-      throw new Error(`unexpected response ${response.statusText}`)
-    }
-    const { versions } = await response.json()
-    version = versions.pop()
+async function resolve (branch) {
+  if (branch == 'main') {
+    branch = 'master'
   }
-
-  core.info(`Resolving version ${version}`)
-  const response = await fetch(`https://artifacts-api.elastic.co/v1/versions/${version}`)
+  const url = `https://artifacts-snapshot.elastic.co/elasticsearch/latest/${branch}.json`
+  const response = await fetch(url)
   if (!response.ok) {
-    throw new Error(`unexpected response ${response.statusText}`)
+    throw new Error(`Unexpected response. Invalid version? ${url}: ${response.statusText}`)
   }
-
   const data = await response.json()
-  const esBuilds = data.version.builds
-    .filter(build => build.projects.elasticsearch != null)
-    .map(build => {
-      return {
-        projects: build.projects.elasticsearch,
-        buildId: build.build_id,
-        date: build.start_time,
-        version: build.version
-      }
-    })
-    .sort((a, b) => {
-      const dA = new Date(a.date)
-      const dB = new Date(b.date)
-      if (dA > dB) return -1
-      if (dA < dB) return 1
-      return 0
-    })
-
-  if (hash != null) {
-    const build = esBuilds.find(build => build.projects.commit_hash === hash)
-    if (!build) {
-      throw new Error(`Can't find any build with hash '${hash}'`)
-    }
-    const zipKey = Object.keys(build.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-    return {
-      url: build.projects.packages[zipKey].url,
-      id: build.buildId,
-      hash: build.projects.commit_hash,
-      version: build.version
-    }
-  }
 
-  const lastBuild = esBuilds[0]
-  const zipKey = Object.keys(lastBuild.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-  return {
-    url: lastBuild.projects.packages[zipKey].url,
-    id: lastBuild.buildId,
-    hash: lastBuild.projects.commit_hash,
-    version: lastBuild.version
+  let manifest_url = data.manifest_url
+  const manifestResponse = await fetch(manifest_url)
+  if (!manifestResponse.ok) {
+    throw new Error(`Unexpected manifestResponse. ${manifest_url}: ${manifestResponse.statusText}`)
   }
-}
+  const manifestData = await manifestResponse.json()
+  const elasticsearch = manifestData.projects.elasticsearch
+  const restResourceName = `rest-resources-zip-${manifestData.version}.zip`
 
-function fromBranch (branch) {
-  if (branch === 'main') {
-    return 'latest'
-  } else if (branch === '7.x') {
-    return '7.x-SNAPSHOT'
-  } else if ((branch.startsWith('7.') || branch.startsWith('8.')) && !isNaN(Number(branch.split('.')[1]))) {
-    return `${branch}-SNAPSHOT`
-  } else {
-    throw new Error(`Cannot derive version from branch '${branch}'`)
+  return {
+    url: elasticsearch.packages[restResourceName].url,
+    commit_url: elasticsearch.commit_url,
   }
 }
 
@@ -172,7 +125,7 @@ async function main (options) {
 }
 
 const options = minimist(process.argv.slice(2), {
-  string: ['id', 'version', 'hash', 'branch']
+  string: ['branch']
 })
 main(options).catch(t => {
   core.error(t)

.github/download-artifacts/package.json

@@ -2,6 +2,7 @@
   "name": "download-artifacts",
   "version": "1.0.0",
   "description": "",
+  "type": "module",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -13,7 +14,7 @@
     "@actions/core": "^1.6.0",
     "cross-zip": "^4.0.0",
     "minimist": "^1.2.5",
-    "node-fetch": "^2.6.7",
-    "rimraf": "^3.0.2"
+    "node-fetch": "^3",
+    "rimraf": "^6"
   }
 }

.github/labeler.yml

@@ -1,12 +1,15 @@
 documentation:
-  - '*.md'
+- changed-files:
+  - any-glob-to-any-file: '*.md'
 
 specification:
-  - 'specification/**/*'
+- changed-files:
+  - any-glob-to-any-file: 'specification/**/*'
 
 generator:typescript:
-  - 'typescript-generator/**/*'
+- changed-files:
+  - any-glob-to-any-file: 'typescript-generator/**/*'
 
 compiler:
-  - 'compiler/**/*'
-  - 'compiler-rs/**/*'
+- changed-files:
+  - any-glob-to-any-file: ['compiler/**/*', 'compiler-rs/**/*']