Merge remote-tracking branch 'origin/main' into rest-api-spec-converter

Author: pquentin

.github/validate-pr/index.js

@@ -22,10 +22,8 @@
 import { dirname } from 'path'
 import { fileURLToPath } from 'url'
 import 'zx/globals'
-import assert from 'assert'
 import * as core from '@actions/core'
 import { copyFile } from 'fs/promises'
-import * as github from '@actions/github'
 import specification from '../../output/schema/schema.json' with { type: 'json' }
 import baselineValidation from '../../../clients-flight-recorder/recordings/types-validation/types-validation.json' with { type: 'json' }
 import { run as getReport } from '../../../clients-flight-recorder/scripts/types-validator/index.js'
@@ -36,7 +34,6 @@ import {
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
 
-const octokit = github.getOctokit(argv.token)
 
 const privateNames = ['_global']
 const tick = '`'
@@ -55,51 +52,16 @@ async function run() {
     path.join(__dirname, '..', '..', 'output', 'typescript', 'types.ts'),
     path.join(tsValidationPath, 'types.ts')
   )
-  const context = github.context
-  assert(context.payload.pull_request, 'We should be in a PR context')
-  const files = []
-  let page = 1
-  while (true) {
-    const { data } = await octokit.rest.pulls.listFiles({
-      owner: 'elastic',
-      repo: 'elasticsearch-specification',
-      pull_number: context.payload.pull_request.number,
-      page,
-      per_page: 100
-    })
-    if (data.length > 0) {
-      files.push(
-        ...data
-          .filter((entry) => entry.status !== 'deleted')
-          .map((entry) => entry.filename)
-      )
-      page += 1
-    } else {
-      break
-    }
-  }
-
-  const specFiles = files.filter(
-    (file) => file.includes('specification') && !file.includes('compiler/test')
-  )
   const reports = new Map()
 
-  cd(tsValidationPath)
 
   // Collect all APIs to validate
   const apisToValidate = new Set()
 
-  for (const file of specFiles) {
+  cd(path.join(__dirname, '..', '..'))
+  for (const file of await glob('specification/**/*.ts')) {
     if (file.startsWith('specification/_types')) continue
-    if (file.startsWith('specification/_spec_utils')) continue
-    if (file.startsWith('specification/_doc_ids')) continue
-    if (file.startsWith('specification/_json_spec')) continue
     if (file.startsWith('specification/node_modules')) continue
-    if (file.endsWith('tsconfig.json')) continue
-    if (file.endsWith('eslint.config.js')) continue
-    if (file.endsWith('package.json')) continue
-    if (file.endsWith('package-lock.json')) continue
-    if (file.endsWith('.md')) continue
     if (getApi(file).endsWith('_types')) {
       const apis = specification.endpoints
         .filter(endpoint => endpoint.name.split('.').filter(s => !privateNames.includes(s))[0] === getApi(file).split('.')[0])
@@ -113,26 +75,27 @@ async function run() {
     }
   }
 
+  cd(tsValidationPath)
+  console.log(`Validating ${apisToValidate.size} APIs...`)
+
   // Call getReport once with all APIs
-  if (apisToValidate.size > 0) {
-    const allApis = Array.from(apisToValidate).join(',')
-    const report = await getReport({
-      api: allApis,
-      'generate-report': false,
-      request: true,
-      response: true,
-      ci: false,
-      verbose: false
-    })
-
-    // Extract individual API reports from the combined result
-    for (const api of apisToValidate) {
-      const namespace = getNamespace(api)
-      if (report.has(namespace)) {
-        const namespaceReport = report.get(namespace).find(r => r.api === getName(api))
-        if (namespaceReport) {
-          reports.set(api, namespaceReport)
-        }
+  const allApis = Array.from(apisToValidate).join(',')
+  const report = await getReport({
+    api: allApis,
+    'generate-report': false,
+    request: true,
+    response: true,
+    ci: false,
+    verbose: false
+  })
+
+  // Extract individual API reports from the combined result
+  for (const api of apisToValidate) {
+    const namespace = getNamespace(api)
+    if (report.has(namespace)) {
+      const namespaceReport = report.get(namespace).find(r => r.api === getName(api))
+      if (namespaceReport) {
+        reports.set(api, namespaceReport)
       }
     }
   }
@@ -170,7 +133,12 @@ function getApi (file) {
 }
 
 function findBaselineReport(apiName, baselineValidation) {
-  const [namespace, method] = apiName.split('.')
+  let namespace, method = [null, null]
+  if (!apiName.includes('.')) {
+    [namespace, method] = ['global', apiName]
+  } else {
+    [namespace, method] = apiName.split('.')
+  }
 
   if (!baselineValidation.namespaces[namespace]) {
     return null

.github/workflows/code-format.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Use Node.js 22
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v5
       with:
         node-version: 22
 

.github/workflows/compiler-rs.yml

@@ -12,10 +12,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Use Node.js 22
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: 22
 

.github/workflows/compiler.test.yml

@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Use Node.js 22
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: 22
 

.github/workflows/generate.yml

@@ -30,13 +30,13 @@ jobs:
     if: github.repository_owner == 'elastic' && github.actor != 'elasticmachine'
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v5
       with:
         token: ${{ secrets.PAT }}
         persist-credentials: true
 
     - name: Setup Node 22
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v5
       with:
         node-version: 22
         cache: npm

.github/workflows/license.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Check license headers
       run: |

.github/workflows/linter.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Use Node.js 22
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v5
       with:
         node-version: 22
 

.github/workflows/metamodel-sync.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Copy compiler's metamodel to typescript-generator
       run: |

.github/workflows/update-rest-api-json.yml

@@ -16,12 +16,12 @@ jobs:
         branch: ['main', '9.1', '9.0', '8.19', '8.18']
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ matrix.branch }}
 
       - name: Use Node.js 22
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: 22
 

.github/workflows/validate-pr.yml .github/workflows/validate-apis.yml.github/workflows/validate-pr.yml renamed to .github/workflows/validate-apis.yml

@@ -5,33 +5,51 @@ on:
     branches:
       - main
       - '[0-9]+.[0-9]+'
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+
+permissions:
+  pull-requests: write    # To create/update PR comments
 
 jobs:
-  validate-pr:
+  validate-apis:
     name: build
     runs-on: ubuntu-latest
 
     steps:
       - name: Check pull request was opened from branch
-        if: github.event.pull_request.head.repo.full_name != 'elastic/elasticsearch-specification'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'elastic/elasticsearch-specification'
         run: echo "Validation is not supported from forks"; exit 1
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           path: ./elasticsearch-specification
+          persist-credentials: false
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           repository: elastic/clients-flight-recorder
           path: ./clients-flight-recorder
           token: ${{ secrets.PAT }}
-          ref: main
+          ref: ${{ github.base_ref }}
+          persist-credentials: false
 
       - name: Use Node.js 22
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: 22
 
+      - name: Cache JSON report
+        id: cache-json-report
+        uses: actions/cache@v4
+        with:
+          path: ./clients-flight-recorder/recordings/types-validation/types-validation.json
+          key: types-validation-json-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.base_ref, github.event.pull_request.base.sha) || format('{0}-{1}', github.ref_name, github.sha) }}
+          restore-keys: |
+            types-validation-json-${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }}-
+
       - name: Install deps 1/2
         working-directory: ./clients-flight-recorder
         run: |
@@ -53,20 +71,30 @@ jobs:
       - name: Download artifacts
         working-directory: ./clients-flight-recorder
         run: |
-          if [[ -n "${GITHUB_BASE_REF+x}" ]]; then
+          if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
             branch=$GITHUB_BASE_REF
           else
             branch=$GITHUB_REF_NAME
           fi
+          echo "Using branch: $branch"
           node scripts/upload-recording/download.js --branch $branch --git
           node scripts/clone-elasticsearch/index.js --branch $branch
 
-      - name: Run validation
+      - name: Run validation (Push)
+        if: github.event_name == 'push'
+        working-directory: ./clients-flight-recorder/scripts/types-validator
+        env:
+          BRANCH: ${{ github.ref_name }}
+        run: node index.js --generate-report --ci --branch $BRANCH
+
+      - name: Run validation (PR)
+        if: github.event_name == 'pull_request'
         id: validation
         working-directory: ./elasticsearch-specification
         run: node .github/validate-pr --token ${{ secrets.GITHUB_TOKEN }}
 
       - name: Find existing comment
+        if: github.event_name == 'pull_request'
         uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
         id: find-comment
         with:
@@ -75,6 +103,7 @@ jobs:
           body-includes: 'Following you can find the validation changes'
 
       - name: Create or update comment
+        if: github.event_name == 'pull_request'
         uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}