Run telemetry automated checks conditionally, use octokit (elastic#240799)

## Summary

Closes elastic#240390

This PR brings 2 important enhancements:
* Use octokit to fetch telemetry JSON schemas from GH (at the moment we
were performing anonymous requests and we have already been
rate-limited).
* Only run "PR automated checks" task if the pull request has modified
the telemetry schemas.

Author: gsoldevila

.buildkite/scripts/steps/checks/telemetry.sh

@@ -9,6 +9,9 @@ echo --- Check Telemetry Schema
 if is_pr && ! is_auto_commit_disabled; then
   node scripts/telemetry_check --baseline "${GITHUB_PR_MERGE_BASE:-}" --fix
   check_for_changed_files "node scripts/telemetry_check" true
-else
+elif is_pr; then
   node scripts/telemetry_check --baseline "${GITHUB_PR_MERGE_BASE:-}"
+else
+  # assume on-merge pipeline
+  node scripts/telemetry_check --baseline "${BUILDKITE_BRANCH:-main}"
 fi

src/platform/packages/private/kbn-telemetry-tools/src/cli/run_telemetry_check.ts

@@ -7,13 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type { ListrTask } from 'listr2';
 import { Listr } from 'listr2';
 import chalk from 'chalk';
 import { createFailError } from '@kbn/dev-cli-errors';
 import { run } from '@kbn/dev-cli-runner';
 
-import { prAutomatedChecks } from '../tools/tasks/pr_automated_checks';
+import { validateSchemaChanges } from '../tools/tasks/validate_schema_changes';
 import type { TaskContext } from '../tools/tasks';
 import {
   createTaskContext,
@@ -119,15 +118,13 @@ export function runTelemetryCheck() {
             title: 'Updating telemetry mapping files',
             task: (context, task) => task.newListr(writeToFileTask(context), { exitOnError: true }),
           },
-          ...(baselineSha && false // temporarily disable (see https://github.com/elastic/kibana/issues/240390)
-            ? [
-                {
-                  title: 'Automated PR review checks',
-                  task: (context, task) =>
-                    task.newListr(prAutomatedChecks(context), { exitOnError: true }),
-                } as ListrTask<TaskContext>,
-              ]
-            : []),
+          {
+            title: 'Validating changes in telemetry schemas',
+            task: (context, task) =>
+              task.newListr(validateSchemaChanges(context), { exitOnError: true }),
+            // only run if on a PR branch
+            enabled: (_) => Boolean(baselineSha),
+          },
         ],
         {
           renderer: process.env.CI ? 'verbose' : ('default' as any),

src/platform/packages/private/kbn-telemetry-tools/src/tools/tasks/git.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { promisify } from 'util';
+import { exec } from 'child_process';
+import { Octokit } from '@octokit/rest';
+import type { TelemetrySchemaObject } from '../../schema_ftr_validations/schema_to_config_schema';
+
+const execAsync = promisify(exec);
+let octokit: null | Octokit;
+let changedFilesCache: null | string[];
+
+function getOctokit() {
+  if (!process.env.GITHUB_TOKEN) {
+    throw new Error('Missing environment variable: GITHUB_TOKEN');
+  }
+  octokit =
+    octokit ??
+    new Octokit({
+      auth: process.env.GITHUB_TOKEN,
+    });
+
+  return octokit;
+}
+
+async function fetchPrChangedFiles(
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+): Promise<string[]> {
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to retrieve PR changes"
+    );
+  }
+
+  const github = getOctokit();
+  const files = await github.paginate(github.pulls.listFiles, {
+    owner,
+    repo,
+    pull_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    per_page: 100,
+  });
+
+  return files.map(({ filename }) => filename);
+}
+
+async function getPrChangedFiles(): Promise<string[]> {
+  changedFilesCache = changedFilesCache || (await fetchPrChangedFiles());
+  return changedFilesCache;
+}
+
+async function getUncommitedChanges(): Promise<string[]> {
+  const { stdout } = await execAsync(
+    `git status --porcelain -- . ':!:config/node.options' ':!config/kibana.yml'`,
+    { maxBuffer: 1024 * 1024 * 128 }
+  );
+
+  return stdout
+    .split('\n')
+    .map((line) =>
+      line
+        .replace('?? ', '')
+        .replace('A ', '')
+        .replace('M ', '')
+        .replace('D ', '')
+        .replace('R ', '')
+        .trim()
+    );
+}
+
+export async function isTelemetrySchemaModified({ path }: { path: string }): Promise<boolean> {
+  const modifiedFiles = await getUncommitedChanges();
+  if (modifiedFiles.includes(path)) {
+    return true;
+  }
+
+  const prChanges = await getPrChangedFiles();
+  return prChanges.length >= 3000 || prChanges.includes(path);
+}
+
+export async function fetchTelemetrySchemaAtRevision({
+  path,
+  ref,
+}: {
+  path: string;
+  ref: string;
+}): Promise<TelemetrySchemaObject> {
+  const github = getOctokit();
+  const res = await github.rest.repos.getContent({
+    mediaType: {
+      format: 'application/vnd.github.VERSION.raw',
+    },
+    owner: 'elastic',
+    repo: 'kibana',
+    path,
+    ref,
+  });
+
+  if (!Array.isArray(res.data) && res.data.type === 'file') {
+    return JSON.parse(Buffer.from(res.data.content!, 'base64').toString()) as TelemetrySchemaObject;
+  } else {
+    throw new Error(`Error retrieving contents of ${path}, unexpected response data.`);
+  }
+}