Merge branch 'main' into xpack-usage-ilm-action

Author: pquentin

.github/validate-pr/index.js

@@ -53,7 +53,7 @@ async function run () {
       per_page: 100
     })
     if (data.length > 0) {
-      files.push(...data.map(entry => entry.filename))
+      files.push(...data.filter(entry => entry.status !== 'deleted').map(entry => entry.filename))
       page += 1
     } else {
       break

.github/workflows/check-backport-labels.yml

@@ -0,0 +1,118 @@
+name: Check
+
+on:
+  pull_request:
+    branches:
+      - main
+    types: [labeled, unlabeled, opened, reopened, synchronize]
+
+permissions:
+  pull-requests: "read"
+
+jobs:
+  check-backport-label:
+    name: backport label 
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: "Check backport label"
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          json_pr_labels='${{ toJSON(github.event.pull_request.labels) }}'
+          readarray -t pr_labels < <(echo "${json_pr_labels}" | jq -r -c '.[].name')
+
+          json_all_labels="$(gh label list --repo ${{ github.repository }} --json name --search "backport" --limit 1000)"
+          readarray -t all_labels < <(echo "${json_all_labels}" | jq -r -c '.[].name')
+
+          declare -A all_backport_labels=()
+          declare -A all_floating_majors=()
+
+          backport_regex="^backport ([0-9])+\.([0-9]+|x)$"
+
+          echo "::group::Available Labels"
+          echo "skip-backport"
+
+          for label in "${all_labels[@]}"; do
+            if [[ "${label}" =~ ${backport_regex} ]]; then
+              major="${BASH_REMATCH[1]}"
+              minor="${BASH_REMATCH[2]}"
+              all_backport_labels["${label}"]=1
+              echo "${label}"
+
+              if [ "${minor}" = "x" ]; then
+                all_floating_majors["${major}"]=1
+              fi
+            fi
+          done
+
+          echo "::endgroup::"
+
+          has_backport_label=false
+          has_skip_backport_label=false
+          declare -A pr_exact_majors=()
+          declare -A pr_floating_majors=()
+
+          echo "::group::Detected Labels"
+
+          for pr_label in "${pr_labels[@]}"; do
+            if [ "${pr_label}" = "skip-backport" ]; then
+              has_skip_backport_label=true
+              echo "${pr_label}"
+              continue
+            fi
+
+            if [ -z "${all_backport_labels[${pr_label}]}" ]; then
+              continue
+            fi
+
+            has_backport_label=true
+
+            if [[ "${pr_label}" =~ ${backport_regex} ]]; then
+              major="${BASH_REMATCH[1]}"
+              minor="${BASH_REMATCH[2]}"
+              if [ "${minor}" != "x" ]; then
+                pr_exact_majors["${major}"]=1
+              else
+                pr_floating_majors["${major}"]=1
+              fi
+            fi
+
+            echo "${pr_label}"
+          done
+
+          echo "::endgroup::"
+
+          if [ "${has_skip_backport_label}" = true ] && [ "${has_backport_label}" = true ]; then
+            echo "::error::The 'skip-backport' not be used in combination with another backport"\
+                  "label."
+            exit 1
+          fi
+
+          if [ "${has_skip_backport_label}" != true ] && [ "${has_backport_label}" != true ]; then
+            echo "::error::No backport label found. Please add at least one of the"\
+                  "'backport {major}.{minor|x}' labels or use 'skip-backport',"\
+                  "if this PR should not be backported."
+            exit 1
+          fi
+
+          # Validate that a floating backport label exists for each exact backport label major
+          # version.
+
+          has_required_floating_labels=true
+
+          for pr_major in "${!pr_exact_majors[@]}"; do
+            if [ -z "${all_floating_majors[${pr_major}]}" ]; then
+              # There is no floating version branch for the given major version.
+              continue
+            fi
+
+            if [ -z "${pr_floating_majors[${pr_major}]}" ]; then
+              has_required_floating_labels=false
+              echo "::error::Missing floating backport label for '${pr_major}.x'"
+            fi
+          done
+
+          if [ "${has_required_floating_labels}" != true ]; then
+            exit 1
+          fi

.github/workflows/gh-pages-report.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: ['main', '8.x', '8.16', '8.17', 7.17']
+        branch: ['main', '8.x', '8.16', '8.17', '7.17']
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/update-rest-api-json.yml

@@ -305,7 +305,7 @@ If you are using MacOS, run the following command to fix the issue:
 brew install coreutils
 ```
 
-### I need to modify che compiler, help!
+### I need to modify the compiler, help!
 
 Take a look at the [compiler documentation](./docs/compiler.md).
 

README.md

@@ -17,6 +17,7 @@ derive_more = "1.0.0-beta.6"
 either_n = "0.2"
 icu_segmenter = "1"
 indexmap = "2"
+itertools = "0.14"
 maplit = "1"
 once_cell = "1.16"
 openapiv3 = "2"

compiler-rs/Cargo.lock

@@ -11,6 +11,8 @@ serde_json = { workspace = true }
 once_cell = { workspace = true }
 anyhow = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
+itertools = { workspace = true }
+
 
 arcstr = { workspace = true, features = ["serde", "substr"] }
 clap = { workspace = true, features = ["derive"] }

compiler-rs/Cargo.toml

@@ -0,0 +1,121 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use clap::Parser;
+use itertools::Itertools;
+
+
+fn main() -> anyhow::Result<()> {
+    let cli = Cli::parse();
+    cli.run()?;
+    Ok(())
+}
+
+// Example usage:
+// (cd compiler-rs; find ../specification -name '*Request.ts' | cargo run --bin add_url_paths ../output/schema/schema.json | sh)
+
+/// Adds url paths to request definitions. Stdin must be a list of files, one per line.
+/// Outputs a shell script that uses ast-grep.
+#[derive(Debug, Parser)]
+#[command(author, version, about, long_about)]
+pub struct Cli {
+    /// input schema file, eg: ../output/schema/schema-no-generics.json
+    schema: PathBuf,
+}
+
+impl Cli {
+    pub fn run(&self) -> anyhow::Result<()> {
+
+        // Canonicalize all file names, so that we can do some suffix mapping from the schema locations.
+        let files: Vec<PathBuf> = std::io::read_to_string(std::io::stdin())?
+            .lines()
+            .flat_map(|line| std::fs::canonicalize(line)
+                .map_err(|e| {
+                    eprintln!("File {} not found", line);
+                    Result::<PathBuf, _>::Err(e)
+                })) // Remove errors
+            .collect();
+
+        let json = std::fs::read_to_string(&self.schema)?;
+        let schema = clients_schema::IndexedModel::from_reader(json.as_bytes())?;
+
+        let mut location_to_request = HashMap::<&Path, &clients_schema::Endpoint>::new();
+        for ep in &schema.endpoints {
+            let Some(req_name) = ep.request.as_ref() else {
+                //eprintln!("Skipping endpoint {} with no request", ep.name);
+                continue;
+            };
+
+            let type_def = schema.types.get(req_name).unwrap();
+            let location = type_def.base().spec_location.as_ref().unwrap();
+            let location = Path::new(location.split_once('#').unwrap().0);
+
+            location_to_request.insert(location, ep);
+        };
+
+        for file in files {
+            if let Some((_, endpoint)) = location_to_request.iter().find(|(location, _)| file.ends_with(location)) {
+                generate_astgrep_command(&file, endpoint);
+            } else {
+                eprintln!("No request found for {:?}", file);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+fn generate_astgrep_command(file: &Path, endpoint: &clients_schema::Endpoint) {
+
+    let text = std::fs::read_to_string(file).unwrap();
+    if text.contains("urls:") {
+        eprintln!("Found an existing 'url' property. Skipping {file:?}");
+        return;
+    }
+
+    // We cannot express conditional parts in the source form of patterns.
+
+    // Requests with generic parameters
+    let request_expr = if text.contains("Request<") {
+        "Request<$$$PARAM>"
+    } else {
+        "Request"
+    };
+
+    // A handful of requests don't have an extends clause
+    let extends_expr = if text.contains(" extends ") {
+        "extends $REQBASE"
+    } else {
+        ""
+    };
+
+    let urls: String = endpoint.urls.iter().map(|url| {
+        let path = &url.path;
+        let methods = url.methods.iter().map(|method| format!("\"{}\"", method)).join(", ");
+        let deprecation = match &url.deprecation {
+            Some(deprecation) => format!("/** @deprecated {} {} */\n      ", deprecation.version, deprecation.description),
+            None => "".to_string(),
+        };
+
+        format!(r#"    {{
+      {deprecation}path: "{path}",
+      methods: [{methods}]
+    }}"#)
+    }).join(",\n");
+
+    let pattern = format!(r#"interface {request_expr} {extends_expr} {{
+  $$$PROPS
+}}"#);
+
+    let fix = format!(r#"interface {request_expr} {extends_expr} {{
+  urls: [
+{urls}
+  ],
+  $$$PROPS
+}}"#);
+
+    let file = file.to_str().unwrap();
+    println!("#----- {file}");
+    println!(r#"ast-grep --update-all --lang ts --pattern '{pattern}' --rewrite '{fix}' "{file}""#);
+
+    println!();
+}