From 86224051931bcee81e6d09530f73ea3c80a71e62 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 17:50:04 +0100 Subject: [PATCH 1/6] added new agent for testing --- buildkite/src/Command/DockerImage.dhall | 1 + buildkite/src/Command/Size.dhall | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 319885bd838a..e5b843da4a44 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -162,6 +162,7 @@ let generateStep = , QA = "--filter until=24h" , Multi = "--filter until=24h" , Perf = "--filter until=24h" + , Dev = "--filter until=24h" } spec.size diff --git a/buildkite/src/Command/Size.dhall b/buildkite/src/Command/Size.dhall index 4ac89d69a1eb..13d5c9fa69ba 100644 --- a/buildkite/src/Command/Size.dhall +++ b/buildkite/src/Command/Size.dhall @@ -1 +1,11 @@ -< XLarge | Large | Medium | Small | Integration | QA | Multi | Perf | Arm64 > +< XLarge +| Large +| Medium +| Small +| Integration +| QA +| Multi +| Perf +| Arm64 +| Dev +> From d05087a546a65f717f07336e753e97e104151588 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 17:53:01 +0100 Subject: [PATCH 2/6] removed old filtering mechanism --- buildkite/scripts/generate-jobs.sh | 9 --- buildkite/src/Pipeline/FilterMode.dhall | 7 +- buildkite/src/Pipeline/JobSelection.dhall | 10 ++- buildkite/src/Pipeline/Scope.dhall | 38 ++------- buildkite/src/Pipeline/ScopeFilter.dhall | 14 ++-- buildkite/src/Pipeline/Tag.dhall | 85 ++------------------- buildkite/src/Pipeline/TagTest.dhall | 70 ----------------- buildkite/src/Pipeline/TriggerCommand.dhall | 7 -- buildkite/src/gen/Jobs.dhall | 3 - 9 files changed, 30 insertions(+), 213 deletions(-) delete mode 100755 buildkite/scripts/generate-jobs.sh delete mode 100644 buildkite/src/Pipeline/TagTest.dhall delete mode 100644 buildkite/src/Pipeline/TriggerCommand.dhall delete mode 100644 buildkite/src/gen/Jobs.dhall diff --git a/buildkite/scripts/generate-jobs.sh b/buildkite/scripts/generate-jobs.sh deleted file mode 100755 index 27adf2d27a2c..000000000000 --- a/buildkite/scripts/generate-jobs.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -ROOT=${1:-"./buildkite"} -PREFIX=${2:-"../../../"} - -echo -n "[ -- Autogenerated do not edit by hand " -find "$ROOT/src/Jobs/" -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c "echo ',' && echo -n \"${PREFIX}{}\"" -echo " ]" - diff --git a/buildkite/src/Pipeline/FilterMode.dhall b/buildkite/src/Pipeline/FilterMode.dhall index a360eb951ed9..1c2c7372b135 100644 --- a/buildkite/src/Pipeline/FilterMode.dhall +++ b/buildkite/src/Pipeline/FilterMode.dhall @@ -2,7 +2,6 @@ let Mode = < Any | All > -in { Type = Mode - , any = \(mode : Mode) -> merge { Any = True, All = False } mode - , all = \(mode : Mode) -> merge { Any = False, All = True } mode - } +let show = \(mode : Mode) -> merge { Any = "any", All = "all" } mode + +in { Type = Mode, show = show } diff --git a/buildkite/src/Pipeline/JobSelection.dhall b/buildkite/src/Pipeline/JobSelection.dhall index 17e6cb697855..837a1f5d1f8a 100644 --- a/buildkite/src/Pipeline/JobSelection.dhall +++ b/buildkite/src/Pipeline/JobSelection.dhall @@ -15,4 +15,12 @@ let isFull = \(selection : Selection) -> merge { Triaged = False, Full = True } selection -in { Type = Selection, capitalName = capitalName, isFull = isFull } +let show = + \(selection : Selection) + -> merge { Triaged = "triaged", Full = "full" } selection + +in { Type = Selection + , capitalName = capitalName + , isFull = isFull + , show = show + } diff --git a/buildkite/src/Pipeline/Scope.dhall b/buildkite/src/Pipeline/Scope.dhall index f0d1e1f4a055..5db7e4b1dec5 100644 --- a/buildkite/src/Pipeline/Scope.dhall +++ b/buildkite/src/Pipeline/Scope.dhall @@ -12,7 +12,7 @@ let Prelude = ../External/Prelude.dhall -let List/any = Prelude.List.any +let Extensions = ../Lib/Extensions.dhall let Scope = < PullRequest | Nightly | MainlineNightly | Release > @@ -36,33 +36,9 @@ let lowerName = } scope -let toNatural = - \(scope : Scope) - -> merge - { PullRequest = 0, Nightly = 1, MainlineNightly = 2, Release = 3 } - scope - -let equal - : Scope -> Scope -> Bool - = \(left : Scope) - -> \(right : Scope) - -> Prelude.Natural.equal (toNatural left) (toNatural right) - -let hasAny - : Scope -> List Scope -> Bool - = \(input : Scope) - -> \(scopes : List Scope) - -> List/any Scope (\(x : Scope) -> equal x input) scopes - -let hasPullRequest - : List Scope -> Bool - = \(scopes : List Scope) -> hasAny Scope.PullRequest scopes - -let contains - : List Scope -> List Scope -> Bool - = \(input : List Scope) - -> \(scopes : List Scope) - -> List/any Scope (\(x : Scope) -> hasAny x scopes) input +let join = + \(scopes : List Scope) + -> Extensions.join "," (Prelude.List.map Scope Text lowerName scopes) let Full = [ Scope.PullRequest, Scope.Nightly, Scope.MainlineNightly, Scope.Release ] @@ -75,11 +51,7 @@ in { Type = Scope , Full = Full , PullRequestOnly = PullRequestOnly , AllButPullRequest = AllButPullRequest - , hasPullRequest = hasPullRequest , capitalName = capitalName , lowerName = lowerName - , toNatural = toNatural - , equal = equal - , hasAny = hasAny - , contains = contains + , join = join } diff --git a/buildkite/src/Pipeline/ScopeFilter.dhall b/buildkite/src/Pipeline/ScopeFilter.dhall index 3d6a70249013..ee9035c32ad7 100644 --- a/buildkite/src/Pipeline/ScopeFilter.dhall +++ b/buildkite/src/Pipeline/ScopeFilter.dhall @@ -8,7 +8,7 @@ let Filter : Type = < PullRequestOnly | StableOnly | Nightly | MainlineNightly | All > -let tags +let scopes : Filter -> List Scope.Type = \(filter : Filter) -> merge @@ -29,12 +29,12 @@ let show : Filter -> Text = \(filter : Filter) -> merge - { PullRequestOnly = "PullRequestOnly" - , StableOnly = "StableOnly" - , Nightly = "Nightly" - , MainlineNightly = "MainlineNightly" - , All = "All" + { PullRequestOnly = "pullrequestonly" + , StableOnly = "stableonly" + , Nightly = "nightly" + , MainlineNightly = "mainlinenightly" + , All = "all" } filter -in { Type = Filter, tags = tags, show = show } +in { Type = Filter, scopes = scopes, show = show } diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall index 7ab06a32a677..b3537d43fd09 100644 --- a/buildkite/src/Pipeline/Tag.dhall +++ b/buildkite/src/Pipeline/Tag.dhall @@ -4,11 +4,7 @@ let Prelude = ../External/Prelude.dhall -let FilterMode = ./FilterMode.dhall - -let List/any = Prelude.List.any - -let List/all = Prelude.List.all +let Extensions = ../Lib/Extensions.dhall let Tag : Type @@ -38,74 +34,6 @@ let Tag | Jammy > -let toNatural - : Tag -> Natural - = \(tag : Tag) - -> merge - { Fast = 1 - , Long = 2 - , VeryLong = 3 - , TearDown = 4 - , Lint = 5 - , Release = 6 - , Test = 7 - , Toolchain = 8 - , Hardfork = 9 - , Stable = 10 - , Promote = 11 - , Debian = 12 - , Docker = 13 - , Rosetta = 14 - , Devnet = 15 - , Lightnet = 16 - , Arm64 = 17 - , Amd64 = 18 - , Mainnet = 19 - , Bullseye = 20 - , Bookworm = 21 - , Noble = 22 - , Focal = 23 - , Jammy = 24 - } - tag - -let equal - : Tag -> Tag -> Bool - = \(left : Tag) - -> \(right : Tag) - -> Prelude.Natural.equal (toNatural left) (toNatural right) - -let hasAny - : Tag -> List Tag -> Bool - = \(input : Tag) - -> \(tags : List Tag) - -> List/any Tag (\(x : Tag) -> equal x input) tags - -let hasAll - : List Tag -> List Tag -> Bool - = \(input : List Tag) - -> \(tags : List Tag) - -> List/all Tag (\(x : Tag) -> hasAny x tags) input == True - -let containsAll - : List Tag -> List Tag -> Bool - = \(input : List Tag) -> \(tags : List Tag) -> hasAll input tags - -let containsAny - : List Tag -> List Tag -> Bool - = \(input : List Tag) - -> \(tags : List Tag) - -> List/any Tag (\(x : Tag) -> hasAny x tags) input - -let contains - : List Tag -> List Tag -> FilterMode.Type -> Bool - = \(input : List Tag) - -> \(tags : List Tag) - -> \(filterMode : FilterMode.Type) - -> merge - { Any = containsAny input tags, All = containsAll input tags } - filterMode - let capitalName = \(tag : Tag) -> merge @@ -166,13 +94,12 @@ let lowerName = } tag +let join = + \(tags : List Tag) + -> Extensions.join "," (Prelude.List.map Tag Text lowerName tags) + in { Type = Tag , capitalName = capitalName , lowerName = lowerName - , toNatural = toNatural - , equal = equal - , hasAny = hasAny - , containsAny = containsAny - , containsAll = containsAll - , contains = contains + , join = join } diff --git a/buildkite/src/Pipeline/TagTest.dhall b/buildkite/src/Pipeline/TagTest.dhall deleted file mode 100644 index f67e99e0039b..000000000000 --- a/buildkite/src/Pipeline/TagTest.dhall +++ /dev/null @@ -1,70 +0,0 @@ --- filepath: /home/darek/work/minaprotocol/mina/buildkite/src/Pipeline/TagTest.dhall -let Tag = ./Tag.dhall - -let fastTag = Tag.Type.Fast - -let longTag = Tag.Type.Long - -let lintTag = Tag.Type.Lint - -let testTag = Tag.Type.Test - -let dockerTag = Tag.Type.Docker - -let emptyTagList = [] : List Tag.Type - -let testContainsAnyTrue = - assert : Tag.containsAny [ fastTag ] [ fastTag, longTag ] === True - -let testContainsAnyFalse = - assert : Tag.containsAny [ testTag ] [ fastTag, longTag ] === False - -let testContainsAnyMultipleTrue = - assert - : Tag.containsAny [ fastTag, testTag ] [ fastTag, longTag ] === True - -let testContainsAnyMultipleFalse = - assert - : Tag.containsAny [ testTag, dockerTag ] [ fastTag, longTag ] === False - -let testContainsAnyEmpty = - assert : Tag.containsAny emptyTagList [ fastTag, longTag ] === False - -let testContainsAnyAgainstEmpty = - assert : Tag.containsAny [ fastTag ] emptyTagList === False - -let testContainsAllTrue = - assert : Tag.containsAll [ fastTag ] [ fastTag, longTag ] === True - -let testContainsAllFalse = - assert - : Tag.containsAll [ fastTag, testTag ] [ fastTag, longTag ] === False - -let testContainsAllMultipleTrue = - assert - : Tag.containsAll [ fastTag, longTag ] [ fastTag, longTag, lintTag ] - === True - -let testContainsAllMultipleFalse = - assert - : Tag.containsAll [ fastTag, testTag ] [ fastTag, longTag ] === False - -let testContainsAllEmpty = - assert : Tag.containsAll emptyTagList [ fastTag, longTag ] === True - -let testContainsAllAgainstEmpty = - assert : Tag.containsAll [ fastTag ] emptyTagList === False - -in { testContainsAnyTrue = testContainsAnyTrue - , testContainsAnyFalse = testContainsAnyFalse - , testContainsAnyMultipleTrue = testContainsAnyMultipleTrue - , testContainsAnyMultipleFalse = testContainsAnyMultipleFalse - , testContainsAnyEmpty = testContainsAnyEmpty - , testContainsAnyAgainstEmpty = testContainsAnyAgainstEmpty - , testContainsAllTrue = testContainsAllTrue - , testContainsAllFalse = testContainsAllFalse - , testContainsAllMultipleTrue = testContainsAllMultipleTrue - , testContainsAllMultipleFalse = testContainsAllMultipleFalse - , testContainsAllEmpty = testContainsAllEmpty - , testContainsAllAgainstEmpty = testContainsAllAgainstEmpty - } diff --git a/buildkite/src/Pipeline/TriggerCommand.dhall b/buildkite/src/Pipeline/TriggerCommand.dhall deleted file mode 100644 index 37bcd389ebc3..000000000000 --- a/buildkite/src/Pipeline/TriggerCommand.dhall +++ /dev/null @@ -1,7 +0,0 @@ -let Cmd = ../Lib/Cmds.dhall - -in ( \(dhallPipelineRelativeToBuildKiteDir : Text) - -> Cmd.quietly - "dhall-to-yaml --quoted <<< '(./buildkite/${dhallPipelineRelativeToBuildKiteDir}).pipeline' | buildkite-agent pipeline upload" - ) - : Text -> Cmd.Type diff --git a/buildkite/src/gen/Jobs.dhall b/buildkite/src/gen/Jobs.dhall deleted file mode 100644 index f3763e5b86d4..000000000000 --- a/buildkite/src/gen/Jobs.dhall +++ /dev/null @@ -1,3 +0,0 @@ --- This file is autogenerated during builds. It remains checked in to ensure --- dhall configuration can still execute locally without running codegen. -let Pipeline = ../Pipeline/Dsl.dhall in [] : List Pipeline.CompoundType From 98042cde14b77b871629ca8eb11465a691758391 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 17:53:26 +0100 Subject: [PATCH 3/6] implement filtering in bash --- buildkite/Makefile | 11 +- .../scripts/dhall/dump_dhall_to_pipelines.sh | 13 +- buildkite/scripts/monorepo.sh | 222 ++++++++++++++++++ buildkite/src/Command/Base.dhall | 1 + buildkite/src/Jobs/Lint/Dhall.dhall | 2 +- buildkite/src/Monorepo.dhall | 114 ++------- buildkite/src/Prepare.dhall | 10 +- 7 files changed, 264 insertions(+), 109 deletions(-) create mode 100755 buildkite/scripts/monorepo.sh diff --git a/buildkite/Makefile b/buildkite/Makefile index 0ce095e47f1b..c04ac28062c2 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -25,16 +25,9 @@ format: find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' $(MAKE) convert_ifs_to_backticks -check_filter: - # Ensure Jobs.dhall is generated before proceeding - (cd ../ && ./buildkite/scripts/generate-jobs.sh > ./buildkite/src/gen/Jobs.dhall) - python3 ./unit-tests/parse_triage.py - # Clean up - git checkout -- ./src/gen/Jobs.dhall - dump_pipelines: $(eval TMP := $(shell mktemp -d)) - scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" + ./buildkite/scripts/dhall/dump_dhall_to_pipelines.sh ./buildkite/src "$(TMP)" check_deps: dump_pipelines python3 scripts/dhall/checker.py --root "$(TMP)" deps @@ -48,4 +41,4 @@ check_dups: dump_pipelines check_names: dump_pipelines python3 scripts/dhall/checker.py --root "$(TMP)" names -all: check_syntax lint format check_deps check_dirty check_dups check_names check_filter +all: check_syntax lint format check_deps check_dirty check_dups check_names diff --git a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh index 84193329b76a..52d1501fd348 100755 --- a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh +++ b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh @@ -1,8 +1,16 @@ #!/bin/bash +# Check for required arguments +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + ROOT=$1 OUTPUT=$2 + + mkdir -p "$OUTPUT" shopt -s globstar nullglob @@ -11,13 +19,16 @@ echo "Dumping pipelines from '$ROOT' to '$OUTPUT'" COUNTER=0 -for file in "$ROOT"/**/*.dhall +for file in "$ROOT"/Jobs/**/*.dhall do filename=$(basename "$file") filename="${filename%.*}" dhall-to-yaml --quoted --file "$file" > "$OUTPUT"/"$filename".yml + dhall-to-yaml <<< "let SelectFiles = $ROOT/Lib/SelectFiles.dhall in SelectFiles.compile (./$file).spec.dirtyWhen " > "$OUTPUT"/"$filename".dirtywhen + + COUNTER=$((COUNTER+1)) done diff --git a/buildkite/scripts/monorepo.sh b/buildkite/scripts/monorepo.sh new file mode 100755 index 000000000000..68307e74201e --- /dev/null +++ b/buildkite/scripts/monorepo.sh @@ -0,0 +1,222 @@ +#!/bin/bash + +# Usage: +# monorepo.sh --scopes --tags --filter-mode --selection --jobs + +set -euo pipefail + +# Default values +SCOPES="" +TAGS="" +FILTER_MODE="" +SELECTION="" +JOBS="" +GIT_DIFF_FILE="" +DRY_RUN=false + +# Parse CLI arguments +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --scopes) + SCOPES="$2" + shift; shift + ;; + --tags) + TAGS="$2" + shift; shift + ;; + --filter-mode) + FILTER_MODE="$2" + shift; shift + ;; + --selection) + SELECTION="$2" + shift; shift + ;; + --jobs) + JOBS="$2" + shift; shift + ;; + --git-diff-file) + GIT_DIFF_FILE="$2" + shift; shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --debug) + set -x + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + + +# Require all arguments +if [[ -z "$SCOPES" || -z "$TAGS" || -z "$FILTER_MODE" || -z "$SELECTION" || -z "$JOBS" || -z "$GIT_DIFF_FILE" ]]; then + echo "Error: All arguments --scopes, --tags, --filter-mode, --selection, --jobs, and --git-diff-file are required." + exit 1 +fi + +# Check if yq is installed, if not install it +if ! command -v yq &> /dev/null; then + echo "yq not found, installing..." + if command -v apt-get &> /dev/null; then + sudo apt-get update && sudo apt-get install -y wget + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq + sudo chmod +x /usr/local/bin/yq + else + echo "Error: yq is not installed and automatic installation is not supported on this system. Please install yq manually." + exit 1 + fi +fi + +IFS=',' read -r -a DESIRED_TAGS <<< "$TAGS" +IFS=',' read -r -a DESIRED_SCOPES <<< "$SCOPES" + +# Set filter flag based on FILTER_MODE +FILTER_ANY=false +FILTER_ALL=false +if [[ "$FILTER_MODE" == "any" ]]; then + FILTER_ANY=true +elif [[ "$FILTER_MODE" == "all" ]]; then + FILTER_ALL=true +else + echo "Error: --filter-mode must be 'any' or 'all'." + exit 1 +fi + +# Validate SELECTION value + +# Set selection flags +SELECTION_TRIAGED=false +SELECTION_FULL=false +if [[ "$SELECTION" == "triaged" ]]; then + SELECTION_TRIAGED=true +elif [[ "$SELECTION" == "full" ]]; then + SELECTION_FULL=true +else + echo "Error: --selection must be 'triaged' or 'full'." + exit 1 +fi +# Find all files in jobs and use yq to get .pipeline.spec.tags + +has_matching_tags() { + local tags="$1" + local filter_any="$2" + local filter_all="$3" + shift 3 + local desired_tags=("$@") + + local match_count=0 + + + + + for want in "${desired_tags[@]}"; do + + if WANT="$want" \ + yq -e '.[] | select((downcase) == (env(WANT) | downcase))' \ + <<< "$tags" \ + >/dev/null 2>&1 + then + match_count=$((match_count+1)) + fi + done + + if $filter_any && [[ $match_count -ge 1 ]]; then + echo 1 + elif $filter_all && [[ $match_count -eq ${#desired_tags[@]} ]]; then + echo 1 + else + echo 0 + fi +} + +scope_matches() { + local scope="$1" + shift + local desired_scopes=("$@") + local match_count=0 + + for want in "${desired_scopes[@]}"; do + # yq v4 NIE ma --arg, więc używamy env(WANT) + if WANT="$want" \ + yq -e '.[] | select((downcase) == (env(WANT) | downcase))' \ + <<< "$scope" \ + >/dev/null 2>&1 + then + match_count=$((match_count+1)) + break + fi + done + + echo $(( match_count == 1 ? 1 : 0 )) +} + +select_job() { + local selection_full="$1" + local selection_triaged="$2" + local file="$3" + local job_name="$4" + local git_diff_file="$5" + + if [[ "$selection_full" == true ]]; then + echo 1 + elif [[ "$selection_triaged" == true ]]; then + dirtyWhen=$(cat "${file%.yml}.dirtywhen") + # Remove quotes from beginning and end of string + dirtyWhen="${dirtyWhen%\"}" + dirtyWhen="${dirtyWhen#\"}" + if cat "$git_diff_file" | grep -E "$dirtyWhen" > /dev/null; then + echo 1 + else + echo 0 + fi + fi +} + +find "$JOBS" -type f -name "*.yml" | while read -r file; do + tags=$(yq .spec.tags "$file") + scope=$(yq .spec.scope "$file") + job_name=$(yq -r .spec.name "$file") + + tag_match=$(has_matching_tags "$tags" "$FILTER_ANY" "$FILTER_ALL" "${DESIRED_TAGS[@]}") + + scope_match=$(scope_matches "$scope" "${DESIRED_SCOPES[@]}") + + if [[ $tag_match -ne 1 ]]; then + echo "🏷️🚫 $job_name rejected job due to tags mismatch: $file" >&2 + continue + fi + if [[ $scope_match -ne 1 ]]; then + echo "🔭🚫 $job_name rejected job due to scope mismatch: $file" >&2 + continue + fi + + job_selected=$(select_job "$SELECTION_FULL" "$SELECTION_TRIAGED" "$file" "$job_name" "$GIT_DIFF_FILE") + + if [[ $job_selected -ne 1 ]]; then + echo "🧹🚫 $job_name rejected job as it does not fall into dirty when: $file" >&2 + continue + fi + + echo "✅ Including job $job_name in build " + + if [[ "$DRY_RUN" == true ]]; then + printf " -> 🛑 Dry run enabled, skipping upload for job: %s\n" "$job_name" + else + job_path=$(yq -r .spec.path "$file") + + dhall-to-yaml --quoted <<< "(./buildkite/src/Jobs/$job_path/$job_name.dhall).pipeline" | buildkite-agent pipeline upload + printf " -> ✅ Uploaded job: %s\n" "$job_name" + fi + +done + diff --git a/buildkite/src/Command/Base.dhall b/buildkite/src/Command/Base.dhall index 3dfe2db27064..2de8b365ef15 100644 --- a/buildkite/src/Command/Base.dhall +++ b/buildkite/src/Command/Base.dhall @@ -133,6 +133,7 @@ let targetToAgent = , Perf = toMap { size = "performance" } , Multi = toMap { size = "generic-multi" } , Arm64 = toMap { size = "arm64-static" } + , Dev = toMap { size = "development-0" } } target diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 2411c52bff11..95b4eaab641e 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -21,7 +21,7 @@ let dump_pipelines_cmd = Cmd.Docker::{ , image = (../../Constants/ContainerImages.dhall).toolchainBase } - "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" + "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh ./buildkite/src/ _pipelines" in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 61d0d77c0270..c996b4c0afbf 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -1,15 +1,9 @@ -let Prelude = ./External/Prelude.dhall - -let List/map = Prelude.List.map - let SelectFiles = ./Lib/SelectFiles.dhall let Cmd = ./Lib/Cmds.dhall let Command = ./Command/Base.dhall -let Docker = ./Command/Docker/Type.dhall - let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall @@ -28,21 +22,13 @@ let PipelineScopeFilter = ./Pipeline/ScopeFilter.dhall let Size = ./Command/Size.dhall -let triggerCommand = ./Pipeline/TriggerCommand.dhall - -let jobs - : List JobSpec.Type - = List/map - Pipeline.CompoundType - JobSpec.Type - (\(composite : Pipeline.CompoundType) -> composite.spec) - ./gen/Jobs.dhall - let prefixCommands = [ Cmd.run "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt" , Cmd.run "./buildkite/scripts/refresh_code.sh" , Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt" + , Cmd.run + "./buildkite/scripts/dhall/dump_dhall_to_pipelines.sh ./buildkite/src buildkite/src/gen" ] let commands @@ -50,67 +36,24 @@ let commands -> PipelineTagFilter.Type -> PipelineFilterMode.Type -> PipelineScopeFilter.Type - -> List Cmd.Type + -> Cmd.Type = \(selection : PipelineJobSelection.Type) - -> \(filter : PipelineTagFilter.Type) + -> \(tagFilter : PipelineTagFilter.Type) -> \(filterMode : PipelineFilterMode.Type) - -> \(scope : PipelineScopeFilter.Type) - -> List/map - JobSpec.Type - Cmd.Type - ( \(job : JobSpec.Type) - -> let targetTags = PipelineTagFilter.tags filter - - let jobsFilter = PipelineTagFilter.show filter - - let isIncludedInTag = - Prelude.Bool.show - (PipelineTag.contains targetTags job.tags filterMode) - - let targetScopes = PipelineScopeFilter.tags scope - - let scopeFilter = PipelineScopeFilter.show scope - - let isIncludedInScope = - Prelude.Bool.show - (PipelineScope.contains job.scope targetScopes) - - let dirtyWhen = SelectFiles.compile job.dirtyWhen - - let trigger = - triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" - - let pipelineHandlers = - { Triaged = - '' - if [ "${isIncludedInTag}" == "False" ]; then - echo "Skipping ${job.name} because this job is not falling under ${jobsFilter} filter " - elif [ "${isIncludedInScope}" == "False" ]; then - echo "Skipping ${job.name} because this is job is not falling under ${scopeFilter} stage" - elif (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then - echo "Triggering ${job.name} for reason:" - cat _computed_diff.txt | egrep '${dirtyWhen}' - ${Cmd.format trigger} - else - echo "Skipping ${job.name} because is irrelevant to PR changes" - fi - '' - , Full = - '' - if [ "${isIncludedInTag}" == "False" ]; then - echo "Skipping ${job.name} because this job is not falling under ${jobsFilter} filter " - elif [ "${isIncludedInScope}" == "False" ]; then - echo "Skipping ${job.name} because this is job is not falling under ${scopeFilter} stage" - else - echo "Triggering ${job.name} because this is a stable buildkite run" - ${Cmd.format trigger} - fi - '' - } - - in Cmd.quietly (merge pipelineHandlers selection) - ) - jobs + -> \(scopeFilter : PipelineScopeFilter.Type) + -> let requestedScopes = PipelineScopeFilter.scopes scopeFilter + + let requestedTags = PipelineTagFilter.tags tagFilter + + in Cmd.run + ( "./buildkite/scripts/monorepo.sh" + ++ " --scopes ${PipelineScope.join requestedScopes} " + ++ " --tags ${PipelineTag.join requestedTags} " + ++ " --filter-mode ${PipelineFilterMode.show filterMode} " + ++ " --selection ${PipelineJobSelection.show selection} " + ++ " --jobs ./buildkite/src/gen" + ++ " --git-diff-file _computed_diff.txt " + ) in \ ( args : { selection : PipelineJobSelection.Type @@ -135,26 +78,19 @@ in \ ( args Command.Config::{ , commands = prefixCommands - # commands - args.selection - args.tagFilter - args.filterMode - args.scopeFilter + # [ commands + args.selection + args.tagFilter + args.filterMode + args.scopeFilter + ] , label = "Monorepo triage ${PipelineTagFilter.show args.tagFilter} ${PipelineScopeFilter.show args.scopeFilter} ${PipelineJobSelection.capitalName args.selection}" , key = "cmds" - , target = Size.Multi - , docker = Some Docker::{ - , image = - (./Constants/ContainerImages.dhall).toolchainBase - , environment = - [ "BUILDKITE_AGENT_ACCESS_TOKEN" - , "BUILDKITE_INCREMENTAL" - ] - } + , target = Size.Dev } ] } diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index b772c74f641a..718af5bc7331 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -7,8 +7,6 @@ let Cmd = ./Lib/Cmds.dhall let Command = ./Command/Base.dhall -let Docker = ./Command/Docker/Type.dhall - let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall @@ -41,18 +39,12 @@ let config , Cmd.run "export BUILDKITE_PIPELINE_FILTER=${tagFilter}" , Cmd.run "export BUILDKITE_PIPELINE_SCOPE=${scopeFilter}" , Cmd.run "export BUILDKITE_PIPELINE_FILTER_MODE=${filterMode}" - , Cmd.run - "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" , Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { selection=(./buildkite/src/Pipeline/JobSelection.dhall).Type.${selection}, tagFilter=(./buildkite/src/Pipeline/TagFilter.dhall).Type.${tagFilter}, scopeFilter=(./buildkite/src/Pipeline/ScopeFilter.dhall).Type.${scopeFilter}, filterMode=(./buildkite/src/Pipeline/FilterMode.dhall).Type.${filterMode} }' | buildkite-agent pipeline upload" ] , label = "Prepare monorepo triage" , key = "monorepo-${selection}-${tagFilter}-${scopeFilter}" - , target = Size.Multi - , docker = Some Docker::{ - , image = (./Constants/ContainerImages.dhall).toolchainBase - , environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ] - } + , target = Size.Dev } ] } From dff1611c3a2c6ca6b9c5f17119d8e7049944f435 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 22:36:51 +0100 Subject: [PATCH 4/6] fix CI jobs --- buildkite/Makefile | 2 +- buildkite/scripts/dhall/checker.py | 2 +- buildkite/scripts/dhall/dump_dhall_to_pipelines.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/Makefile b/buildkite/Makefile index c04ac28062c2..75e236e6f2cd 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -27,7 +27,7 @@ format: dump_pipelines: $(eval TMP := $(shell mktemp -d)) - ./buildkite/scripts/dhall/dump_dhall_to_pipelines.sh ./buildkite/src "$(TMP)" + ./scripts/dhall/dump_dhall_to_pipelines.sh ./src "$(TMP)" check_deps: dump_pipelines python3 scripts/dhall/checker.py --root "$(TMP)" deps diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py index 50ad95acd8c7..cd4f0c518497 100755 --- a/buildkite/scripts/dhall/checker.py +++ b/buildkite/scripts/dhall/checker.py @@ -149,7 +149,7 @@ def keys(self): args = parser.parse_args() pipelinesInfo = [PipelineInfoBuilder(args.root, file).build() - for file in os.listdir(path=args.root)] + for file in os.listdir(path=args.root) if file.endswith(".yml")] if args.cmd == "deps": diff --git a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh index 52d1501fd348..8330c8566800 100755 --- a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh +++ b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Check for required arguments if [ "$#" -ne 2 ]; then From fea85b58defee0b41b7bb64c4779a06d1591bcd0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 22:43:26 +0100 Subject: [PATCH 5/6] fix shebang --- buildkite/scripts/dhall/dump_dhall_to_pipelines.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh index 8330c8566800..52d1501fd348 100755 --- a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh +++ b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Check for required arguments if [ "$#" -ne 2 ]; then From 3c66f82af111845660ae7875d06d40f80d96c57f Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 26 Nov 2025 22:53:22 +0100 Subject: [PATCH 6/6] remove parse_traige.py --- buildkite/unit-tests/parse_triage.py | 82 ---------------------------- 1 file changed, 82 deletions(-) delete mode 100755 buildkite/unit-tests/parse_triage.py diff --git a/buildkite/unit-tests/parse_triage.py b/buildkite/unit-tests/parse_triage.py deleted file mode 100755 index 5026261b4f34..000000000000 --- a/buildkite/unit-tests/parse_triage.py +++ /dev/null @@ -1,82 +0,0 @@ -import yaml -import sys -import re -import subprocess -import io - -# Generate YAML content from dhall-to-yaml command and load it directly - -tag_filters = [ - "FastOnly", - "Long", - "LongAndVeryLong", - "TearDownOnly", - "ToolchainsOnly", - "AllTests", - "Release", - "Promote", - "DebianBuild", - "DockerBuild" -] - -yaml_contents = [] -for tag in tag_filters: - dhall_cmd = f"""dhall-to-yaml --quoted <<< '(./src/Monorepo.dhall) {{ selection=(./src/Pipeline/JobSelection.dhall).Type.Full, tagFilter=(./src/Pipeline/TagFilter.dhall).Type.{tag}, filterMode=(./src/Pipeline/FilterMode.dhall).Type.Any, scopeFilter=(./src/Pipeline/ScopeFilter.dhall).Type.All }}'""" - result = subprocess.run(dhall_cmd, shell=True, capture_output=True, text=True, executable="/bin/bash") - if result.returncode != 0: - print(f"Failed to generate YAML from dhall-to-yaml for tagFilter {tag}") - print(result.stderr) - sys.exit(1) - yaml_contents.append(yaml.safe_load(io.StringIO(result.stdout))) - -job_names_hit = [] -missing_job_names = [] - -for yaml_content in yaml_contents: - steps = yaml_content.get('steps', []) - - for step in steps: - commands = step.get('commands', []) - block = [] - - for command in commands: - if isinstance(command, str) and '\n' in command: - lines = command.splitlines() - temp_block = [] - for line in lines: - if line.strip() == '': - # End of a logical block - block_text = '\n'.join(temp_block) - if "dhall-to-yaml --quoted" in block_text: - block_text = re.sub(r'dhall-to-yaml --quoted.*', '', block_text) - temp_block = [] - else: - temp_block.append(line) - # Catch any remaining block - if temp_block: - block_text = '\n'.join(temp_block) - if "dhall-to-yaml --quoted" in block_text: - block_text = re.sub(r'dhall-to-yaml --quoted.*', '', block_text) - result = subprocess.run(block_text, shell=True, capture_output=True, text=True, executable="/bin/bash") - if result.stdout.startswith("Triggering"): - match = re.search(r'Triggering (\w+)', result.stdout) - if match: - job_names_hit.append(match.group(1)) - if result.stdout.startswith("Skipping"): - match = re.search(r'Skipping (\w+)', result.stdout) - if match: - missing_job_names.append(match.group(1)) - if result.stderr: - print("== Errors ==") - print(result.stderr) - -# Remove any intersected items from missing_job_names -missing_job_names = [job for job in missing_job_names if job not in job_names_hit] - -if not missing_job_names: - print("All jobs were found.") -else: - print("Some jobs are missing:") - for job in missing_job_names: - print(f"- {job}") - sys.exit(1) \ No newline at end of file