simplify filter helper structs

buildkite/Makefile

@@ -25,16 +25,9 @@ format:
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255'
 	$(MAKE) convert_ifs_to_backticks
 
-check_filter:
-	# Ensure Jobs.dhall is generated before proceeding
-	(cd ../ && ./buildkite/scripts/generate-jobs.sh > ./buildkite/src/gen/Jobs.dhall)
-	python3 ./unit-tests/parse_triage.py
-	# Clean up
-	git checkout -- ./src/gen/Jobs.dhall
-
 dump_pipelines:
 	$(eval TMP := $(shell mktemp -d))
-	scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)"
+	./scripts/dhall/dump_dhall_to_pipelines.sh ./src "$(TMP)"
 
 check_deps: dump_pipelines
 	python3 scripts/dhall/checker.py --root "$(TMP)" deps
@@ -48,4 +41,4 @@ check_dups: dump_pipelines
 check_names: dump_pipelines
 	python3 scripts/dhall/checker.py --root "$(TMP)" names
 
-all: check_syntax lint format check_deps check_dirty check_dups check_names check_filter
+all: check_syntax lint format check_deps check_dirty check_dups check_names

buildkite/scripts/dhall/checker.py

@@ -149,7 +149,7 @@ def keys(self):
 args = parser.parse_args()
 
 pipelinesInfo = [PipelineInfoBuilder(args.root, file).build()
-                 for file in os.listdir(path=args.root)]
+                 for file in os.listdir(path=args.root) if file.endswith(".yml")]
 
 if args.cmd == "deps":
 

buildkite/scripts/dhall/dump_dhall_to_pipelines.sh

@@ -1,8 +1,16 @@
 #!/bin/bash
 
+# Check for required arguments
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <dhall_root_dir> <output_dir>"
+    exit 1
+fi
+
 ROOT=$1
 OUTPUT=$2
 
+
+
 mkdir -p "$OUTPUT"
 
 shopt -s globstar nullglob
@@ -11,13 +19,16 @@ echo "Dumping pipelines from '$ROOT' to '$OUTPUT'"
 
 COUNTER=0
 
-for file in "$ROOT"/**/*.dhall
+for file in "$ROOT"/Jobs/**/*.dhall
 do
     filename=$(basename "$file")
     filename="${filename%.*}"
 
     dhall-to-yaml --quoted --file "$file" > "$OUTPUT"/"$filename".yml
 
+    dhall-to-yaml <<< "let SelectFiles = $ROOT/Lib/SelectFiles.dhall in SelectFiles.compile (./$file).spec.dirtyWhen " > "$OUTPUT"/"$filename".dirtywhen
+
+
     COUNTER=$((COUNTER+1))
 done
 

buildkite/scripts/monorepo.sh

@@ -0,0 +1,222 @@
+#!/bin/bash
+
+# Usage:
+# monorepo.sh --scopes <scopes> --tags <tags> --filter-mode <filter_mode> --selection <selection> --jobs <jobs>
+
+set -euo pipefail
+
+# Default values
+SCOPES=""
+TAGS=""
+FILTER_MODE=""
+SELECTION=""
+JOBS=""
+GIT_DIFF_FILE=""
+DRY_RUN=false
+
+# Parse CLI arguments
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  case $key in
+    --scopes)
+      SCOPES="$2"
+      shift; shift
+      ;;
+    --tags)
+      TAGS="$2"
+      shift; shift
+      ;;
+    --filter-mode)
+      FILTER_MODE="$2"
+      shift; shift
+      ;;
+    --selection)
+      SELECTION="$2"
+      shift; shift
+      ;;
+    --jobs)
+      JOBS="$2"
+      shift; shift
+      ;;
+    --git-diff-file)
+      GIT_DIFF_FILE="$2"
+      shift; shift
+      ;;
+    --dry-run)
+      DRY_RUN=true
+      shift
+      ;;
+    --debug)
+      set -x
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+
+# Require all arguments
+if [[ -z "$SCOPES" || -z "$TAGS" || -z "$FILTER_MODE" || -z "$SELECTION" || -z "$JOBS" || -z "$GIT_DIFF_FILE" ]]; then
+  echo "Error: All arguments --scopes, --tags, --filter-mode, --selection, --jobs, and --git-diff-file are required."
+  exit 1
+fi
+
+# Check if yq is installed, if not install it
+if ! command -v yq &> /dev/null; then
+  echo "yq not found, installing..."
+  if command -v apt-get &> /dev/null; then
+    sudo apt-get update && sudo apt-get install -y wget
+    sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
+    sudo chmod +x /usr/local/bin/yq
+  else
+    echo "Error: yq is not installed and automatic installation is not supported on this system. Please install yq manually."
+    exit 1
+  fi
+fi
+
+IFS=',' read -r -a DESIRED_TAGS <<< "$TAGS"
+IFS=',' read -r -a DESIRED_SCOPES <<< "$SCOPES"
+
+# Set filter flag based on FILTER_MODE
+FILTER_ANY=false
+FILTER_ALL=false
+if [[ "$FILTER_MODE" == "any" ]]; then
+  FILTER_ANY=true
+elif [[ "$FILTER_MODE" == "all" ]]; then
+  FILTER_ALL=true
+else
+  echo "Error: --filter-mode must be 'any' or 'all'."
+  exit 1
+fi
+
+# Validate SELECTION value
+
+# Set selection flags
+SELECTION_TRIAGED=false
+SELECTION_FULL=false
+if [[ "$SELECTION" == "triaged" ]]; then
+  SELECTION_TRIAGED=true
+elif [[ "$SELECTION" == "full" ]]; then
+  SELECTION_FULL=true
+else
+  echo "Error: --selection must be 'triaged' or 'full'."
+  exit 1
+fi
+# Find all files in jobs and use yq to get .pipeline.spec.tags
+
+has_matching_tags() {
+  local tags="$1"
+  local filter_any="$2"
+  local filter_all="$3"
+  shift 3
+  local desired_tags=("$@")
+
+  local match_count=0
+
+
+
+
+  for want in "${desired_tags[@]}"; do
+
+    if WANT="$want" \
+       yq -e '.[] | select((downcase) == (env(WANT) | downcase))' \
+       <<< "$tags" \
+       >/dev/null 2>&1
+    then
+      match_count=$((match_count+1))
+    fi
+  done
+
+  if $filter_any && [[ $match_count -ge 1 ]]; then
+    echo 1
+  elif $filter_all && [[ $match_count -eq ${#desired_tags[@]} ]]; then
+    echo 1
+  else
+    echo 0
+  fi
+}
+
+scope_matches() {
+  local scope="$1"
+  shift
+  local desired_scopes=("$@")
+  local match_count=0
+
+ for want in "${desired_scopes[@]}"; do
+    # yq v4 NIE ma --arg, więc używamy env(WANT)
+    if WANT="$want" \
+       yq -e '.[] | select((downcase) == (env(WANT) | downcase))' \
+       <<< "$scope" \
+         >/dev/null 2>&1
+    then
+      match_count=$((match_count+1))
+      break
+    fi
+  done
+
+  echo $(( match_count == 1 ? 1 : 0 ))
+}
+
+select_job() {
+  local selection_full="$1"
+  local selection_triaged="$2"
+  local file="$3"
+  local job_name="$4"
+  local git_diff_file="$5"
+
+  if [[ "$selection_full" == true ]]; then
+    echo 1
+  elif [[ "$selection_triaged" == true ]]; then
+    dirtyWhen=$(cat "${file%.yml}.dirtywhen")
+    # Remove quotes from beginning and end of string
+    dirtyWhen="${dirtyWhen%\"}"
+    dirtyWhen="${dirtyWhen#\"}"
+    if cat "$git_diff_file" | grep -E "$dirtyWhen" > /dev/null; then
+      echo 1
+    else
+      echo 0
+    fi
+  fi
+}
+
+find "$JOBS" -type f -name "*.yml" | while read -r file; do
+  tags=$(yq .spec.tags "$file")
+  scope=$(yq .spec.scope "$file")
+  job_name=$(yq -r .spec.name "$file")
+
+  tag_match=$(has_matching_tags "$tags" "$FILTER_ANY" "$FILTER_ALL" "${DESIRED_TAGS[@]}")
+
+  scope_match=$(scope_matches "$scope" "${DESIRED_SCOPES[@]}")
+
+  if [[ $tag_match -ne 1 ]]; then
+    echo "🏷️🚫 $job_name rejected job due to tags mismatch: $file" >&2
+    continue
+  fi
+  if [[ $scope_match -ne 1 ]]; then
+    echo "🔭🚫 $job_name rejected job due to scope mismatch: $file" >&2
+    continue
+  fi
+
+  job_selected=$(select_job "$SELECTION_FULL" "$SELECTION_TRIAGED" "$file" "$job_name" "$GIT_DIFF_FILE")
+
+  if [[ $job_selected -ne 1 ]]; then
+    echo "🧹🚫 $job_name rejected job as it does not fall into dirty when: $file" >&2
+    continue
+  fi
+
+  echo "✅ Including job $job_name in build "
+
+  if [[ "$DRY_RUN" == true ]]; then
+      printf " -> 🛑 Dry run enabled, skipping upload for job: %s\n" "$job_name"
+  else
+    job_path=$(yq -r .spec.path "$file")
+
+    dhall-to-yaml --quoted <<< "(./buildkite/src/Jobs/$job_path/$job_name.dhall).pipeline" | buildkite-agent pipeline upload
+    printf " -> ✅ Uploaded job: %s\n" "$job_name"
+  fi
+
+done
+

buildkite/src/Command/Base.dhall

@@ -133,6 +133,7 @@ let targetToAgent =
             , Perf = toMap { size = "performance" }
             , Multi = toMap { size = "generic-multi" }
             , Arm64 = toMap { size = "arm64-static" }
+            , Dev = toMap { size = "development-0" }
             }
             target
 

buildkite/src/Command/DockerImage.dhall

@@ -162,6 +162,7 @@ let generateStep =
                       , QA = "--filter until=24h"
                       , Multi = "--filter until=24h"
                       , Perf = "--filter until=24h"
+                      , Dev = "--filter until=24h"
                       }
                       spec.size
 

buildkite/src/Command/Size.dhall

@@ -1 +1,11 @@
-< XLarge | Large | Medium | Small | Integration | QA | Multi | Perf | Arm64 >
+< XLarge
+| Large
+| Medium
+| Small
+| Integration
+| QA
+| Multi
+| Perf
+| Arm64
+| Dev
+>

buildkite/src/Jobs/Lint/Dhall.dhall

@@ -21,7 +21,7 @@ let dump_pipelines_cmd =
         Cmd.Docker::{
         , image = (../../Constants/ContainerImages.dhall).toolchainBase
         }
-        "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines"
+        "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh ./buildkite/src/ _pipelines"
 
 in  Pipeline.build
       Pipeline.Config::{