Merge pull request #321 from smvv/try-event-path

Compute job matrix distribution using extbuild matrix

Author: samansmink

.github/workflows/_extension_distribution.yml

@@ -197,8 +197,72 @@ jobs:
           ref: ${{ inputs.ci_tools_version }}
           repository: ${{ inputs.override_ci_tools_repository }}
 
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: extension-ci-tools/scripts/extbuild/go.mod
+          cache: true
+          cache-dependency-path: extension-ci-tools/scripts/extbuild/go.sum
+
+      - name: Dump event path file
+        run: |
+          echo "$GITHUB_EVENT_PATH"
+          cat "$GITHUB_EVENT_PATH"
+
+      - name: Compute extension build matrix
+        continue-on-error: true
+        run: |
+          make -C extension-ci-tools/scripts/extbuild test build -sj4
+          mkdir -p build
+          extension-ci-tools/scripts/extbuild/build/extbuild matrix \
+            --input extension-ci-tools/config/distribution_matrix.json \
+            --exclude "${{ inputs.exclude_archs }}" \
+            --opt-in "${{ inputs.opt_in_archs }}" \
+            --reduced-ci-mode "${{ inputs.reduced_ci_mode }}" \
+            --out build/extbuild_matrix.env
+          echo "extbuild matrix output:"
+          cat build/extbuild_matrix.env
+
+      - name: Compare extbuild vs python matrices
+        continue-on-error: true
+        run: |
+          mkdir -p build/python_ref build/extbuild_ref
+
+          python3 extension-ci-tools/scripts/modify_distribution_matrix.py --input extension-ci-tools/config/distribution_matrix.json --select_os linux   --output build/python_ref/linux_matrix.json   --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
+          python3 extension-ci-tools/scripts/modify_distribution_matrix.py --input extension-ci-tools/config/distribution_matrix.json --select_os osx     --output build/python_ref/osx_matrix.json     --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
+          python3 extension-ci-tools/scripts/modify_distribution_matrix.py --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/python_ref/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
+          python3 extension-ci-tools/scripts/modify_distribution_matrix.py --input extension-ci-tools/config/distribution_matrix.json --select_os wasm    --output build/python_ref/wasm_matrix.json    --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
+
+          for platform in linux osx windows wasm; do
+            grep "^${platform}_matrix=" build/extbuild_matrix.env | sed "s/^${platform}_matrix=//" > "build/extbuild_ref/${platform}_matrix.json"
+          done
+
+          has_diff=0
+          for platform in linux osx windows wasm; do
+            echo "=== ${platform} matrix diff (extbuild vs python) ==="
+            if ! diff -u \
+              <(jq -S 'walk(if type == "object" then del(.opt_in, .run_in_reduced_ci_mode) else . end)' "build/extbuild_ref/${platform}_matrix.json") \
+              <(jq -S 'walk(if type == "object" then del(.opt_in, .run_in_reduced_ci_mode) else . end)' "build/python_ref/${platform}_matrix.json"); then
+              has_diff=1
+            else
+              echo "No differences."
+            fi
+            echo
+          done
+
+          if [ "$has_diff" -ne 0 ]; then
+            echo "Matrix differences detected."
+            echo "::warning title=Matrix differences detected::extbuild and python matrix outputs differ. See diff logs in this step."
+            {
+              echo "### Matrix differences detected"
+              echo "extbuild and python matrix outputs differ. See the diff logs in this step."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+
       - id: parse-matrices
         run: |
+          rm -rf build
           mkdir build
           python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux   --output build/linux_matrix.json   --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}
           python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx     --output build/osx_matrix.json     --exclude "${{ inputs.exclude_archs }}" --opt_in "${{ inputs.opt_in_archs }}" --pretty --reduced_ci_mode ${{ inputs.reduced_ci_mode }}

scripts/extbuild/.gitignore

@@ -0,0 +1,2 @@
+build/
+coverage.out

scripts/extbuild/Makefile

@@ -0,0 +1,21 @@
+GO_TEST_FLAGS ?= -coverprofile=coverage.out -timeout 2s
+
+.PHONY: test cov build clean
+
+test:
+	go test ${GO_TEST_FLAGS} ./...
+
+cov: test
+	go tool cover -html=coverage.out
+
+build:
+	mkdir -p build
+	go build -o build/extbuild ./cmd/extbuild
+
+clean:
+	rm -rf build
+
+.PHONY: tidy
+
+tidy:
+	go mod tidy

scripts/extbuild/README.md

@@ -0,0 +1,34 @@
+# Extbuild
+
+A small CLI tool for iterating quickly on DuckDB extension CI pipelines.
+
+For now, it can compute the job matrix. Later, the tool can be extended to
+generate the command list used.
+
+# Development
+
+## Install dependencies
+
+```shell
+brew install go
+```
+
+## Build extbuild
+
+Build and test `extbuild`:
+
+```shell
+make build test -sj4
+```
+
+or from the repo root directory:
+
+```shell
+make -C scripts/extbuild build test -sj4
+```
+
+## How to use
+
+The target `build` creates `./scripts/extbuild/build/extbuild`.
+
+See `./scripts/extbuild/build/extbuild --help` for local usage and subcommands.

scripts/extbuild/cmd/extbuild/main.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	if err := newRootCommand().Execute(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

scripts/extbuild/cmd/extbuild/matrix.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/duckdb/extension-ci-tools/internal/distmatrix"
+	"github.com/spf13/cobra"
+)
+
+func newMatrixCommand() *cobra.Command {
+	var (
+		inputPath     string
+		platformsRaw  string
+		archsRaw      string
+		excludeRaw    string
+		optInRaw      string
+		reducedCIMode string
+		outPath       string
+	)
+
+	cmd := &cobra.Command{
+		Use:   "matrix",
+		Short: "Compute distribution matrices and emit GitHub output lines",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			data, err := os.ReadFile(inputPath)
+			if err != nil {
+				return fmt.Errorf("read input matrix %q: %w", inputPath, err)
+			}
+			matrix, err := distmatrix.ParseMatrixFile(data)
+			if err != nil {
+				return fmt.Errorf("parse input matrix %q: %w", inputPath, err)
+			}
+
+			result, err := distmatrix.ComputePlatformMatrices(matrix, distmatrix.ComputeOptions{
+				Platform:      platformsRaw,
+				Arch:          archsRaw,
+				Exclude:       excludeRaw,
+				OptIn:         optInRaw,
+				ReducedCIMode: distmatrix.ReducedCIMode(reducedCIMode),
+			})
+			if err != nil {
+				return fmt.Errorf("compute platform matrices: %w", err)
+			}
+
+			content, err := distmatrix.RenderGitHubOutputLines(result, distmatrix.MachineReadable)
+			if err != nil {
+				return fmt.Errorf("render GitHub output lines: %w", err)
+			}
+			readable, err := distmatrix.RenderGitHubOutputLines(result, distmatrix.HumanReadable)
+			if err != nil {
+				return fmt.Errorf("render readable output: %w", err)
+			}
+
+			if outPath != "" {
+				if err := os.WriteFile(outPath, []byte(content), 0o644); err != nil {
+					return fmt.Errorf("write output file %q: %w", outPath, err)
+				}
+			}
+
+			_, _ = fmt.Fprint(cmd.OutOrStdout(), readable)
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&inputPath, "input", "config/distribution_matrix.json", "Input distribution matrix JSON file")
+	cmd.Flags().StringVar(&platformsRaw, "platform", "", "Comma-separated list of platforms")
+	cmd.Flags().StringVar(&archsRaw, "arch", "", "Comma-separated list of arch tokens (amd64;arm64)")
+	cmd.Flags().StringVar(&excludeRaw, "exclude", "", "Comma-separated list of duckdb_arch values to exclude")
+	cmd.Flags().StringVar(&optInRaw, "opt-in", "", "Comma-separated list of opt-in duckdb_arch values")
+	cmd.Flags().StringVar(&reducedCIMode, "reduced-ci-mode", "", "Reduced CI mode: auto|enabled|disabled")
+	cmd.Flags().StringVar(&outPath, "out", "", "Path to write GitHub output lines")
+
+	return cmd
+}