re-write CI analysis command, separate steps/jobs, tests

.github/workflows/framework-cli-tests.yml

@@ -0,0 +1,56 @@
+name: Framework CLI Tests
+on:
+  push:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-framework-cli
+  cancel-in-progress: true
+jobs:
+  framework-component-tests:
+    defaults:
+      run:
+        working-directory: framework/cmd
+    env:
+      CTF_JD_IMAGE: ${{ secrets.CTF_JD_IMAGE }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.PUBLIC_AWS_ECR_ROLE }}
+          aws-region: us-east-1
+      - name: Authenticate to ECR Public
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+        with:
+          registry-type: public
+      - name: Check for changes in Framework
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'framework/cmd/**'
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}-framework
+          restore-keys: |
+            go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}-framework
+      - name: Install dependencies
+        run: go mod download
+      - name: Run Docker Component Tests
+        if: steps.changes.outputs.src == 'true'
+        run: |
+          go test -race -timeout 2m -v -count 1 -run TestCLI

.nancy-ignore

@@ -15,4 +15,5 @@ CVE-2024-10086 # Improper Neutralization of Input During Web Page Generation ('C
 CVE-2024-51744 # CWE-755: Improper Handling of Exceptional Conditions
 CVE-2024-45338 # CWE-770: Allocation of Resources Without Limits or Throttling
 CVE-2024-45337 # CWE-863: Incorrect Authorization in golang.org/x/crypto@v0.28.0
-CVE-2025-24883 # CWE-248: Uncaught Exception
+CVE-2025-24883 # CWE-248: Uncaught Exception
+CVE-2025-22869 # CWE-770: Allocation of Resources Without Limits or Throttling

book/src/framework/components/analyze_ci.md

@@ -2,14 +2,36 @@
 
 We offer a straightforward CLI tool designed to analyze CI runs, focusing on Jobs and Steps, to provide deeper insights into system-level tests.
 
+![img_1.png](img_1.png)
+
+## What to look for
+
+Check execution times, results are sorted by `median`.
+
+We exclude any runs without `conclusion` or if they have `conclusion: "skipped"`.
+
+🔄 - if you see high numbers here it means your pipeline is flaky, and it does not pass from the first attempt on some commits. It may indicate test flakiness!
+
+🚫 - if numbers are high there it means your pipeline is constantly `cancelled` so it wastes time without bringing any results/value!
+
 ## Examples
 ```
 # GITHUB_TOKEN must have access to "actions" API
 export GITHUB_TOKEN=...
 
-# E2E tests from core, the last day
-ctf ci -r "smartcontractkit/chainlink" -w "Integration Tests"
+# E2E test jobs from core for the last day
+ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs
+
+# E2E test steps from core for the last day
+ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t steps
+
+# E2E tests from core for the last week (1k jobs limit!)
+ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs -s 7
+
+# E2E tests from core for some specific days from 5 days ago to 3 days ago
+ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs -s 5 -e 3
 
 # Last 3 days runs for e2e framework tests
-ctf ci -r "smartcontractkit/chainlink-testing-framework" -w "Framework Golden Tests Examples" -d 3
+ctf ci -r "smartcontractkit/chainlink-testing-framework" -w "Framework Golden Tests Examples" -t jobs -s 3
 ```
+You can also use `-debug` flag to dump all the workflow runs/jobs to `ctf-ci-debug` folder.

framework/cmd/blockscout.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"fmt"
-	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"os"
 	"path/filepath"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 )
 
 func blockscoutUp(url string) error {
@@ -35,12 +36,11 @@ func blockscoutDown(url string) error {
 	if err != nil {
 		return err
 	}
-	err = runCommand("bash", "-c", fmt.Sprintf(`
+	return runCommand("bash", "-c", fmt.Sprintf(`
 		cd %s && \
 		rm -rf blockscout-db-data && \
 		rm -rf logs && \
 		rm -rf redis-data && \
 		rm -rf stats-db-data
 	`, filepath.Join("blockscout", "services")))
-	return nil
 }