test(skills-generator): cache remote schema across parameterized cases #2229

Workflow file for this run

	name: CI Builds

	on:
	push:
	branches:
	- main
	pull_request:

	jobs:
	detect-validation-scope:
	name: Detect Validation Scope
	runs-on: ubuntu-latest
	outputs:
	markdown_changed: ${{ steps.detect.outputs.markdown_changed }}
	generated_skills_changed: ${{ steps.detect.outputs.generated_skills_changed }}
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	submodules: true
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'
	- name: Detect validation scope
	id: detect
	shell: bash
	run: \|
	set -euo pipefail

	resolve_skills_module() {
	if grep -q '<module>plinth-skills-generator</module>' pom.xml; then
	echo plinth-skills-generator
	else
	echo skills-generator
	fi
	}

	if [ "${{ github.event_name }}" = "pull_request" ]; then
	BASE_SHA="${{ github.event.pull_request.base.sha }}"
	else
	BASE_SHA="${{ github.event.before }}"
	fi

	if [ -z "$BASE_SHA" ] \|\| [[ "$BASE_SHA" =~ ^0+$ ]]; then
	BASE_SHA="$(git rev-list --max-parents=0 HEAD)"
	fi

	HEAD_SHA="$(git rev-parse HEAD)"

	if git diff --name-only "$BASE_SHA" "$HEAD_SHA" \| grep -qE '(^\|/)[^/]+\.md$'; then
	echo "markdown_changed=true" >> "$GITHUB_OUTPUT"
	else
	echo "markdown_changed=false" >> "$GITHUB_OUTPUT"
	fi

	git checkout "$BASE_SHA"
	./mvnw --batch-mode --no-transfer-progress clean install -pl "$(resolve_skills_module)" -am
	cp -R .agents/skills /tmp/base-skills

	git checkout "$HEAD_SHA"
	./mvnw --batch-mode --no-transfer-progress clean install -pl "$(resolve_skills_module)" -am
	cp -R .agents/skills /tmp/head-skills

	if diff -qr /tmp/base-skills /tmp/head-skills >/dev/null; then
	echo "generated_skills_changed=false" >> "$GITHUB_OUTPUT"
	else
	echo "generated_skills_changed=true" >> "$GITHUB_OUTPUT"
	fi

	validate-markdown:
	name: Validate Markdown Files
	needs: detect-validation-scope
	if: needs.detect-validation-scope.outputs.markdown_changed == 'true'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true # Fetches all submodules
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm' # See 'Supported distributions' for available options
	java-version: '25'
	- name: Install JBang
	run: \|
	curl -Ls https://sh.jbang.dev \| bash -s - app setup
	echo "$HOME/.jbang/bin" >> $GITHUB_PATH
	- name: Validate Markdown Files
	run: jbang markdown-validator/src/main/java/info/jab/mv/MarkdownValidator.java .

	validate-skill-check:
	name: Validate Agent Skills with skill-check
	needs: detect-validation-scope
	if: needs.detect-validation-scope.outputs.generated_skills_changed == 'true'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'
	- name: Generate current agent skills
	run: ./mvnw --batch-mode --no-transfer-progress clean verify -pl plinth-skills-generator -am
	- name: Validate generated SKILL.md files
	run: npx skill-check@latest .agents/skills --no-security-scan --format github
	env:
	SKILL_CHECK_NO_BANNER: 1

	validate-skill-scanner:
	name: Validate Agent Skills with Skill Scanner
	needs: detect-validation-scope
	if: needs.detect-validation-scope.outputs.generated_skills_changed == 'true'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'
	- name: Generate current agent skills
	run: ./mvnw --batch-mode --no-transfer-progress clean verify -pl plinth-skills-generator -am
	- uses: actions/setup-python@v7
	with:
	python-version: "3.12"
	- name: Install Skill Scanner
	run: python -m pip install --upgrade cisco-ai-skill-scanner
	- name: Scan generated skills in .agents/skills with Skill Scanner
	run: skill-scanner scan-all .agents/skills --recursive --use-behavioral --policy strict --fail-on-severity high

	validate-skillspector:
	name: Validate Agent Skills with SkillSpector
	needs: detect-validation-scope
	if: needs.detect-validation-scope.outputs.generated_skills_changed == 'true'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'
	- name: Generate current agent skills
	run: ./mvnw --batch-mode --no-transfer-progress clean verify -pl plinth-skills-generator -am
	- uses: actions/setup-python@v7
	with:
	python-version: "3.12"
	- name: Install SkillSpector
	run: python -m pip install "git+https://github.com/NVIDIA/SkillSpector.git"
	- name: Generate SkillSpector Markdown report
	run: skillspector scan .agents/skills --no-llm --verbose --format markdown --output skillspector-report.md
	- name: Display SkillSpector report
	if: always()
	run: \|
	if [ -f skillspector-report.md ]; then
	cat skillspector-report.md
	else
	echo "SkillSpector Markdown report was not generated."
	fi
	- name: Upload SkillSpector reports
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: skillspector-reports
	path: skillspector-report.md
	if-no-files-found: error

	validate-snyk-agent-scan:
	name: Validate Agent Skills with Snyk Agent Scan
	needs: detect-validation-scope
	if: >
	needs.detect-validation-scope.outputs.generated_skills_changed == 'true' &&
	(github.event_name == 'push' \|\| github.event.pull_request.head.repo.full_name == github.repository)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'
	- name: Generate current agent skills
	run: ./mvnw --batch-mode --no-transfer-progress clean verify -pl plinth-skills-generator -am
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: false
	- name: Verify SNYK_TOKEN is configured
	run: \|
	if [ -z "${SNYK_TOKEN}" ]; then
	echo "SNYK_TOKEN secret is required to run Snyk Agent Scan."
	exit 1
	fi
	env:
	SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
	- name: Scan generated skills with Snyk Agent Scan
	run: \|
	set +e
	uvx snyk-agent-scan@latest scan .agents/skills \
	--ci \
	--no-bootstrap \
	--dangerously-run-mcp-servers \
	--suppress-mcpserver-io=true \
	--verbose \
	--print-errors \
	--print-full-descriptions \
	2>&1 \| tee snyk-agent-scan-report.txt
	scan_status=${PIPESTATUS[0]}
	set -e

	if [ "${scan_status}" -eq 0 ]; then
	exit 0
	fi

	if grep -Eq "X007\|Could not reach analysis server\|API timeout while scanning discovered servers" snyk-agent-scan-report.txt; then
	echo "::warning title=Snyk Agent Scan unavailable::Snyk Agent Scan could not reach the analysis server. Treating this infrastructure timeout as non-blocking; review the uploaded report."
	exit 0
	fi

	exit "${scan_status}"
	env:
	SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
	- name: Upload Snyk Agent Scan report
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: snyk-agent-scan-report
	path: snyk-agent-scan-report.txt
	if-no-files-found: ignore

	pre-commit:
	name: Pre-commit
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- uses: actions/setup-python@v7
	with:
	python-version: "3.x"
	- name: Install pre-commit
	run: pip install pre-commit
	- name: Run pre-commit (all files)
	run: pre-commit run --all-files
	- name: Validate HEAD commit message
	run: \|
	git log -1 --pretty=%B > /tmp/commit-msg.txt
	pre-commit run conventional-pre-commit --hook-stage commit-msg --commit-msg-filename /tmp/commit-msg.txt

	validate-virustotal-skills:
	name: Validate Agent Skills with VirusTotal
	needs: detect-validation-scope
	if: >
	needs.detect-validation-scope.outputs.generated_skills_changed == 'true' &&
	(github.event_name == 'push' \|\| github.event.pull_request.head.repo.full_name == github.repository)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true

	- uses: actions/setup-java@v5
	with:
	distribution: 'graalvm'
	java-version: '25'

	- name: Generate current agent skills
	run: ./mvnw --batch-mode --no-transfer-progress clean verify -pl plinth-skills-generator -am

	- name: Verify VT_API_KEY is configured
	run: \|
	if [ -z "${VT_API_KEY}" ]; then
	echo "VT_API_KEY secret is required to run VirusTotal scan."
	exit 1
	fi
	env:
	VT_API_KEY: ${{ secrets.VT_API_KEY }}

	- name: Package generated skills
	run: zip -r generated-agent-skills.zip .agents/skills

	- name: Upload skills archive to VirusTotal
	run: \|
	set -euo pipefail

	RESPONSE="$(curl --fail --silent --show-error \
	--request POST \
	--url https://www.virustotal.com/api/v3/files \
	--header "x-apikey: ${VT_API_KEY}" \
	--form "file=@generated-agent-skills.zip")"

	echo "$RESPONSE" \| jq . > virustotal-upload-response.json
	ANALYSIS_ID="$(echo "$RESPONSE" \| jq -r '.data.id')"

	echo "VirusTotal analysis id: $ANALYSIS_ID"
	echo "ANALYSIS_ID=$ANALYSIS_ID" >> "$GITHUB_ENV"
	env:
	VT_API_KEY: ${{ secrets.VT_API_KEY }}

	- name: Wait for VirusTotal analysis result
	run: \|
	set -euo pipefail

	for i in {1..50}; do
	RESPONSE="$(curl --fail --silent --show-error \
	--request GET \
	--url "https://www.virustotal.com/api/v3/analyses/${ANALYSIS_ID}" \
	--header "x-apikey: ${VT_API_KEY}")"

	echo "$RESPONSE" \| jq . > virustotal-analysis-response.json

	STATUS="$(echo "$RESPONSE" \| jq -r '.data.attributes.status')"
	echo "VirusTotal status: $STATUS"

	if [ "$STATUS" = "completed" ]; then
	MALICIOUS="$(echo "$RESPONSE" \| jq -r '.data.attributes.stats.malicious')"
	SUSPICIOUS="$(echo "$RESPONSE" \| jq -r '.data.attributes.stats.suspicious')"

	echo "Malicious: $MALICIOUS"
	echo "Suspicious: $SUSPICIOUS"

	if [ "$MALICIOUS" -gt 0 ] \|\| [ "$SUSPICIOUS" -gt 0 ]; then
	echo "VirusTotal detected malicious or suspicious results."
	exit 1
	fi

	exit 0
	fi

	sleep 20
	done

	echo "VirusTotal analysis did not complete in time."
	exit 1
	env:
	VT_API_KEY: ${{ secrets.VT_API_KEY }}

	- name: Upload VirusTotal reports
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: virustotal-skills-report
	path: \|
	virustotal-upload-response.json
	virustotal-analysis-response.json
	if-no-files-found: ignore

	package-agent-artifacts:
	name: Package Commands, Agents, and Skills
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	submodules: true
	- name: Create commands archive
	run: cd .cursor && zip -r ../commands.zip commands
	- name: Create agents archive
	run: cd .cursor && zip -r ../agents.zip agents
	- name: Create skills archive
	run: zip -r skills.zip skills
	- name: Upload commands.zip
	uses: actions/upload-artifact@v7
	with:
	name: commands.zip
	path: commands.zip
	if-no-files-found: error
	- name: Upload agents.zip
	uses: actions/upload-artifact@v7
	with:
	name: agents.zip
	path: agents.zip
	if-no-files-found: error
	- name: Upload skills.zip
	uses: actions/upload-artifact@v7
	with:
	name: skills.zip
	path: skills.zip
	if-no-files-found: error

Navigation Menu

Uh oh!

test(skills-generator): cache remote schema across parameterized cases #2229

Workflow file

test(skills-generator): cache remote schema across parameterized cases #2229

Uh oh!

Workflow file for this run