refactor(action.yml, README.md, workflows, scripts): transition from … #116

Workflow file for this run

	name: Test Action

	# This workflow tests the Usable PR Validator action
	# - Lints and validates scripts, templates, and documentation
	# - Runs integration tests on PRs using the action itself
	# - Demonstrates multi-stage validation with separate comments

	on:
	push:
	branches: [main, develop]
	pull_request:
	branches: [main, develop]
	workflow_dispatch:

	jobs:
	lint-scripts:
	name: Lint Shell Scripts
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Run ShellCheck
	uses: ludeeus/action-shellcheck@master
	with:
	scandir: './scripts'
	severity: warning

	validate-action:
	name: Validate action.yml
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Validate Action Metadata
	run: \|
	# Check if action.yml exists
	if [ ! -f "action.yml" ]; then
	echo "::error::action.yml not found"
	exit 1
	fi

	# Validate YAML syntax
	if ! command -v yq &> /dev/null; then
	echo "Installing yq..."
	sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
	sudo chmod +x /usr/local/bin/yq
	fi

	yq eval '.' action.yml > /dev/null
	echo "✅ action.yml is valid YAML"

	# Check required fields
	if ! yq eval '.name' action.yml \| grep -q "Usable PR Validator"; then
	echo "::error::Missing or invalid 'name' field"
	exit 1
	fi

	echo "✅ All required fields present"

	test-script-execution:
	name: Test Script Execution
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Check Script Permissions
	run: \|
	for script in scripts/*.sh; do
	if [ ! -x "$script" ]; then
	echo "::error::$script is not executable"
	exit 1
	fi
	echo "✅ $script is executable"
	done

	- name: Test Script Syntax
	run: \|
	for script in scripts/*.sh; do
	bash -n "$script"
	echo "✅ $script syntax is valid"
	done

	test-templates:
	name: Validate Templates
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Check Templates Exist
	run: \|
	if [ ! -d "templates" ]; then
	echo "::error::templates directory not found"
	exit 1
	fi

	if [ ! -f "templates/basic-validation.md" ]; then
	echo "::error::basic-validation.md template not found"
	exit 1
	fi

	if [ ! -f "templates/mcp-integration.md" ]; then
	echo "::error::mcp-integration.md template not found"
	exit 1
	fi

	echo "✅ All required templates present"

	- name: Validate Template Content
	run: \|
	for template in templates/*.md; do
	# Check for required placeholders
	if ! grep -q "{{PR_CONTEXT}}" "$template"; then
	echo "::warning::$template missing {{PR_CONTEXT}} placeholder"
	fi

	if ! grep -q "{{BASE_BRANCH}}" "$template"; then
	echo "::warning::$template missing {{BASE_BRANCH}} placeholder"
	fi

	echo "✅ $template validated"
	done

	documentation-check:
	name: Check Documentation
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Check README
	run: \|
	if [ ! -f "README.md" ]; then
	echo "::error::README.md not found"
	exit 1
	fi

	# Check for required sections
	if ! grep -q "## ✨ Features" README.md; then
	echo "::error::README missing Features section"
	exit 1
	fi

	if ! grep -q "## 🚀 Quick Start" README.md; then
	echo "::error::README missing Quick Start section"
	exit 1
	fi

	echo "✅ README.md is complete"

	- name: Check Other Docs
	run: \|
	for file in LICENSE CHANGELOG.md CONTRIBUTING.md; do
	if [ ! -f "$file" ]; then
	echo "::error::$file not found"
	exit 1
	fi
	echo "✅ $file exists"
	done

	security-check:
	name: Security Checks
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Check for Secrets in Code
	run: \|
	# Look for potential secrets or credentials (excluding comments, variable names, and GitHub Actions secret references)
	if grep -rn "AKIA\\|password\\|secret\\|token" --include=".sh" --include=".yml" scripts/ .github/ 2>/dev/null \| \
	grep -v "^[^:]:[^:]:#" \| \
	grep -v "SECRET_NAME\\|MCP_SECRET\\|secret-name" \| \
	grep -v "echo.secret\\|echo.token" \| \
	grep -v "grep -rn.secret\\|grep -rn.token" \| \
	grep -v "grep -v.*secret" \| \
	grep -v 'secrets\.' \| \
	grep -v "- No hardcoded\\|None found" \| \
	grep -v "# Get the\\|# Look for\\|# This workflow"; then
	echo "::error::Potential hardcoded secrets found in code"
	exit 1
	fi
	echo "✅ No hardcoded secrets found"

	- name: Verify Cleanup Steps
	run: \|
	# Check that action.yml has cleanup steps
	if ! grep -q "rm -f /tmp/forge-config.yaml" action.yml; then
	echo "::error::Missing cleanup for forge-config.yaml"
	exit 1
	fi

	if ! grep -q "rm -f /tmp/validation-.*\.md" action.yml; then
	echo "::error::Missing cleanup for validation files"
	exit 1
	fi

	echo "✅ Cleanup steps verified"

	test-mcp-connection:
	name: Test MCP Connection
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'
	steps:
	- uses: actions/checkout@v4

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '20'

	- name: Install ForgeCode
	run: \|
	echo "Installing ForgeCode CLI..."
	npm install -g forgecode@latest
	forge --version

	echo ""
	echo "Node.js version:"
	node --version
	echo "npm version:"
	npm --version

	- name: Setup MCP Server
	env:
	USABLE_URL: ${{ vars.USABLE_URL \|\| 'https://usable.dev' }}
	USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }}
	WORKSPACE_ID: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938'
	run: \|
	./scripts/setup-mcp.sh

	- name: Verify MCP Server Package
	run: \|
	echo "Testing if @usabledev/mcp-server can be executed..."
	npx --yes @usabledev/mcp-server@latest --version \|\| echo "::warning::MCP server version check failed"

	echo ""
	echo "Checking .mcp.json configuration..."
	echo "File exists: $([ -f .mcp.json ] && echo 'YES' \|\| echo 'NO')"

	if [ -f .mcp.json ]; then
	echo ""
	echo "Full .mcp.json content (with token partially masked for security):"
	cat .mcp.json \| sed -E 's/(mmesh_[a-zA-Z0-9_]{6})[a-zA-Z0-9_]/\1**/g'

	echo ""
	echo "Parsed configuration:"
	jq '.' .mcp.json 2>&1 \|\| echo "::warning::Failed to parse JSON"

	echo ""
	echo "Command that will be executed:"
	jq -r '.mcpServers.usable.command + " " + (.mcpServers.usable.args \| join(" "))' .mcp.json 2>&1 \|\| echo "::warning::Failed to extract command"

	echo ""
	echo "Environment variables configured in .mcp.json:"
	jq -r '.mcpServers.usable.env \| keys[]' .mcp.json 2>&1 \|\| echo "::warning::Failed to extract env vars"
	else
	echo "::error::.mcp.json file not found!"
	fi

	echo ""
	echo "ForgeCode MCP servers:"
	forge mcp list

	- name: Test MCP Connection
	env:
	OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
	USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }}
	USABLE_URL: ${{ vars.USABLE_URL \|\| 'https://usable.dev' }}
	PROVIDER: ${{ vars.PROVIDER \|\| 'auto' }}
	MODEL: ${{ vars.MODEL \|\| 'anthropic/claude-haiku-4.5' }}
	run: \|
	./scripts/test-mcp-connection.sh

	- name: Upload MCP Test Results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: mcp-test-results
	path: \|
	/tmp/mcp-test-*.txt
	/tmp/mcp-test-*.json
	retention-days: 7
	if-no-files-found: warn

	integration-test:
	name: Integration Test
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'
	needs: test-mcp-connection
	permissions:
	contents: read
	pull-requests: write
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Determine validation base ref
	id: base-ref
	run: \|
	if [[ "${{ github.head_ref }}" == release-please--* ]]; then
	# For release-please branches, validate since last release
	git fetch --tags
	LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null \|\| echo "")
	if [ -z "$LAST_TAG" ]; then
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	else
	echo "base-ref=$LAST_TAG" >> $GITHUB_OUTPUT
	fi
	else
	# For regular PRs, use the base branch
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	fi

	- name: Create Test Validation Prompt
	run: \|
	mkdir -p .github/prompts
	cat > .github/prompts/test-validation.md <<'EOF'
	# PR Validation Instructions

	## CRITICAL OUTPUT INSTRUCTION
	START YOUR OUTPUT DIRECTLY WITH: `# PR Validation Report`

	## PR Context
	{{PR_CONTEXT}}

	## Your Task
	Validate this PR for the usable-pr-validator GitHub Action repository.

	### Step 0: Verify MCP Connection (REQUIRED - CRITICAL)
	YOU MUST TEST THE USABLE MCP CONNECTION FIRST:

	1. Call the `mcp_usable_list-workspaces` tool with `{"includeArchived": false}`
	2. In your report, include the EXACT output showing:
	- Total number of workspaces found
	- Name of each workspace
	- Whether the expected workspace "Flowcore" (ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938) was found

	CRITICAL: If you do not call this tool and show the results, your report will be considered INVALID.

	Expected result: Should find at least the "Flowcore" workspace.

	If the MCP connection fails, report this as a CRITICAL VIOLATION with the exact error message.

	### Step 1: Get PR Changes
	```bash
	# Compare base ref (branch or tag) with HEAD
	# If BASE_BRANCH is a tag, use it directly; if it's a branch, try origin/ prefix
	git diff {{BASE_BRANCH}}...HEAD 2>/dev/null \|\| git diff origin/{{BASE_BRANCH}}...HEAD
	```

	### Step 2: Search Usable Knowledge Base
	Use `agentic-search-fragments` to find relevant standards:
	- Query: "GitHub Actions best practices and standards"
	- Query: "Shell script security and validation standards"
	- Tags: `repo:usable-pr-validator`
	- Workspace: Check for coding standards and conventions

	Use `get-memory-fragment-content` to read full details of relevant fragments.

	### Step 3: Validation Checks
	1. Code Quality
	- Shell scripts follow best practices
	- Proper error handling
	- Clear variable naming

	2. Security
	- No hardcoded secrets
	- Proper cleanup of sensitive data
	- Secure file permissions

	3. Documentation
	- Changes are documented
	- README updates if needed
	- Comments are clear

	4. Action Structure
	- Inputs are properly defined
	- Outputs are correctly set
	- Error handling is present

	## Output Format

	# PR Validation Report

	## MCP Connection Verification ✅ (REQUIRED)

	MCP Server: https://usable.dev/api/mcp

	### Connection Test Results:
	- Status: Connected ✅ or Failed ❌
	- Tool Called: `mcp_usable_list-workspaces`
	- Total Workspaces Found: [exact number]
	- Workspaces:
	- [Workspace 1 name] (ID: [uuid])
	- [Workspace 2 name] (ID: [uuid])
	- ...
	- Target Workspace Found: ✅ Flowcore (60c10ca2-4115-4c1a-b6d7-04ac39fd3938) or ❌ Not Found

	Verification Stamp: 🔐 MCP connection verified on [timestamp]

	## Summary
	[Brief overview of the changes and validation results]

	## Critical Violations ❌
	[Must-fix issues that would break functionality or security]
	- If none, state "None found"

	## Important Issues ⚠️
	[Should-fix issues for code quality and best practices]
	- If none, state "None found"

	## Suggestions ℹ️
	[Nice-to-have improvements and enhancements]
	- If none, state "None found"

	## Validation Outcome
	- Status: PASS ✅ or FAIL ❌
	- Critical Issues: [count]
	- Important Issues: [count]
	- Suggestions: [count]
	EOF

	- name: Run Action Test
	id: test-action
	continue-on-error: true
	uses: ./
	with:
	prompt-file: '.github/prompts/test-validation.md'
	workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938'
	base-ref: ${{ steps.base-ref.outputs.base-ref }}
	comment-title: 'Integration Test Validation'
	comment-mode: 'update'
	fail-on-critical: false
	env:
	OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
	USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }}

	- name: Debug Test Results
	if: always()
	run: \|
	echo "Test result: ${{ steps.test-action.outcome }}"
	if [ -f "/tmp/validation-full-output.md" ]; then
	echo "::group::Full ForgeCode Output"
	cat /tmp/validation-full-output.md
	echo "::endgroup::"
	else
	echo "⚠️ No full output file found"
	fi

	integration-test-documentation:
	name: Integration Test (Documentation)
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'
	needs: test-mcp-connection
	permissions:
	contents: read
	pull-requests: write
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Determine validation base ref
	id: base-ref
	run: \|
	if [[ "${{ github.head_ref }}" == release-please--* ]]; then
	# For release-please branches, validate since last release
	git fetch --tags
	LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null \|\| echo "")
	if [ -z "$LAST_TAG" ]; then
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	else
	echo "base-ref=$LAST_TAG" >> $GITHUB_OUTPUT
	fi
	else
	# For regular PRs, use the base branch
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	fi

	- name: Create Documentation Validation Prompt
	run: \|
	mkdir -p .github/prompts
	cat > .github/prompts/test-docs-validation.md <<'EOF'
	# Documentation Validation Instructions

	## CRITICAL OUTPUT INSTRUCTION
	START YOUR OUTPUT DIRECTLY WITH: `# PR Validation Report`

	## PR Context
	{{PR_CONTEXT}}

	## Your Task
	Validate documentation changes in this PR.

	### Step 0: Verify MCP Connection (REQUIRED)
	YOU MUST call `mcp_usable_list-workspaces` tool first and show the results in your report with:
	- Total workspaces found
	- List of workspace names and IDs
	- Confirmation that "Flowcore" workspace was found

	### Get Changes
	```bash
	# Compare base ref (branch or tag) with HEAD for markdown files
	# If BASE_BRANCH is a tag, use it directly; if it's a branch, try origin/ prefix
	git diff {{BASE_BRANCH}}...HEAD -- '.md' 2>/dev/null \|\| git diff origin/{{BASE_BRANCH}}...HEAD -- '.md'
	```

	### Validation Checks
	1. Completeness
	- All new features are documented
	- Examples are provided
	- Configuration options explained

	2. Clarity
	- Instructions are clear
	- Code examples are correct
	- Links are valid

	3. Consistency
	- Follows existing documentation style
	- Terminology is consistent
	- Formatting is proper

	## Output Format

	# PR Validation Report

	## MCP Connection Verification ✅
	- Status: Connected ✅ or Failed ❌
	- Workspaces Found: [count]
	- Target Workspace: ✅ Flowcore found or ❌ Not Found
	- Verification Stamp: 🔐 MCP verified

	## Summary
	[Overview of documentation changes]

	## Critical Violations ❌
	- If none, state "None found"

	## Important Issues ⚠️
	- If none, state "None found"

	## Suggestions ℹ️
	- If none, state "None found"

	## Validation Outcome
	- Status: PASS ✅ or FAIL ❌
	- Critical Issues: [count]
	- Important Issues: [count]
	- Suggestions: [count]
	EOF

	- name: Run Documentation Validation
	id: docs-action
	continue-on-error: true
	uses: ./
	with:
	prompt-file: '.github/prompts/test-docs-validation.md'
	workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938'
	base-ref: ${{ steps.base-ref.outputs.base-ref }}
	comment-title: 'Documentation Validation'
	comment-mode: 'update'
	fail-on-critical: false
	env:
	OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
	USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }}

	- name: Debug Documentation Results
	if: always()
	run: \|
	echo "Docs test result: ${{ steps.docs-action.outcome }}"
	if [ -f "/tmp/validation-full-output.md" ]; then
	echo "::group::Full ForgeCode Output"
	cat /tmp/validation-full-output.md
	echo "::endgroup::"
	else
	echo "⚠️ No full output file found"
	fi

	integration-test-comment-revalidation:
	name: Integration Test (Comment Revalidation)
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request'
	needs: test-mcp-connection
	permissions:
	contents: read
	pull-requests: write
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Determine validation base ref
	id: base-ref
	run: \|
	if [[ "${{ github.head_ref }}" == release-please--* ]]; then
	# For release-please branches, validate since last release
	git fetch --tags
	LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null \|\| echo "")
	if [ -z "$LAST_TAG" ]; then
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	else
	echo "base-ref=$LAST_TAG" >> $GITHUB_OUTPUT
	fi
	else
	# For regular PRs, use the base branch
	echo "base-ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT
	fi

	- name: Create Comment Revalidation Prompt
	run: \|
	mkdir -p .github/prompts
	cat > .github/prompts/test-comment-revalidation.md <<'EOF'
	# Comment-Triggered Revalidation Test

	## CRITICAL OUTPUT INSTRUCTION
	START YOUR OUTPUT DIRECTLY WITH: `# PR Validation Report`

	## PR Context
	{{PR_CONTEXT}}

	## Your Task
	Test the comment revalidation feature.

	### Step 0: Verify MCP Connection (REQUIRED)
	YOU MUST call `mcp_usable_list-workspaces` tool first and show the results in your report with:
	- Total workspaces found
	- Verification that "Flowcore" workspace is accessible

	### Step 1: Verify Override Comment

	Check if there is an override/clarification comment in the PR context above (marked with 🔄).

	If present:
	- Acknowledge it in your report
	- Parse what the user is requesting
	- Test the deviation documentation feature

	### Step 2: Get PR Changes
	```bash
	# Compare base ref (branch or tag) with HEAD
	# If BASE_BRANCH is a tag, use it directly; if it's a branch, try origin/ prefix
	git diff {{BASE_BRANCH}}...HEAD 2>/dev/null \|\| git diff origin/{{BASE_BRANCH}}...HEAD
	```

	### Step 3: Simulate Deviation Approval

	If the override comment requests approval of a deviation:
	- Test creating a fragment (but mark it as a test)
	- Include fragment creation in your report

	## Output Format

	# PR Validation Report

	## MCP Connection Verification ✅
	- Status: Connected ✅ or Failed ❌
	- Workspaces Found: [count]
	- Verification Stamp: 🔐 MCP verified

	## Summary
	[Mention this is a comment revalidation test]
	[Note if override comment was detected]

	## Override Comment Test Results
	- Override Detected: Yes/No
	- Comment Author: @[author or "none"]
	- Request Type: [Type of request if detected]

	## Critical Violations ❌
	None found (test scenario)

	## Important Issues ⚠️
	None found (test scenario)

	## Suggestions ℹ️
	- Comment revalidation feature is working correctly

	## Validation Outcome
	- Status: PASS ✅
	- Test Type: Comment Revalidation
	- Override Support: Tested
	- Critical Issues: 0
	- Important Issues: 0
	- Suggestions: 1
	EOF

	- name: Simulate Override Comment
	id: override
	run: \|
	# Simulate a user comment requesting deviation approval
	cat <<'OVERRIDE_EOF' >> $GITHUB_OUTPUT
	override-comment<<COMMENT_END
	@usable This PR is a test.

	Please approve this as a test of the comment revalidation feature.

	This simulates a developer requesting approval for a deviation from standards.
	COMMENT_END
	OVERRIDE_EOF

	- name: Run Comment Revalidation Test
	id: comment-test
	continue-on-error: true
	uses: ./
	with:
	prompt-file: '.github/prompts/test-comment-revalidation.md'
	workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938'
	base-ref: ${{ steps.base-ref.outputs.base-ref }}
	override-comment: ${{ steps.override.outputs.override-comment }}
	comment-title: 'Comment Revalidation Test'
	comment-mode: 'update'
	fail-on-critical: false
	env:
	OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
	USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }}
	COMMENT_AUTHOR: 'test-user'

	- name: Verify Override Context Was Included
	if: always()
	run: \|
	echo "Comment revalidation test result: ${{ steps.comment-test.outcome }}"

	# Check if the override comment was processed
	if [ -f "/tmp/validation-prompt.txt" ]; then
	echo "::group::Checking Override Comment in Prompt"
	if grep -q "🔄 Override/Clarification Comment" /tmp/validation-prompt.txt; then
	echo "✅ Override comment marker found in prompt"
	else
	echo "::warning::Override comment marker NOT found in prompt"
	fi

	if grep -q "@usable This PR is a test" /tmp/validation-prompt.txt; then
	echo "✅ Override comment content found in prompt"
	else
	echo "::warning::Override comment content NOT found in prompt"
	fi

	if grep -q "test-user" /tmp/validation-prompt.txt; then
	echo "✅ Comment author found in prompt"
	else
	echo "::warning::Comment author NOT found in prompt"
	fi
	echo "::endgroup::"
	else
	echo "⚠️ Prompt file not found - cannot verify override context"
	fi

	# Check the validation output
	if [ -f "/tmp/validation-full-output.md" ]; then
	echo "::group::Validation Output"
	cat /tmp/validation-full-output.md
	echo "::endgroup::"
	fi

	all-checks-passed:
	name: All Checks Passed
	runs-on: ubuntu-latest
	needs: [lint-scripts, validate-action, test-script-execution, test-templates, documentation-check, security-check]
	steps:
	- name: Summary
	run: \|
	echo "🎉 All checks passed successfully!"
	echo "✅ Scripts are valid and executable"
	echo "✅ Action metadata is correct"
	echo "✅ Templates are present"
	echo "✅ Documentation is complete"
	echo "✅ Security checks passed"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

refactor(action.yml, README.md, workflows, scripts): transition from … #116

Workflow file

refactor(action.yml, README.md, workflows, scripts): transition from … #116

Uh oh!

Workflow file for this run