From 449831a0aa0292f61bea0a6c18eeebd81be40c10 Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Tue, 14 Oct 2025 17:10:23 +0530
Subject: [PATCH 1/6] CLI: Add gen-test-instructions command

---
 tools/cli/cliRouter.js                      |   2 +
 tools/cli/commands/gen-test-instructions.js | 572 ++++++++++++++++++++
 2 files changed, 574 insertions(+)
 create mode 100644 tools/cli/commands/gen-test-instructions.js

diff --git a/tools/cli/cliRouter.js b/tools/cli/cliRouter.js
index 67f6d9788db81..7b43b1ce26e31 100644
--- a/tools/cli/cliRouter.js
+++ b/tools/cli/cliRouter.js
@@ -8,6 +8,7 @@ import * as dependenciesCommand from './commands/dependencies.js';
 import { dockerDefine } from './commands/docker.js';
 import { docsDefine } from './commands/docs.js';
 import { draftDefine } from './commands/draft.js';
+import { genTestInstructionsDefine } from './commands/gen-test-instructions.js';
 import { generateDefine } from './commands/generate.js';
 import * as installCommand from './commands/install.js';
 import * as noopCommand from './commands/noop.js';
@@ -49,6 +50,7 @@ export async function cli() {
 	argv = releaseDefine( argv );
 	argv = rsyncDefine( argv );
 	argv.command( testCommand );
+	argv = genTestInstructionsDefine( argv );
 	argv = watchDefine( argv );
 
 	// This adds usage information on failure and demands that a subcommand must be passed.
diff --git a/tools/cli/commands/gen-test-instructions.js b/tools/cli/commands/gen-test-instructions.js
new file mode 100644
index 0000000000000..718e785d0621d
--- /dev/null
+++ b/tools/cli/commands/gen-test-instructions.js
@@ -0,0 +1,572 @@
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import chalk from 'chalk';
+import enquirer from 'enquirer';
+import { chalkJetpackGreen } from '../helpers/styling.js';
+
+/**
+ * Command definition for the gen-test-instructions subcommand.
+ *
+ * @param {object} yargs - The Yargs dependency.
+ * @return {object} Yargs with the gen-test-instructions commands defined.
+ */
+export function genTestInstructionsDefine( yargs ) {
+	yargs.command(
+		'gen-test-instructions [version]',
+		'Generates consolidated test instructions from changelog entries',
+		yarg => {
+			yarg
+				.positional( 'version', {
+					describe: 'Start after this version (e.g., 15.1). Defaults to last stable release.',
+					type: 'string',
+				} )
+				.option( 'changelog', {
+					alias: 'c',
+					describe: 'Path to CHANGELOG.md file',
+					type: 'string',
+					default: 'projects/plugins/jetpack/CHANGELOG.md',
+				} )
+				.option( 'output', {
+					alias: 'o',
+					describe: 'Output file path for test instructions',
+					type: 'string',
+				} )
+				.option( 'since-date', {
+					describe: 'Include changelog entries since this date (YYYY-MM-DD)',
+					type: 'string',
+				} )
+				.option( 'api-key', {
+					describe: 'Anthropic API key for AI consolidation (or set ANTHROPIC_API_KEY env var)',
+					type: 'string',
+				} )
+				.option( 'skip-ai', {
+					describe: 'Skip AI consolidation and just output raw test instructions',
+					type: 'boolean',
+					default: false,
+				} );
+		},
+		async argv => {
+			await genTestInstructionsCli( argv );
+		}
+	);
+
+	return yargs;
+}
+
+/**
+ * Main CLI handler for gen-test-instructions command.
+ *
+ * @param {object} argv - Command line arguments.
+ */
+async function genTestInstructionsCli( argv ) {
+	try {
+		console.log( chalkJetpackGreen( '🧪 Generating Test Instructions...\n' ) );
+
+		// Validate changelog file exists
+		const changelogPath = path.resolve( argv.changelog );
+		if ( ! fs.existsSync( changelogPath ) ) {
+			throw new Error( `Changelog file not found at: ${ changelogPath }` );
+		}
+
+		// Parse changelog to extract entries
+		const relativeChangelogPath = path.relative( process.cwd(), changelogPath );
+		console.log( chalk.blue( `📖 Reading changelog from: ${ relativeChangelogPath }` ) );
+		const parseResult = parseChangelog( changelogPath, argv.version, argv.sinceDate );
+		const entries = parseResult.entries;
+
+		if ( entries.length === 0 ) {
+			throw new Error( 'No changelog entries found for the specified criteria.' );
+		}
+
+		console.log(
+			chalk.green(
+				`✓ Found ${ entries.length } changelog entries since version ${ parseResult.startVersion }\n`
+			)
+		);
+
+		// Extract PR numbers
+		const prNumbers = extractPRNumbers( entries );
+		console.log( chalk.blue( `🔍 Identified ${ prNumbers.length } unique PRs\n` ) );
+
+		// Fetch PR details from GitHub
+		console.log( chalk.blue( '📥 Fetching PR details from GitHub...' ) );
+		const prDetails = await fetchPRDetails( prNumbers );
+		console.log( chalk.green( `✓ Fetched details for ${ prDetails.length } PRs\n` ) );
+
+		// Generate test instructions
+		let testInstructions;
+		if ( argv.skipAi ) {
+			testInstructions = generateRawTestInstructions( entries, prDetails );
+		} else {
+			const apiKey = argv.apiKey || process.env.ANTHROPIC_API_KEY;
+			if ( ! apiKey ) {
+				console.log(
+					chalk.yellow(
+						'⚠️  No API key provided. Using raw output mode. Pass --api-key or set ANTHROPIC_API_KEY env var for AI consolidation.\n'
+					)
+				);
+				testInstructions = generateRawTestInstructions( entries, prDetails );
+			} else {
+				console.log( chalk.blue( '🤖 Using AI to consolidate test instructions...' ) );
+				testInstructions = await generateAIConsolidatedInstructions(
+					entries,
+					prDetails,
+					apiKey,
+					argv.version
+				);
+			}
+		}
+
+		// Determine output path
+		const outputPath = argv.output || ( await promptForOutputPath( argv.version ) );
+
+		// Write to file
+		fs.writeFileSync( outputPath, testInstructions );
+		console.log( chalkJetpackGreen( `\n✅ Test guide generated successfully!\n` ) );
+		console.log( chalk.cyan( `📄 Output file: ${ outputPath }` ) );
+		console.log(
+			chalk.dim( `\nYou can now review and edit the test instructions before sharing.\n` )
+		);
+	} catch ( error ) {
+		console.error( chalk.red( `\n❌ Error: ${ error.message }` ) );
+		if ( argv.v ) {
+			console.error( error );
+		}
+		process.exit( 1 );
+	}
+}
+
+/**
+ * Parse changelog file and extract entries since a specific version or date.
+ *
+ * @param {string} changelogPath - Path to CHANGELOG.md file.
+ * @param {string} sinceVersion  - Start from entries after this version (optional).
+ * @param {string} sinceDate     - Date to filter from (optional).
+ * @return {object} Object with entries array and metadata.
+ */
+function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
+	const content = fs.readFileSync( changelogPath, 'utf-8' );
+	const lines = content.split( '\n' );
+
+	const entries = [];
+	const versions = [];
+	let currentVersion = null;
+	let currentDate = null;
+	let currentSection = null;
+	let collectingEntries = false;
+	let lastStableVersion = null;
+
+	const versionRegex = /^## ([\d.]+(?:-[a-z]+\.\d+)?)\s*-\s*(\d{4}-\d{2}-\d{2})/i;
+	const sectionRegex = /^### (.+)/;
+	const entryRegex = /^- (.+?) \[#(\d+)\]/;
+
+	// First pass: collect all versions and find last stable
+	for ( const line of lines ) {
+		const versionMatch = line.match( versionRegex );
+		if ( versionMatch ) {
+			const ver = versionMatch[ 1 ];
+			const date = versionMatch[ 2 ];
+			versions.push( { version: ver, date } );
+
+			// A stable version doesn't have -a., -b., -rc. suffixes (alpha, beta, rc)
+			// Examples: 15.1 and 15.1.1 are stable, but 15.2-a.1 is not
+			if ( ! lastStableVersion && ! ver.match( /-[a-z]+\./i ) ) {
+				lastStableVersion = ver;
+			}
+		}
+	}
+
+	// Determine the version to start from
+	const startVersion = sinceVersion || lastStableVersion;
+
+	if ( ! startVersion && ! sinceDate ) {
+		throw new Error( 'Could not determine last stable version. Please specify a version or date.' );
+	}
+
+	// Validate that the specified version exists
+	if ( sinceVersion && ! versions.find( v => v.version === sinceVersion ) ) {
+		throw new Error(
+			`Version "${ sinceVersion }" not found in changelog. Available versions: ${ versions
+				.slice( 0, 10 )
+				.map( v => v.version )
+				.join( ', ' ) }...`
+		);
+	}
+
+	// Second pass: collect entries after the start version
+	// Note: Changelog is in reverse chronological order (newest first)
+	// So we collect entries BEFORE we find the start version
+	for ( const line of lines ) {
+		const versionMatch = line.match( versionRegex );
+		if ( versionMatch ) {
+			currentVersion = versionMatch[ 1 ];
+			currentDate = versionMatch[ 2 ];
+
+			// If we find the start version, STOP collecting (we've reached the cutoff)
+			if ( startVersion && currentVersion === startVersion ) {
+				collectingEntries = false;
+				break; // Stop processing, we've reached our cutoff
+			} else if ( sinceDate && currentDate < sinceDate ) {
+				// If using date filter and current date is before sinceDate, stop
+				collectingEntries = false;
+				break;
+			} else {
+				// Haven't reached the cutoff yet, so collect entries
+				collectingEntries = true;
+			}
+
+			currentSection = null;
+			continue;
+		}
+
+		// Check for section header
+		const sectionMatch = line.match( sectionRegex );
+		if ( sectionMatch ) {
+			currentSection = sectionMatch[ 1 ];
+			continue;
+		}
+
+		// Check for changelog entry with PR number
+		if ( collectingEntries && currentVersion ) {
+			const entryMatch = line.match( entryRegex );
+			if ( entryMatch ) {
+				entries.push( {
+					text: entryMatch[ 1 ],
+					prNumber: entryMatch[ 2 ],
+					section: currentSection,
+					version: currentVersion,
+					date: currentDate,
+				} );
+			}
+		}
+	}
+
+	return {
+		entries,
+		startVersion: startVersion || 'date: ' + sinceDate,
+		versions: versions.slice( 0, Math.min( versions.length, 20 ) ), // Return first 20 versions for reference
+	};
+}
+
+/**
+ * Extract unique PR numbers from changelog entries.
+ *
+ * @param {Array} entries - Changelog entries.
+ * @return {Array} Array of unique PR numbers.
+ */
+function extractPRNumbers( entries ) {
+	const prNumbers = new Set();
+	entries.forEach( entry => {
+		if ( entry.prNumber ) {
+			prNumbers.add( entry.prNumber );
+		}
+	} );
+	return Array.from( prNumbers ).sort( ( a, b ) => parseInt( a ) - parseInt( b ) );
+}
+
+/**
+ * Fetch PR details from GitHub using gh CLI.
+ *
+ * @param {Array} prNumbers - Array of PR numbers to fetch.
+ * @return {Promise<Array>} Array of PR details objects.
+ */
+async function fetchPRDetails( prNumbers ) {
+	const prDetails = [];
+
+	for ( const prNumber of prNumbers ) {
+		try {
+			// Fetch PR details using gh CLI
+			const prData = execSync(
+				`gh pr view ${ prNumber } --json number,title,body,labels,author --repo Automattic/jetpack`,
+				{ encoding: 'utf-8' }
+			);
+
+			const pr = JSON.parse( prData );
+
+			// Extract testing instructions from PR body
+			const testingInstructions = extractTestingInstructions( pr.body );
+
+			prDetails.push( {
+				number: pr.number,
+				title: pr.title,
+				body: pr.body,
+				testingInstructions,
+				labels: pr.labels.map( l => l.name ),
+				author: pr.author.login,
+			} );
+
+			// Add a small delay to avoid rate limiting
+			await sleep( 100 );
+		} catch ( error ) {
+			console.warn( chalk.yellow( `⚠️  Could not fetch PR #${ prNumber }: ${ error.message }` ) );
+		}
+	}
+
+	return prDetails;
+}
+
+/**
+ * Extract testing instructions from PR body.
+ *
+ * @param {string} prBody - PR description body.
+ * @return {string|null} Extracted testing instructions or null.
+ */
+function extractTestingInstructions( prBody ) {
+	if ( ! prBody ) {
+		return null;
+	}
+
+	// Common patterns for testing instructions sections
+	const patterns = [
+		/## Testing [Ii]nstructions[\s\S]*?(?=\n## |$)/,
+		/### Testing [Ii]nstructions[\s\S]*?(?=\n## |$)/,
+		/## Test [Pp]lan[\s\S]*?(?=\n## |$)/,
+		/### Test [Pp]lan[\s\S]*?(?=\n## |$)/,
+		/## How to [Tt]est[\s\S]*?(?=\n## |$)/,
+		/### How to [Tt]est[\s\S]*?(?=\n## |$)/,
+	];
+
+	for ( const pattern of patterns ) {
+		const match = prBody.match( pattern );
+		if ( match ) {
+			return match[ 0 ].trim();
+		}
+	}
+
+	return null;
+}
+
+/**
+ * Generate raw (non-AI) test instructions output.
+ *
+ * @param {Array} entries   - Changelog entries.
+ * @param {Array} prDetails - PR details with testing instructions.
+ * @return {string} Markdown formatted test instructions.
+ */
+function generateRawTestInstructions( entries, prDetails ) {
+	let output = '# Test Instructions\n\n';
+	output += `Generated on: ${ new Date().toISOString().split( 'T' )[ 0 ] }\n\n`;
+	output += '## Overview\n\n';
+	output +=
+		'This document contains testing instructions for changes in the current release cycle.\n\n';
+
+	// Group PRs by section
+	const prsBySection = {};
+	const prMap = new Map( prDetails.map( pr => [ pr.number.toString(), pr ] ) );
+
+	entries.forEach( entry => {
+		const section = entry.section || 'Other';
+		if ( ! prsBySection[ section ] ) {
+			prsBySection[ section ] = [];
+		}
+
+		const pr = prMap.get( entry.prNumber );
+		if ( pr && ! prsBySection[ section ].some( p => p.number.toString() === entry.prNumber ) ) {
+			prsBySection[ section ].push( pr );
+		}
+	} );
+
+	// Output each section
+	Object.keys( prsBySection )
+		.sort()
+		.forEach( section => {
+			output += `## ${ section }\n\n`;
+
+			prsBySection[ section ].forEach( pr => {
+				// Make the PR title itself a hyperlink
+				output += `### [${ pr.title }](https://github.com/Automattic/jetpack/pull/${ pr.number }) (#${ pr.number })\n\n`;
+
+				if ( pr.testingInstructions ) {
+					// Convert any PR numbers in the testing instructions to links
+					const linkedInstructions = convertPRNumbersToLinks( pr.testingInstructions );
+					output += `${ linkedInstructions }\n\n`;
+				} else {
+					output += '_No specific testing instructions provided._\n\n';
+					if ( pr.body ) {
+						const linkedBody = convertPRNumbersToLinks( pr.body.substring( 0, 300 ) + '...' );
+						output += `**PR Description:**\n${ linkedBody }\n\n`;
+					} else {
+						output += '**PR Description:** N/A\n\n';
+					}
+				}
+
+				output += '---\n\n';
+			} );
+		} );
+
+	return output;
+}
+
+/**
+ * Convert PR number references to clickable GitHub links.
+ *
+ * @param {string} text - Text containing PR references.
+ * @return {string} Text with PR numbers converted to links.
+ */
+function convertPRNumbersToLinks( text ) {
+	// Pattern 1: [#12345] (not already a link) -> [#12345](https://github.com/Automattic/jetpack/pull/12345)
+	text = text.replace( /\[#(\d+)\](?!\()/g, ( _match, prNum ) => {
+		return `[#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
+	} );
+
+	// Pattern 2: PR #12345 at word boundaries -> PR [#12345](https://github.com/Automattic/jetpack/pull/12345)
+	// Only match if not already inside a markdown link
+	text = text.replace( /(?<!\[)(?<!\()PR #(\d+)\b/g, ( _match, prNum ) => {
+		return `PR [#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
+	} );
+
+	// Pattern 3: Standalone #12345 at word boundaries -> [#12345](https://github.com/Automattic/jetpack/pull/12345)
+	// Avoid markdown headings (###), already linked numbers, and matches inside brackets
+	text = text.replace( /(?<!\[)(?<!#)(?<!\()(?<!\/)#(\d{4,})\b/g, ( _match, prNum ) => {
+		return `[#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
+	} );
+
+	return text;
+}
+
+/**
+ * Generate AI-consolidated test instructions using Claude API.
+ *
+ * @param {Array}  entries   - Changelog entries.
+ * @param {Array}  prDetails - PR details with testing instructions.
+ * @param {string} apiKey    - Anthropic API key.
+ * @param {string} version   - Version being tested.
+ * @return {Promise<string>} Markdown formatted consolidated test instructions.
+ */
+async function generateAIConsolidatedInstructions( entries, prDetails, apiKey, version ) {
+	// Prepare the data for AI processing
+	const prMap = new Map( prDetails.map( pr => [ pr.number.toString(), pr ] ) );
+
+	const prsBySection = {};
+	entries.forEach( entry => {
+		const section = entry.section || 'Other';
+		if ( ! prsBySection[ section ] ) {
+			prsBySection[ section ] = [];
+		}
+
+		const pr = prMap.get( entry.prNumber );
+		if ( pr && ! prsBySection[ section ].some( p => p.number.toString() === entry.prNumber ) ) {
+			prsBySection[ section ].push( {
+				number: pr.number,
+				title: pr.title,
+				changelogText: entry.text,
+				testingInstructions: pr.testingInstructions || 'No testing instructions provided.',
+			} );
+		}
+	} );
+
+	const prompt = `You are helping to create a consolidated testing guide for Jetpack plugin version ${
+		version || 'upcoming release'
+	}.
+
+I have changelog entries grouped by feature area/section. Each entry includes:
+- The PR number and title
+- The changelog entry text
+- Testing instructions from the PR (if available)
+
+Your task is to:
+1. Analyze the testing instructions for each section
+2. Consolidate similar or overlapping test steps
+3. Remove redundant instructions
+4. Organize tests in a logical order within each section
+5. Provide clear, actionable testing steps
+6. Note which features/areas need the most attention
+7. Identify any changes without test instructions that might need manual testing
+
+IMPORTANT REQUIREMENTS FOR PR REFERENCES:
+- ALWAYS reference PR numbers when discussing changes
+- Format PR numbers as markdown links: [#12345](https://github.com/Automattic/jetpack/pull/12345)
+- List all related PR numbers at the beginning of each section
+- Do NOT omit or skip PR numbers - they are critical for tracking
+- When combining multiple PRs into one testing section, list ALL PR numbers involved as links
+
+Output format should be a well-structured markdown document with:
+- A summary section highlighting key areas to test
+- Each feature area as a heading with PR numbers listed as clickable links
+- Consolidated test steps (not just copying individual PR instructions)
+- All PR number references formatted as: [#12345](https://github.com/Automattic/jetpack/pull/12345)
+- A section for changes without specific test instructions (with PR links)
+
+Here is the data:
+
+${ JSON.stringify( prsBySection, null, 2 ) }
+
+Generate the consolidated test guide now. Remember to format ALL PR numbers as markdown links!`;
+
+	try {
+		// Call Claude API
+		const response = await fetch( 'https://api.anthropic.com/v1/messages', {
+			method: 'POST',
+			headers: {
+				'Content-Type': 'application/json',
+				'x-api-key': apiKey,
+				'anthropic-version': '2023-06-01',
+			},
+			body: JSON.stringify( {
+				model: 'claude-3-5-sonnet-20241022',
+				max_tokens: 4096,
+				messages: [
+					{
+						role: 'user',
+						content: prompt,
+					},
+				],
+			} ),
+		} );
+
+		if ( ! response.ok ) {
+			throw new Error( `API request failed: ${ response.status } ${ response.statusText }` );
+		}
+
+		const data = await response.json();
+		const consolidatedGuide = data.content[ 0 ].text;
+
+		// Note: PR numbers should already be formatted as links by the AI
+		// but we could add a fallback conversion here if needed
+
+		// Add metadata header
+		let output = `# Test Instructions for Jetpack ${ version || 'Release' }\n\n`;
+		output += `Generated on: ${ new Date().toISOString().split( 'T' )[ 0 ] }\n`;
+		output += `Total PRs: ${ prDetails.length }\n\n`;
+		output += '---\n\n';
+		output += consolidatedGuide;
+
+		return output;
+	} catch ( error ) {
+		console.warn(
+			chalk.yellow(
+				`\n⚠️  AI consolidation failed: ${ error.message }. Falling back to raw output.\n`
+			)
+		);
+		return generateRawTestInstructions( entries, prDetails );
+	}
+}
+
+/**
+ * Prompt user for output file path.
+ *
+ * @param {string} version - Version string.
+ * @return {Promise<string>} Output file path.
+ */
+async function promptForOutputPath( version ) {
+	const defaultName = `test-instructions-${ version || 'latest' }.md`;
+	const response = await enquirer.prompt( {
+		type: 'input',
+		name: 'outputPath',
+		message: 'Where should the test guide be saved?',
+		initial: defaultName,
+	} );
+
+	return response.outputPath;
+}
+
+/**
+ * Simple sleep utility.
+ *
+ * @param {number} ms - Milliseconds to sleep.
+ * @return {Promise} Promise that resolves after ms milliseconds.
+ */
+function sleep( ms ) {
+	return new Promise( resolve => setTimeout( resolve, ms ) );
+}

From 20e89211d37187ffde3552a927f656fdc9bb3dd5 Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Wed, 15 Oct 2025 11:28:29 +0530
Subject: [PATCH 2/6] Move the tool from CLI to /tools

---
 tools/cli/cliRouter.js                        |   2 -
 tools/gen-test-instructions.sh                | 212 ++++++++
 .../gen-test-instructions.mjs}                | 462 +++++++++---------
 3 files changed, 442 insertions(+), 234 deletions(-)
 create mode 100755 tools/gen-test-instructions.sh
 rename tools/{cli/commands/gen-test-instructions.js => js-tools/gen-test-instructions.mjs} (55%)

diff --git a/tools/cli/cliRouter.js b/tools/cli/cliRouter.js
index 7b43b1ce26e31..67f6d9788db81 100644
--- a/tools/cli/cliRouter.js
+++ b/tools/cli/cliRouter.js
@@ -8,7 +8,6 @@ import * as dependenciesCommand from './commands/dependencies.js';
 import { dockerDefine } from './commands/docker.js';
 import { docsDefine } from './commands/docs.js';
 import { draftDefine } from './commands/draft.js';
-import { genTestInstructionsDefine } from './commands/gen-test-instructions.js';
 import { generateDefine } from './commands/generate.js';
 import * as installCommand from './commands/install.js';
 import * as noopCommand from './commands/noop.js';
@@ -50,7 +49,6 @@ export async function cli() {
 	argv = releaseDefine( argv );
 	argv = rsyncDefine( argv );
 	argv.command( testCommand );
-	argv = genTestInstructionsDefine( argv );
 	argv = watchDefine( argv );
 
 	// This adds usage information on failure and demands that a subcommand must be passed.
diff --git a/tools/gen-test-instructions.sh b/tools/gen-test-instructions.sh
new file mode 100755
index 0000000000000..d4adc9b39cefe
--- /dev/null
+++ b/tools/gen-test-instructions.sh
@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+
+##
+## Generate test instructions from changelog for Jetpack releases.
+##
+## This script automates the creation of consolidated test instructions
+## by parsing the CHANGELOG.md, fetching PR details from GitHub, and
+## optionally using AI to consolidate the instructions.
+##
+
+set -eo pipefail
+
+# Get the base directory
+BASE=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+# Source common includes
+. "$BASE/tools/includes/check-osx-bash-version.sh"
+. "$BASE/tools/includes/chalk-lite.sh"
+
+# Default values
+CHANGELOG_PATH="projects/plugins/jetpack/CHANGELOG.md"
+OUTPUT_FILE=""
+SINCE_VERSION=""
+SINCE_DATE=""
+SKIP_AI=false
+API_KEY="${ANTHROPIC_API_KEY:-}"
+VERBOSE=false
+
+##
+## Print usage information and exit
+##
+function usage {
+	cat <<-'EOH'
+		usage: gen-test-instructions.sh [options]
+
+		Generate consolidated test instructions from Jetpack changelog entries.
+
+		OPTIONS:
+		  -c, --changelog <path>    Path to CHANGELOG.md file (default: projects/plugins/jetpack/CHANGELOG.md)
+		  -o, --output <file>       Output file path for test instructions
+		  -v, --version <version>   Start from this version (e.g., 15.1). Defaults to last stable release
+		  -d, --since-date <date>   Include entries since this date (YYYY-MM-DD)
+		  -k, --api-key <key>       Anthropic API key for AI consolidation (or set ANTHROPIC_API_KEY env var)
+		  -s, --skip-ai             Skip AI consolidation and output raw format
+		  -h, --help                Show this help message
+
+		EXAMPLES:
+		  # Generate since last stable release (default)
+		  tools/gen-test-instructions.sh
+
+		  # Generate since specific version
+		  tools/gen-test-instructions.sh --version 15.1
+
+		  # Specify output file
+		  tools/gen-test-instructions.sh --output test-guide.md
+
+		  # Skip AI consolidation
+		  tools/gen-test-instructions.sh --skip-ai
+
+		  # With API key
+		  tools/gen-test-instructions.sh --api-key sk-ant-...
+
+		REQUIREMENTS:
+		  - Node.js (available in monorepo)
+		  - GitHub CLI (gh) - must be installed and authenticated
+		  - Anthropic API Key (optional, for AI consolidation)
+	EOH
+	exit 1
+}
+
+##
+## Parse command line arguments
+##
+while [[ $# -gt 0 ]]; do
+	case "$1" in
+		-c|--changelog)
+			CHANGELOG_PATH="$2"
+			shift 2
+			;;
+		-o|--output)
+			OUTPUT_FILE="$2"
+			shift 2
+			;;
+		-v|--version)
+			SINCE_VERSION="$2"
+			shift 2
+			;;
+		-d|--since-date)
+			SINCE_DATE="$2"
+			shift 2
+			;;
+		-k|--api-key)
+			API_KEY="$2"
+			shift 2
+			;;
+		-s|--skip-ai)
+			SKIP_AI=true
+			shift
+			;;
+		--verbose)
+			VERBOSE=true
+			shift
+			;;
+		-h|--help)
+			usage
+			;;
+		*)
+			error "Unknown option: $1"
+			usage
+			;;
+	esac
+done
+
+# Main execution starts here
+info "🧪 Generating Testing Instructions..."
+echo ""
+
+# Check prerequisites
+if ! command -v gh &> /dev/null; then
+	error "GitHub CLI (gh) is not installed. Please install it first:"
+	echo "  macOS: brew install gh"
+	echo "  See: https://github.com/cli/cli/blob/trunk/docs/install_linux.md for other platforms"
+	exit 1
+fi
+
+# Check if gh is authenticated
+if ! gh auth status &> /dev/null; then
+	error "GitHub CLI is not authenticated. Please run: gh auth login"
+	exit 1
+fi
+
+# Check if Node.js is available
+if ! command -v node &> /dev/null; then
+	error "Node.js is not installed"
+	exit 1
+fi
+
+# Resolve changelog path
+if [[ ! "$CHANGELOG_PATH" = /* ]]; then
+	CHANGELOG_PATH="$BASE/$CHANGELOG_PATH"
+fi
+
+# Check if changelog exists
+if [[ ! -f "$CHANGELOG_PATH" ]]; then
+	error "Changelog file not found at: $CHANGELOG_PATH"
+	exit 1
+fi
+
+# Set default output file if not specified
+if [[ -z "$OUTPUT_FILE" ]]; then
+	if [[ -n "$SINCE_VERSION" ]]; then
+		OUTPUT_FILE="test-instructions-${SINCE_VERSION}.md"
+	else
+		OUTPUT_FILE="test-instructions-latest.md"
+	fi
+
+	# Prompt for confirmation
+	info "Output file will be: $OUTPUT_FILE"
+	read -p "Press Enter to continue or provide a name, or press Ctrl+C to cancel:"
+fi
+
+# Build arguments for the Node.js script
+NODE_ARGS=()
+NODE_ARGS+=("--changelog" "$CHANGELOG_PATH")
+NODE_ARGS+=("--output" "$OUTPUT_FILE")
+
+if [[ -n "$SINCE_VERSION" ]]; then
+	NODE_ARGS+=("--version" "$SINCE_VERSION")
+fi
+
+if [[ -n "$SINCE_DATE" ]]; then
+	NODE_ARGS+=("--since-date" "$SINCE_DATE")
+fi
+
+if [[ -n "$API_KEY" ]]; then
+	NODE_ARGS+=("--api-key" "$API_KEY")
+fi
+
+if [[ "$SKIP_AI" = true ]]; then
+	NODE_ARGS+=("--skip-ai")
+fi
+
+if [[ "$VERBOSE" = true ]]; then
+	NODE_ARGS+=("--verbose")
+fi
+
+# Run the Node.js script
+NODE_SCRIPT="$BASE/tools/js-tools/gen-test-instructions.mjs"
+
+if [[ ! -f "$NODE_SCRIPT" ]]; then
+	error "Node.js script not found at: $NODE_SCRIPT"
+	exit 1
+fi
+
+# Execute the Node.js script
+if [[ "$VERBOSE" = true ]]; then
+	info "Running: node $NODE_SCRIPT ${NODE_ARGS[*]}"
+fi
+
+node "$NODE_SCRIPT" "${NODE_ARGS[@]}"
+
+# Check if the script succeeded
+if [[ $? -eq 0 ]]; then
+	success "✅ Test guide generated successfully!"
+	echo ""
+	info "📄 Output file: $(cyan "$OUTPUT_FILE")"
+	echo ""
+	echo "You can now review and edit the test instructions before sharing."
+else
+	error "Failed to generate test instructions"
+	exit 1
+fi
diff --git a/tools/cli/commands/gen-test-instructions.js b/tools/js-tools/gen-test-instructions.mjs
similarity index 55%
rename from tools/cli/commands/gen-test-instructions.js
rename to tools/js-tools/gen-test-instructions.mjs
index 718e785d0621d..e71dd65ca859c 100644
--- a/tools/cli/commands/gen-test-instructions.js
+++ b/tools/js-tools/gen-test-instructions.mjs
@@ -1,149 +1,100 @@
-import { execSync } from 'child_process';
-import fs from 'fs';
-import path from 'path';
-import chalk from 'chalk';
-import enquirer from 'enquirer';
-import { chalkJetpackGreen } from '../helpers/styling.js';
+#!/usr/bin/env node
 
 /**
- * Command definition for the gen-test-instructions subcommand.
+ * Generate Test Instructions Tool
  *
- * @param {object} yargs - The Yargs dependency.
- * @return {object} Yargs with the gen-test-instructions commands defined.
- */
-export function genTestInstructionsDefine( yargs ) {
-	yargs.command(
-		'gen-test-instructions [version]',
-		'Generates consolidated test instructions from changelog entries',
-		yarg => {
-			yarg
-				.positional( 'version', {
-					describe: 'Start after this version (e.g., 15.1). Defaults to last stable release.',
-					type: 'string',
-				} )
-				.option( 'changelog', {
-					alias: 'c',
-					describe: 'Path to CHANGELOG.md file',
-					type: 'string',
-					default: 'projects/plugins/jetpack/CHANGELOG.md',
-				} )
-				.option( 'output', {
-					alias: 'o',
-					describe: 'Output file path for test instructions',
-					type: 'string',
-				} )
-				.option( 'since-date', {
-					describe: 'Include changelog entries since this date (YYYY-MM-DD)',
-					type: 'string',
-				} )
-				.option( 'api-key', {
-					describe: 'Anthropic API key for AI consolidation (or set ANTHROPIC_API_KEY env var)',
-					type: 'string',
-				} )
-				.option( 'skip-ai', {
-					describe: 'Skip AI consolidation and just output raw test instructions',
-					type: 'boolean',
-					default: false,
-				} );
-		},
-		async argv => {
-			await genTestInstructionsCli( argv );
-		}
-	);
-
-	return yargs;
-}
-
-/**
- * Main CLI handler for gen-test-instructions command.
+ * This tool automates the generation of test instructions for Jetpack releases by:
+ * 1. Parsing the CHANGELOG.md to extract entries since a specified version
+ * 2. Fetching PR details from GitHub using the gh CLI
+ * 3. Extracting testing instructions from PR descriptions
+ * 4. Optionally consolidating instructions using Claude AI
+ * 5. Generating a markdown document with all PR numbers as clickable links
  *
- * @param {object} argv - Command line arguments.
+ * Usage node gen-test-instructions.mjs --changelog <path> --output <file> [options]
  */
-async function genTestInstructionsCli( argv ) {
-	try {
-		console.log( chalkJetpackGreen( '🧪 Generating Test Instructions...\n' ) );
 
-		// Validate changelog file exists
-		const changelogPath = path.resolve( argv.changelog );
-		if ( ! fs.existsSync( changelogPath ) ) {
-			throw new Error( `Changelog file not found at: ${ changelogPath }` );
-		}
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
 
-		// Parse changelog to extract entries
-		const relativeChangelogPath = path.relative( process.cwd(), changelogPath );
-		console.log( chalk.blue( `📖 Reading changelog from: ${ relativeChangelogPath }` ) );
-		const parseResult = parseChangelog( changelogPath, argv.version, argv.sinceDate );
-		const entries = parseResult.entries;
+// ============================================================================
+// CONFIGURATION & CONSTANTS
+// ============================================================================
 
-		if ( entries.length === 0 ) {
-			throw new Error( 'No changelog entries found for the specified criteria.' );
-		}
-
-		console.log(
-			chalk.green(
-				`✓ Found ${ entries.length } changelog entries since version ${ parseResult.startVersion }\n`
-			)
-		);
+const GITHUB_REPO = 'Automattic/jetpack';
+const CLAUDE_API_URL = 'https://api.anthropic.com/v1/messages';
+const CLAUDE_MODEL = 'claude-3-5-sonnet-20241022';
 
-		// Extract PR numbers
-		const prNumbers = extractPRNumbers( entries );
-		console.log( chalk.blue( `🔍 Identified ${ prNumbers.length } unique PRs\n` ) );
+// ============================================================================
+// COMMAND LINE ARGUMENT PARSING
+// ============================================================================
 
-		// Fetch PR details from GitHub
-		console.log( chalk.blue( '📥 Fetching PR details from GitHub...' ) );
-		const prDetails = await fetchPRDetails( prNumbers );
-		console.log( chalk.green( `✓ Fetched details for ${ prDetails.length } PRs\n` ) );
+/**
+ * Parse command line arguments into an options object.
+ *
+ * @return {object} Parsed options
+ */
+function parseArguments() {
+	const args = process.argv.slice( 2 );
+	const options = {
+		changelog: null,
+		output: null,
+		version: null,
+		sinceDate: null,
+		apiKey: process.env.ANTHROPIC_API_KEY || null,
+		skipAi: false,
+	};
 
-		// Generate test instructions
-		let testInstructions;
-		if ( argv.skipAi ) {
-			testInstructions = generateRawTestInstructions( entries, prDetails );
-		} else {
-			const apiKey = argv.apiKey || process.env.ANTHROPIC_API_KEY;
-			if ( ! apiKey ) {
-				console.log(
-					chalk.yellow(
-						'⚠️  No API key provided. Using raw output mode. Pass --api-key or set ANTHROPIC_API_KEY env var for AI consolidation.\n'
-					)
-				);
-				testInstructions = generateRawTestInstructions( entries, prDetails );
-			} else {
-				console.log( chalk.blue( '🤖 Using AI to consolidate test instructions...' ) );
-				testInstructions = await generateAIConsolidatedInstructions(
-					entries,
-					prDetails,
-					apiKey,
-					argv.version
-				);
-			}
+	for ( let i = 0; i < args.length; i++ ) {
+		switch ( args[ i ] ) {
+			case '--changelog':
+				options.changelog = args[ ++i ];
+				break;
+			case '--output':
+				options.output = args[ ++i ];
+				break;
+			case '--version':
+				options.version = args[ ++i ];
+				break;
+			case '--since-date':
+				options.sinceDate = args[ ++i ];
+				break;
+			case '--api-key':
+				options.apiKey = args[ ++i ];
+				break;
+			case '--skip-ai':
+				options.skipAi = true;
+				break;
+			default:
+				throw new Error( `Unknown option: ${ args[ i ] }` );
 		}
+	}
 
-		// Determine output path
-		const outputPath = argv.output || ( await promptForOutputPath( argv.version ) );
-
-		// Write to file
-		fs.writeFileSync( outputPath, testInstructions );
-		console.log( chalkJetpackGreen( `\n✅ Test guide generated successfully!\n` ) );
-		console.log( chalk.cyan( `📄 Output file: ${ outputPath }` ) );
-		console.log(
-			chalk.dim( `\nYou can now review and edit the test instructions before sharing.\n` )
-		);
-	} catch ( error ) {
-		console.error( chalk.red( `\n❌ Error: ${ error.message }` ) );
-		if ( argv.v ) {
-			console.error( error );
-		}
-		process.exit( 1 );
+	// Validate required options
+	if ( ! options.changelog ) {
+		throw new Error( 'Missing required option: --changelog' );
 	}
+	if ( ! options.output ) {
+		throw new Error( 'Missing required option: --output' );
+	}
+
+	return options;
 }
 
+// ============================================================================
+// CHANGELOG PARSING
+// ============================================================================
+
 /**
- * Parse changelog file and extract entries since a specific version or date.
+ * Parse the changelog file and extract entries since a specific version or date.
+ *
+ * The changelog is organized in reverse chronological order (newest first).
+ * This function collects all entries from the top until it reaches the cutoff version.
  *
- * @param {string} changelogPath - Path to CHANGELOG.md file.
- * @param {string} sinceVersion  - Start from entries after this version (optional).
- * @param {string} sinceDate     - Date to filter from (optional).
- * @return {object} Object with entries array and metadata.
+ * @param {string} changelogPath - Absolute path to CHANGELOG.md
+ * @param {string} sinceVersion  - Start from entries after this version (optional)
+ * @param {string} sinceDate     - Start from entries after this date (optional)
+ * @return {object} Object with entries, startVersion, and versions array
  */
 function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	const content = fs.readFileSync( changelogPath, 'utf-8' );
@@ -157,6 +108,7 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	let collectingEntries = false;
 	let lastStableVersion = null;
 
+	// Regular expressions for matching changelog format
 	const versionRegex = /^## ([\d.]+(?:-[a-z]+\.\d+)?)\s*-\s*(\d{4}-\d{2}-\d{2})/i;
 	const sectionRegex = /^### (.+)/;
 	const entryRegex = /^- (.+?) \[#(\d+)\]/;
@@ -169,8 +121,7 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 			const date = versionMatch[ 2 ];
 			versions.push( { version: ver, date } );
 
-			// A stable version doesn't have -a., -b., -rc. suffixes (alpha, beta, rc)
-			// Examples: 15.1 and 15.1.1 are stable, but 15.2-a.1 is not
+			// A stable version doesn't have -a., -b., -rc. suffixes
 			if ( ! lastStableVersion && ! ver.match( /-[a-z]+\./i ) ) {
 				lastStableVersion = ver;
 			}
@@ -187,32 +138,26 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	// Validate that the specified version exists
 	if ( sinceVersion && ! versions.find( v => v.version === sinceVersion ) ) {
 		throw new Error(
-			`Version "${ sinceVersion }" not found in changelog. Available versions: ${ versions
+			`Version "${ sinceVersion }" not found in changelog.\nAvailable versions: ${ versions
 				.slice( 0, 10 )
 				.map( v => v.version )
 				.join( ', ' ) }...`
 		);
 	}
 
-	// Second pass: collect entries after the start version
-	// Note: Changelog is in reverse chronological order (newest first)
-	// So we collect entries BEFORE we find the start version
+	// Second pass: collect entries (changelog is reverse chronological)
 	for ( const line of lines ) {
 		const versionMatch = line.match( versionRegex );
 		if ( versionMatch ) {
 			currentVersion = versionMatch[ 1 ];
 			currentDate = versionMatch[ 2 ];
 
-			// If we find the start version, STOP collecting (we've reached the cutoff)
+			// Stop when we reach the cutoff version
 			if ( startVersion && currentVersion === startVersion ) {
-				collectingEntries = false;
-				break; // Stop processing, we've reached our cutoff
+				break;
 			} else if ( sinceDate && currentDate < sinceDate ) {
-				// If using date filter and current date is before sinceDate, stop
-				collectingEntries = false;
 				break;
 			} else {
-				// Haven't reached the cutoff yet, so collect entries
 				collectingEntries = true;
 			}
 
@@ -220,14 +165,12 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 			continue;
 		}
 
-		// Check for section header
 		const sectionMatch = line.match( sectionRegex );
 		if ( sectionMatch ) {
 			currentSection = sectionMatch[ 1 ];
 			continue;
 		}
 
-		// Check for changelog entry with PR number
 		if ( collectingEntries && currentVersion ) {
 			const entryMatch = line.match( entryRegex );
 			if ( entryMatch ) {
@@ -245,15 +188,15 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	return {
 		entries,
 		startVersion: startVersion || 'date: ' + sinceDate,
-		versions: versions.slice( 0, Math.min( versions.length, 20 ) ), // Return first 20 versions for reference
+		versions: versions.slice( 0, 20 ),
 	};
 }
 
 /**
  * Extract unique PR numbers from changelog entries.
  *
- * @param {Array} entries - Changelog entries.
- * @return {Array} Array of unique PR numbers.
+ * @param {Array} entries - Changelog entries
+ * @return {Array} Sorted array of unique PR numbers
  */
 function extractPRNumbers( entries ) {
 	const prNumbers = new Set();
@@ -265,26 +208,30 @@ function extractPRNumbers( entries ) {
 	return Array.from( prNumbers ).sort( ( a, b ) => parseInt( a ) - parseInt( b ) );
 }
 
+// ============================================================================
+// GITHUB PR FETCHING
+// ============================================================================
+
 /**
  * Fetch PR details from GitHub using gh CLI.
  *
- * @param {Array} prNumbers - Array of PR numbers to fetch.
- * @return {Promise<Array>} Array of PR details objects.
+ * @param {Array} prNumbers - Array of PR numbers to fetch
+ * @return {Promise<Array>} Array of PR detail objects
  */
 async function fetchPRDetails( prNumbers ) {
 	const prDetails = [];
 
-	for ( const prNumber of prNumbers ) {
+	for ( let i = 0; i < prNumbers.length; i++ ) {
+		const prNumber = prNumbers[ i ];
+		process.stdout.write( `\r  Fetching PR #${ prNumber } (${ i + 1 }/${ prNumbers.length })...` );
+
 		try {
-			// Fetch PR details using gh CLI
 			const prData = execSync(
-				`gh pr view ${ prNumber } --json number,title,body,labels,author --repo Automattic/jetpack`,
-				{ encoding: 'utf-8' }
+				`gh pr view ${ prNumber } --json number,title,body,labels,author --repo ${ GITHUB_REPO }`,
+				{ encoding: 'utf-8', stdio: [ 'pipe', 'pipe', 'ignore' ] }
 			);
 
 			const pr = JSON.parse( prData );
-
-			// Extract testing instructions from PR body
 			const testingInstructions = extractTestingInstructions( pr.body );
 
 			prDetails.push( {
@@ -296,28 +243,28 @@ async function fetchPRDetails( prNumbers ) {
 				author: pr.author.login,
 			} );
 
-			// Add a small delay to avoid rate limiting
+			// Small delay to avoid rate limiting
 			await sleep( 100 );
 		} catch ( error ) {
-			console.warn( chalk.yellow( `⚠️  Could not fetch PR #${ prNumber }: ${ error.message }` ) );
+			console.warn( `\n⚠️  Could not fetch PR #${ prNumber }: ${ error.message }` );
 		}
 	}
 
+	process.stdout.write( '\r' + ' '.repeat( 80 ) + '\r' ); // Clear the line
 	return prDetails;
 }
 
 /**
  * Extract testing instructions from PR body.
  *
- * @param {string} prBody - PR description body.
- * @return {string|null} Extracted testing instructions or null.
+ * @param {string} prBody - PR description
+ * @return {string|null} Testing instructions or null
  */
 function extractTestingInstructions( prBody ) {
 	if ( ! prBody ) {
 		return null;
 	}
 
-	// Common patterns for testing instructions sections
 	const patterns = [
 		/## Testing [Ii]nstructions[\s\S]*?(?=\n## |$)/,
 		/### Testing [Ii]nstructions[\s\S]*?(?=\n## |$)/,
@@ -337,12 +284,45 @@ function extractTestingInstructions( prBody ) {
 	return null;
 }
 
+// ============================================================================
+// PR NUMBER LINKING
+// ============================================================================
+
+/**
+ * Convert PR number references to clickable GitHub links.
+ *
+ * @param {string} text - Text containing PR references
+ * @return {string} Text with PR numbers converted to links
+ */
+function convertPRNumbersToLinks( text ) {
+	// Pattern 1: [#12345] (not already a link) -> [#12345](url)
+	text = text.replace( /\[#(\d+)\](?!\()/g, ( _match, prNum ) => {
+		return `[#${ prNum }](https://github.com/${ GITHUB_REPO }/pull/${ prNum })`;
+	} );
+
+	// Pattern 2: PR #12345 at word boundaries -> PR [#12345](url)
+	text = text.replace( /(?<!\[)(?<!\()PR #(\d+)\b/g, ( _match, prNum ) => {
+		return `PR [#${ prNum }](https://github.com/${ GITHUB_REPO }/pull/${ prNum })`;
+	} );
+
+	// Pattern 3: Standalone #12345 (4+ digits, not headings) -> [#12345](url)
+	text = text.replace( /(?<!\[)(?<!#)(?<!\()(?<!\/)#(\d{4,})\b/g, ( _match, prNum ) => {
+		return `[#${ prNum }](https://github.com/${ GITHUB_REPO }/pull/${ prNum })`;
+	} );
+
+	return text;
+}
+
+// ============================================================================
+// OUTPUT GENERATION - RAW MODE
+// ============================================================================
+
 /**
  * Generate raw (non-AI) test instructions output.
  *
- * @param {Array} entries   - Changelog entries.
- * @param {Array} prDetails - PR details with testing instructions.
- * @return {string} Markdown formatted test instructions.
+ * @param {Array} entries   - Changelog entries
+ * @param {Array} prDetails - PR details with testing instructions
+ * @return {string} Markdown formatted test instructions
  */
 function generateRawTestInstructions( entries, prDetails ) {
 	let output = '# Test Instructions\n\n';
@@ -374,11 +354,10 @@ function generateRawTestInstructions( entries, prDetails ) {
 			output += `## ${ section }\n\n`;
 
 			prsBySection[ section ].forEach( pr => {
-				// Make the PR title itself a hyperlink
-				output += `### [${ pr.title }](https://github.com/Automattic/jetpack/pull/${ pr.number }) (#${ pr.number })\n\n`;
+				// Make the PR title a clickable link
+				output += `### [${ pr.title }](https://github.com/${ GITHUB_REPO }/pull/${ pr.number }) (#${ pr.number })\n\n`;
 
 				if ( pr.testingInstructions ) {
-					// Convert any PR numbers in the testing instructions to links
 					const linkedInstructions = convertPRNumbersToLinks( pr.testingInstructions );
 					output += `${ linkedInstructions }\n\n`;
 				} else {
@@ -398,47 +377,24 @@ function generateRawTestInstructions( entries, prDetails ) {
 	return output;
 }
 
-/**
- * Convert PR number references to clickable GitHub links.
- *
- * @param {string} text - Text containing PR references.
- * @return {string} Text with PR numbers converted to links.
- */
-function convertPRNumbersToLinks( text ) {
-	// Pattern 1: [#12345] (not already a link) -> [#12345](https://github.com/Automattic/jetpack/pull/12345)
-	text = text.replace( /\[#(\d+)\](?!\()/g, ( _match, prNum ) => {
-		return `[#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
-	} );
-
-	// Pattern 2: PR #12345 at word boundaries -> PR [#12345](https://github.com/Automattic/jetpack/pull/12345)
-	// Only match if not already inside a markdown link
-	text = text.replace( /(?<!\[)(?<!\()PR #(\d+)\b/g, ( _match, prNum ) => {
-		return `PR [#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
-	} );
-
-	// Pattern 3: Standalone #12345 at word boundaries -> [#12345](https://github.com/Automattic/jetpack/pull/12345)
-	// Avoid markdown headings (###), already linked numbers, and matches inside brackets
-	text = text.replace( /(?<!\[)(?<!#)(?<!\()(?<!\/)#(\d{4,})\b/g, ( _match, prNum ) => {
-		return `[#${ prNum }](https://github.com/Automattic/jetpack/pull/${ prNum })`;
-	} );
-
-	return text;
-}
+// ============================================================================
+// OUTPUT GENERATION - AI CONSOLIDATED MODE
+// ============================================================================
 
 /**
  * Generate AI-consolidated test instructions using Claude API.
  *
- * @param {Array}  entries   - Changelog entries.
- * @param {Array}  prDetails - PR details with testing instructions.
- * @param {string} apiKey    - Anthropic API key.
- * @param {string} version   - Version being tested.
- * @return {Promise<string>} Markdown formatted consolidated test instructions.
+ * @param {Array}  entries   - Changelog entries
+ * @param {Array}  prDetails - PR details with testing instructions
+ * @param {string} apiKey    - Anthropic API key
+ * @param {string} version   - Version being tested
+ * @return {Promise<string>} Markdown formatted consolidated test instructions
  */
 async function generateAIConsolidatedInstructions( entries, prDetails, apiKey, version ) {
-	// Prepare the data for AI processing
+	// Prepare data for AI processing
 	const prMap = new Map( prDetails.map( pr => [ pr.number.toString(), pr ] ) );
-
 	const prsBySection = {};
+
 	entries.forEach( entry => {
 		const section = entry.section || 'Other';
 		if ( ! prsBySection[ section ] ) {
@@ -456,6 +412,7 @@ async function generateAIConsolidatedInstructions( entries, prDetails, apiKey, v
 		}
 	} );
 
+	// Construct the AI prompt
 	const prompt = `You are helping to create a consolidated testing guide for Jetpack plugin version ${
 		version || 'upcoming release'
 	}.
@@ -476,7 +433,7 @@ Your task is to:
 
 IMPORTANT REQUIREMENTS FOR PR REFERENCES:
 - ALWAYS reference PR numbers when discussing changes
-- Format PR numbers as markdown links: [#12345](https://github.com/Automattic/jetpack/pull/12345)
+- Format PR numbers as markdown links: [#12345](https://github.com/${ GITHUB_REPO }/pull/12345)
 - List all related PR numbers at the beginning of each section
 - Do NOT omit or skip PR numbers - they are critical for tracking
 - When combining multiple PRs into one testing section, list ALL PR numbers involved as links
@@ -485,7 +442,7 @@ Output format should be a well-structured markdown document with:
 - A summary section highlighting key areas to test
 - Each feature area as a heading with PR numbers listed as clickable links
 - Consolidated test steps (not just copying individual PR instructions)
-- All PR number references formatted as: [#12345](https://github.com/Automattic/jetpack/pull/12345)
+- All PR number references formatted as: [#12345](https://github.com/${ GITHUB_REPO }/pull/12345)
 - A section for changes without specific test instructions (with PR links)
 
 Here is the data:
@@ -496,7 +453,7 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 
 	try {
 		// Call Claude API
-		const response = await fetch( 'https://api.anthropic.com/v1/messages', {
+		const response = await fetch( CLAUDE_API_URL, {
 			method: 'POST',
 			headers: {
 				'Content-Type': 'application/json',
@@ -504,14 +461,9 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 				'anthropic-version': '2023-06-01',
 			},
 			body: JSON.stringify( {
-				model: 'claude-3-5-sonnet-20241022',
+				model: CLAUDE_MODEL,
 				max_tokens: 4096,
-				messages: [
-					{
-						role: 'user',
-						content: prompt,
-					},
-				],
+				messages: [ { role: 'user', content: prompt } ],
 			} ),
 		} );
 
@@ -522,9 +474,6 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 		const data = await response.json();
 		const consolidatedGuide = data.content[ 0 ].text;
 
-		// Note: PR numbers should already be formatted as links by the AI
-		// but we could add a fallback conversion here if needed
-
 		// Add metadata header
 		let output = `# Test Instructions for Jetpack ${ version || 'Release' }\n\n`;
 		output += `Generated on: ${ new Date().toISOString().split( 'T' )[ 0 ] }\n`;
@@ -535,38 +484,87 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 		return output;
 	} catch ( error ) {
 		console.warn(
-			chalk.yellow(
-				`\n⚠️  AI consolidation failed: ${ error.message }. Falling back to raw output.\n`
-			)
+			`\n⚠️  AI consolidation failed: ${ error.message }. Falling back to raw output.\n`
 		);
 		return generateRawTestInstructions( entries, prDetails );
 	}
 }
 
-/**
- * Prompt user for output file path.
- *
- * @param {string} version - Version string.
- * @return {Promise<string>} Output file path.
- */
-async function promptForOutputPath( version ) {
-	const defaultName = `test-instructions-${ version || 'latest' }.md`;
-	const response = await enquirer.prompt( {
-		type: 'input',
-		name: 'outputPath',
-		message: 'Where should the test guide be saved?',
-		initial: defaultName,
-	} );
-
-	return response.outputPath;
-}
+// ============================================================================
+// UTILITIES
+// ============================================================================
 
 /**
  * Simple sleep utility.
  *
- * @param {number} ms - Milliseconds to sleep.
- * @return {Promise} Promise that resolves after ms milliseconds.
+ * @param {number} ms - Milliseconds to sleep
+ * @return {Promise} Promise that resolves after ms milliseconds
  */
 function sleep( ms ) {
 	return new Promise( resolve => setTimeout( resolve, ms ) );
 }
+
+// ============================================================================
+// MAIN EXECUTION
+// ============================================================================
+
+/**
+ * Main function that orchestrates the entire process.
+ */
+async function main() {
+	try {
+		// Parse command line arguments
+		const options = parseArguments();
+
+		// Step 1: Parse changelog
+		const relativeChangelogPath = path.relative( process.cwd(), options.changelog );
+		console.log( `\n📖 Reading changelog from: ${ relativeChangelogPath }` );
+
+		const parseResult = parseChangelog( options.changelog, options.version, options.sinceDate );
+		const entries = parseResult.entries;
+
+		if ( entries.length === 0 ) {
+			throw new Error( 'No changelog entries found for the specified criteria.' );
+		}
+
+		console.log(
+			`✓ Found ${ entries.length } changelog entries since version ${ parseResult.startVersion }\n`
+		);
+
+		// Step 2: Extract PR numbers
+		const prNumbers = extractPRNumbers( entries );
+		console.log( `🔍 Identified ${ prNumbers.length } unique PRs\n` );
+
+		// Step 3: Fetch PR details from GitHub
+		console.log( '📥 Fetching PR details from GitHub...' );
+		const prDetails = await fetchPRDetails( prNumbers );
+		console.log( `✓ Fetched details for ${ prDetails.length } PRs\n` );
+
+		// Step 4: Generate test instructions
+		let testInstructions;
+
+		if ( options.skipAi || ! options.apiKey ) {
+			if ( ! options.skipAi ) {
+				console.log( '⚠️  No API key provided. Using raw output mode.\n' );
+			}
+			testInstructions = generateRawTestInstructions( entries, prDetails );
+		} else {
+			console.log( '🤖 Using AI to consolidate test instructions...' );
+			testInstructions = await generateAIConsolidatedInstructions(
+				entries,
+				prDetails,
+				options.apiKey,
+				options.version || parseResult.startVersion
+			);
+		}
+
+		// Step 5: Write to file
+		fs.writeFileSync( options.output, testInstructions );
+	} catch ( error ) {
+		console.error( `\n❌ Error: ${ error.message }` );
+		process.exit( 1 );
+	}
+}
+
+// Run the main function
+main();

From 3234481763bfddaf87c366121d69a8f42a27b748 Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Wed, 15 Oct 2025 12:08:10 +0530
Subject: [PATCH 3/6] Address Copilot feedback

---
 tools/gen-test-instructions.sh           |  6 +++++-
 tools/js-tools/gen-test-instructions.mjs | 14 +++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/tools/gen-test-instructions.sh b/tools/gen-test-instructions.sh
index d4adc9b39cefe..41e400109a91f 100755
--- a/tools/gen-test-instructions.sh
+++ b/tools/gen-test-instructions.sh
@@ -156,7 +156,11 @@ if [[ -z "$OUTPUT_FILE" ]]; then
 
 	# Prompt for confirmation
 	info "Output file will be: $OUTPUT_FILE"
-	read -p "Press Enter to continue or provide a name, or press Ctrl+C to cancel:"
+	read -p "Press Enter to continue, provide a different name, or press Ctrl+C to cancel: " USER_INPUT
+	if [[ -n "$USER_INPUT" ]]; then
+		OUTPUT_FILE="$USER_INPUT"
+		info "Output file updated to: $OUTPUT_FILE"
+	fi
 fi
 
 # Build arguments for the Node.js script
diff --git a/tools/js-tools/gen-test-instructions.mjs b/tools/js-tools/gen-test-instructions.mjs
index e71dd65ca859c..2cab7f0236582 100644
--- a/tools/js-tools/gen-test-instructions.mjs
+++ b/tools/js-tools/gen-test-instructions.mjs
@@ -10,7 +10,7 @@
  * 4. Optionally consolidating instructions using Claude AI
  * 5. Generating a markdown document with all PR numbers as clickable links
  *
- * Usage node gen-test-instructions.mjs --changelog <path> --output <file> [options]
+ * Usage: node gen-test-instructions.mjs --changelog <path> --output <file> [options]
  */
 
 import { execSync } from 'child_process';
@@ -146,18 +146,23 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	}
 
 	// Second pass: collect entries (changelog is reverse chronological)
+	// We collect entries from the top (newest) until we reach the startVersion (cutoff point)
+	// Example: If startVersion = "15.1", we collect 15.2-a.1, 15.1.1, etc. until we hit 15.1
 	for ( const line of lines ) {
 		const versionMatch = line.match( versionRegex );
 		if ( versionMatch ) {
 			currentVersion = versionMatch[ 1 ];
 			currentDate = versionMatch[ 2 ];
 
-			// Stop when we reach the cutoff version
+			// Stop when we reach the cutoff version (we want entries AFTER this version, not including it)
 			if ( startVersion && currentVersion === startVersion ) {
+				collectingEntries = false;
 				break;
 			} else if ( sinceDate && currentDate < sinceDate ) {
+				collectingEntries = false;
 				break;
 			} else {
+				// We're still in the "newer than cutoff" range, so collect entries
 				collectingEntries = true;
 			}
 
@@ -413,9 +418,8 @@ async function generateAIConsolidatedInstructions( entries, prDetails, apiKey, v
 	} );
 
 	// Construct the AI prompt
-	const prompt = `You are helping to create a consolidated testing guide for Jetpack plugin version ${
-		version || 'upcoming release'
-	}.
+	const releaseVersion = version || 'upcoming release';
+	const prompt = `You are helping to create a consolidated testing guide for Jetpack plugin version ${ releaseVersion }.
 
 I have changelog entries grouped by feature area/section. Each entry includes:
 - The PR number and title

From 96eb7ad027307882ca7db65ee620bcc66796eb68 Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Sun, 19 Oct 2025 11:59:46 +0700
Subject: [PATCH 4/6] Address PR feedback

---
 tools/gen-test-instructions.sh           | 13 +++-
 tools/js-tools/gen-test-instructions.mjs | 82 +++++++++++++++++++-----
 2 files changed, 75 insertions(+), 20 deletions(-)

diff --git a/tools/gen-test-instructions.sh b/tools/gen-test-instructions.sh
index 41e400109a91f..1686fb9542dfe 100755
--- a/tools/gen-test-instructions.sh
+++ b/tools/gen-test-instructions.sh
@@ -3,9 +3,16 @@
 ##
 ## Generate test instructions from changelog for Jetpack releases.
 ##
-## This script automates the creation of consolidated test instructions
-## by parsing the CHANGELOG.md, fetching PR details from GitHub, and
-## optionally using AI to consolidate the instructions.
+## This bash script serves as a user-friendly wrapper around the Node.js
+## implementation at tools/js-tools/gen-test-instructions.mjs. It provides:
+## - Prerequisite checking (gh CLI, Node.js, authentication)
+## - Path resolution and validation
+## - User-friendly prompts and error messages
+## - Integration with standard Jetpack tooling (chalk-lite, etc.)
+##
+## The actual logic for parsing, fetching, and generating test instructions
+## is in the JavaScript module, while this script handles the CLI interface
+## and environment setup.
 ##
 
 set -eo pipefail
diff --git a/tools/js-tools/gen-test-instructions.mjs b/tools/js-tools/gen-test-instructions.mjs
index 2cab7f0236582..2fe54c44ce21e 100644
--- a/tools/js-tools/gen-test-instructions.mjs
+++ b/tools/js-tools/gen-test-instructions.mjs
@@ -10,7 +10,18 @@
  * 4. Optionally consolidating instructions using Claude AI
  * 5. Generating a markdown document with all PR numbers as clickable links
  *
- * Usage: node gen-test-instructions.mjs --changelog <path> --output <file> [options]
+ * Usage: node gen-test-instructions.mjs [options]
+ *
+ * Required Options:
+ *   --changelog <path>    Path to CHANGELOG.md file
+ *   --output <file>       Output file path for generated test instructions
+ *
+ * Optional:
+ *   --version <version>   Start from this version (e.g., 15.1). Defaults to last stable release
+ *   --since-date <date>   Include entries since this date (YYYY-MM-DD format)
+ *   --api-key <key>       Anthropic API key for AI consolidation (or use ANTHROPIC_API_KEY env var)
+ *   --skip-ai             Skip AI consolidation and output raw format
+ *   --verbose             Enable verbose output for debugging
  */
 
 import { execSync } from 'child_process';
@@ -43,6 +54,7 @@ function parseArguments() {
 		sinceDate: null,
 		apiKey: process.env.ANTHROPIC_API_KEY || null,
 		skipAi: false,
+		verbose: false,
 	};
 
 	for ( let i = 0; i < args.length; i++ ) {
@@ -65,6 +77,9 @@ function parseArguments() {
 			case '--skip-ai':
 				options.skipAi = true;
 				break;
+			case '--verbose':
+				options.verbose = true;
+				break;
 			default:
 				throw new Error( `Unknown option: ${ args[ i ] }` );
 		}
@@ -111,7 +126,9 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 	// Regular expressions for matching changelog format
 	const versionRegex = /^## ([\d.]+(?:-[a-z]+\.\d+)?)\s*-\s*(\d{4}-\d{2}-\d{2})/i;
 	const sectionRegex = /^### (.+)/;
-	const entryRegex = /^- (.+?) \[#(\d+)\]/;
+	// Match the entry text and extract all PR numbers (handles single or multiple PRs)
+	const entryRegex = /^- (.+?)(?:\s+\[#\d+\])+/;
+	const prNumberRegex = /\[#(\d+)\]/g;
 
 	// First pass: collect all versions and find last stable
 	for ( const line of lines ) {
@@ -179,12 +196,25 @@ function parseChangelog( changelogPath, sinceVersion, sinceDate ) {
 		if ( collectingEntries && currentVersion ) {
 			const entryMatch = line.match( entryRegex );
 			if ( entryMatch ) {
-				entries.push( {
-					text: entryMatch[ 1 ],
-					prNumber: entryMatch[ 2 ],
-					section: currentSection,
-					version: currentVersion,
-					date: currentDate,
+				const text = entryMatch[ 1 ].trim();
+
+				// Extract all PR numbers from the line (handles single or multiple PRs)
+				const prNumbers = [];
+				let prMatch;
+				while ( ( prMatch = prNumberRegex.exec( line ) ) !== null ) {
+					prNumbers.push( prMatch[ 1 ] );
+				}
+
+				// Create an entry for each PR number
+				// This allows us to fetch details for all related PRs
+				prNumbers.forEach( prNumber => {
+					entries.push( {
+						text,
+						prNumber,
+						section: currentSection,
+						version: currentVersion,
+						date: currentDate,
+					} );
 				} );
 			}
 		}
@@ -363,16 +393,18 @@ function generateRawTestInstructions( entries, prDetails ) {
 				output += `### [${ pr.title }](https://github.com/${ GITHUB_REPO }/pull/${ pr.number }) (#${ pr.number })\n\n`;
 
 				if ( pr.testingInstructions ) {
+					// We have explicit testing instructions
 					const linkedInstructions = convertPRNumbersToLinks( pr.testingInstructions );
 					output += `${ linkedInstructions }\n\n`;
+				} else if ( pr.body ) {
+					// No explicit testing instructions, include full PR description for context
+					output +=
+						'_No specific testing instructions section found. Full PR description below:_\n\n';
+					const linkedBody = convertPRNumbersToLinks( pr.body );
+					output += `${ linkedBody }\n\n`;
 				} else {
-					output += '_No specific testing instructions provided._\n\n';
-					if ( pr.body ) {
-						const linkedBody = convertPRNumbersToLinks( pr.body.substring( 0, 300 ) + '...' );
-						output += `**PR Description:**\n${ linkedBody }\n\n`;
-					} else {
-						output += '**PR Description:** N/A\n\n';
-					}
+					// No PR description at all
+					output += '_No testing instructions or PR description available._\n\n';
 				}
 
 				output += '---\n\n';
@@ -466,7 +498,9 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 			},
 			body: JSON.stringify( {
 				model: CLAUDE_MODEL,
-				max_tokens: 4096,
+				// Max tokens for the response. 8192 allows for comprehensive test guides
+				// with many PRs. Can be increased if needed for very large releases.
+				max_tokens: 8192,
 				messages: [ { role: 'user', content: prompt } ],
 			} ),
 		} );
@@ -520,9 +554,15 @@ async function main() {
 		// Parse command line arguments
 		const options = parseArguments();
 
+		console.log( '🧪 Generating Test Instructions Guide...\n' );
+
+		if ( options.verbose ) {
+			console.log( 'Options:', JSON.stringify( options, null, 2 ) );
+		}
+
 		// Step 1: Parse changelog
 		const relativeChangelogPath = path.relative( process.cwd(), options.changelog );
-		console.log( `\n📖 Reading changelog from: ${ relativeChangelogPath }` );
+		console.log( `📖 Reading changelog from: ${ relativeChangelogPath }` );
 
 		const parseResult = parseChangelog( options.changelog, options.version, options.sinceDate );
 		const entries = parseResult.entries;
@@ -535,6 +575,14 @@ async function main() {
 			`✓ Found ${ entries.length } changelog entries since version ${ parseResult.startVersion }\n`
 		);
 
+		if ( options.verbose ) {
+			console.log(
+				`Available versions in changelog: ${ parseResult.versions
+					.map( v => v.version )
+					.join( ', ' ) }`
+			);
+		}
+
 		// Step 2: Extract PR numbers
 		const prNumbers = extractPRNumbers( entries );
 		console.log( `🔍 Identified ${ prNumbers.length } unique PRs\n` );

From 1a6de0ce14c38e6ad8473bb5ccc5781e92569689 Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Thu, 23 Oct 2025 14:42:54 +0530
Subject: [PATCH 5/6] Update Claude model version

---
 tools/js-tools/gen-test-instructions.mjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/js-tools/gen-test-instructions.mjs b/tools/js-tools/gen-test-instructions.mjs
index 2fe54c44ce21e..8f9ec5d6d3ad7 100644
--- a/tools/js-tools/gen-test-instructions.mjs
+++ b/tools/js-tools/gen-test-instructions.mjs
@@ -34,7 +34,7 @@ import path from 'path';
 
 const GITHUB_REPO = 'Automattic/jetpack';
 const CLAUDE_API_URL = 'https://api.anthropic.com/v1/messages';
-const CLAUDE_MODEL = 'claude-3-5-sonnet-20241022';
+const CLAUDE_MODEL = 'claude-sonnet-4-5-20250929';
 
 // ============================================================================
 // COMMAND LINE ARGUMENT PARSING

From 1297299a6b4cce0e93d75524d3811d5aa9f6b42d Mon Sep 17 00:00:00 2001
From: Manzoor Wani <manzoorwani.jk@gmail.com>
Date: Thu, 23 Oct 2025 15:29:52 +0530
Subject: [PATCH 6/6] Update instructions for headings

---
 tools/js-tools/gen-test-instructions.mjs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/js-tools/gen-test-instructions.mjs b/tools/js-tools/gen-test-instructions.mjs
index 8f9ec5d6d3ad7..57e2811d632f7 100644
--- a/tools/js-tools/gen-test-instructions.mjs
+++ b/tools/js-tools/gen-test-instructions.mjs
@@ -475,11 +475,13 @@ IMPORTANT REQUIREMENTS FOR PR REFERENCES:
 - When combining multiple PRs into one testing section, list ALL PR numbers involved as links
 
 Output format should be a well-structured markdown document with:
+- Start with ### (heading level 3) for all top-level sections
 - A summary section highlighting key areas to test
 - Each feature area as a heading with PR numbers listed as clickable links
 - Consolidated test steps (not just copying individual PR instructions)
 - All PR number references formatted as: [#12345](https://github.com/${ GITHUB_REPO }/pull/12345)
 - A section for changes without specific test instructions (with PR links)
+- Do NOT use # (heading level 1) or ## (heading level 2) - all headings should be ### (level 3) or deeper
 
 Here is the data:
 
@@ -513,7 +515,7 @@ Generate the consolidated test guide now. Remember to format ALL PR numbers as m
 		const consolidatedGuide = data.content[ 0 ].text;
 
 		// Add metadata header
-		let output = `# Test Instructions for Jetpack ${ version || 'Release' }\n\n`;
+		let output = `## Test Instructions for Jetpack ${ version || 'Release' }\n\n`;
 		output += `Generated on: ${ new Date().toISOString().split( 'T' )[ 0 ] }\n`;
 		output += `Total PRs: ${ prDetails.length }\n\n`;
 		output += '---\n\n';