More progress

Author: cte

benchmark/harness/package-lock.json

@@ -4,14 +4,15 @@
 	"private": true,
 	"scripts": {
 		"build": "cd ../.. && npm run build",
-		"compile": "tsc -p tsconfig.json",
+		"compile": "rm -rf out && tsc -p tsconfig.json",
 		"lint": "eslint src --ext ts",
 		"check-types": "tsc --noEmit",
 		"dev": "npm run compile && npx dotenvx run -f .env.integration -- node ./out/run.js"
 	},
 	"devDependencies": {
 		"@vscode/test-electron": "^2.4.0",
 		"tsx": "^4.19.3",
-		"typescript": "^5.4.5"
+		"typescript": "^5.4.5",
+		"yargs": "^17.7.2"
 	}
 }

benchmark/harness/package.json

@@ -1,15 +1,40 @@
 import * as path from "path"
 
+import yargs from "yargs"
+import { hideBin } from "yargs/helpers"
 import { runTests } from "@vscode/test-electron"
 
 async function main() {
+	const argv = await yargs(hideBin(process.argv))
+		.option("exercise", {
+			alias: "e",
+			type: "string",
+			description: "Path to the exercise directory",
+			required: true,
+		})
+		.help().argv
+
 	const extensionDevelopmentPath = path.resolve(__dirname, "../../../")
 	const extensionTestsPath = path.resolve(__dirname, "./runExercise")
+	const testWorkspace = path.resolve(__dirname, "..", argv.exercise)
 
 	await runTests({
 		extensionDevelopmentPath: extensionDevelopmentPath,
 		extensionTestsPath: extensionTestsPath,
-		launchArgs: ["foo", "bar", "baz"],
+		launchArgs: [testWorkspace, "--disable-extensions"],
+		extensionTestsEnv: {
+			prompt: `
+				Your job is to complete a coding exercise described by \`.docs/instructions.md\`.
+				A file with the implementation stubbed out has been created for you, along with a test file.
+				To successfully complete the exercise, you must pass all the tests in the test file.
+				To confirm that your solution is correct, run the tests with \`yarn test\`.
+				Before running the test make sure your environment is set up by running \`corepack enable\` to enable yarn and \`yarn install\` to install the dependencies.
+			`
+				.trim()
+				.split("\n")
+				.map((line) => line.trim())
+				.join("\n"),
+		},
 	})
 }