Use a task to build the GNATtest harness project

Author: eliericha

CHANGELOG.md

@@ -7,6 +7,7 @@ section below it for the last release. -->
 * Support single-line (`//`) and multi-line (`/* */`) comments in ALS JSON configuration files
 * Fix the reporting of test results in the Testing view when the test tree is not expanded
 * Fix sluggish completion while editing GPR files
+* Provide a task `ada: Build GNATtest test harness project` allowing to customize the build step of test execution in `tasks.json`
 
 ## 26.0.202412190
 

README.md

@@ -291,14 +291,18 @@ For example, if the project defines a `main1.adb` and `main2.adb` located under
 
 If you install GNATtest, the Ada & SPARK extension for VS Code will provide the following functionalities:
 
-* The task `ada: Create/update test skeletons for the project` will call `gnattest` to create test skeletons for your project automatically. You can use standard VS Code task customization to configure command line arguments to your liking in a `tasks.json` file.
+* The task `ada: Create or update GNATtest test framework` will call `gnattest` to create a test harness and test skeletons for your project automatically. You can use standard VS Code task customization to configure command line arguments to your liking in a `tasks.json` file.
+
+* Once the test harness project is created, the task `ada: Build GNATtest test harness project` is provided automatically for building it. Command line arguments can be customized by configuring the task in a `tasks.json` file.
 
 * Tests created with GNATtest will be loaded in the VS Code **Testing** view as follows.
 
   <img src="doc/gnattest-test-tree.png" width="650" alt="GNATtest Test Tree">
 
 * Tests can be executed individually or in batch through the available buttons in the interface, or through the `Test: Run All Tests` command or related commands.
 
+* Test execution always starts by executing the `ada: Build GNATtest test harness project` task to make sure that test executables are up to date.
+
 * Test execution results are reflected in the test tree.
 
   <img src="doc/gnattest-results.png" width="500" alt="GNATtest Test Results">
@@ -308,8 +312,6 @@ GNATtest support has the following known limitations:
 * The extension relies on the default conventions of GNATtest such as the naming, location and object directory of the test harness project.
   If those aspects are configured or altered manually, the features may no longer work.
 
-* Test execution always starts with a `gprbuild` call on the test harness project. It is not possible to disable that call or customize its arguments. This limitation will be lifted in future releases.
-
 * Language support such as navigation and auto-completion is limited when editing test sources. A workaround is to modify the `ada.projectFile` setting to point to the test harness project created by GNATtest. That should restore language support when developing tests.
 
 * Sections of test sources delimited by `begin read only` and `end read only` comments are not really protected from inadvertant edits.

integration/vscode/ada/src/commands.ts

@@ -656,16 +656,25 @@ async function buildAndDebugSpecifiedMain(main: vscode.Uri): Promise<void> {
  * for use with GPR-based tools.
  */
 export async function gprProjectArgs(): Promise<string[]> {
+    const scenarioArgs = gprScenarioArgs();
+
+    return ['-P', await getProjectFromConfigOrALS()].concat(scenarioArgs);
+}
+
+export const PROJECT_FROM_CONFIG = '${config:ada.projectFile}';
+
+/**
+ * @returns an array of -X scenario command lines arguments for use with
+ * GPR-based tools.
+ */
+export function gprScenarioArgs() {
     const vars: string[][] = Object.entries(
         vscode.workspace.getConfiguration('ada').get('scenarioVariables') ?? [],
     );
-    return ['-P', await getProjectFromConfigOrALS()].concat(
-        vars.map(([key, value]) => `-X${key}=${value}`),
-    );
+    const scenarioArgs = vars.map(([key, value]) => `-X${key}=${value}`);
+    return scenarioArgs;
 }
 
-export const PROJECT_FROM_CONFIG = '${config:ada.projectFile}';
-
 /**
  * @returns `"$\{config:ada.projectFile\}"` if that setting has a value, or else
  * queries the ALS for the current project and returns the full path.

integration/vscode/ada/src/gnattest.ts

@@ -8,6 +8,12 @@ import { TestItem } from 'vscode';
 import { CancellationToken } from 'vscode-languageclient';
 import { adaExtState } from './extension';
 import { escapeRegExp, exe, getObjectDir } from './helpers';
+import {
+    findTaskByName,
+    runTaskAndGetResult,
+    TASK_BUILD_TEST_DRIVER,
+    TASK_TYPE_ADA,
+} from './taskProviders';
 
 export let controller: vscode.TestController;
 export let testRunProfile: vscode.TestRunProfile;
@@ -582,18 +588,22 @@ async function handleRunRequestedTests(request: vscode.TestRunRequest, token?: C
  * reported on those tests.
  */
 async function buildTestDriverAndReportErrors(run: vscode.TestRun, testsToRun: vscode.TestItem[]) {
-    try {
-        await buildTestDriver(run);
-    } catch (error) {
+    const task = await findTaskByName(`${TASK_TYPE_ADA}: ${TASK_BUILD_TEST_DRIVER}`);
+    const result = await runTaskAndGetResult(task);
+    if (result != 0) {
+        const msg =
+            `Task '${TASK_BUILD_TEST_DRIVER}' failed.` +
+            ` Check the [Problems](command:workbench.panel.markers.view.focus) view` +
+            ` and the [Terminal](command:terminal.focus) view for more information.`;
+        const md = new vscode.MarkdownString(msg);
+        md.isTrusted = true;
+        const testMsg = new vscode.TestMessage(md);
         /**
-         * In case of failure, report all tests as errored.
+         * Mark each test as errored, not failed, since the tests can't run
+         * because of the build error.
          */
-        const md = getBuildErrorMessage();
-        for (const test of testsToRun) {
-            run.errored(test, new vscode.TestMessage(md));
-        }
-        run.end();
-        throw error;
+        testsToRun.forEach((test) => run.errored(test, testMsg));
+        throw Error(msg);
     }
 }
 
@@ -617,6 +627,15 @@ function prepareAndAppendOutput(run: vscode.TestRun, out: string) {
 async function handleRunAll(request: vscode.TestRunRequest, token?: CancellationToken) {
     const run = controller.createTestRun(request, undefined, false);
     try {
+        /**
+         * If the run request was created with the 'Tests: Run All Tests'
+         * command before activating the Testing view, then the controller is
+         * still empty. In that case let's refresh it to load the tests.
+         */
+        if (controller.items.size == 0) {
+            await controller.refreshHandler!(token ?? new vscode.CancellationTokenSource().token);
+        }
+
         /**
          * Collect all tests, i.e. all leafs of the TestItem tree.
          */
@@ -673,54 +692,6 @@ async function handleRunAll(request: vscode.TestRunRequest, token?: Cancellation
     }
 }
 
-/**
- * Failures to build the test driver are reported as test errors to make them
- * clearly visible.
- *
- * @returns the message to be used as a test failure when the test driver fails
- * to build.
- */
-function getBuildErrorMessage() {
-    const md = new vscode.MarkdownString(
-        'Failed to build the test driver, [view output](command:testing.showMostRecentOutput)',
-    );
-    md.isTrusted = true;
-    return md;
-}
-
-/**
- * Invoke gprbuild on the test driver project and pipe the output into the given
- * TestRun object.
- *
- * @param run - the TestRun object hosting this execution
- * @throws an Error if the process ends with an error code
- */
-async function buildTestDriver(run: vscode.TestRun) {
-    /**
-     * TODO replace this with a task invocation to capture problems
-     */
-    const driverPrjPath = await getGnatTestDriverProjectPath();
-    run.appendOutput(`Building the test harness project\r\n`);
-    /**
-     * The following arguments are passed directly to the spawned subprocess.
-     * It not necessary nor appropriate to apply shell quoting here.
-     */
-    const gprbuild = logAndRun(run, ['gprbuild', '-P', driverPrjPath, '-cargs:ada', '-gnatef']);
-
-    prepareAndAppendOutput(run, gprbuild.stdout.toLocaleString());
-    prepareAndAppendOutput(run, gprbuild.stderr.toLocaleString());
-
-    if (gprbuild.status !== 0) {
-        throw Error(
-            'Error while building the test driver:\n' +
-                gprbuild.output
-                    .filter((x) => x != null)
-                    .map((x) => x.toLocaleString())
-                    .join('\n'),
-        );
-    }
-}
-
 /**
  * Decide the outcome of the given test based on the test driver output, and
  * report it to the TestRun object.

integration/vscode/ada/src/taskProviders.ts

@@ -16,10 +16,12 @@
 ----------------------------------------------------------------------------*/
 
 import assert from 'assert';
+import { existsSync } from 'fs';
 import path, { basename } from 'path';
 import * as vscode from 'vscode';
 import { CMD_GPR_PROJECT_ARGS } from './commands';
 import { adaExtState, logger } from './extension';
+import { getGnatTestDriverProjectPath } from './gnattest';
 import { AdaMain, getAdaMains, showErrorMessageWithOpenLogButton } from './helpers';
 
 export const TASK_TYPE_ADA = 'ada';
@@ -90,6 +92,7 @@ export const TASK_PROVE_SUPB_PLAIN_NAME = 'Prove subprogram';
 export const TASK_PROVE_REGION_PLAIN_NAME = 'Prove selected region';
 export const TASK_PROVE_LINE_PLAIN_NAME = 'Prove line';
 export const TASK_PROVE_FILE_PLAIN_NAME = 'Prove file';
+export const TASK_BUILD_TEST_DRIVER = 'Build GNATtest test harness project';
 
 /**
  * Predefined tasks offered by the extension. Both 'ada' and 'spark' tasks are
@@ -230,7 +233,7 @@ const predefinedTasks: PredefinedTask[] = [
         problemMatchers: '',
     },
     {
-        label: 'Create/update test skeletons for the project',
+        label: 'Create or update GNATtest test framework',
         taskDef: {
             type: TASK_TYPE_ADA,
             command: 'gnattest',
@@ -454,6 +457,23 @@ export class SimpleTaskProvider implements vscode.TaskProvider {
                     return [buildTask, runTask, buildAndRunTask];
                 }),
             );
+
+            /**
+             * If a test harness project exists, provide a task to build it.
+             */
+            const harnessPrj = await getGnatTestDriverProjectPath();
+            if (existsSync(harnessPrj)) {
+                taskDeclsToOffer.push({
+                    label: TASK_BUILD_TEST_DRIVER,
+                    taskDef: {
+                        type: TASK_TYPE_ADA,
+                        command: 'gprbuild',
+                        args: ['-P', harnessPrj, "'-cargs:ada'", '-gnatef'],
+                    },
+                    problemMatchers: DEFAULT_PROBLEM_MATCHER,
+                    taskGroup: vscode.TaskGroup.Build,
+                });
+            }
         }
 
         /**
@@ -1157,3 +1177,63 @@ function isAlire(command: string | vscode.ShellQuotedString): boolean {
     const commandBasename = basename(value);
     return commandBasename.match(/^alr(\.exe)?$/) != null;
 }
+
+/**
+ * Execute the given task, wait until it finishes and return the underlying
+ * process exit code.
+ *
+ * @param task - a {@link vscode.Task}
+ * @returns a Promise that resolves to the underlying process exit code when the
+ * task finishes execution.
+ */
+export async function runTaskAndGetResult(task: vscode.Task): Promise<number | undefined> {
+    return await new Promise<number | undefined>((resolve, reject) => {
+        let started = false;
+
+        const startDisposable = vscode.tasks.onDidStartTask((e) => {
+            if (e.execution.task == task) {
+                /**
+                 * Task was started, let's listen to the end.
+                 */
+                started = true;
+                startDisposable.dispose();
+            }
+        });
+
+        const endDisposable = vscode.tasks.onDidEndTaskProcess((e) => {
+            if (e.execution.task == task) {
+                endDisposable.dispose();
+                resolve(e.exitCode);
+            }
+        });
+
+        setTimeout(() => {
+            /**
+             * If the task has not started within the timeout below, it means an
+             * error occured during startup. Reject the promise.
+             */
+            if (!started) {
+                const msg = `The task '${getConventionalTaskLabel(
+                    task,
+                )}' was not started, likely due to an error.\n`;
+                reject(Error(msg));
+            }
+        }, 3000);
+
+        void vscode.tasks.executeTask(task);
+    }).catch(async (reason) => {
+        if (reason instanceof Error) {
+            let msg = 'The current list of tasks is:\n';
+            msg += await vscode.tasks.fetchTasks({ type: task.definition.type }).then(
+                (list) => list.map(getConventionalTaskLabel).join('\n'),
+
+                (reason) => `fetchTasks promise was rejected: ${reason}`,
+            );
+
+            reason.message += '\n' + msg;
+        }
+
+        // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
+        return Promise.reject(reason);
+    });
+}

integration/vscode/ada/test/general/tasks.test.ts

@@ -21,9 +21,9 @@ import {
     isCoreTask,
     isGNATSASTask,
     negate,
-    runTaskAndGetResult,
     testTask,
 } from '../utils';
+import { runTaskAndGetResult } from '../../src/taskProviders';
 
 suite('Task Providers', function () {
     this.timeout('15s');