Fix issue with tool cache on self-hosted runners (#152)

* Add failing test

* Always call mpm install even if MATLAB is in toolcache

* Disable Windows installation tests due to symlink issues with toolcache on Microsoft-hosted agents

* Add test running InstallMATLAB twice with no additional products

* Enhance MATLAB installation handling to ignore 'already installed' messages and update test cases accordingly

Author: mcafaro

integ-test-promote-template.yml

@@ -60,6 +60,53 @@ jobs:
           inputs:
             command: assert(strcmp(version('-release'),'2023a'))
 
+    - job: test_install_with_toolcache_v${{ version }}
+      condition: not(eq(${{ version }}, '0'))
+      strategy:
+        matrix:
+          microsoft_hosted_linux:
+            poolName: Azure Pipelines
+            vmImage: ubuntu-22.04
+          microsoft_hosted_macos:
+            poolName: Azure Pipelines
+            vmImage: macOS-latest
+          # Installing with toolcache on Windows cannot be tested on Microsoft-hosted agents
+          # because InstallMATLAB sets up a symlinked toolcache for performance reasons and MPM
+          # does not play well with symlinks on Windows.
+          # microsoft_hosted_windows:
+          #   poolName: Azure Pipelines
+          #   vmImage: windows-latest
+      pool:
+        name: $(poolName)
+        vmImage: $(vmImage)
+      steps:
+        - checkout: none
+        - task: MathWorks.matlab-azure-devops-extension-dev.InstallMATLAB.InstallMATLAB@${{ version }}
+          displayName: Install MATLAB
+          inputs:
+            release: R2023a
+        - task: MathWorks.matlab-azure-devops-extension-dev.RunMATLABCommand.RunMATLABCommand@${{ version }}
+          displayName: Check Image Processing Toolbox is NOT installed
+          inputs:
+            command: assert(~any(strcmp({ver().Name},'Image Processing Toolbox')))
+        - task: MathWorks.matlab-azure-devops-extension-dev.InstallMATLAB.InstallMATLAB@${{ version }}
+          displayName: Install MATLAB no-ops on second run without additional products
+          inputs:
+            release: R2023a
+        - task: MathWorks.matlab-azure-devops-extension-dev.RunMATLABCommand.RunMATLABCommand@${{ version }}
+          displayName: Check Image Processing Toolbox is NOT installed
+          inputs:
+            command: assert(~any(strcmp({ver().Name},'Image Processing Toolbox')))
+        - task: MathWorks.matlab-azure-devops-extension-dev.InstallMATLAB.InstallMATLAB@${{ version }}
+          displayName: Install MATLAB installs additional products
+          inputs:
+            release: R2023a
+            products: Image_Processing_Toolbox
+        - task: MathWorks.matlab-azure-devops-extension-dev.RunMATLABCommand.RunMATLABCommand@${{ version }}
+          displayName: Check Image Processing Toolbox is installed
+          inputs:
+            command: assert(any(strcmp({ver().Name},'Image Processing Toolbox')))
+
     - job: test_install_latest_including_prerelease_v${{ version }}
       condition: not(eq(${{ version }}, '0'))
       strategy:

tasks/install-matlab/v1/src/install.ts

@@ -28,10 +28,8 @@ export async function install(platform: string, architecture: string, release: s
     const mpmPath: string = await mpm.setup(platform, matlabArch);
 
     // install MATLAB using mpm
-    const [toolpath, alreadyExists] = await matlab.makeToolcacheDir(parsedRelease, platform);
-    if (!alreadyExists) {
-        await mpm.install(mpmPath, parsedRelease, toolpath, products);
-    }
+    const [toolpath] = await matlab.makeToolcacheDir(parsedRelease, platform);
+    await mpm.install(mpmPath, parsedRelease, toolpath, products);
 
     // add MATLAB to system path
     try {

tasks/install-matlab/v1/src/mpm.ts

@@ -1,6 +1,8 @@
 // Copyright 2023-2024 The MathWorks, Inc.
 
 import * as taskLib from "azure-pipelines-task-lib/task";
+import * as trm from "azure-pipelines-task-lib/toolrunner";
+import { Writable } from "stream";
 import * as matlab from "./matlab";
 import { downloadToolWithRetries } from "./utils";
 
@@ -66,8 +68,25 @@ export async function install(
     mpmArguments = mpmArguments.concat("--products");
     mpmArguments = mpmArguments.concat(parsedProducts);
 
-    const exitCode = await taskLib.exec(mpmPath, mpmArguments);
-    if (exitCode !== 0) {
+    let output = "";
+    const captureStream = new Writable({
+        write(chunk, encoding, callback) {
+            const text = chunk.toString();
+            output += text;
+            process.stdout.write(text);
+            callback();
+        },
+    });
+
+    const options: trm.IExecOptions = {
+        outStream: captureStream,
+        errStream: captureStream,
+        ignoreReturnCode: true,
+    };
+
+    const exitCode = await taskLib.exec(mpmPath, mpmArguments, options);
+
+    if (exitCode !== 0 && !output.toLowerCase().includes("already installed")) {
         return Promise.reject(Error(`Failed to install MATLAB.`));
     }
     return;

tasks/install-matlab/v1/test/install.test.ts

@@ -83,14 +83,14 @@ export default function suite() {
             assert(stubInstallSystemDependencies.notCalled);
         });
 
-        it("does not install if MATLAB already exists in toolcache", async () => {
+        it("re-calls MPM install even if MATLAB already exists in toolcache", async () => {
             stubMakeToolcacheDir.callsFake((rel) => {
                 return [toolcacheDir, true];
             });
             await assert.doesNotReject(async () => {
                 await install.install(platform, architecture, release, products);
             });
-            assert(stubMpmInstall.notCalled);
+            assert(stubMpmInstall.calledWith(sinon.match.any, releaseInfo, toolcacheDir, products));
         });
 
         it("fails if add to path fails", async () => {

tasks/install-matlab/v1/test/mpm.test.ts

@@ -162,7 +162,27 @@ export default function suite() {
                 // non-zero exit code
                 return Promise.resolve(1);
             });
-            assert.rejects(async () => mpm.install(mpmPath, releaseInfo, destination, products));
+            await assert.rejects(async () => mpm.install(mpmPath, releaseInfo, destination, products));
+        });
+
+        it("install does not fail when mpm reports products already installed", async () => {
+            const mpmPath = "mpm";
+            const releaseInfo = {name: "r2022b", version: "9.13.0", update: "Latest", isPrerelease: false};
+            const destination = "/opt/matlab";
+            const products = "MATLAB Compiler";
+
+            // Simulate mpm returning a non-zero exit code but writing the
+            // "already installed" message to the outStream so install should succeed.
+            stubExec.callsFake((bin, args?, options?) => {
+                if (options && options.outStream && typeof options.outStream.write === "function") {
+                    options.outStream.write("All specified products are already installed.\n");
+                }
+                return Promise.resolve(1);
+            });
+
+            // Should not reject
+            await mpm.install(mpmPath, releaseInfo, destination, products);
+            assert(stubExec.called);
         });
     });
 }