Merge pull request #6626 from dibarbet/speed_up_build

Build VSIXs on multiple platforms to parallelize build

Author: dibarbet

.vscode/launch.json

@@ -131,6 +131,17 @@
             ],
             "preLaunchTask": "buildDev"
         },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Debug gulp task",
+            "preLaunchTask": "build",
+            "program": "${workspaceFolder}/node_modules/gulp/bin/gulp.js",
+            "args": [
+                "${input:gulpTaskName}"
+            ],
+            "cwd": "${workspaceFolder}"
+        },
         {
             "type": "node",
             "request": "launch",
@@ -214,7 +225,11 @@
                 "slnFilterWithCsproj",
                 "slnWithGenerator"
             ]
-            // type specific configuration attributes
+        },
+        {
+            "id": "gulpTaskName",
+            "description": "The name of the gulp task to debug",
+            "type": "promptString",
         }
     ]
 }

azure-pipelines-official.yml

@@ -13,7 +13,7 @@ parameters:
     default: 'default'
 
 stages:
-- template: azure-pipelines/build.yml
+- template: azure-pipelines/build-all.yml
   parameters:
     versionNumberOverride: ${{ parameters.versionNumberOverride }}
 

azure-pipelines.yml

@@ -12,7 +12,7 @@ pr:
 - main
 
 stages:
-- template: azure-pipelines/build.yml
+- template: azure-pipelines/build-all.yml
 
 - stage: Test
   displayName: Test

azure-pipelines/build-all.yml

@@ -0,0 +1,25 @@
+parameters:
+- name: versionNumberOverride
+  type: string
+  default: 'default'
+
+stages:
+- stage: Build
+  displayName: 'Build VSIXs'
+  dependsOn: []
+  jobs:
+  - template: build.yml
+    parameters:
+      versionNumberOverride: ${{ parameters.versionNumberOverride }}
+      vmImageName: ubuntu-latest
+      platform: linux
+  - template: build.yml
+    parameters:
+      versionNumberOverride: ${{ parameters.versionNumberOverride }}
+      vmImageName: windows-latest
+      platform: windows
+  - template: build.yml
+    parameters:
+      versionNumberOverride: ${{ parameters.versionNumberOverride }}
+      vmImageName: macOS-latest
+      platform: darwin

azure-pipelines/build.yml

@@ -2,62 +2,70 @@ parameters:
 - name: versionNumberOverride
   type: string
   default: 'default'
+- name: vmImageName
+  type: string
+  default: ''
+- name: platform
+  type: string
+
+jobs:
+- job:
+  displayName: 'Build ${{ parameters.platform }} prerelease vsixs'
+  pool:
+    name: Azure Pipelines
+    vmImage: ${{ parameters.vmImageName }}
+  steps:
+  - checkout: self
+    clean: true
+    submodules: true
+    fetchTags: false
+    fetchDepth: 0
+
+  - template: prereqs.yml
+    parameters:
+      versionNumberOverride: ${{ parameters.versionNumberOverride }}
+
+  - script: gulp vsix:release:package:${{ parameters.platform }} --prerelease
+    displayName: 'Build VSIXs'
+
+  - task: PublishBuildArtifacts@1
+    # Run the publish step so we have vsix's even if the tests fail.
+    condition: succeededOrFailed()
+    displayName: 'Publish VSIXs'
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/vsix'
+      ArtifactName: 'VSIX_Prerelease_$(System.JobAttempt)'
+
+  - script: npm run test:artifacts
+    displayName: 'Run artifacts tests'
+
+- job:
+  displayName: 'Build ${{ parameters.platform }} release vsixs'
+  pool:
+    name: Azure Pipelines
+    vmImage: ${{ parameters.vmImageName }}
+  steps:
+  - checkout: self
+    clean: true
+    submodules: true
+    fetchTags: false
+    fetchDepth: 0
+
+  - template: prereqs.yml
+    parameters:
+      versionNumberOverride: ${{ parameters.versionNumberOverride }}
+
+  - script: gulp vsix:release:package:${{ parameters.platform }}
+    displayName: 'Build VSIXs'
+
+  - task: PublishBuildArtifacts@1
+    # Run the publish step so we have vsix's even if the tests fail.
+    condition: succeededOrFailed()
+    displayName: 'Publish VSIXs'
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/vsix'
+      ArtifactName: 'VSIX_Release_$(System.JobAttempt)'
 
-stages:
-- stage: Build
-  displayName: 'Build VSIXs'
-  jobs:
-  - job:
-    displayName: 'Build Prerelease VSIXs'
-    steps:
-    - checkout: self
-      clean: true
-      submodules: true
-      fetchTags: false
-      fetchDepth: 0
-
-    - template: prereqs.yml
-      parameters:
-        versionNumberOverride: ${{ parameters.versionNumberOverride }}
-
-    - script: gulp 'vsix:release:package' --prerelease
-      displayName: 'Build VSIXs'
-
-    - task: PublishPipelineArtifact@1
-      # Run the publish step so we have vsix's even if the tests fail.
-      condition: succeededOrFailed()
-      displayName: 'Publish VSIXs'
-      inputs:
-        targetPath: '$(Build.SourcesDirectory)/vsix'
-        artifactName: 'VSIX_Prerelease_$(System.JobAttempt)'
-
-    - script: npm run test:artifacts
-      displayName: 'Run artifacts tests'
-
-  - job:
-    displayName: 'Build Release VSIXs'
-    steps:
-    - checkout: self
-      clean: true
-      submodules: true
-      fetchTags: false
-      fetchDepth: 0
-
-    - template: prereqs.yml
-      parameters:
-        versionNumberOverride: ${{ parameters.versionNumberOverride }}
-
-    - script: gulp 'vsix:release:package'
-      displayName: 'Build VSIXs'
-
-    - task: PublishPipelineArtifact@1
-      # Run the publish step so we have vsix's even if the tests fail.
-      condition: succeededOrFailed()
-      displayName: 'Publish VSIXs'
-      inputs:
-        targetPath: '$(Build.SourcesDirectory)/vsix'
-        artifactName: 'VSIX_Release_$(System.JobAttempt)'
-
-    - script: npm run test:artifacts
-      displayName: 'Run artifacts tests'
+  - script: npm run test:artifacts
+    displayName: 'Run artifacts tests'
 

tasks/commandLineArguments.ts

@@ -6,12 +6,9 @@
 import * as minimist from 'minimist';
 import * as path from 'path';
 
-const argv = minimist(process.argv.slice(2), {
-    boolean: ['retainVsix'],
-});
+const argv = minimist(process.argv.slice(2));
 
 export const commandLineOptions = {
-    retainVsix: !!argv['retainVsix'],
     outputFolder: makePathAbsolute(argv['o']),
     codeExtensionPath: makePathAbsolute(argv['codeExtensionPath']),
 };

tasks/offlinePackagingTasks.ts

@@ -17,7 +17,6 @@ import NetworkSettings from '../src/networkSettings';
 import { downloadAndInstallPackages } from '../src/packageManager/downloadAndInstallPackages';
 import { getRuntimeDependenciesPackages } from '../src/tools/runtimeDependencyPackageUtils';
 import { getAbsolutePathPackagesToInstall } from '../src/packageManager/getAbsolutePathPackagesToInstall';
-import { commandLineOptions } from '../tasks/commandLineArguments';
 import {
     codeExtensionPath,
     packedVsixOutputRoot,
@@ -33,8 +32,14 @@ import path = require('path');
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const argv = require('yargs').argv;
 
+interface VSIXPlatformInfo {
+    vsceTarget: string;
+    rid: string;
+    platformInfo: PlatformInformation;
+}
+
 // Mapping of vsce vsix packaging target to the RID used to build the server executable
-export const platformSpecificPackages = [
+export const platformSpecificPackages: VSIXPlatformInfo[] = [
     { vsceTarget: 'win32-x64', rid: 'win-x64', platformInfo: new PlatformInformation('win32', 'x86_64') },
     { vsceTarget: 'win32-ia32', rid: 'win-x86', platformInfo: new PlatformInformation('win32', 'x86') },
     { vsceTarget: 'win32-arm64', rid: 'win-arm64', platformInfo: new PlatformInformation('win32', 'arm64') },
@@ -46,16 +51,46 @@ export const platformSpecificPackages = [
     { vsceTarget: 'darwin-arm64', rid: 'osx-arm64', platformInfo: new PlatformInformation('darwin', 'arm64') },
 ];
 
-gulp.task('vsix:release:package', async () => {
-    //if user does not want to clean up the existing vsix packages
-    await cleanAsync(/* deleteVsix: */ !commandLineOptions.retainVsix);
+const vsixTasks: string[] = [];
+for (const p of platformSpecificPackages) {
+    let platformName: string;
+    if (p.platformInfo.isWindows()) {
+        platformName = 'windows';
+    } else if (p.platformInfo.isLinux()) {
+        platformName = 'linux';
+    } else if (p.platformInfo.isMacOS()) {
+        platformName = 'darwin';
+    } else {
+        throw new Error(`Unexpected platform ${p.platformInfo.platform}`);
+    }
+
+    const taskName = `vsix:release:package:${platformName}:${p.vsceTarget}`;
+    vsixTasks.push(taskName);
+    gulp.task(taskName, async () => {
+        await doPackageOffline(p);
+    });
+}
 
-    await doPackageOffline();
+gulp.task('vsix:release:package:windows', gulp.series(...vsixTasks.filter((t) => t.includes('windows'))));
+gulp.task('vsix:release:package:linux', gulp.series(...vsixTasks.filter((t) => t.includes('linux'))));
+gulp.task('vsix:release:package:darwin', gulp.series(...vsixTasks.filter((t) => t.includes('darwin'))));
+gulp.task('vsix:release:package:neutral', async () => {
+    await doPackageOffline(undefined);
 });
 
+gulp.task(
+    'vsix:release:package',
+    gulp.series(
+        'vsix:release:package:windows',
+        'vsix:release:package:linux',
+        'vsix:release:package:darwin',
+        'vsix:release:package:neutral'
+    )
+);
+
 // Downloads dependencies for local development.
 gulp.task('installDependencies', async () => {
-    await cleanAsync(/* deleteVsix: */ false);
+    await cleanAsync();
 
     const packageJSON = getPackageJSON();
 
@@ -210,7 +245,8 @@ async function acquireNugetPackage(packageName: string, packageVersion: string,
     return packageOutputPath;
 }
 
-async function doPackageOffline() {
+async function doPackageOffline(vsixPlatform: VSIXPlatformInfo | undefined) {
+    await cleanAsync();
     // Set the package.json version based on the value in version.json.
     const versionInfo = await nbgv.getVersion();
     console.log(versionInfo.npmPackageVersion);
@@ -229,38 +265,37 @@ async function doPackageOffline() {
         // Now that we've updated the version, get the package.json.
         const packageJSON = getPackageJSON();
 
-        for (const p of platformSpecificPackages) {
-            try {
-                if (process.platform === 'win32' && !p.rid.startsWith('win')) {
-                    console.warn(
-                        `Skipping packaging for ${p.rid} on Windows since runtime executables will not be marked executable in *nix packages.`
-                    );
-                    continue;
-                }
-
-                await buildVsix(packageJSON, packedVsixOutputRoot, prerelease, p.vsceTarget, p.platformInfo);
-            } catch (err) {
-                const message = (err instanceof Error ? err.stack : err) ?? '<unknown error>';
-                // NOTE: Extra `\n---` at the end is because gulp will print this message following by the
-                // stack trace of this line. So that seperates the two stack traces.
-                throw Error(`Failed to create package ${p.vsceTarget}. ${message}\n---`);
-            }
+        if (process.platform === 'win32' && !vsixPlatform?.rid.startsWith('win')) {
+            console.warn(
+                `Skipping packaging for ${vsixPlatform?.rid} on Windows since runtime executables will not be marked executable in *nix packages.`
+            );
+            return;
         }
 
-        // Also output the platform neutral VSIX using the platform neutral server bits we created before.
-        await buildVsix(packageJSON, packedVsixOutputRoot, prerelease);
+        if (vsixPlatform === undefined) {
+            await buildVsix(packageJSON, packedVsixOutputRoot, prerelease);
+        } else {
+            await buildVsix(
+                packageJSON,
+                packedVsixOutputRoot,
+                prerelease,
+                vsixPlatform.vsceTarget,
+                vsixPlatform.platformInfo
+            );
+        }
+    } catch (err) {
+        const message = (err instanceof Error ? err.stack : err) ?? '<unknown error>';
+        // NOTE: Extra `\n---` at the end is because gulp will print this message following by the
+        // stack trace of this line. So that seperates the two stack traces.
+        throw Error(`Failed to create package ${vsixPlatform?.vsceTarget ?? 'neutral'}. ${message}\n---`);
     } finally {
         // Reset package version to the placeholder value.
         await nbgv.resetPackageVersionPlaceholder();
     }
 }
 
-async function cleanAsync(deleteVsix: boolean) {
+async function cleanAsync() {
     await del(['install.*', '.omnisharp*', '.debugger', '.razor', languageServerDirectory]);
-
-    if (deleteVsix) {
-        await del('*.vsix');
-    }
 }
 
 async function buildVsix(
@@ -270,8 +305,6 @@ async function buildVsix(
     vsceTarget?: string,
     platformInfo?: PlatformInformation
 ) {
-    await cleanAsync(false);
-
     await installRoslyn(packageJSON, platformInfo);
 
     if (platformInfo != null) {