Log more information about disk space and skip the MSVC builds on public (#3090)

mattleibow · web-flow · commit 9d7e8b05d54e · 2024-12-02T22:44:19.000+08:00
* Try and get more information as to disk space
* Only build MSVC on the internal build

The MSVC compiler is so heavy that the outputs are over 10.8GB
whereas the LLVM compiler is just under 10.1GB.

The public agent only starts with 12GB and after the checkout we are
left with 10.67GB which is smaller than the 10.8GB required. The LLVM
compiler just manages to pass under the bar with about 600MB spare.

If the SkiaSharp artifacts get any bigger, then we will need custom
machines for the public build.
diff --git a/scripts/azure-templates-bootstrapper.yml b/scripts/azure-templates-bootstrapper.yml
@@ -82,11 +82,19 @@ jobs:
               targetPath: ${{ additionalArtifact.path }}
 
     steps:
+      - pwsh: if ($IsLinux -or $IsMacOS) { df -h } else { Get-Volume }
+        displayName: Get the volume information before the run
+        condition: always()
+
       # prepare
       - checkout: self
         submodules: recursive
       - template: /scripts/azure-templates-variables.yml@self
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information after the checkout
+        condition: always()
+
       # # checkout required skia PR
       # - pwsh: .\scripts\checkout-skia.ps1 -GitHubToken $(GitHub.Token.PublicAccess)
       #   displayName: Checkout required skia PR
@@ -101,6 +109,10 @@ jobs:
       - pwsh: .\scripts\get-build-type.ps1 -ExternalsBuildId '${{ parameters.buildExternals }}'
         displayName: Determine build type
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information before the provisioning
+        condition: always()
+
       # provisioning steps
       - ${{ if ne(parameters.buildAgent.provisioningSteps, '') }}:
         - ${{ parameters.buildAgent.provisioningSteps }}
@@ -273,6 +285,10 @@ jobs:
             retryCountOnTaskFailure: 3
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information before downloading artifacts
+        condition: always()
+
       # download artifacts
       - template: /scripts/azure-templates-download-artifacts.yml@self
         parameters:
@@ -285,12 +301,20 @@ jobs:
           artifacts:
             - name: ${{ parameters.name }}
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information after downloading artifacts
+        condition: always()
+
       # pre-build steps
       - ${{ if ne(parameters.buildAgent.preBuildSteps, '') }}:
         - ${{ parameters.buildAgent.preBuildSteps }}
       - ${{ if ne(length(parameters.preBuildSteps), 0) }}:
         - ${{ parameters.preBuildSteps }}
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information before the build
+        condition: always()
+
       # actual build
       - ${{ if ne(parameters.skipSteps, 'true') }}:
         - ${{ if eq(parameters.docker, '') }}:
@@ -359,8 +383,16 @@ jobs:
             retryCountOnTaskFailure: ${{ parameters.retryCount }}
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
 
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information after the build
+        condition: always()
+
       # post-build steps
       - ${{ parameters.postBuildSteps }}
 
       - ${{ if eq(parameters.buildPipelineType, 'tests') }}:
         - template: /scripts/azure-templates-github-status.yml@self
+
+      - pwsh: .\scripts\get-free-space.ps1
+        displayName: Get the volume information after the entire run
+        condition: always()
diff --git a/scripts/azure-templates-stages.yml b/scripts/azure-templates-stages.yml
@@ -145,39 +145,40 @@ stages:
             buildAgent: ${{ parameters.buildAgentWindowsNative }}
             target: externals-windows
             additionalArgs: --buildarch=arm64
-        - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x86 (Win + MSVC)
-          parameters:
-            name: native_win32_x86_msvc_windows
-            displayName: Win32 x86 [MSVC]
-            sdl: ${{ parameters.sdl }}
-            buildExternals: ${{ parameters.buildExternals }}
-            buildPipelineType: ${{ parameters.buildPipelineType }}
-            buildAgent: ${{ parameters.buildAgentWindowsNative }}
-            target: externals-windows
-            additionalArgs: --buildarch=x86 --llvm="msvc"
-            installLlvm: false
-        - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x64 (Win + MSVC)
-          parameters:
-            name: native_win32_x64_msvc_windows
-            displayName: Win32 x64 [MSVC]
-            sdl: ${{ parameters.sdl }}
-            buildExternals: ${{ parameters.buildExternals }}
-            buildPipelineType: ${{ parameters.buildPipelineType }}
-            buildAgent: ${{ parameters.buildAgentWindowsNative }}
-            target: externals-windows
-            additionalArgs: --buildarch=x64 --llvm="msvc"
-            installLlvm: false
-        - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|arm64 (Win + MSVC)
-          parameters:
-            name: native_win32_arm64_msvc_windows
-            displayName: Win32 arm64 [MSVC]
-            sdl: ${{ parameters.sdl }}
-            buildExternals: ${{ parameters.buildExternals }}
-            buildPipelineType: ${{ parameters.buildPipelineType }}
-            buildAgent: ${{ parameters.buildAgentWindowsNative }}
-            target: externals-windows
-            additionalArgs: --buildarch=arm64 --llvm="msvc"
-            installLlvm: false
+        - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:
+          - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x86 (Win + MSVC)
+            parameters:
+              name: native_win32_x86_msvc_windows
+              displayName: Win32 x86 [MSVC]
+              sdl: ${{ parameters.sdl }}
+              buildExternals: ${{ parameters.buildExternals }}
+              buildPipelineType: ${{ parameters.buildPipelineType }}
+              buildAgent: ${{ parameters.buildAgentWindowsNative }}
+              target: externals-windows
+              additionalArgs: --buildarch=x86 --llvm="msvc"
+              installLlvm: false
+          - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x64 (Win + MSVC)
+            parameters:
+              name: native_win32_x64_msvc_windows
+              displayName: Win32 x64 [MSVC]
+              sdl: ${{ parameters.sdl }}
+              buildExternals: ${{ parameters.buildExternals }}
+              buildPipelineType: ${{ parameters.buildPipelineType }}
+              buildAgent: ${{ parameters.buildAgentWindowsNative }}
+              target: externals-windows
+              additionalArgs: --buildarch=x64 --llvm="msvc"
+              installLlvm: false
+          - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|arm64 (Win + MSVC)
+            parameters:
+              name: native_win32_arm64_msvc_windows
+              displayName: Win32 arm64 [MSVC]
+              sdl: ${{ parameters.sdl }}
+              buildExternals: ${{ parameters.buildExternals }}
+              buildPipelineType: ${{ parameters.buildPipelineType }}
+              buildAgent: ${{ parameters.buildAgentWindowsNative }}
+              target: externals-windows
+              additionalArgs: --buildarch=arm64 --llvm="msvc"
+              installLlvm: false
         - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native WinUI|x86 (Win)
           parameters:
             name: native_winui_x86_windows
@@ -527,17 +528,18 @@ stages:
             matrixArtifacts:
               - name: native_wasm
                 jobs: $(nativeWasmJobs)
-        - template: /scripts/azure-templates-merger.yml@self # Merge Native MSVC Artifacts
-          parameters:
-            name: native_msvc
-            displayName: Merge Native MSVC Artifacts
-            sdl: ${{ parameters.sdl }}
-            buildPipelineType: ${{ parameters.buildPipelineType }}
-            buildAgent: ${{ parameters.buildAgentHost }}
-            requiredArtifacts:
-              - name: native_win32_x86_msvc_windows
-              - name: native_win32_x64_msvc_windows
-              - name: native_win32_arm64_msvc_windows
+        - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:
+          - template: /scripts/azure-templates-merger.yml@self # Merge Native MSVC Artifacts
+            parameters:
+              name: native_msvc
+              displayName: Merge Native MSVC Artifacts
+              sdl: ${{ parameters.sdl }}
+              buildPipelineType: ${{ parameters.buildPipelineType }}
+              buildAgent: ${{ parameters.buildAgentHost }}
+              requiredArtifacts:
+                - name: native_win32_x86_msvc_windows
+                - name: native_win32_x64_msvc_windows
+                - name: native_win32_arm64_msvc_windows
 
   - ${{ if ne(parameters.buildPipelineType, 'build') }}:
     - stage: managed
diff --git a/scripts/get-free-space.ps1 b/scripts/get-free-space.ps1
@@ -1,7 +1,7 @@
 $ErrorActionPreference = 'Stop'
 
 if ($IsLinux -or $IsMacOS) {
-    Write-Host "Not supported yet."
+    df -h
 } else {
     Get-Volume
 }

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`$ErrorActionPreference = 'Stop'`
`2`	`2`
`3`	`3`	`if ($IsLinux -or $IsMacOS) {`
`4`		`- Write-Host "Not supported yet."`
	`4`	`+ df -h`
`5`	`5`	`} else {`
`6`	`6`	`Get-Volume`
`7`	`7`	`}`