Merge pull request #513 from SteveL-MSFT/publish-token

Default build to using CFS and include musl

Author: SteveL-MSFT

.cargo/config.toml

@@ -3,7 +3,7 @@
 registry-auth = true
 
 [registries]
-powershell = { index = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/Cargo/index/" }
+POWERSHELL = { index = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/Cargo/index/" }
 
 [registry]
 global-credential-providers = ["cargo:token"]
@@ -17,4 +17,10 @@ rustflags = ["-Ccontrol-flow-guard", "-Ctarget-feature=+crt-static", "-Clink-arg
 
 # The following is only needed for release builds
 [source.crates-io]
-replace-with = "powershell"
+replace-with = "POWERSHELL"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"

.github/workflows/rust.yml

@@ -28,6 +28,22 @@ jobs:
       shell: pwsh
       run: ./build.ps1 -test
 
+  build-musl:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - run: rustup target add x86_64-unknown-linux-musl
+    - name: Install musl build tools
+      run: sudo apt update && sudo apt install musl-tools -y
+    - name: Build
+      shell: pwsh
+      run: ./build.ps1 -clippy -target x86_64-unknown-linux-musl
+    - name: Run tests
+      shell: pwsh
+      run: ./build.ps1 -test -target x86_64-unknown-linux-musl
+
   build-windows:
 
     runs-on: windows-latest

.pipelines/DSC-Official.yml

@@ -1,12 +1,6 @@
 name: DSC-Release-$(Build.BuildId)
 trigger: none
 
-parameters:
-- name: 'debugConsole'
-  displayName: 'Enable debug console'
-  type: boolean
-  default: false
-
 pr:
   branches:
     include:
@@ -16,8 +10,9 @@ pr:
 variables:
   BuildConfiguration: 'release'
   PackageRoot: '$(System.ArtifactsDirectory)/Packages'
-  LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0'
-  WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2019/vse2022:latest
+#  LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0'
+  LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest'
+  WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
 
 resources:
   repositories:
@@ -30,7 +25,9 @@ extends:
   template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
   parameters:
     featureFlags:
-      debugConsole: ${{ parameters.debugConsole }}
+      WindowsHostVersion:
+        Disk: Large
+        Version: 2022    
     customTags: 'ES365AIMigrationTooling'
     globalSdl:
       disableLegacyManifest: true
@@ -60,6 +57,7 @@ extends:
         displayName: Set PackageVersion
         pool:
           type: windows
+          vmImage: windows-latest
         variables:
           repoRoot: $(Build.SourcesDirectory)\DSC
           ob_sdl_tsa_configFile: $(Build.SourcesDirectory)\DSC\.config\tsaoptions.json
@@ -124,24 +122,15 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
             Set-Location "$(Build.SourcesDirectory)/DSC"
-            ./build.ps1 -Release -Architecture $(buildName) -SkipLinkCheck -UseCFS
+            $LLVMBIN = "$($env:PROGRAMFILES)\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm\bin"
+            if (!(Test-Path $LLVMBIN)) {
+              throw "LLVM path '$LLVMBIN' does not exist"
+            }
+            $env:PATH += ";$LLVMBIN"
+            write-verbose -verbose (gcm clang.exe | out-string)
+            ./build.ps1 -Release -Architecture $(buildName) -SkipLinkCheck
           displayName: 'Build $(buildName)'
           env:
             ob_restore_phase: true
@@ -257,23 +246,8 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture x86_64-unknown-linux-gnu -UseCFS
+            ./build.ps1 -Release -Architecture x86_64-unknown-linux-gnu
             ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-gnu -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
           displayName: 'Build x86_64-unknown-linux-gnu'
@@ -287,7 +261,6 @@ extends:
         displayName: Linux-ARM64-gnu
         pool:
           type: linux
-          hostArchitecture: arm64
         steps:
         - task: RustInstaller@1
           inputs:
@@ -297,26 +270,69 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
+        - pwsh: |
+            apt -y install gcc-aarch64-linux-gnu
+            ./build.ps1 -Release -Architecture aarch64-unknown-linux-gnu
+            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-gnu -Release
+            Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
+          displayName: 'Build aarch64-unknown-linux-gnu'
+          condition: succeeded()
+
+      - job: BuildLinuxMusl
+        dependsOn: SetPackageVersion
+        variables:
+          PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
+          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: Linux-x64-musl
+        pool:
+          type: linux
+        steps:
+        - task: RustInstaller@1
           inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
+            rustVersion: ms-stable
+            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            additionalTargets: x86_64-unknown-linux-musl
+          displayName: Install Rust
+          env:
+            ob_restore_phase: true
+        - pwsh: |
+            apt -y install musl-tools
+            ./build.ps1 -Release -Architecture x86_64-unknown-linux-musl
+            ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-musl -Release
+            Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
+          displayName: 'Build x86_64-unknown-linux-musl'
+          condition: succeeded()
 
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
+      - job: BuildLinuxArm64Musl
+        dependsOn: SetPackageVersion
+        variables:
+          PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
+          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: Linux-ARM64-musl
+        pool:
+          type: linux
+        steps:
+        - task: RustInstaller@1
+          inputs:
+            rustVersion: ms-stable
+            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            additionalTargets: aarch64-unknown-linux-musl
+          displayName: Install Rust
           env:
             ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture aarch64-unknown-linux-gnu -UseCFS
-            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-gnu -Release
+            $env:CC_aarch64_unknown_linux_musl='clang'
+            $env:AR_aarch64_unknown_linux_musl='llvm-ar'
+            $env:CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
+            apt -y install clang
+            apt -y install llvm
+            apt -y install musl-tools
+            apt -y install gcc-multilib
+            rustup default stable-aarch64-unknown-linux-musl
+            ./build.ps1 -Release -Architecture aarch64-unknown-linux-musl
+            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-musl -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
-          displayName: 'Build aarch64-unknown-linux-gnu'
+          displayName: 'Build aarch64-unknown-linux-musl'
           condition: succeeded()
 
       - job: BuildMac
@@ -345,23 +361,8 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture $(buildName) -UseCFS
+            ./build.ps1 -Release -Architecture $(buildName)
             ./build.ps1 -PackageType tgz -Architecture $(buildName) -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
             Write-Host "##vso[artifact.upload containerfolder=release;artifactname=release]$(ob_outputDirectory)/DSC-$(PackageVersion)-$(buildName).tar.gz"
@@ -372,6 +373,9 @@ extends:
       dependsOn: BuildAndSign
       variables:
         PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
+        ob_sdl_sbom_enabled: false
+        ob_signing_setup_enabled: false
+        ob_sdl_codeql_compiled_enabled: false
         drop: $(Pipeline.Workspace)/drop_build_main
       jobs:
       - job: Validation

build.ps1

@@ -13,7 +13,7 @@ param(
     [switch]$GetPackageVersion,
     [switch]$SkipLinkCheck,
     [switch]$UseX64MakeAppx,
-    [switch]$UseCFS
+    [switch]$UseCratesIO
 )
 
 if ($GetPackageVersion) {
@@ -168,7 +168,7 @@ if (!$SkipBuild) {
     }
     New-Item -ItemType Directory $target -ErrorAction Ignore > $null
 
-    if (!$UseCFS) {
+    if ($UseCratesIO) {
         # this will override the config.toml
         Write-Host "Setting CARGO_SOURCE_crates-io_REPLACE_WITH to 'crates-io'"
         ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = 'CRATESIO'
@@ -177,11 +177,28 @@ if (!$SkipBuild) {
         Write-Host "Using CFS for cargo source replacement"
         ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = $null
         $env:CARGO_REGISTRIES_CRATESIO_INDEX = $null
+
+        if ($null -eq (Get-Command 'az' -ErrorAction Ignore)) {
+            throw "Azure CLI not found"
+        }
+
+        if ($null -ne $env:CARGO_REGISTRIES_POWERSHELL_TOKEN) {
+            Write-Host "Using existing token"
+        } else {
+            Write-Host "Getting token"
+            $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
+            if ($LASTEXITCODE -ne 0) {
+                Write-Warning "Failed to get access token, use 'az login' first, or use '-useCratesIO' to use crates.io.  Proceeding with anonymous access."
+            } else {
+                $header = "Bearer $accessToken"
+                $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header
+                $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token'
+            }
+        }
     }
 
     # make sure dependencies are built first so clippy runs correctly
     $windows_projects = @("pal", "registry", "reboot_pending", "wmi-adapter")
-
     $macOS_projects = @("resources/brew")
     $linux_projects = @("resources/apt")