diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 89e502cd7f..725471befd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,38 +7,38 @@ - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b - + https://github.com/dotnet/arcade - 35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3 + d27184f7cb92b4abb4b20e91ecb5c43bc43de65b diff --git a/eng/Versions.props b/eng/Versions.props index f7f879a844..c7af00a839 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -82,7 +82,6 @@ 4.13.0 4.7.0 4.3.6 - 9.0.4 6.0.0 @@ -95,7 +94,7 @@ 9.0.0-beta.24212.4 5.0.0-preview.5.20278.1 8.0.0-beta.24525.2 - 10.0.0-beta.25268.1 + 10.0.0-beta.25408.3 9.0.4 0.0.6-test 0.0.13-test diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 6b3be1916f..8cfee107e7 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -21,6 +21,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('pb')]$productBuild, + [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -74,6 +75,7 @@ function Print-Usage() { Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" Write-Host " -buildCheck Sets /check msbuild parameter" + Write-Host " -fromVMR Set when building from within the VMR" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -127,7 +129,8 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` - /p:DotNetBuildRepo=$productBuild ` + /p:DotNetBuild=$productBuild ` + /p:DotNetBuildFromVMR=$fromVMR ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index 08f99154b7..9767bb411a 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -43,6 +43,7 @@ usage() echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo " --buildCheck Sets /check msbuild parameter" + echo " --fromVMR Set when building from within the VMR" echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -64,6 +65,7 @@ restore=false build=false source_build=false product_build=false +from_vmr=false rebuild=false test=false integration_test=false @@ -89,7 +91,7 @@ verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -properties='' +properties=() while [[ $# > 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in @@ -142,6 +144,9 @@ while [[ $# > 0 ]]; do restore=true pack=true ;; + -fromvmr|-from-vmr) + from_vmr=true + ;; -test|-t) test=true ;; @@ -187,7 +192,7 @@ while [[ $# > 0 ]]; do shift ;; *) - properties="$properties $1" + properties+=("$1") ;; esac @@ -221,7 +226,7 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" + properties+=("/p:Projects=$projects") fi local bl="" @@ -241,8 +246,9 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ - /p:DotNetBuildRepo=$product_build \ + /p:DotNetBuild=$product_build \ /p:DotNetBuildSourceOnly=$source_build \ + /p:DotNetBuildFromVMR=$from_vmr \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ @@ -251,7 +257,7 @@ function Build { /p:Sign=$sign \ /p:Publish=$publish \ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ - $properties + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 6badecba7b..d901325154 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -20,6 +20,7 @@ parameters: artifacts: '' enableMicrobuild: false enableMicrobuildForMacAndLinux: false + microbuildUseESRP: true enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false @@ -128,6 +129,7 @@ jobs: parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} + microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index 00feec8ebb..8bf7d23355 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -4,7 +4,7 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: '' - + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) @@ -27,7 +27,7 @@ parameters: is1ESPipeline: '' jobs: - job: OneLocBuild${{ parameters.JobNameSuffix }} - + dependsOn: ${{ parameters.dependsOn }} displayName: OneLocBuild${{ parameters.JobNameSuffix }} @@ -86,8 +86,7 @@ jobs: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} ${{ if eq(parameters.CreatePr, true) }}: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} + isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: @@ -100,22 +99,20 @@ jobs: mirrorBranch: ${{ parameters.MirrorBranch }} condition: ${{ parameters.condition }} - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish Localization Files - pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} + # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact + - task: CopyFiles@2 + displayName: Copy LocProject.json + inputs: + SourceFolder: '$(Build.SourcesDirectory)/eng/Localize/' + Contents: 'LocProject.json' + TargetFolder: '$(Build.ArtifactStagingDirectory)/loc' + condition: ${{ parameters.condition }} - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: - displayName: Publish LocProject.json - pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + targetPath: '$(Build.ArtifactStagingDirectory)/loc' + artifactName: 'Loc' + displayName: 'Publish Localization Files' + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 4f1dc42e02..d5303229c9 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -32,6 +32,12 @@ parameters: # Optional: 🌤️ or not the build has assets it wants to publish to BAR isAssetlessBuild: false + # Optional, publishing version + publishingVersion: 3 + + # Optional: A minimatch pattern for the asset manifests to publish to BAR + assetManifestsPattern: '*/manifests/**/*.xml' + jobs: - job: Asset_Registry_Publish @@ -77,13 +83,31 @@ jobs: clean: true - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Asset Manifests - inputs: - artifactName: AssetManifests - targetPath: '$(Build.StagingDirectory)/AssetManifests' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.publishingVersion, 3) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Asset Manifests + inputs: + artifactName: AssetManifests + targetPath: '$(Build.StagingDirectory)/AssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.publishingVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download V4 asset manifests + inputs: + itemPattern: '*/manifests/**/*.xml' + targetPath: '$(Build.StagingDirectory)/AllAssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: CopyFiles@2 + displayName: Copy V4 asset manifests to AssetManifests + inputs: + SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests' + Contents: ${{ parameters.assetManifestsPattern }} + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + flattenFolders: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} - task: NuGetAuthenticate@1 @@ -120,6 +144,17 @@ jobs: Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs" } + - ${{ if eq(parameters.publishingVersion, 4) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' + artifactName: AssetManifests + displayName: 'Publish Merged Manifest' + retryCountOnTaskFailure: 10 # for any logs being locked + sbomEnabled: false # we don't need SBOM for logs + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index bf35b78faa..2f992b2c6e 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -83,7 +83,6 @@ jobs: - template: /eng/common/core-templates/jobs/source-build.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - allCompletedJobId: Source_Build_Complete ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} @@ -108,8 +107,6 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete runAsPublic: ${{ parameters.runAsPublic }} publishAssetsImmediately: ${{ or(parameters.publishAssetsImmediately, parameters.isAssetlessBuild) }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml index df24c948ba..d92860cba2 100644 --- a/eng/common/core-templates/jobs/source-build.yml +++ b/eng/common/core-templates/jobs/source-build.yml @@ -2,12 +2,6 @@ parameters: # This template adds arcade-powered source-build to CI. A job is created for each platform, as # well as an optional server job that completes when all platform jobs complete. - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - # See /eng/common/core-templates/job/source-build.yml jobNamePrefix: 'Source_Build' @@ -31,16 +25,6 @@ parameters: jobs: -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - - ${{ each platform in parameters.platforms }}: - template: /eng/common/core-templates/job/source-build.yml parameters: diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 5757915edb..a151fd811e 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -193,9 +193,6 @@ stages: buildId: $(AzDOBuildId) artifactName: PackageArtifacts checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index 2bcf974ee1..da30e67bc3 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -4,8 +4,16 @@ parameters: # Enable install tasks for MicroBuild on Mac and Linux # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' enableMicrobuildForMacAndLinux: false + # Determines whether the ESRP service connection information should be passed to the signing plugin. + # This overlaps with _SignType to some degree. We only need the service connection for real signing. + # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. + # Doing so will cause the service connection to be authorized for the pipeline, which isn't allowed and won't work for non-prod. + # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The + # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough. + microbuildUseESRP: true # Location of the MicroBuild output folder microBuildOutputFolder: '$(Build.SourcesDirectory)' + continueOnError: false steps: @@ -21,15 +29,37 @@ steps: workingDirectory: ${{ parameters.microBuildOutputFolder }} condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + - script: | + REM Check if ESRP is disabled while SignType is real + if /I "${{ parameters.microbuildUseESRP }}"=="false" if /I "$(_SignType)"=="real" ( + echo Error: ESRP must be enabled when SignType is real. + exit /b 1 + ) + displayName: 'Validate ESRP usage (Windows)' + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) + - script: | + # Check if ESRP is disabled while SignType is real + if [ "${{ parameters.microbuildUseESRP }}" = "false" ] && [ "$(_SignType)" = "real" ]; then + echo "Error: ESRP must be enabled when SignType is real." + exit 1 + fi + displayName: 'Validate ESRP usage (Non-Windows)' + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) zipSources: false feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}: - azureSubscription: 'MicroBuild Signing Task (DevDiv)' - useEsrpCli: true + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}: + azureSubscription: 'MicroBuild Signing Task (DevDiv)' + useEsrpCli: true + ${{ elseif eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea + ${{ else }}: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca env: TeamName: $(_TeamName) MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} @@ -48,38 +78,3 @@ steps: eq(variables['_SignType'], 'real') ) )) - - # Workaround for ESRP CLI on Linux - https://github.com/dotnet/source-build/issues/4964 - - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}: - - task: UseDotNet@2 - displayName: Install .NET 9.0 SDK for ESRP CLI Workaround - inputs: - packageType: sdk - version: 9.0.x - installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet - workingDirectory: ${{ parameters.microBuildOutputFolder }} - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - - - task: PowerShell@2 - displayName: Workaround for ESRP CLI on Linux - inputs: - targetType: 'inline' - script: | - Write-Host "Copying Linux Path" - $MBSIGN_APPFOLDER = '$(MBSIGN_APPFOLDER)' - $MBSIGN_APPFOLDER = ($MBSIGN_APPFOLDER -replace '/build', '') - - $versionRegex = '\d+\.\d+\.\d+' - $package = Get-ChildItem -Path $MBSIGN_APPFOLDER -Directory | - Where-Object { $_.Name -match $versionRegex } - - if ($package.Count -ne 1) { - Write-Host "There should be exactly one matching subfolder, but found $($package.Count)." - exit 1 - } - - $MBSIGN_APPFOLDER = $package[0].FullName + '/build' - $MBSIGN_APPFOLDER | Write-Host - $SignConfigPath = $MBSIGN_APPFOLDER + '/signconfig.xml' - Copy-Item -Path "$(MBSIGN_APPFOLDER)/signconfig.xml" -Destination $SignConfigPath -Force - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 0dde553c3e..acf16ed349 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -38,11 +38,6 @@ steps: targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi - baseRidArgs= - if [ '${{ parameters.platform.baseRID }}' != '' ]; then - baseRidArgs='/p:BaseRid=${{ parameters.platform.baseRID }}' - fi - portableBuildArgs= if [ '${{ parameters.platform.portableBuild }}' != '' ]; then portableBuildArgs='/p:PortableBuild=${{ parameters.platform.portableBuild }}' @@ -55,7 +50,6 @@ steps: ${{ parameters.platform.buildArguments }} \ $internalRuntimeDownloadArgs \ $targetRidArgs \ - $baseRidArgs \ $portableBuildArgs \ displayName: Build diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 99c2326fc1..c2917c1efc 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,15 +1,15 @@ parameters: sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250515.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: - task: UseDotNet@2 - displayName: "Source Index: Use .NET 8 SDK" + displayName: "Source Index: Use .NET 9 SDK" inputs: packageType: sdk - version: 8.0.x + version: 9.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d6f005b5da..8abfb71f72 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -295,8 +295,8 @@ while :; do ;; noble) # Ubuntu 24.04 __CodeName=noble - if [[ -n "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-18-dev" + if [[ -z "$__LLDB_Package" ]]; then + __LLDB_Package="liblldb-19-dev" fi ;; stretch) # Debian 9 diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd new file mode 100644 index 0000000000..527fa4bb38 --- /dev/null +++ b/eng/common/dotnet.cmd @@ -0,0 +1,7 @@ +@echo off + +:: This script is used to install the .NET SDK. +:: It will also invoke the SDK with any provided arguments. + +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*" +exit /b %ErrorLevel% diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1 new file mode 100644 index 0000000000..45e5676c9e --- /dev/null +++ b/eng/common/dotnet.ps1 @@ -0,0 +1,11 @@ +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +. $PSScriptRoot\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true + +# Invoke acquired SDK with args if they are provided +if ($args.count -gt 0) { + $env:DOTNET_NOLOGO=1 + & "$dotnetRoot\dotnet.exe" $args +} diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh new file mode 100644 index 0000000000..2ef6823567 --- /dev/null +++ b/eng/common/dotnet.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +source $scriptroot/tools.sh +InitializeDotNetCli true # install + +# Invoke acquired SDK with args if they are provided +if [[ $# > 0 ]]; then + __dotnetDir=${_InitializeDotNetCli} + dotnetPath=${__dotnetDir}/dotnet + ${dotnetPath} "$@" +fi diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config index 19d3d311b1..f70261ed68 100644 --- a/eng/common/internal/NuGet.config +++ b/eng/common/internal/NuGet.config @@ -4,4 +4,7 @@ + + + diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml index 670cf32c3b..ce3c29a62f 100644 --- a/eng/common/templates/vmr-build-pr.yml +++ b/eng/common/templates/vmr-build-pr.yml @@ -1,14 +1,23 @@ +# This pipeline is used for running the VMR verification of the PR changes in repo-level PRs. +# +# It will run a full set of verification jobs defined in: +# https://github.com/dotnet/dotnet/blob/10060d128e3f470e77265f8490f5e4f72dae738e/eng/pipelines/templates/stages/vmr-build.yml#L27-L38 +# +# For repos that do not need to run the full set, you would do the following: +# +# 1. Copy this YML file to a repo-specific location, i.e. outside of eng/common. +# +# 2. Add `verifications` parameter to VMR template reference +# +# Examples: +# - For source-build stage 1 verification, add the following: +# verifications: [ "source-build-stage1" ] +# +# - For Windows only verifications, add the following: +# verifications: [ "unified-build-windows-x64", "unified-build-windows-x86" ] + trigger: none -pr: - branches: - include: - - main - - release/* - paths: - exclude: - - documentation/* - - README.md - - CODEOWNERS +pr: none variables: - template: /eng/common/templates/variables/pool-providers.yml@self diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5f40a3f823..d4cfd9ccd8 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,8 +65,8 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# True if the build is a product build -[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } +# True when the build is running within the VMR. +[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false } function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null @@ -257,7 +257,20 @@ function Retry($downloadBlock, $maxRetries = 5) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' + $shouldDownload = $false + if (!(Test-Path $installScript)) { + $shouldDownload = $true + } else { + # Check if the script is older than 30 days + $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime + if ($fileAge.Days -gt 30) { + Write-Host "Existing install script is too old, re-downloading..." + $shouldDownload = $true + } + } + + if ($shouldDownload) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" @@ -414,7 +427,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null) { + if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] @@ -644,7 +657,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } @@ -766,28 +778,13 @@ function MSBuild() { $toolsetBuildProject = InitializeToolset $basePath = Split-Path -parent $toolsetBuildProject - $possiblePaths = @( - # new scripts need to work with old packages, so we need to look for the old names/versions - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - - # This list doesn't need to be updated anymore and can eventually be removed. - (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll')) - ) - $selectedPath = $null - foreach ($path in $possiblePaths) { - if (Test-Path $path -PathType Leaf) { - $selectedPath = $path - break - } - } + $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') + if (-not $selectedPath) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' + Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 } + $args += "/logger:$selectedPath" } @@ -850,8 +847,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild) { + # Skip this when the build is a child of the VMR build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 25f5932eee..c1841c9dfd 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -72,8 +72,8 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# True if the build is a product build -product_build=${product_build:-false} +# True when the build is running within the VMR. +from_vmr=${from_vmr:-false} # Resolve any symlinks in the given path. function ResolvePath { @@ -300,8 +300,29 @@ function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" + local timestamp_file="$root/.dotnet-install.timestamp" + local should_download=false if [[ ! -a "$install_script" ]]; then + should_download=true + elif [[ -f "$timestamp_file" ]]; then + # Check if the script is older than 30 days using timestamp file + local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0") + local current_time=$(date +%s) + local age_seconds=$((current_time - download_time)) + + # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds + if [[ $age_seconds -gt 2592000 ]]; then + echo "Existing install script is too old, re-downloading..." + should_download=true + fi + else + # No timestamp file exists, assume script is old and re-download + echo "No timestamp found for existing install script, re-downloading..." + should_download=true + fi + + if [[ "$should_download" == true ]]; then mkdir -p "$root" echo "Downloading '$install_script_url'" @@ -328,6 +349,9 @@ function GetDotNetInstallScript { ExitWithExitCode $exit_code } fi + + # Create timestamp file to track download time in seconds from epoch + date +%s > "$timestamp_file" fi # return value _GetDotNetInstallScript="$install_script" @@ -345,14 +369,12 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi @@ -449,27 +471,13 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - # new scripts need to work with old packages, so we need to look for the old names/versions - local selectedPath= - local possiblePaths=() - possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.Arcade.Sdk.dll" ) - - # This list doesn't need to be updated anymore and can eventually be removed. - possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - for path in "${possiblePaths[@]}"; do - if [[ -f $path ]]; then - selectedPath=$path - break - fi - done + local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" + if [[ -z "$selectedPath" ]]; then - Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." + Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 fi + args+=( "-logger:$selectedPath" ) fi @@ -506,8 +514,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true ]]; then + # Skip this when the build is a child of the VMR build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error @@ -530,6 +538,7 @@ function GetDarc { fi "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version + darc_tool="$darc_path/darc" } # Returns a full path to an Arcade SDK task project file. diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 index 8c3c91ce8d..97302f3205 100644 --- a/eng/common/vmr-sync.ps1 +++ b/eng/common/vmr-sync.ps1 @@ -103,14 +103,14 @@ Set-StrictMode -Version Latest Highlight 'Installing .NET, preparing the tooling..' . .\eng\common\tools.ps1 $dotnetRoot = InitializeDotNetCli -install:$true +$darc = Get-Darc $dotnet = "$dotnetRoot\dotnet.exe" -& "$dotnet" tool restore Highlight "Starting the synchronization of VMR.." # Synchronize the VMR $darcArgs = ( - "darc", "vmr", "forwardflow", + "vmr", "forwardflow", "--tmp", $tmpDir, "--$verbosity", $vmrDir @@ -124,7 +124,7 @@ if ($azdevPat) { $darcArgs += ("--azdev-pat", $azdevPat) } -& "$dotnet" $darcArgs +& "$darc" $darcArgs if ($LASTEXITCODE -eq 0) { Highlight "Synchronization succeeded" diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh index 86d77ccf5b..44239e331c 100644 --- a/eng/common/vmr-sync.sh +++ b/eng/common/vmr-sync.sh @@ -164,9 +164,9 @@ set -e highlight 'Installing .NET, preparing the tooling..' source "./eng/common/tools.sh" InitializeDotNetCli true +GetDarc dotnetDir=$( cd ./.dotnet/; pwd -P ) dotnet=$dotnetDir/dotnet -"$dotnet" tool restore highlight "Starting the synchronization of VMR.." set +e @@ -186,7 +186,9 @@ fi # Synchronize the VMR -"$dotnet" darc vmr forwardflow \ +export DOTNET_ROOT="$dotnetDir" + +"$darc_tool" vmr forwardflow \ --tmp "$tmp_dir" \ $azdev_pat \ --$verbosity \ diff --git a/global.json b/global.json index 768272c719..f70e9d2c73 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "10.0.100-preview.3.25201.16", + "dotnet": "10.0.100-preview.7.25372.107", "runtimes": { "dotnet": [ "$(DotNetRuntime60Version)", @@ -13,8 +13,8 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25268.1", - "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25268.1", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25408.3", + "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25408.3", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.2.0" }