diff --git a/NuGet.config b/NuGet.config index 780fb0ea1..b1204a1ae 100644 --- a/NuGet.config +++ b/NuGet.config @@ -6,12 +6,8 @@ - - - - diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..32a4b1f03 --- /dev/null +++ b/build.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +"$scriptroot/eng/common/build.sh" --build --restore --pack $@ diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 26b90bf96..d21c8a00d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ - + https://github.com/dotnet/arcade - 49750c02e63d0ad3a77d035bba7498a0b1acd218 + a8aae93d709533604b73c619268415c6ccc71292 - + https://github.com/dotnet/arcade - 49750c02e63d0ad3a77d035bba7498a0b1acd218 + a8aae93d709533604b73c619268415c6ccc71292 diff --git a/eng/Versions.props b/eng/Versions.props index 496c1c0d6..002f48b29 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,21 +11,21 @@ --> 1.4.1 - 16.7 - 5.0.0-rc.1.20451.14 - 12.0.3 - 5.8.0-preview.3.6823 - 5.0.0-rc.1.20451.14 - 2.0.0-beta1.20371.2 - 5.0.0-rc.1.20451.14 - 5.0.0-rc.1.20451.14 + 16.10 + 5.0.0 + 13.0.1 + 6.0.0 + 6.0.0 + 2.0.0-beta3.22114.1 + 6.0.0 + 5.0.0 - 1.3.0 - 4.14.6 + 3.1.2 + 4.18.1 true diff --git a/eng/azure-pipelines.yml b/eng/azure-pipelines.yml index aadc3b880..0c5d4aa79 100644 --- a/eng/azure-pipelines.yml +++ b/eng/azure-pipelines.yml @@ -12,7 +12,7 @@ pr: variables: - name: _TeamName value: DotNetCore - - name: _BuildConfig + - name: _BuildConfig value: Release - name: _PublishUsingPipelines value: true @@ -31,7 +31,7 @@ jobs: helixRepo: dotnet/standard jobs: - job: Windows_NT - pool: + pool: ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: name: NetCore1ESPool-Public demands: ImageOverride -equals build.windows.10.amd64.vs2019.pre.open @@ -50,7 +50,7 @@ jobs: - name: _BuildArgs value: /p:SignType=$(_SignType) /p:DotNetSignType=$(_SignType) - /p:MicroBuild_SigningEnabled=true + /p:MicroBuild_SigningEnabled=true /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /p:TeamName=$(_TeamName) /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) @@ -78,8 +78,15 @@ jobs: - checkout: self clean: true - script: eng\common\cibuild.cmd - -configuration $(_BuildConfig) + -configuration $(_BuildConfig) -prepareMachine -warnaserror:0 $(_BuildArgs) displayName: Windows Build / Publish + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: TestResults' + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + ArtifactName: 'Windows $(_BuildConfig) Test Logs' + continueOnError: true + condition: always() \ No newline at end of file diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 7e4be9a0c..5a59dcff2 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -4,12 +4,13 @@ set -e usage() { - echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd12, freebsd13" echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" + echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 @@ -48,6 +49,7 @@ __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" +__AlpinePackages+=" compiler-rt-static" # CoreFX dependencies __UbuntuPackages+=" libcurl4-openssl-dev" @@ -164,6 +166,15 @@ while :; do no-lldb) unset __LLDB_Package ;; + llvm*) + version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" + parts=(${version//./ }) + __LLVM_MajorVersion="${parts[0]}" + __LLVM_MinorVersion="${parts[1]}" + if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then + __LLVM_MinorVersion=0; + fi + ;; xenial) # Ubuntu 16.04 if [ "$__CodeName" != "jessie" ]; then __CodeName=xenial @@ -194,8 +205,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for arm, armel and arm64." + if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ] && [ "$__BuildArch" != "x86" ] ; then + echo "Tizen is available only for arm, armel, arm64 and x86." usage; exit 1; fi @@ -261,6 +272,10 @@ if [ "$__BuildArch" == "armel" ]; then fi __UbuntuPackages+=" ${__LLDB_Package:-}" +if [ ! -z "$__LLVM_MajorVersion" ]; then + __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" +fi + if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then __RootfsDir=$ROOTFS_DIR fi diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 9fd345bde..eaeeab38f 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -63,6 +63,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "s390x") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + endif() elseif (FREEBSD) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(triple "x86_64-unknown-freebsd12") @@ -91,6 +94,10 @@ if(TIZEN) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) endif() + if(TARGET_ARCH_NAME STREQUAL "x86") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) + endif() endif() if(ANDROID) @@ -197,6 +204,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_toolchain_linker_flag(-m32) + + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") @@ -232,7 +246,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() if(TIZEN) - if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|x86)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 5c94e9863..abd045a32 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -52,7 +52,7 @@ done # Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples cpuname=$(uname -m) case $cpuname in - aarch64) + arm64|aarch64) buildarch=arm64 ;; loongarch64) @@ -64,7 +64,7 @@ case $cpuname in armv*l) buildarch=arm ;; - i686) + i[3-6]86) buildarch=x86 ;; *) diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a..24a5e65de 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,98 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0] + if ([string]::IsNullOrWhiteSpace($ToolDirectory)) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + } + } + exit 0 + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +189,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index beb9c4648..7f5ce6d60 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -8,6 +8,9 @@ + + + diff --git a/eng/common/retain-build.ps1 b/eng/common/retain-build.ps1 index e08fc227b..e7ba975ad 100644 --- a/eng/common/retain-build.ps1 +++ b/eng/common/retain-build.ps1 @@ -8,8 +8,6 @@ Param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 -. $PSScriptRoot\tools.ps1 - function Get-AzDOHeaders( [string] $token) @@ -38,10 +36,10 @@ function Update-BuildRetention( Write-Host "Updated retention settings for build ${buildId}." } catch { - Write-PipelineTelemetryError -Category "Build" -Message "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" - ExitWithExitCode 1 + Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" + exit 1 } } Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token -ExitWithExitCode 0 +exit 0 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b1bca63ab..119a6c660 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.1.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 9d1e3042d..3bcd243c4 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -72,8 +72,8 @@ jobs: lclSource: ${{ parameters.LclSource }} lclPackageId: ${{ parameters.LclPackageId }} isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} ${{ if eq(parameters.CreatePr, true) }}: - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index d91bf9147..1cbb6a0c5 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -23,23 +23,33 @@ parameters: # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishUsingPipelines: false + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + jobs: - job: Asset_Registry_Publish dependsOn: ${{ parameters.dependsOn }} - displayName: Publish to Build Asset Registry + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry pool: ${{ parameters.pool }} variables: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _BuildConfig - value: ${{ parameters.configuration }} - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates/post-build/common-variables.yml steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -52,14 +62,13 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@0 - - task: PowerShell@2 - displayName: Enable cross-org NuGet feed authentication - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) + - task: PowerShell@2 + displayName: Enable cross-org NuGet feed authentication + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) - task: PowerShell@2 displayName: Publish Build Assets @@ -70,7 +79,6 @@ jobs: /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:Configuration=$(_BuildConfig) /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} @@ -114,7 +122,25 @@ jobs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container ArtifactName: ReleaseConfigs - + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 4af724eb1..c2d51098d 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -7,6 +7,7 @@ parameters: binlogPath: artifacts/log/Debug/Build.binlog condition: '' dependsOn: '' + pool: '' jobs: - job: SourceIndexStage1 @@ -22,13 +23,17 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: source-dot-net stage1 variables - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 554e71cfc..2cca53c2d 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -27,6 +27,13 @@ parameters: # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + # Optional: should run as a public build even in the internal project # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false @@ -68,7 +75,6 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml parameters: @@ -94,4 +100,7 @@ jobs: runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2f176571f..5a9056f6b 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -82,6 +82,11 @@ parameters: default: - Validate + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + stages: - ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - stage: Validate @@ -99,7 +104,7 @@ stages: name: VSEngSS-MicroBuild2022-1ES demands: Cmd # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + ${{ else }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 @@ -136,7 +141,7 @@ stages: name: VSEngSS-MicroBuild2022-1ES demands: Cmd # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + ${{ else }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: @@ -196,7 +201,7 @@ stages: name: VSEngSS-MicroBuild2022-1ES demands: Cmd # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + ${{ else }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: @@ -235,43 +240,44 @@ stages: artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} -- stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + ${{ else }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@0 - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f1e1cb539..423bd962e 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -365,8 +365,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview - $defaultXCopyMSBuildVersion = '16.10.0-preview2' + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.1.0&view=overview + $defaultXCopyMSBuildVersion = '17.1.0' if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } @@ -635,6 +635,10 @@ function InitializeNativeTools() { InstallDirectory = "$ToolsDir" } } + if (Test-Path variable:NativeToolsOnMachine) { + Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." + $nativeArgs += @{ PathPromotion = $true } + } & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } diff --git a/global.json b/global.json index 3ab69dbaa..79a374b0d 100644 --- a/global.json +++ b/global.json @@ -1,20 +1,21 @@ { "tools": { - "dotnet": "6.0.100", + "dotnet": "7.0.100-preview.2.22153.17", "runtimes": { "dotnet": [ - "5.0.14", - "3.1.7", - "2.1.11" + "6.0.5", + "5.0.17", + "3.1.25", + "2.1.30" ] } }, "sdk": { - "version": "6.0.100", + "version": "7.0.100-preview.2.22153.17", "rollForward": "major" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22117.2", - "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.22117.2" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22275.2", + "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.22275.2" } } \ No newline at end of file diff --git a/src/try-convert/Program.cs b/src/try-convert/Program.cs index 561dc6e55..195f89ebe 100644 --- a/src/try-convert/Program.cs +++ b/src/try-convert/Program.cs @@ -1,7 +1,6 @@ using System; using System.CommandLine; using System.CommandLine.Builder; -using System.CommandLine.Invocation; using System.CommandLine.Parsing; using System.IO; using System.Threading.Tasks; @@ -14,58 +13,88 @@ namespace MSBuild.Conversion { internal class Program { + private static Task ErrorResult => Task.FromResult(-1); + private static Task SuccessResult => Task.FromResult(0); + + private static Option ProjectOption => new Option(new[] { "-p", "--project" }, "The path to a project to convert"); + private static Option WorkspaceOption => new Option(new[] { "-w", "--workspace" }, "The solution or project file to operate on. If a project is not specified, the command will search the current directory for one."); + private static Option MSBuildPathOption => new Option(new[] { "-m", "--msbuild-path" }, "The path to an MSBuild.exe, if you prefer to use that"); + private static Option TargetFrameworkOption => new Option(new[] { "-tfm", "--target-framework" }, "The name of the framework you would like to upgrade to. If unspecified, the default TFM for apps chosen will be the highest available one found on your machine, and the default TFM for libraries will be .NET Standard 2.0."); + private static Option ForceWebConversionOption => new Option(new[] { "--force-web-conversion" }, "Attempt to convert MVC and WebAPI projects even though significant manual work is necessary after migrating such projects."); + private static Option PreviewOption => new Option(new[] { "--preview" }, "Use preview SDKs as part of conversion"); + private static Option DiffOnlyOption => new Option(new[] { "--diff-only" }, "Produces a diff of the project to convert; no conversion is done"); + private static Option NoBackupOption => new Option(new[] { "--no-backup" }, "Converts projects, does not create a backup of the originals and removes packages.config file."); + private static Option KeepCurrentTfmsOption => new Option(new[] { "--keep-current-tfms" }, "Converts project files but does not change any TFMs. If unspecified, TFMs may change."); + private static Option MauiConverionOption => new Option(new[] { "--maui-conversion" }, "Attempt to convert Xamarin.Forms Projects to .NET MAUI projects. There may be additional manual work necessary after migrating such projects."); + private static Option ForceRemoveCustomImportsOption => new Option(new[] { "--force-remove-custom-imports" }, "Force remove custom imports from the project file if set to true."); + private static Option UpdateOption => new Option(new[] { "-u", "--update" }, "Updates the try-convert tool to the latest available version"); + private static async Task Main(string[] args) { var rootCommand = new RootCommand { - Name = "try-convert", - Handler = CommandHandler.Create(typeof(Program).GetMethod(nameof(Run))!) + ProjectOption, + WorkspaceOption, + MSBuildPathOption, + TargetFrameworkOption, + ForceWebConversionOption, + PreviewOption, + DiffOnlyOption, + NoBackupOption, + KeepCurrentTfmsOption, + MauiConverionOption, + ForceRemoveCustomImportsOption, + UpdateOption }; + Func> handler = Run; + + rootCommand.SetHandler( + handler, + ProjectOption, + WorkspaceOption, + MSBuildPathOption, + TargetFrameworkOption, + ForceWebConversionOption, + PreviewOption, + DiffOnlyOption, + NoBackupOption, + KeepCurrentTfmsOption, + MauiConverionOption, + ForceRemoveCustomImportsOption, + UpdateOption); + var parser = new CommandLineBuilder(rootCommand) .UseParseDirective() .UseHelp() - .UseDebugDirective() .UseSuggestDirective() .RegisterWithDotnetSuggest() .UseParseErrorReporting() .UseExceptionHandler() - .AddOption(new Option(new[] { "-p", "--project" }, "The path to a project to convert") { Argument = new Argument(() => null) }) - .AddOption(new Option(new[] { "-w", "--workspace" }, "The solution or project file to operate on. If a project is not specified, the command will search the current directory for one.") { Argument = new Argument(() => null) }) - .AddOption(new Option(new[] { "-m", "--msbuild-path" }, "The path to an MSBuild.exe, if you prefer to use that") { Argument = new Argument(() => null) }) - .AddOption(new Option(new[] { "-tfm", "--target-framework" }, "The name of the framework you would like to upgrade to. If unspecified, the default TFM for apps chosen will be the highest available one found on your machine, and the default TFM for libraries will be .NET Standard 2.0.") { Argument = new Argument(() => null) }) - .AddOption(new Option(new[] { "--force-web-conversion" }, "Attempt to convert MVC and WebAPI projects even though significant manual work is necessary after migrating such projects.") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--preview" }, "Use preview SDKs as part of conversion") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--diff-only" }, "Produces a diff of the project to convert; no conversion is done") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--no-backup" }, "Converts projects, does not create a backup of the originals and removes packages.config file.") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--keep-current-tfms" }, "Converts project files but does not change any TFMs. If unspecified, TFMs may change.") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--maui-conversion" }, "Attempt to convert Xamarin.Forms Projects to .NET MAUI projects. There may be additional manual work necessary after migrating such projects.") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "--force-remove-custom-imports" }, "Force remove custom imports from the project file if set to true.") { Argument = new Argument(() => false) }) - .AddOption(new Option(new[] { "-u", "--update" }, "Updates the try-convert tool to the latest available version") { Argument = new Argument(() => false) }) .Build(); return await parser.InvokeAsync(args).ConfigureAwait(false); } - public static int Run(string? project, string? workspace, string? msbuildPath, string? tfm, bool forceWebConversion, bool preview, bool diffOnly, bool noBackup, bool keepCurrentTfms, bool update, bool mauiConversion, bool forceRemoveCustomImports) + public static Task Run(string? project, string? workspace, string? msbuildPath, string? tfm, bool forceWebConversion, bool preview, bool diffOnly, bool noBackup, bool keepCurrentTfms, bool update, bool mauiConversion, bool forceRemoveCustomImports) { if (update) { UpdateTryConvert.Update(); - return 0; + return SuccessResult; } if (!string.IsNullOrWhiteSpace(project) && !string.IsNullOrWhiteSpace(workspace)) { Console.WriteLine("Cannot specify both a project and a workspace."); - return -1; + return ErrorResult; } if (!string.IsNullOrWhiteSpace(tfm) && keepCurrentTfms) { Console.WriteLine($"Both '{nameof(tfm)}' and '{nameof(keepCurrentTfms)}' cannot be specified. Please pick one."); - return -1; + return ErrorResult; } try @@ -86,14 +115,14 @@ public static int Run(string? project, string? workspace, string? msbuildPath, s else { Console.WriteLine("Error locating VS Install Directory. Try setting Environment Variable VSINSTALLDIR."); - return -1; + return ErrorResult; } } if (string.IsNullOrWhiteSpace(msbuildPath)) { Console.WriteLine("Could not find an MSBuild."); - return -1; + return ErrorResult; } if (!string.IsNullOrWhiteSpace(tfm)) @@ -102,7 +131,7 @@ public static int Run(string? project, string? workspace, string? msbuildPath, s if (!TargetFrameworkHelper.IsValidTargetFramework(tfm)) { Console.WriteLine($"Invalid framework specified for --target-framework: '{tfm}'"); - return -1; + return ErrorResult; } } else @@ -133,7 +162,7 @@ public static int Run(string? project, string? workspace, string? msbuildPath, s if (msbuildWorkspace.WorkspaceItems.Length is 0) { Console.WriteLine("No projects converted."); - return 0; + return SuccessResult; } foreach (var item in msbuildWorkspace.WorkspaceItems) @@ -161,11 +190,11 @@ public static int Run(string? project, string? workspace, string? msbuildPath, s catch (Exception e) { Console.WriteLine(e.ToString()); - return -1; + return ErrorResult; } Console.WriteLine("Conversion complete!"); - return 0; + return SuccessResult; } } } diff --git a/test.sh b/test.sh new file mode 100755 index 000000000..637dec360 --- /dev/null +++ b/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +"$scriptroot/eng/common/build.sh" --test $@