Merge branch 'master' into johluo/migrating-extensions

Author: John Luo

.azure/pipelines/ci.yml

@@ -32,13 +32,30 @@ variables:
 - name: _DotNetValidationArtifactsCategory
   value: .NETCORE
 - ${{ if ne(variables['System.TeamProject'], 'internal') }}:
+  - name: _UseHelixOpenQueues
+    value: 'true'
   - name: _BuildArgs
     value: ''
   - name: _PublishArgs
     value: ''
   - name: _SignType
     value: ''
+  - name: _InternalRuntimeDownloadArgs
+    value: ''
+  - name: _InternalRuntimeDownloadCodeSignArgs
+    value: ''
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - group: DotNet-MSRC-Storage
+  - name: _InternalRuntimeDownloadArgs
+    value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
+  # The code signing doesn't use the aspnet build scripts, so the msbuild parameers have
+  # to be passed directly. This is awkward, since we pass the same info above, but we have
+  # to have it in two different forms
+  - name: _InternalRuntimeDownloadCodeSignArgs
+    value: /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
+  - group: DotNet-HelixApi-Access
+  - name: _UseHelixOpenQueues
+    value: 'false'
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
     # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
@@ -63,7 +80,7 @@ variables:
     - name: _BuildArgs
       value: ''
     - name: _SignType
-      valule: test
+      value: test
     - name: _PublishArgs
       value: ''
   # used for post-build phases, internal builds only
@@ -81,7 +98,15 @@ stages:
       jobDisplayName: Code check
       agentOs: Windows
       steps:
-      - powershell: ./eng/scripts/CodeCheck.ps1 -ci
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs)
         displayName: Run eng/scripts/CodeCheck.ps1
       artifacts:
       - name: Code_Check_Logs
@@ -108,6 +133,14 @@ stages:
       # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even
       # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing.
       # The sign settings have been configured to
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
       - script: ./build.cmd
                 -ci
@@ -117,6 +150,7 @@ stages:
                 -buildNative
                 /bl:artifacts/log/build.x64.binlog
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         displayName: Build x64
 
       # Build the x86 shared framework
@@ -132,6 +166,7 @@ stages:
                 /p:OnlyPackPlatformSpecificPackages=true
                 /bl:artifacts/log/build.x86.binlog
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         displayName: Build x86
 
       # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53
@@ -140,6 +175,7 @@ stages:
                 -pack
                 -noBuildDeps
                 $(_BuildArgs)
+                $(_InternalRuntimeDownloadArgs)
         condition: ne(variables['Build.Reason'], 'PullRequest')
         displayName: Build SiteExtension
 
@@ -166,6 +202,7 @@ stages:
                 /p:AssetManifestFileName=aspnetcore-win-x64-x86.xml
                 $(_BuildArgs)
                 $(_PublishArgs)
+                $(_InternalRuntimeDownloadArgs)
                 /p:PublishInstallerBaseVersion=true
         displayName: Build Installers
 
@@ -206,6 +243,7 @@ stages:
         /p:AssetManifestFileName=aspnetcore-win-arm.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -232,6 +270,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -252,6 +291,14 @@ stages:
       jobDisplayName: "Build: Linux x64"
       agentOs: Linux
       steps:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
       - script: ./build.sh
             --ci
             --arch x64
@@ -262,6 +309,7 @@ stages:
             -p:OnlyPackPlatformSpecificPackages=true
             -bl:artifacts/log/build.linux-x64.binlog
             $(_BuildArgs)
+            $(_InternalRuntimeDownloadArgs)
         displayName: Run build.sh
       - script: |
           git clean -xfd src/**/obj/
@@ -275,7 +323,8 @@ stages:
             -p:BuildRuntimeArchive=false \
             -p:LinuxInstallerType=deb \
             -bl:artifacts/log/build.deb.binlog \
-            $(_BuildArgs)
+            $(_BuildArgs) \
+            $(_InternalRuntimeDownloadArgs)
         displayName: Build Debian installers
       - script: |
           git clean -xfd src/**/obj/
@@ -291,7 +340,8 @@ stages:
             -bl:artifacts/log/build.rpm.binlog \
             -p:AssetManifestFileName=aspnetcore-Linux_x64.xml \
             $(_BuildArgs) \
-            $(_PublishArgs)
+            $(_PublishArgs) \
+            $(_InternalRuntimeDownloadArgs)
         displayName: Build RPM installers
       installNodeJs: false
       installJdk: false
@@ -323,6 +373,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_arm.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -353,6 +404,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -386,6 +438,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -419,6 +472,7 @@ stages:
         -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadArgs)
       installNodeJs: false
       installJdk: false
       artifacts:
@@ -440,7 +494,7 @@ stages:
       jobDisplayName: "Test: Windows Server 2016 x64"
       agentOs: Windows
       isTestingJob: true
-      buildArgs: -all -pack -test -BuildNative "/p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false"
+      buildArgs: -all -pack -test -BuildNative "/p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
         displayName: Setup IISExpress test certificates and schema
@@ -476,7 +530,15 @@ stages:
       agentOs: Windows
       isTestingJob: true
       steps:
-      - script: ./build.cmd -ci -all -pack
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: PowerShell@2
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
+      - script: ./build.cmd -ci -all -pack $(_InternalRuntimeDownloadArgs)
         displayName: Build Repo
       - script: ./src/ProjectTemplates/build.cmd -ci -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.pack.binlog"
         displayName: Pack Templates
@@ -503,7 +565,7 @@ stages:
       jobDisplayName: "Test: macOS 10.13"
       agentOs: macOS
       isTestingJob: true
-      buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true"
+      buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-mac.sh"
         displayName: Installing Nginx
@@ -538,7 +600,7 @@ stages:
       jobDisplayName: "Test: Ubuntu 16.04 x64"
       agentOs: Linux
       isTestingJob: true
-      buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true"
+      buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-linux.sh"
         displayName: Installing Nginx
@@ -582,6 +644,7 @@ stages:
       - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
+          HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
           SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
       artifacts:
       - name: Helix_logs
@@ -591,6 +654,7 @@ stages:
 
   - template: jobs/default-build.yml
     parameters:
+      condition: ne(variables['Build.Reason'], 'PullRequest')
       jobName: Helix_x64_daily
       jobDisplayName: 'Tests: Helix x64 Daily'
       agentOs: Windows
@@ -601,6 +665,7 @@ stages:
       - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
+          HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
           SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
       artifacts:
       - name: Helix_logs
@@ -622,6 +687,7 @@ stages:
       - script: ./build.sh -ci --arch arm64 -test --no-build-nodejs -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.sh helix arm64 target
         env:
+          HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
           SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
       installNodeJs: false
       artifacts:
@@ -647,6 +713,14 @@ stages:
         chmod +x $HOME/bin/jq
         echo "##vso[task.prependpath]$HOME/bin"
       displayName: Install jq
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - task: Bash@3
+          displayName: Setup Private Feeds Credentials
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+          env:
+            Token: $(dn-bot-dnceng-artifact-feeds-rw)
     - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false
       displayName: Run ci-source-build.sh
     - task: PublishBuildArtifacts@1

.azure/pipelines/helix-test.yml

@@ -11,6 +11,14 @@ pr:
     include:
     - '*'
 
+variables:
+- ${{ if ne(variables['System.TeamProject'], 'internal') }}:
+  - name: _UseHelixOpenQueues
+    value: 'true'
+- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - name: _UseHelixOpenQueues
+    value: 'false'
+
 jobs:
 - template: jobs/default-build.yml
   parameters:

.azure/pipelines/jobs/codesign-xplat.yml

@@ -28,6 +28,14 @@ jobs:
         contents: '**/*.nupkg'
         targetFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/shipping/
         flattenFolders: true
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - task: PowerShell@2
+        displayName: Setup Private Feeds Credentials
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+          arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+        env:
+          Token: $(dn-bot-dnceng-artifact-feeds-rw)
     - powershell: .\eng\common\build.ps1
         -ci
         -restore
@@ -39,6 +47,7 @@ jobs:
         /p:DotNetSignType=$(_SignType)
         $(_BuildArgs)
         $(_PublishArgs)
+        $(_InternalRuntimeDownloadCodeSignArgs)
       displayName: Sign and publish packages
     artifacts:
     - name: CodeSign_Xplat_${{ parameters.inputName }}_Logs

.azure/pipelines/jobs/default-build.yml

@@ -161,6 +161,24 @@ jobs:
     - ${{ if ne(parameters.steps, '')}}:
       - ${{ parameters.steps }}
     - ${{ if eq(parameters.steps, '')}}:
+      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        - ${{ if eq(parameters.agentOs, 'Windows') }}:
+          - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+            - task: PowerShell@2
+              displayName: Setup Private Feeds Credentials
+              inputs:
+                filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+                arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+              env:
+                Token: $(dn-bot-dnceng-artifact-feeds-rw)
+        - ${{ if ne(parameters.agentOs, 'Windows') }}:
+          - task: Bash@3
+            displayName: Setup Private Feeds Credentials
+            inputs:
+              filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+              arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+            env:
+              Token: $(dn-bot-dnceng-artifact-feeds-rw)
       - ${{ if eq(parameters.buildScript, '') }}:
         - ${{ if eq(parameters.agentOs, 'Windows') }}:
           - script: .\$(BuildDirectory)\build.cmd -ci /p:DotNetSignType=$(_SignType) -Configuration $(BuildConfiguration) $(BuildScriptArgs)
@@ -233,7 +251,7 @@ jobs:
         condition: always()
         inputs:
           testRunner: junit
-          testResultsFiles: '**/TEST-com.microsoft.signalr*.xml'
+          testResultsFiles: '**/TEST-junit-jupiter.xml'
           buildConfiguration: $(BuildConfiguration)
           buildPlatform: $(AgentOsName)
           mergeTestResults: true

.gitattributes

@@ -8,6 +8,11 @@
 ###############################################################################
 *.sh eol=lf
 
+###############################################################################
+# Make gradlew always have LF as line endings
+###############################################################################
+gradlew eol=lf
+
 ###############################################################################
 # Set default behavior for command prompt diff.
 #

Directory.Build.props

@@ -91,8 +91,11 @@
     <RuntimeInstallerBaseName>aspnetcore-runtime</RuntimeInstallerBaseName>
     <TargetingPackInstallerBaseName>aspnetcore-targeting-pack</TargetingPackInstallerBaseName>
 
-    <!-- Used to only produce targeting pack installers/packages once per major.minor. -->
-    <IsTargetingPackBuilding Condition="'$(AspNetCorePatchVersion)' != '0' OR '$(DotNetBuildFromSource)' == 'true'">false</IsTargetingPackBuilding>
+    <!-- Produce targeting pack installers/packages once per major.minor. -->
+    <IsTargetingPackBuilding Condition=" '$(DotNetBuildFromSource)' == 'true' ">false</IsTargetingPackBuilding>
+    <IsTargetingPackBuilding
+        Condition=" '$(IsTargetingPackBuilding)' == '' AND '$(AspNetCorePatchVersion)' != '0' ">false</IsTargetingPackBuilding>
+    <IsTargetingPackBuilding Condition=" '$(IsTargetingPackBuilding)' == '' ">true</IsTargetingPackBuilding>
 
     <!--
       Archives and installers using this prefix are intended for internal-use only.
@@ -179,13 +182,13 @@
 
   <Import Project="eng\Workarounds.props" />
   <Import Project="eng\Dependencies.props" />
-  <Import Project="eng\PatchConfig.props" />
   <Import Project="eng\ProjectReferences.props" />
   <Import Project="eng\SharedFramework.Local.props" />
   <Import Project="eng\SharedFramework.External.props" />
   <Import Project="eng\targets\Cpp.Common.props"  Condition="'$(MSBuildProjectExtension)' == '.vcxproj'" />
   <Import Project="eng\targets\CSharp.Common.props"  Condition="'$(MSBuildProjectExtension)' == '.csproj'" />
   <Import Project="eng\targets\Wix.Common.props"  Condition="'$(MSBuildProjectExtension)' == '.wixproj'" />
   <Import Project="eng\targets\Npm.Common.props"  Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
+  <Import Project="eng\targets\Helix.props" Condition="'$(IsTestProject)' == 'true'" />
 
 </Project>