Merge branch 'master' into johluo/tooling-consolidation-fixup

Author: John Luo

.azure/pipelines/ci.yml

@@ -19,8 +19,6 @@ pr:
     - '*'
 
 variables:
-- name: DOTNET_CLI_HOME
-  value: $(Agent.BuildDirectory)
 - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
   value: true
 - name: _TeamName
@@ -626,7 +624,6 @@ stages:
   # Helix x64
   - template: jobs/default-build.yml
     parameters:
-      condition: in(variables['Build.Reason'], 'PullRequest')
       jobName: Helix_x64
       jobDisplayName: 'Tests: Helix x64'
       agentOs: Windows
@@ -648,54 +645,6 @@ stages:
         publishOnError: true
         includeForks: true
 
-  - template: jobs/default-build.yml
-    parameters:
-      condition: notin(variables['Build.Reason'], 'PullRequest')
-      jobName: Helix_x64_daily
-      jobDisplayName: 'Tests: Helix x64 Daily'
-      agentOs: Windows
-      timeoutInMinutes: 180
-      steps:
-      # Build the shared framework
-      - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
-        displayName: Build shared fx
-      - script: .\restore.cmd -ci -nobl /p:BuildInteropProjects=true
-        displayName: Restore interop projects
-      - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
-        displayName: Run build.cmd helix target
-        env:
-          HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
-          SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
-      artifacts:
-      - name: Helix_logs
-        path: artifacts/log/
-        publishOnError: true
-        includeForks: true
-
-  # Helix ARM64
-  - template: jobs/default-build.yml
-    parameters:
-      condition: and(eq(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))
-      jobName: Helix_arm64_daily
-      jobDisplayName: "Tests: Helix ARM64 Daily"
-      agentOs: Linux
-      timeoutInMinutes: 180
-      steps:
-      # Build the shared framework
-      - script: ./restore.sh -ci -nobl
-        displayName: Restore
-      - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
-        displayName: Run build.sh helix arm64 target
-        env:
-          HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
-          SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
-      installNodeJs: false
-      artifacts:
-      - name: Helix_arm64_logs
-        path: artifacts/log/
-        publishOnError: true
-        includeForks: true
-
   # Source build
   - job: Source_Build
     displayName: 'Test: Linux Source Build'
@@ -704,14 +653,16 @@ stages:
       vmImage: 'ubuntu-16.04'
     variables:
       DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
+      # This isn't needed in the path because build does not need to _use_ global tools.
+      DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
       DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
     steps:
     - script: |
         source eng/common/native/common-library.sh
-        mkdir -p $HOME/bin
-        GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
-        chmod +x $HOME/bin/jq
-        echo "##vso[task.prependpath]$HOME/bin"
+        mkdir -p $(System.DefaultWorkingDirectory)/.tools
+        GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $(System.DefaultWorkingDirectory)/.tools/jq
+        chmod +x $(System.DefaultWorkingDirectory)/.tools/jq
+        echo "##vso[task.prependpath]$(System.DefaultWorkingDirectory)/.tools"
       displayName: Install jq
     - ${{ if ne(variables['System.TeamProject'], 'public') }}:
         - task: Bash@3

.azure/pipelines/devBuilds.yml

@@ -29,8 +29,6 @@ stages:
       jobDisplayName: "Build: Components"
       agentOs: Windows
       steps:
-      - script: git submodule init
-      - script: git submodule update --recursive
       - script: ./build.cmd
                 -ci
                 -arch x64
@@ -54,8 +52,6 @@ stages:
       jobDisplayName: "Build: Servers"
       agentOs: Windows
       steps:
-      - script: git submodule init
-      - script: git submodule update --recursive
       - script: ./build.cmd
                 -ci
                 -arch x64
@@ -79,8 +75,6 @@ stages:
       jobDisplayName: "Build: Project Templates"
       agentOs: Windows
       steps:
-      - script: git submodule init
-      - script: git submodule update --recursive
       - script: ./build.cmd
                 -ci
                 -arch x64
@@ -104,8 +98,6 @@ stages:
       jobDisplayName: "Build: Everything"
       agentOs: Windows
       steps:
-      - script: git submodule init
-      - script: git submodule update --recursive
       - script: ./build.cmd
                 -ci
                 -arch x64

.azure/pipelines/helix-matrix.yml

@@ -0,0 +1,63 @@
+# We only want to run full helix matrix on master
+pr: none
+trigger: none
+schedules:
+- cron: "0 */12 * * *"
+  branches:
+    include:
+    - master
+  always: true
+    
+variables:
+- ${{ if ne(variables['System.TeamProject'], 'internal') }}:
+  - name: _UseHelixOpenQueues
+    value: 'true'
+- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - group: DotNet-HelixApi-Access
+  - name: _UseHelixOpenQueues
+    value: 'false'    
+    
+jobs:
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Helix_matrix_x64
+    jobDisplayName: 'Tests: Helix full matrix x64'
+    agentOs: Windows
+    timeoutInMinutes: 240
+    steps:
+    # Build the shared framework
+    - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+      displayName: Build shared fx
+    - script: .\restore.cmd -ci /p:BuildInteropProjects=true
+      displayName: Restore interop projects
+    - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+      displayName: Run build.cmd helix target
+      env:
+        HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
+    artifacts:
+    - name: Helix_logs
+      path: artifacts/log/
+      publishOnError: true
+      
+# Helix ARM64
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Helix_matrix_arm64
+    jobDisplayName: "Tests: Helix ARM64 matrix"
+    agentOs: Linux
+    timeoutInMinutes: 180
+    steps:
+    - script: ./restore.sh -ci -nobl
+      displayName: Restore
+    - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+      displayName: Run build.sh helix arm64 target
+      env:
+        HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
+    installNodeJs: false
+    artifacts:
+    - name: Helix_arm64_logs
+      path: artifacts/log/
+      publishOnError: true
+      includeForks: true

.azure/pipelines/jobs/default-build.yml

@@ -52,7 +52,7 @@ parameters:
   condition: ''
   # jobName: '' - use agentOs by default.
   # jobDisplayName: '' - use agentOs by default.
-  artifacts:  []
+  artifacts: []
   buildDirectory: ''
   buildScript: ''
   installTar: true
@@ -80,7 +80,7 @@ jobs:
       enableMicrobuild: true
       enablePublishBuildAssets: true
       enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }}
-    enablePublishTestResults: true # publish test results to AzDO (populates AzDO Tests tab)
+    enablePublishTestResults: ${{ eq(parameters.isTestingJob, 'true') }} # publish test results to AzDO (populates AzDO Tests tab)
     enableTelemetry: true
     helixRepo: dotnet/aspnetcore
     helixType: build.product/
@@ -116,12 +116,13 @@ jobs:
     - BuildScriptArgs: ${{ parameters.buildArgs }}
     - _BuildConfig: ${{ parameters.configuration }}
     - BuildConfiguration: ${{ parameters.configuration }}
-    - BuildDirectory: ${{ parameters.buildDirectory }}
-    - DOTNET_CLI_HOME: $(Agent.BuildDirectory)
+    - ${{ if eq(parameters.buildDirectory, '') }}:
+      - BuildDirectory: $(System.DefaultWorkingDirectory)
+    - ${{ if ne(parameters.buildDirectory, '') }}:
+      - BuildDirectory: ${{ parameters.buildDirectory }}
+    - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
     - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     - TeamName: AspNetCore
-    - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-      - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
     - ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}:
       - _SignType: ''
     - ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}:
@@ -164,6 +165,12 @@ jobs:
     - ${{ if and(eq(parameters.installTar, 'true'), eq(parameters.agentOs, 'Windows')) }}:
       - powershell: ./eng/scripts/InstallTar.ps1
         displayName: Find or install Tar
+    - ${{ if eq(parameters.agentOs, 'Windows') }}:
+      - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
+        displayName: Add dotnet tools to path
+    - ${{ if ne(parameters.agentOs, 'Windows') }}:
+      - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools"
+        displayName: Add dotnet tools to path
 
     - ${{ parameters.beforeBuild }}
 
@@ -191,10 +198,10 @@ jobs:
     - ${{ if eq(parameters.steps, '')}}:
       - ${{ if eq(parameters.buildScript, '') }}:
         - ${{ if eq(parameters.agentOs, 'Windows') }}:
-          - script: .\$(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType)
+          - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType)
             displayName: Run build.cmd
         - ${{ if ne(parameters.agentOs, 'Windows') }}:
-          - script: ./$(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
+          - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
             displayName: Run build.sh
       - ${{ if ne(parameters.buildScript, '') }}:
         - script: $(BuildScript) -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
@@ -232,29 +239,15 @@ jobs:
         condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true')))
         continueOnError: true
         inputs:
-          ${{ if eq(parameters.buildDirectory, '') }}:
-            pathtoPublish: ${{ artifact.path }}
-          ${{ if ne(parameters.buildDirectory, '') }}:
-            pathtoPublish: ${{ parameters.buildDirectory }}\${{ artifact.path }}
+          pathtoPublish: $(BuildDirectory)/${{ artifact.path }}
           ${{ if eq(artifact.name, '') }}:
             artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
           ${{ if ne(artifact.name, '') }}:
             artifactName: ${{ artifact.name }}
           artifactType: Container
           parallel: true
 
-    - ${{ if eq(parameters.isTestingJob, true) }}:
-      - task: PublishTestResults@2
-        displayName: Publish VSTest test results
-        condition: always()
-        continueOnError: true
-        inputs:
-          testRunTitle: $(AgentOsName)-$(BuildConfiguration)
-          testRunner: vstest
-          testResultsFiles: '**/artifacts/**/*.trx'
-          mergeTestResults: true
-          buildConfiguration: $(BuildConfiguration)
-          buildPlatform: $(AgentOsName)
+    - ${{ if and(eq(parameters.isTestingJob, true), ne(parameters.jobName, 'Windows_Templates_Test')) }}:
       - task: PublishTestResults@2
         displayName: Publish js test results
         condition: always()
@@ -263,12 +256,3 @@ jobs:
           testResultsFiles: '**/artifacts/log/**/*.junit.xml'
           buildConfiguration: $(BuildConfiguration)
           buildPlatform: $(AgentOsName)
-      - task: PublishTestResults@2
-        displayName: Publish Java test results
-        condition: always()
-        inputs:
-          testRunner: junit
-          testResultsFiles: '**/TEST-junit-jupiter.xml'
-          buildConfiguration: $(BuildConfiguration)
-          buildPlatform: $(AgentOsName)
-          mergeTestResults: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Issue with .NET runtime or core .NET libraries
+    url:  https://github.com/dotnet/runtime/issues/new/choose
+    about: Please open issues relating to the .NET runtime or core .NET libraries in dotnet/runtime.
+  - name: Issue with .NET SDK
+    url:  https://github.com/dotnet/sdk/issues/new/choose
+    about: Please open issues relating to the .NET SDK itself in dotnet/sdk.
+  - name: Issue with Entity Framework Core
+    url:  https://github.com/dotnet/efcore/issues/new/choose
+    about: Please open issues relating to Entity Framework Core in dotnet/efcore.
+  - name: Issue with Roslyn compiler
+    url:  https://github.com/dotnet/roslyn/issues/new/choose
+    about: Please open issues relating to the Roslyn .NET compiler in dotnet/roslyn.

NuGet.config

@@ -2,14 +2,12 @@
 <configuration>
   <packageSources>
     <clear />
-    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="dotnet5" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
     <add key="dotnet5-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json" />
     <add key="roslyn" value="https://dotnet.myget.org/F/roslyn/api/v3/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <add key="aspnetcore-dev" value="https://dotnet.myget.org/F/aspnetcore-dev/api/v3/index.json" />
-    <add key="aspnetcore-tools" value="https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json" />
     <add key="roslyn-tools" value="https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json" />
     <!-- Used for the SiteExtension 3.1 bits that are included in the 5.0 build -->
     <add key="dotnet31-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json" />

build.ps1

@@ -253,9 +253,7 @@ $RunRestore = if ($NoRestore) { $false }
 # Target selection
 $MSBuildArguments += "/p:Restore=$RunRestore"
 $MSBuildArguments += "/p:Build=$RunBuild"
-if (-not $RunBuild) {
-    $MSBuildArguments += "/p:NoBuild=true"
-}
+if (-not $RunBuild) { $MSBuildArguments += "/p:NoBuild=true" }
 $MSBuildArguments += "/p:Pack=$Pack"
 $MSBuildArguments += "/p:Test=$Test"
 $MSBuildArguments += "/p:Sign=$Sign"
@@ -364,18 +362,23 @@ Remove-Item variable:global:_MSBuildExe -ea Ignore
 # Import Arcade
 . "$PSScriptRoot/eng/common/tools.ps1"
 
+# Add default .binlog location if not already on the command line. tools.ps1 does not handle this; it just checks
+# $BinaryLog, $CI and $ExcludeCIBinarylog values for an error case. But tools.ps1 provides a nice function to help.
+if ($BinaryLog) {
+    $bl = GetMSBuildBinaryLogCommandLineArgument($MSBuildArguments)
+    if (-not $bl) {
+        $MSBuildArguments += "/bl:" + (Join-Path $LogDir "Build.binlog")
+    }
+} elseif ($CI) {
+    # Ensure the artifacts/log directory isn't empty to avoid warnings.
+    New-Item (Join-Path $LogDir "empty.log") -ItemType File -ErrorAction SilentlyContinue >$null
+}
+
 # Capture MSBuild crash logs
 $env:MSBUILDDEBUGPATH = $LogDir
 
 $local:exit_code = $null
 try {
-    # Import custom tools configuration, if present in the repo.
-    # Note: Import in global scope so that the script set top-level variables without qualification.
-    $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
-    if (Test-Path $configureToolsetScript) {
-      . $configureToolsetScript
-    }
-
     # Set this global property so Arcade will always initialize the toolset. The error message you get when you build on a clean machine
     # with -norestore is not obvious about what to do to fix it. As initialization takes very little time, we think always initializing
     # the toolset is a better default behavior.