Merge pull request #1975 from dseefeld/5.0.102

Update to 5.0.102

Author: dseefeld

.vsts.pipelines/jobs/ci-linux.yml

@@ -39,6 +39,16 @@ jobs:
     # Default type, can be overridden by matrix legs.
     type: ${{ coalesce(parameters.type, 'Production') }}
 
+    ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
+      SetInternalPackageFeedPatDockerArg: ''
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      # In the internal build, we have one definition that provides dn-bot-dnceng-artifact-feeds-rw,
+      # and another that doesn't (to be more like a public build). For the definition that doesn't
+      # define dn-bot-dnceng-artifact-feeds-rw, this line relies on the subshell to fail when it
+      # tries to find a command by that name and set internalPackageFeedPat to nothing.
+      SetInternalPackageFeedPatDockerArg: >-
+        -e internalPackageFeedPat=$(dn-bot-dnceng-artifact-feeds-rw)
+
   steps:
   - template: ../steps/cleanup-unneeded-files.yml
   - template: ../steps/docker-cleanup-linux.yml
@@ -62,7 +72,7 @@ jobs:
       if [ "$(sb.tarball)" != "true" ]; then
         failOnBaselineError=true
       fi
-      $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+      $(docker.run) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build.sh \
         /p:Configuration=$(sb.configuration) \
         /p:BuildPortableRuntime=$(sb.portable) \
         /p:BuildPortableSdk=$(sb.portable) \
@@ -90,7 +100,7 @@ jobs:
   - script: |
       set -ex
       df -h
-      $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+      $(docker.run) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build.sh \
         --run-smoke-test \
         /p:Configuration=$(sb.configuration) \
         /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
@@ -119,7 +129,7 @@ jobs:
       if [ "$(reportPrebuiltLeaks)" = "true" ]; then
         args="$args --enable-leak-detection"
       fi
-      $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build-source-tarball.sh \
+      $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build-source-tarball.sh \
         "/tb/$(tarballName)" \
         $args
       du -h $(rootDirectory) | sort -h | tail -n 50
@@ -224,10 +234,9 @@ jobs:
   - script: |
       set -ex
       df -h
-      $(docker.run) $(docker.tb.map) $(docker.tb.work) $(imageName) "$(tarballName)/smoke-test.sh" \
-        --minimal \
-        --projectOutput \
-        --configuration $(sb.configuration)
+      $(docker.run) $(docker.tb.map) $(docker.tb.work) $(imageName) "$(tarballName)/build.sh" \
+        --run-smoke-test -- \
+        /p:Configuration=$(sb.configuration)
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Run smoke-test in tarball
     condition: and(succeeded(), eq(variables['sb.tarball'], true))

.vsts.pipelines/jobs/ci-local.yml

@@ -42,6 +42,15 @@ jobs:
     # Default type, can be overridden by matrix legs.
     type: Production
 
+    ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
+      SetInternalPackageFeedPatBashCommand: ''
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      # In the internal build, we have one definition that provides dn-bot-dnceng-artifact-feeds-rw,
+      # and another that doesn't (to be more like a public build). For the definition that doesn't
+      # define dn-bot-dnceng-artifact-feeds-rw, this line relies on the subshell to fail when it
+      # tries to find a command by that name and set internalPackageFeedPat to nothing.
+      SetInternalPackageFeedPatBashCommand: 'export internalPackageFeedPat=$(dn-bot-dnceng-artifact-feeds-rw)'
+
   steps:
   - checkout: self
     clean: true
@@ -71,6 +80,9 @@ jobs:
 
   # Build source-build.
   - script: |
+      set -x
+      $(SetInternalPackageFeedPatBashCommand)
+      set -e
       ${{ format('{0}build{1} $(args.build)', parameters.scriptPrefix, parameters.scriptSuffix) }}
     displayName: Build source-build
     timeoutInMinutes: 150
@@ -79,7 +91,11 @@ jobs:
 
   # Run smoke tests.
   - ${{ if ne(parameters.skipSmokeTest, true) }}:
-    - bash: ${{ format('{0}build{1} $(args.smokeTest)', parameters.scriptPrefix, parameters.scriptSuffix) }}
+    - bash: |
+        set -x
+        $(SetInternalPackageFeedPatBashCommand)
+        set -e
+        ${{ format('{0}build{1} $(args.smokeTest)', parameters.scriptPrefix, parameters.scriptSuffix) }}
       displayName: Run smoke-test
 
   # Gather artifacts. Uses git bash on Windows.

.vsts.pipelines/steps/run-bootstrap.yml

@@ -163,14 +163,14 @@ steps:
   condition: always()
   continueOnError: true
 
-# Copy final-stage tarball artifactss and sdk tarball to drop directory.
+# Copy final-stage tarball artifacts and sdk tarball to drop directory.
 - script: |
     set -ex
     df -h
     $(docker.run) $(docker.bst.map) $(docker.drop.map) $(docker.bst.work) $(imageName) /bin/bash -c "
       mkdir -p /drop/tarball/final
       pushd /bst/bootstrap_dir/final-sdk
-      find ./bin \( \
+      find ./artifacts \( \
         -iname 'dotnet-sdk*.tar.gz' -o \
         -iname 'Private.SourceBuilt.Artifacts*.tar.gz' \) \
         -exec cp {} /drop/tarball/final \;"
@@ -195,7 +195,10 @@ steps:
       df -h 
       $(docker.run) $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c "
       pushd /bst/bootstrap_dir/final-sdk
-      ./smoke-test.sh --prodConBlobFeedUrl ''"
+      ./build.sh \
+        --run-smoke-test \
+        --with-packages /bst/bootstrap_dir/stage1-source-built-artifacts-install/ \
+        --with-sdk /bst/bootstrap_dir/stage1-sdk-install/"
   displayName: Smoke-test final SDK
   condition: always()
 

Directory.Build.props

@@ -155,6 +155,7 @@
     <OfflinePrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData-offline.csv</OfflinePrebuiltBurndownDataFile>
     <OnlinePrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData-online.csv</OnlinePrebuiltBurndownDataFile>
     <ReferencePackagesBaseDir>$(IntermediatePath)reference-packages/</ReferencePackagesBaseDir>
+    <TextOnlyPackageBaseDir>$(IntermediatePath)text-only-packages/</TextOnlyPackageBaseDir>
     <ExternalTarballsDir>$(IntermediatePath)external-tarballs/</ExternalTarballsDir>
     <!--
       Change ReferencePackagesBaseDir & ExternalTarballsDir conditionally in offline build.
@@ -163,6 +164,7 @@
     -->
     <ExternalTarballsDir Condition="'$(OfflineBuild)' == 'true'">$(ProjectDir)packages/archive/</ExternalTarballsDir>
     <ReferencePackagesBaseDir Condition="'$(OfflineBuild)' == 'true'">$(ProjectDir)packages/reference/</ReferencePackagesBaseDir>
+    <TextOnlyPackageBaseDir Condition="'$(OfflineBuild)' == 'true'">$(ProjectDir)packages/text-only/</TextOnlyPackageBaseDir>
     <ReferencePackagesDir>$(ReferencePackagesBaseDir)packages/</ReferencePackagesDir>
     <ReferencePackagesDir Condition="'$(CustomReferencePackagesPath)' != ''">$(CustomReferencePackagesPath)/</ReferencePackagesDir>
     <SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>

NuGet.config

@@ -4,9 +4,7 @@
     <clear />
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
     <add key="darc-pub-dotnet-templating-3198063" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-31980633/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-runtime-cf258a1-1" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-runtime-cf258a14-1/nuget/v3/index.json" />
     <add key="darc-pub-dotnet-msbuild-126527f" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-msbuild-126527ff/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-aspnetcore-371a26f" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-371a26f0/nuget/v3/index.json" />
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="nuget-build" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/nuget-build/nuget/v3/index.json" />

build.proj

@@ -153,7 +153,9 @@
       This is also a problem when passing CLI feeds: https://github.com/dotnet/source-build/issues/561
     -->
     <Exec Command="$(SmokeTestCommand)"
-          EnvironmentVariables="prodConBlobFeedUrl=$(ProdConBlobFeedUrl)" />
+          EnvironmentVariables="
+            targetRid=$(TargetRid);
+            prodConBlobFeedUrl=$(ProdConBlobFeedUrl)" />
   </Target>
 
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UploadToAzure" />

build.sh

@@ -102,6 +102,20 @@ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 export DOTNET_MULTILEVEL_LOOKUP=0
 export NUGET_PACKAGES="$scriptroot/packages/restored/"
 
+if [ "${internalPackageFeedPat:-}" ]; then
+  echo "Setting up NuGet credential provider using PAT from env var 'internalPackageFeedPat'..."
+  . "$scriptroot/eng/install-nuget-credprovider.sh"
+  # TODO: Read these from nuget.config
+  # The internal transport isn't added by Darc, though, so it will still need special-casing.
+  export VSS_NUGET_EXTERNAL_FEED_ENDPOINTS='{"endpointCredentials": [
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-runtime-cb5f173b/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-aspnetcore-2670c128/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-installer-71365b4d/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/internal/_packaging/5.0.102-servicing.20614.17/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"}
+  ]}'
+fi
+
 set -x
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 

eng/SourceBuild.Tarball.AllowedPrebuilts.targets

@@ -3,7 +3,8 @@
   <Target Name="AllowWorkInProgressPrebuilts"
           BeforeTargets="ForbidPrebuilts">
     <ItemGroup>
-      <AllowedPrebuiltPackageFile/>
+      <!-- Example: $(TarballPrebuiltPackageDir)microsoft.dotnet.web.itemtemplates.5.0.2.nupkg -->
+      <AllowedPrebuiltPackageFile />
     </ItemGroup>
   </Target>
 

eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets

@@ -31,6 +31,7 @@
         $(TarballPrebuiltPackageDir)microsoft.dotnet.arcade.sdk.5.0.0-beta.20426.4.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.dotnet.signtool.5.0.0-beta.20426.4.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.extensions.dependencymodel.3.1.6.nupkg;
+        $(TarballPrebuiltPackageDir)microsoft.extensions.dependencyinjection.5.0.0.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.netcore.app.host.linux-x64.3.1.7.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.symboluploader.build.task.1.1.141804.nupkg;
         $(TarballPrebuiltPackageDir)netstandard.library.1.6.0.nupkg;
@@ -89,12 +90,15 @@
         $(TarballPrebuiltPackageDir)system.composition.hosting.1.0.31.nupkg;
         $(TarballPrebuiltPackageDir)system.composition.runtime.1.0.31.nupkg;
         $(TarballPrebuiltPackageDir)system.composition.typedparts.1.0.31.nupkg;
+        $(TarballPrebuiltPackageDir)system.io.pipelines.5.0.0.nupkg;
         $(TarballPrebuiltPackageDir)system.reflection.metadata.5.0.0-preview.8.20407.11.nupkg;
         $(TarballPrebuiltPackageDir)system.runtime.compilerservices.unsafe.5.0.0-rc.1.20451.14.nupkg;
+        $(TarballPrebuiltPackageDir)system.security.cryptography.pkcs.5.0.0.nupkg;
         $(TarballPrebuiltPackageDir)system.text.encodings.web.4.7.1.nupkg;
         $(TarballPrebuiltPackageDir)system.text.encodings.web.5.0.0-rc.1.20451.14.nupkg;
         $(TarballPrebuiltPackageDir)system.text.json.4.7.2.nupkg;
         $(TarballPrebuiltPackageDir)system.text.json.5.0.0-rc.1.20451.14.nupkg;
+        $(TarballPrebuiltPackageDir)system.text.json.5.0.0.nupkg;
         $(TarballPrebuiltPackageDir)system.valuetuple.4.3.0.nupkg;
         $(TarballPrebuiltPackageDir)system.xml.xpath.xmldocument.4.0.1.nupkg;
         $(TarballPrebuiltPackageDir)vswhere.2.6.7.nupkg;

eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets

@@ -0,0 +1,92 @@
+<Project>
+
+  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceTextInFile" />
+
+  <!--
+    Find text-only nupkgs and decompose them into text files to include in the tarball as source
+    code. This simplifies maintenance by removing the need to put a new version of these files in
+    dotnet/source-build-reference-packages for potentially every single servicing release.
+
+    'dotnet new' template packages are a prominent example of text-only packages that change often.
+  -->
+  <Target Name="AddAllUnpackedTextOnlyPackageSource"
+          DependsOnTargets="GetTarballDirProps"
+          BeforeTargets="AddTarballPackages">
+    <PropertyGroup>
+      <TarballUnpackedTextOnlyPackageSourceDir>$([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'packages', 'text-only'))</TarballUnpackedTextOnlyPackageSourceDir>
+    </PropertyGroup>
+
+    <!--
+      Find .nupkg files in package cache in order to locate the extraction directory, then filter
+      which files are copied to the tarball to exclude the nupkg itself and unnecessary metadata.
+    -->
+    <ItemGroup>
+      <TextOnlyPrebuiltNupkgFile Include="
+        $(ProjectDir)packages\restored\**\microsoft.dotnet.web.itemtemplates.5.0.*.nupkg;
+        $(ProjectDir)packages\restored\**\microsoft.dotnet.web.projecttemplates.5.0.5.0.*.nupkg;
+        $(ProjectDir)packages\restored\**\microsoft.dotnet.web.spa.projecttemplates.5.0.5.0.*.nupkg;
+        " />
+
+      <!-- Make prebuilt nupkg copy logic ignore these text-only nupkgs. -->
+      <RefOnlyPackageFilename Include="@(TextOnlyPrebuiltNupkgFile->'%(Filename)')" />
+
+      <TextOnlyPrebuiltNupkgProject
+        Include="$(MSBuildProjectFullPath)"
+        Properties="
+          TextOnlyPackageRootDir=%(TextOnlyPrebuiltNupkgFile.RootDir)%(TextOnlyPrebuiltNupkgFile.Directory)" />
+    </ItemGroup>
+
+    <MSBuild
+      Projects="@(TextOnlyPrebuiltNupkgProject)"
+      Targets="AddUnpackedTextOnlyPackageSource">
+      <Output TaskParameter="TargetOutputs" ItemName="TarballCopyFile" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="AddUnpackedTextOnlyPackageSource"
+          DependsOnTargets="GetTarballDirProps"
+          Returns="@(TarballCopyFile)">
+    <PropertyGroup>
+      <PackageVersionDirFile>$([System.IO.Path]::GetDirectoryName('$(TextOnlyPackageRootDir)'))</PackageVersionDirFile>
+      <PackageIdDirFile>$([System.IO.Path]::GetDirectoryName('$(PackageVersionDirFile)'))</PackageIdDirFile>
+      <PackageVersion>$([System.IO.Path]::GetFileName('$(PackageVersionDirFile)'))</PackageVersion>
+      <PackageId>$([System.IO.Path]::GetFileName('$(PackageIdDirFile)'))</PackageId>
+
+      <PackageIntermediateDir>$([MSBuild]::NormalizeDirectory('$(TextOnlyPackageBaseDir)', '$(PackageId)', '$(PackageVersion)'))</PackageIntermediateDir>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(PackageIntermediateDir)" />
+
+    <ItemGroup>
+      <NuspecFile Include="$(TextOnlyPackageRootDir)*.nuspec" />
+      <PackableNuspecFile Include="@(NuspecFile->'$(PackageIntermediateDir)%(Filename)%(Extension)')" />
+    </ItemGroup>
+
+    <Error Condition="@(NuspecFile->Count()) != 1" Text="Expected exactly one nuspec file in $(TextOnlyPackageRootDir)" />
+
+    <Copy SourceFiles="@(NuspecFile)" DestinationFiles="@(PackableNuspecFile)" />
+
+    <ReplaceTextInFile
+      InputFile="@(PackableNuspecFile)"
+      OldText="&lt;/package&gt;"
+      NewText="&lt;files&gt;&lt;file src=&quot;.\**\*&quot;/&gt;&lt;/files&gt;&lt;/package&gt;" />
+
+    <ItemGroup>
+      <TextOnlyPackageContentFile
+        Include="
+          $(TextOnlyPackageRootDir)**;
+          $(PackageIntermediateDir)**"
+        Exclude="
+          @(NuspecFile);
+          $(TextOnlyPackageRootDir)**\.nupkg.metadata;
+          $(TextOnlyPackageRootDir)**\.signature.p7s;
+          $(TextOnlyPackageRootDir)**\*.nupkg;
+          $(TextOnlyPackageRootDir)**\*.nupkg.sha512" />
+
+      <TarballCopyFile
+        Include="@(TextOnlyPackageContentFile)"
+        RelativeDestination="packages\text-only\$(PackageId)\$(PackageVersion)\%(RecursiveDir)%(Filename)%(Extension)" />
+    </ItemGroup>
+  </Target>
+
+</Project>