adding crossgen to build

Author: brettsam

appveyor.yml

@@ -25,7 +25,7 @@ install:
     Install-Product node 10.0.0 x86      
     Invoke-WebRequest -Uri 'https://dot.net/v1/dotnet-install.ps1' -UseBasicParsing -OutFile "$env:temp\dotnet-install.ps1"      
     & $env:temp\dotnet-install.ps1 -Architecture x64 -Version '3.0.100-preview9-014004' -InstallDir "$env:ProgramFiles\dotnet"
-
+    
 build_script:
 - ps: |
     $hasTag = Test-Path env:APPVEYOR_REPO_TAG_NAME

build.ps1

@@ -24,136 +24,52 @@ function ZipContent([string] $sourceDirectory, [string] $target)
   [IO.Compression.ZipFile]::CreateFromDirectory($sourceDirectory, $target)
 }
 
-function CrossGen([string] $runtime, [string] $publishTarget, [string] $privateSiteExtensionPath)
-{
-    Write-Host "publishTarget: " $publishTarget
-    Write-Host "privateSiteExtensionPath: " $privateSiteExtensionPath
-
-    $selfContained = Join-Path $publishTarget "self-contained"
-    $crossGen = "$publishTarget\download\crossgen\crossgen.exe"
-    $symbolsPath = Join-Path $publishTarget "Symbols"
-    new-item -itemtype directory -path $symbolsPath
-
-    #https://dotnet.myget.org/feed/dotnet-core/package/nuget/runtime.win-x86.Microsoft.NETCore.Jit
-    DownloadNupkg "https://dotnet.myget.org/F/dotnet-core/api/v2/package/runtime.$runtime.Microsoft.NETCore.Jit/2.2.0-servicing-26820-03" @("runtimes\$runtime\native\clrjit.dll")  @("$publishTarget\download\clrjit")
-    #https://dotnet.myget.org/feed/dotnet-core/package/nuget/runtime.win-x86.Microsoft.NETCore.Runtime.CoreCLR
-    DownloadNupkg "https://dotnet.myget.org/F/dotnet-core/api/v2/package/runtime.$runtime.Microsoft.NETCore.Runtime.CoreCLR/2.2.0-servicing-26820-03"  @("tools\crossgen.exe")  @("$publishTarget\download\crossgen")
-
-    # Publish self-contained app with all required dlls for crossgen
-    dotnet publish .\src\WebJobs.Script.WebHost\WebJobs.Script.WebHost.csproj -r $runtime -o "$selfContained" -v q /p:BuildNumber=$buildNumber    
-
-    # Modify web.config for inproc
-    dotnet tool install -g dotnet-xdt --version 2.1.0 2> $null
-    dotnet-xdt -s "$privateSiteExtensionPath\web.config" -t "$privateSiteExtensionPath\web.InProcess.$runtime.xdt" -o "$privateSiteExtensionPath\web.config"
-
-    $successfullDlls =@()
-    $failedDlls = @()
-    
-    Get-ChildItem $privateSiteExtensionPath -Filter *.dll | 
-    Foreach-Object {       
-        $prm = "/JITPath", "$publishTarget\download\clrjit\clrjit.dll", "/Platform_Assemblies_Paths", "$selfContained", "/nologo", "/in", $_.FullName
-        # output for Microsoft.Azure.WebJobs.Script.WebHost.dll is Microsoft.Azure.WebJobs.Script.WebHost.exe.dll by default
-        if ($_.FullName -like "*Microsoft.Azure.WebJobs.Script.WebHost.dll") {
-            $prm += "/out"        
-            $prm += Join-Path $privateSiteExtensionPath "Microsoft.Azure.WebJobs.Script.WebHost.ni.dll"
-        }
-
-        & $crossGen $prm >> $buildOutput\crossgenout.$runtime.txt  2>&1
-
-        $niDll = Join-Path $privateSiteExtensionPath $([io.path]::GetFileNameWithoutExtension($_.FullName) + ".ni.dll")
-        if ([System.IO.File]::Exists($niDll)) {
-            Remove-Item $_.FullName
-            Rename-Item -Path $niDll -NewName $_.FullName
-
-            & $crossGen "/Platform_Assemblies_Paths", "$selfContained", "/CreatePDB", "$symbolsPath", $_.FullName >> $buildOutput\crossgenout-PDBs.$runtime.txt 2>&1
-
-            $successfullDlls+=[io.path]::GetFileName($_.FullName)
-        } else {
-            $failedDlls+=[io.path]::GetFileName($_.FullName)
-        }                
-    }
-    
-    # print results of crossgen process
-    $successfullDllsCount = $successfullDlls.length
-    $failedDllsCount = $failedDlls.length
-    Write-Host "CrossGen($runtime) results: Successfull: $successfullDllsCount, Failed: $failedDllsCount"
-    if ($failedDlls.length -gt 0) {
-        Write-Host "Failed CrossGen dlls:"
-        Write-Host $failedDlls
-    }
-    
-    
-    if ($runtime -eq "win-x86") {
-        Copy-Item -Path $privateSiteExtensionPath\runtimes\win\native\*  -Destination $privateSiteExtensionPath -Force
-    }
-
-    #read-host "Press ENTER to continue..."
-    Remove-Item -Recurse -Force $selfContained -ErrorAction SilentlyContinue
-    Remove-Item -Recurse -Force $publishTarget\download -ErrorAction SilentlyContinue
-}
-
-function AddDiaSymReaderToPath()
-{
-    $infoContent = dotnet --info
-    $sdkBasePath = $infoContent  |
-        Where-Object {$_ -match 'Base Path:'} |
-        ForEach-Object {
-            $_ -replace '\s+Base Path:',''
-        }
-
-    $diaSymPath = Join-Path $sdkBasePath.Trim() "Roslyn\bincore\runtimes\win\native"
-
-    Write-Host "Adding DiaSymReader location to path ($diaSymPath)" -ForegroundColor Yellow
-    $env:Path = "$diaSymPath;$env:Path"
-}
-
-function DownloadNupkg([string] $nupkgPath, [string[]]$from, [string[]]$to) {
-    $tempFolderName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName())
-    $tempFolder = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath() ,$tempFolderName)
-    [System.IO.Directory]::CreateDirectory($tempFolder)
-    Remove-Item $tempFolder
-    $tempFile = [System.IO.Path]::GetTempFileName() |
-        Rename-Item -NewName { $_ -replace 'tmp$', 'zip' } -PassThru
-
-    Write-Host "Downloading '$nupkgPath' to '$tempFile'"
-    Invoke-WebRequest -Uri $nupkgPath -OutFile $tempFile
-    Write-Host "Extracting from '$tempFile' to '$tempFolder'"
-    Add-Type -AssemblyName System.IO.Compression.FileSystem
-    [System.IO.Compression.ZipFile]::ExtractToDirectory($tempFile, $tempFolder)
-    
-    #copy nupkg files
-    for ($i=0; $i -lt $from.length; $i++) {
-        New-Item $to -Type Directory -Force
-        Copy-Item -Path $("$tempFolder\" + $from[$i]) -Destination $($to[$i] + "\") -Force -Verbose
-    }
-}
-
 function BuildPackages([bool] $isNoRuntime) {
     if($isNoRuntime) {
         BuildOutput ""
         $applicationHost = Get-Content $buildOutput\publish.no-runtime\SiteExtensions\Functions\applicationHost.xdt
         $applicationHost -replace "\\%XDT_BITNESS%","" | Out-File $buildOutput\publish.no-runtime\SiteExtensions\Functions\applicationHost.xdt
 
         CreateZips ".no-runtime"
-    } else {
-        BuildOutput "win-x86"
-        BuildOutput "win-x64"
-
+    } else {        
+        BuildOutput "win-x86" 1
+        BuildOutput "win-x64" 1
+        BuildOutput "win-x86" 0
+        BuildOutput "win-x64" 0
+                
+        # Use self-contained as the site extension until Antares has .NET Core 3 deployed.
+        # Ignore 64bit folder
         New-Item -Itemtype directory -path $buildOutput\publish.runtime\SiteExtensions\Functions
-        Move-Item -Path $buildOutput\publish.win-x86\SiteExtensions\Functions -Destination $buildOutput\publish.runtime\SiteExtensions\Functions\32bit -Force
-        Move-Item -Path $buildOutput\publish.win-x64\SiteExtensions\Functions -Destination $buildOutput\publish.runtime\SiteExtensions\Functions\64bit -Force
-        Move-Item -Path $buildOutput\publish.runtime\SiteExtensions\Functions\32bit\applicationHost.xdt -Destination $buildOutput\publish.runtime\SiteExtensions\Functions -Force
-        Remove-Item -Path $buildOutput\publish.runtime\SiteExtensions\Functions\64bit\applicationHost.xdt
+        Copy-Item -Path $buildOutput\publish.win-x86.self-contained\SiteExtensions\Functions -Destination $buildOutput\publish.runtime\SiteExtensions\Functions\32bit -Recurse -Force
+        # Move-Item -Path $buildOutput\publish.win-x64\SiteExtensions\Functions -Destination $buildOutput\publish.runtime\SiteExtensions\Functions\64bit -Force
+        Copy-Item -Path $buildOutput\publish.runtime\SiteExtensions\Functions\32bit\applicationHost.xdt -Destination $buildOutput\publish.runtime\SiteExtensions\Functions -Force
+
+        # To minimize size skip 64bit folder.
+        # Until site extension is in Antares, use self-contained.
+        New-Item -Itemtype directory -path $buildOutput\publish.self-contained\SiteExtensions\Functions
+        Move-Item -Path $buildOutput\publish.win-x86.self-contained\SiteExtensions\Functions -Destination $buildOutput\publish.self-contained\SiteExtensions\Functions\32bit -Force
+        Copy-Item -Path $buildOutput\publish.runtime\SiteExtensions\Functions\applicationHost.xdt -Destination $buildOutput\publish.self-contained\SiteExtensions\Functions -Force
+        
+        #Remove-Item -Path $buildOutput\publish.runtime\SiteExtensions\Functions\64bit\applicationHost.xdt
 
         CreateZips ".runtime"
     }
 }
 
 
-function BuildOutput([string] $runtime) {
+function BuildOutput([string] $runtime, [bool] $isSelfContained) {
     $runtimeSuffix = ""
-    if (![string]::IsNullOrEmpty($runtime)) {
+    $ridSwitch = ""
+    $hasRuntime = ![string]::IsNullOrEmpty($runtime)
+
+    if ($hasRuntime -and !$isSelfContained) {
+        Write-Host "Building $runtime"
         $runtimeSuffix = ".$runtime"
+    		$ridSwitch = "-r", "$runtime", "--self-contained", "false", "/p:PublishReadyToRun=true"
+    } elseif ($hasRuntime -and $isSelfContained) {
+        Write-Host "Building $runtime self-contained"
+        $runtimeSuffix = ".$runtime.self-contained"        
+		    $ridSwitch = "-r", "$runtime", "--self-contained", "true", "/p:PublishReadyToRun=true", "/p:PublishReadyToRunEmitSymbols=true"
     } else {
         $runtimeSuffix = ".no-runtime"
     }
@@ -162,16 +78,17 @@ function BuildOutput([string] $runtime) {
     $siteExtensionPath = "$publishTarget\SiteExtensions"
     $privateSiteExtensionPath = "$siteExtensionPath\Functions"
 
-    dotnet publish .\src\WebJobs.Script.WebHost\WebJobs.Script.WebHost.csproj -o "$privateSiteExtensionPath" -v q /p:BuildNumber=$buildNumber /p:IsPackable=false -c Release
-
-    # replace IL dlls with crossgen dlls
-    # TODO: Re-add this.
-    #if (![string]::IsNullOrEmpty($runtime)) {
-    #    CrossGen $runtime $publishTarget $privateSiteExtensionPath
-    #}
+    New-Item -Itemtype directory -path $privateSiteExtensionPath
+    
+	  dotnet publish .\src\WebJobs.Script.WebHost\WebJobs.Script.WebHost.csproj $ridSwitch -o "$privateSiteExtensionPath" -v q /p:BuildNumber=$buildNumber /p:IsPackable=false -c Release
+    
+    if ($hasRuntime -and $isSelfContained) {
+        Write-Host "Moving symbols"
+        New-Item -Itemtype directory -path $publishTarget\Symbols
+        Move-Item -Path $privateSiteExtensionPath\*.pdb -Destination $publishTarget\Symbols
+    }
 }
 
-
 function CreateZips([string] $runtimeSuffix) {
     $publishTarget = "$buildOutput\publish$runtimeSuffix"
     $siteExtensionPath = "$publishTarget\SiteExtensions"
@@ -188,8 +105,6 @@ function CreateZips([string] $runtimeSuffix) {
         # Project cleanup (trim some project files - this should be revisited)
         cleanExtension ""
 
-
-
         # Make the zip
         ZipContent $publishTarget "$buildOutput\Functions.Private.$extensionVersion$runtimeSuffix.zip"
 
@@ -198,17 +113,15 @@ function CreateZips([string] $runtimeSuffix) {
         cleanExtension "32bit"
         cleanExtension "64bit"
 
-        # Create private extension for internal usage. To minimize size remove 64bit folder.
+        # Create private extension for internal usage.
         $tempPath = "$buildOutput\win-x32.inproc.temp\SiteExtensions"
 
         # Make a temp location
         New-Item -Itemtype directory -path $tempPath -ErrorAction SilentlyContinue
 
         # Copy all files to temp folder
-        Copy-Item -Path $privateSiteExtensionPath -Destination $tempPath -Recurse
-
-        # Delete x64 folder to reduce size
-        Remove-Item "$tempPath\Functions\64bit" -Recurse
+        $selfContainedPath = "$buildOutput\publish.self-contained\SiteExtensions\Functions"
+        Copy-Item -Path $selfContainedPath -Destination $tempPath -Recurse
 
         # Make the zip
         ZipContent "$buildOutput\win-x32.inproc.temp" "$buildOutput\Functions.Private.$extensionVersion.win-x32.inproc.zip"
@@ -218,16 +131,16 @@ function CreateZips([string] $runtimeSuffix) {
 
     # Zip up symbols for builds with runtime embedded
     if ($runtimeSuffix -eq  "") {
-        ZipContent "$buildOutput\publish.win-x86\Symbols" "$buildOutput\Functions.Symbols.$extensionVersion.win-x86.zip"
-        ZipContent "$buildOutput\publish.win-x64\Symbols" "$buildOutput\Functions.Symbols.$extensionVersion.win-x64.zip"
+        ZipContent "$buildOutput\publish.win-x86.self-contained\Symbols" "$buildOutput\Functions.Symbols.$extensionVersion.win-x86.zip"
+        ZipContent "$buildOutput\publish.win-x64.self-contained\Symbols" "$buildOutput\Functions.Symbols.$extensionVersion.win-x64.zip"
     }
 
     #Build site extension
     Write-Host "privateSiteExtensionPath: " $privateSiteExtensionPath
     Rename-Item "$privateSiteExtensionPath" "$siteExtensionPath\$extensionVersion"
     Copy-Item .\src\WebJobs.Script.WebHost\extension.xml "$siteExtensionPath"
-    ZipContent $siteExtensionPath "$buildOutput\Functions.$extensionVersion$runtimeSuffix.zip"
 
+    ZipContent $siteExtensionPath "$buildOutput\Functions.$extensionVersion$runtimeSuffix.zip"
 }
 
 function cleanExtension([string] $bitness) {
@@ -254,12 +167,12 @@ $projects =
   "WebJobs.Script",
   "WebJobs.Script.WebHost",
   "WebJobs.Script.Grpc"
-  
+
 foreach ($project in $projects)
 {
 
   $cmd = "pack", "src\$project\$project.csproj", "-o", "$buildOutput", "--no-build" , "-p:PackageVersion=$extensionVersion"
-  
+
   & dotnet $cmd  
 }
 
@@ -274,9 +187,7 @@ $bypassPackaging = $env:APPVEYOR_PULL_REQUEST_NUMBER -and -not $env:APPVEYOR_PUL
 if ($bypassPackaging){
     Write-Host "Bypassing artifact packaging and CrossGen for pull request." -ForegroundColor Yellow
 } else {
-    AddDiaSymReaderToPath
-
-    # build no-runntime extension
+    # build no-runtime extension
     BuildPackages 1
 
     #build win-x86 and win-x64 extension

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

@@ -6,8 +6,28 @@
     <RootNamespace>Microsoft.Azure.WebJobs.Script.WebHost</RootNamespace>
     <PackageId>Microsoft.Azure.WebJobs.Script.WebHost</PackageId>
     <IsPackable Condition="'$(IsPackable)' != ''">true</IsPackable>
-    <AspNetCoreHostingModel>InProcess</AspNetCoreHostingModel>
+    <AspNetCoreHostingModel>InProcess</AspNetCoreHostingModel>    
   </PropertyGroup>
+  <ItemGroup>    
+    <PublishReadyToRunExclude Include="Microsoft.PackageManagement.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PackageManagement.ArchiverProviders.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PackageManagement.CoreProviders.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PackageManagement.MetaProvider.PowerShell.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PackageManagement.NuGetProvider.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.Management.Infrastructure.dll" />    
+    <PublishReadyToRunExclude Include="Microsoft.Management.Infrastructure.Native.dll" />    
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.ConsoleHost.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.PackageManagement.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.Commands.Diagnostics.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.Commands.Utility.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.MarkdownRender.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.Commands.Management.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.PowerShell.Security.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.CodeAnalysis.CSharp.dll" />
+    <PublishReadyToRunExclude Include="System.Management.Automation.dll" />
+    <PublishReadyToRunExclude Include="Microsoft.WSMan.Management.dll" />
+    <PublishReadyToRunExclude Include="System.DirectoryServices.AccountManagement.dll" />
+  </ItemGroup>
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <LangVersion>7.3</LangVersion>

test/WebJobs.Script.Scaling.Tests/PingWorkerTests.cs

@@ -90,7 +90,7 @@ public async Task PingWorkerIntervalTests()
 
                 // assert
                 scaleManager.VerifyAll();
-                scaleManager.MockScaleHandler.Verify(h => h.PingWorker(activityId, workerInfo), Times.Between(2, loop - 1, Range.Inclusive));
+                scaleManager.MockScaleHandler.Verify(h => h.PingWorker(activityId, workerInfo), Times.Between(2, loop - 1, Moq.Range.Inclusive));
                 scaleManager.MockWorkerTable.Verify(t => t.AddOrUpdate(workerInfo), Times.Exactly(loop));
                 scaleManager.MockScaleTracer.Verify(t => t.TraceUpdateWorker(activityId, workerInfo, It.Is<string>(s => s.Contains("updated"))), Times.Exactly(loop));
             }

test/WebJobs.Script.Scaling.Tests/WebJobs.Script.Scaling.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.2</TargetFramework>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Script.Scaling.Tests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Script.Scaling.Tests</RootNamespace>

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_PowerShell.cs

@@ -41,7 +41,7 @@ private async Task InvokeHttpTrigger(string functionName)
             request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/plain"));
 
             var response = await _fixture.Host.HttpClient.SendAsync(request);
-            Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+            Assert.True(response.StatusCode == HttpStatusCode.OK, _fixture.Host.GetLog());
         }
 
         public class TestFixture : EndToEndTestFixture