Merge pull request #3043 from gkhanna79/FixNugetPkg

 Generate Microsoft.NETCore.Runtime.CoreCLR nuget packages

Author: gkhanna79

.gitignore

@@ -144,6 +144,7 @@ publish/
 # NuGet Packages
 *.nupkg
 **/packages/*
+project.lock.json
 
 # Windows Azure Build Output
 csx/

build.cmd

@@ -449,6 +449,31 @@ if errorlevel 1 (
     exit /b 1
 )
 
+:GenerateNuget
+if /i "%__BuildArch%" =="arm64" goto :SkipNuget
+
+set "__BuildLog=%__LogsDir%\Nuget_%__BuildOS%__%__BuildArch%__%__BuildType%.log"
+set "__BuildWrn=%__LogsDir%\Nuget_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn"
+set "__BuildErr=%__LogsDir%\Nuget_%__BuildOS%__%__BuildArch%__%__BuildType%.err"
+set __msbuildLogArgs=^
+/fileloggerparameters:Verbosity=normal;LogFile="%__BuildLog%" ^
+/fileloggerparameters1:WarningsOnly;LogFile="%__BuildWrn%" ^
+/fileloggerparameters2:ErrorsOnly;LogFile="%__BuildErr%" ^
+/consoleloggerparameters:Summary ^
+/verbosity:minimal
+
+set __msbuildArgs="%__ProjectFilesDir%\src\.nuget\Microsoft.NETCore.Runtime.CoreClr\Microsoft.NETCore.Runtime.CoreCLR.builds" /p:Platform=%__BuildArch%
+%_msbuildexe% %__msbuildArgs% %__msbuildLogArgs%
+if errorlevel 1 (
+    echo %__MsgPrefix%Error: Nuget package generation failed build failed. Refer to the build log files for details:
+    echo     %__BuildLog%
+    echo     %__BuildWrn%
+    echo     %__BuildErr%
+    exit /b 1
+)
+
+:SkipNuget
+
 :SkipCrossGenBuild
 
 REM endlocal to rid us of environment changes from vsdevenv.bat

build.proj

@@ -15,4 +15,9 @@
   <Target Name="Clean">
     <Delete Files="$(BinDir)mscorlib.*" />
   </Target>
+
+  <Target Name="RestoreNETCorePlatforms" AfterTargets="Build">
+    <Exec Command="$(DnuRestoreCommand) $(SourceDir).nuget/init/project.json --source https://www.myget.org/F/dotnet-core/" />
+  </Target>
+  
 </Project>

build.sh

@@ -23,6 +23,24 @@ usage()
     exit 1
 }
 
+initDistroName()
+{
+    if [ "$__BuildOS" == "Linux" ]; then
+        # Detect Distro
+        if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
+            export __DistroName=ubuntu
+        elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
+            export __DistroName=rhel
+        elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then
+            export __DistroName=rhel
+        elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then
+            export __DistroName=debian
+        else
+            export __DistroName=""
+        fi
+    fi
+}
+
 setup_dirs()
 {
     echo Setting up directories for build
@@ -162,50 +180,49 @@ build_coreclr()
     fi
 }
 
-build_mscorlib()
+restoreBuildTools()
 {
-    hash mono 2> /dev/null || { echo >&2 "Skipping mscorlib.dll build since Mono is not installed."; return; }
-
-    if [ $__SkipMSCorLib == 1 ]; then
-        echo "Skipping mscorlib.dll build."
-        return
+    echo "Restoring BuildTools..."
+    $__ProjectRoot/init-tools.sh
+    if [ $? -ne 0 ]; then
+        echo "Failed to restore BuildTools."
+        exit 1
     fi
+}
 
-    # Temporary hack to make dnu restore more reliable. This is specifically for dnu beta 5 since this issue should already be addressed in later versions of dnu.
-    export MONO_THREADS_PER_CPU=2000
+isMSBuildOnNETCoreSupported()
+{
+    __isMSBuildOnNETCoreSupported=0
 
-    echo "Commencing build of mscorlib components for $__BuildOS.$__BuildArch.$__BuildType"
+    if [ "$__BuildOS" == "Linux" ]; then
+        if [ "$__DistroName" == "ubuntu" ]; then
+            __isMSBuildOnNETCoreSupported=1
+        fi
+    elif [ "$__BuildOS" == "OSX" ]; then
+        __isMSBuildOnNETCoreSupported=1
+    fi 
+}
 
-    # Pull NuGet.exe down if we don't have it already
-    if [ ! -e "$__NuGetPath" ]; then
-        hash curl 2>/dev/null || hash wget 2>/dev/null || { echo >&2 echo "cURL or wget is required to build mscorlib." ; exit 1; }
+build_mscorlib()
+{
 
-        echo "Restoring NuGet.exe..."
+    if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
+        echo "Mscorlib.dll build unsupported."
+        return
+    fi
 
-        # curl has HTTPS CA trust-issues less often than wget, so lets try that first.
-        which curl > /dev/null 2> /dev/null
-        if [ $? -ne 0 ]; then
-           mkdir -p $__PackagesDir
-           wget -q -O $__NuGetPath https://api.nuget.org/downloads/nuget.exe
-        else
-           curl -sSL --create-dirs -o $__NuGetPath https://api.nuget.org/downloads/nuget.exe
-        fi
+    # CI_TODO: Until we switch CI to stop building mscorlib for platforms supported by isMSBuildOnNETCoreSupported function,
+    # we should ignore skipping building mscorlib.
+    # if [ $__SkipMSCorLib == 1 ]; then
+    #    echo "Skipping building mscorlib."
+    #    return
+    # fi
 
-        if [ $? -ne 0 ]; then
-            echo "Failed to restore NuGet.exe."
-            exit 1
-        fi
-    fi
+    # Restore buildTools
 
-    # Grab the MSBuild package if we don't have it already
-    if [ ! -e "$__MSBuildPath" ]; then
-        echo "Restoring MSBuild..."
-        $__ProjectRoot/init-tools.sh
-        if [ $? -ne 0 ]; then
-            echo "Failed to restore MSBuild."
-            exit 1
-        fi
-    fi
+    restoreBuildTools
+
+    echo "Commencing build of mscorlib components for $__BuildOS.$__BuildArch.$__BuildType"
 
     # Invoke MSBuild
     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/build.proj" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/MSCorLib_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false
@@ -225,6 +242,38 @@ build_mscorlib()
     fi
 }
 
+generate_NugetPackages()
+{
+    # We can only generate nuget package if we also support building mscorlib as part of this build.
+    if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
+        echo "Microsoft.NETCore.Runtime.CoreCLR nuget package generation unsupported."
+        return
+    fi
+
+    # Since we can build mscorlib for this OS, did we build the native components as well?
+    if [ $__SkipCoreCLR == 1 ]; then
+        echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since native components were not built."
+        return
+    fi
+
+    # CI_TODO: Until we switch CI to stop building mscorlib for platforms supported by isMSBuildOnNETCoreSupported function,
+    # we should ignore skipping building mscorlib.
+    # if [ $__SkipMSCorLib == 1 ]; then
+    #    echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since mscorlib was not built."
+    #    return
+    # fi
+
+    echo "Generating nuget packages for "$__BuildOS
+
+    # Invoke MSBuild
+    $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Runtime.CoreCLR/Microsoft.NETCore.Runtime.CoreCLR.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false
+
+    if [ $? -ne 0 ]; then
+        echo "Failed to generate Nuget packages."
+        exit 1
+    fi
+}
+
 echo "Commencing CoreCLR Repo build"
 
 # Argument types supported by this script:
@@ -240,7 +289,7 @@ __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # Use uname to determine what the CPU is.
 CPUName=$(uname -p)
 # Some Linux platforms report unknown for platform, but the arch for machine.
-if [ $CPUName = "unknown" ]; then
+if [ $CPUName == "unknown" ]; then
     CPUName=$(uname -m)
 fi
 
@@ -327,6 +376,7 @@ __ClangMajorVersion=3
 __ClangMinorVersion=5
 __MSBuildPath=$__ProjectRoot/Tools/MSBuild.exe
 __NuGetPath="$__PackagesDir/NuGet.exe"
+__DistroName=""
 
 for i in "$@"
 do
@@ -443,13 +493,30 @@ if [[ $__ConfigureOnly == 1 && $__SkipConfigure == 1 ]]; then
     exit 1
 fi
 
+# init the distro name
+initDistroName
+
 # Set the remaining variables based upon the determined build configuration
 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 __PackagesBinDir="$__BinDir/.nuget"
 __ToolsDir="$__RootBinDir/tools"
 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
+__isMSBuildOnNETCoreSupported=0
+
+# Init if MSBuild for .NET Core is supported for this platform
+isMSBuildOnNETCoreSupported
+
+# CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
+# This is needed by CLI to function.
+if [ -z "$HOME" ]; then
+    if [ ! -d "$__ProjectDir/temp_home" ]; then
+        mkdir temp_home
+    fi
+    export HOME=$__ProjectDir/temp_home
+    echo "HOME not defined; setting it to $HOME"
+fi
 
 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 # This is where all built CoreClr libraries will copied to.
@@ -489,6 +556,11 @@ build_coreclr
 
 build_mscorlib
 
+# Generate nuget packages
+
+generate_NugetPackages
+
+
 # Build complete
 
 echo "Repo successfully built."

dir.props

@@ -59,7 +59,7 @@
          fails to write packages to it if the path contains the forward slash.
     -->
     <PackagesBinDir>$(__PackagesBinDir)</PackagesBinDir>
-    <PackagesBinDir Condition="'$(__PackagesBinDir)'==''">$(BinDir).nuget</PackagesBinDir>
+    <PackagesBinDir Condition="'$(__PackagesBinDir)'==''">$(BinDir).nuget\</PackagesBinDir>
 
     <ToolsDir Condition="'$(ToolsDir)'==''">$(ProjectDir)Tools/</ToolsDir> 
     <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(ToolsDir)dotnetcli/bin/</DotnetCliPath> 
@@ -103,17 +103,11 @@
 
   <!-- Setup Nuget properties -->
   <ItemGroup>
-    <NuSpecSrcs Include="$(SourceDir)\.nuget\Microsoft.DotNet.CoreCLR.nuspec" />
     <NuSpecSrcs Include="$(SourceDir)\.nuget\toolchain.win7-x64.Microsoft.DotNet.RyuJit.nuspec" />
-    <NuSpecSrcs Condition="'$(Configuration)'=='Release'" Include="$(SourceDir)\.nuget\Microsoft.DotNet.CoreCLR.Development.nuspec" />
-    <NuSpecSrcs Condition="'$(Configuration)'=='Debug'" Include="$(SourceDir)\.nuget\Microsoft.DotNet.CoreCLR.Debug.Development.nuspec" />
   </ItemGroup>
   <ItemGroup>
     <!-- Backslash appended, see note in dir.props about the PackagesBinDir property -->
-    <NuSpecs Include="$(PackagesBinDir)\Microsoft.DotNet.CoreCLR.nuspec" />
     <NuSpecs Include="$(PackagesBinDir)\toolchain.win7-x64.Microsoft.DotNet.RyuJit.nuspec" />
-    <NuSpecs Condition="'$(Configuration)'=='Release'" Include="$(PackagesBinDir)\Microsoft.DotNet.CoreCLR.Development.nuspec" />
-    <NuSpecs Condition="'$(Configuration)'=='Debug'" Include="$(PackagesBinDir)\Microsoft.DotNet.CoreCLR.Debug.Development.nuspec" />
   </ItemGroup>
 
   <!--
@@ -132,5 +126,31 @@
     <TargetsWindows Condition="'$(BuildOS)' == 'Windows_NT'">true</TargetsWindows>
 
     <TargetsUnix Condition="'$(TargetsFreeBSD)' == 'true' or '$(TargetsLinux)' == 'true' or '$(TargetsOSX)' == 'true'">true</TargetsUnix>
+
+    <!-- We are only tracking Linux Distributions for Nuget RID mapping -->
+    <DistroName Condition="'$(TargetsLinux)' == 'true'">$(__DistroName)</DistroName>
+
   </PropertyGroup>
+
+  <!-- Packaging properties -->
+  <PropertyGroup>
+    <PreReleaseLabel>rc3</PreReleaseLabel>
+    <PackageDescriptionFile>$(SourceDir).nuget/descriptions.json</PackageDescriptionFile>
+    <PackageLicenseFile>$(SourceDir).nuget/dotnet_library_license.txt</PackageLicenseFile>
+    <PackageThirdPartyNoticesFile>$(SourceDir).nuget/ThirdPartyNotices.txt</PackageThirdPartyNoticesFile>
+    
+    <!-- This should be kept in sync with package details in src/.nuget/init/project.json -->
+    <RuntimeIdGraphDefinitionFile>$(PackagesDir)/Microsoft.NETCore.Platforms/1.0.1-rc2-23712/runtime.json</RuntimeIdGraphDefinitionFile>
+
+    <!-- On Windows, MSbuild still runs against Desktop FX while it runs on .NET Core on non-Windows. this requires
+         pulling in different packaging dependencies.
+     -->
+    <PackagingTaskDir Condition="'$(TargetsWindows)' == 'true'">$(ToolsDir)net45/</PackagingTaskDir>
+    <BuildNumberMajor Condition="'$(BuildNumberMajor)' == ''">00001</BuildNumberMajor>
+    <!-- defined in buildtools packaging.targets, but we need this before targets are imported -->
+    <PackagePlatform Condition="'$(PackagePlatform)' == ''">$(BuildArch)</PackagePlatform>
+    <PackagePlatform Condition="'$(PackagePlatform)' == 'amd64'">x64</PackagePlatform>
+    <PackageOutputPath>$(PackagesBinDir)/pkg/</PackageOutputPath>
+  </PropertyGroup>
+
 </Project>

init-tools.sh

@@ -1,6 +1,35 @@
 #!/usr/bin/env bash
 
+initDistroName()
+{
+    if [ "$1" == "Linux" ]; then
+        # Detect Distro
+        if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
+            export __DistroName=ubuntu
+        elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
+            export __DistroName=centos
+        elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then
+            export __DistroName=rhel
+        elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then
+            export __DistroName=debian
+        else
+            export __DistroName=""
+        fi
+    fi
+}
+
 __scriptpath=$(cd "$(dirname "$0")"; pwd -P)
+
+# CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
+# This is needed by CLI to function.
+if [ -z "$HOME" ]; then
+    if [ ! -d "$__scriptpath/temp_home" ]; then
+        mkdir temp_home
+    fi
+    export HOME=$__scriptpath/temp_home
+    echo "HOME not defined; setting it to $HOME"
+fi
+
 __PACKAGES_DIR=$__scriptpath/packages
 __TOOLRUNTIME_DIR=$__scriptpath/Tools
 __DOTNET_PATH=$__TOOLRUNTIME_DIR/dotnetcli
@@ -12,6 +41,7 @@ __BUILD_TOOLS_PATH=$__PACKAGES_DIR/Microsoft.DotNet.BuildTools/$__BUILD_TOOLS_PA
 __PROJECT_JSON_PATH=$__TOOLRUNTIME_DIR/$__BUILD_TOOLS_PACKAGE_VERSION
 __PROJECT_JSON_FILE=$__PROJECT_JSON_PATH/project.json
 __PROJECT_JSON_CONTENTS="{ \"dependencies\": { \"Microsoft.DotNet.BuildTools\": \"$__BUILD_TOOLS_PACKAGE_VERSION\" }, \"frameworks\": { \"dnxcore50\": { } } }"
+__DistroName=""
 
 OSName=$(uname -s)
 case $OSName in
@@ -32,6 +62,16 @@ case $OSName in
         ;;
 esac
 
+# Initialize Linux Distribution name and .NET CLI package name.
+
+initDistroName $OS
+if [ "$__DistroName" == "centos" ]; then
+    __DOTNET_PKG=dotnet-centos-x64
+fi
+
+__CLIDownloadURL=https://dotnetcli.blob.core.windows.net/dotnet/dev/Binaries/${__DOTNET_TOOLS_VERSION}/${__DOTNET_PKG}.${__DOTNET_TOOLS_VERSION}.tar.gz
+echo ".NET CLI will be downloaded from $__CLIDownloadURL"
+
 if [ ! -e $__PROJECT_JSON_FILE ]; then
  if [ -e $__TOOLRUNTIME_DIR ]; then rm -rf -- $__TOOLRUNTIME_DIR; fi
 
@@ -40,9 +80,9 @@ if [ ! -e $__PROJECT_JSON_FILE ]; then
     which curl > /dev/null 2> /dev/null
     if [ $? -ne 0 ]; then
       mkdir -p "$__DOTNET_PATH"
-      wget -q -O $__DOTNET_PATH/dotnet.tar https://dotnetcli.blob.core.windows.net/dotnet/dev/Binaries/${__DOTNET_TOOLS_VERSION}/${__DOTNET_PKG}.${__DOTNET_TOOLS_VERSION}.tar.gz
+      wget -q -O $__DOTNET_PATH/dotnet.tar $__CLIDownloadURL
     else
-      curl -sSL --create-dirs -o $__DOTNET_PATH/dotnet.tar https://dotnetcli.blob.core.windows.net/dotnet/dev/Binaries/${__DOTNET_TOOLS_VERSION}/${__DOTNET_PKG}.${__DOTNET_TOOLS_VERSION}.tar.gz
+      curl -sSL --create-dirs -o $__DOTNET_PATH/dotnet.tar $__CLIDownloadURL
     fi
     cd $__DOTNET_PATH
     tar -xf $__DOTNET_PATH/dotnet.tar