Consume optimization data during release builds

Restore optimization data from nuget prior to the native build. Update
PGO implementation to consume the data from the restored package.

Author: Daniel Podder

build.cmd

@@ -189,6 +189,15 @@ call                                 "%__VSToolsRoot%\VsDevCmd.bat"
 
 @call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj -generateHeaderWindows -NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h" %__RunArgs% %__UnprocessedBuildArgs% 
 
+REM =========================================================================================
+REM ===
+REM === Restore optimization profile data
+REM ===
+REM =========================================================================================
+
+echo %__MsgPrefix%Restoring the OptimizationData Package
+@call %__ProjectDir%\run.cmd sync -optdata
+
 REM =========================================================================================
 REM ===
 REM === Build the CLR VM

build.proj

@@ -24,6 +24,10 @@
     <Delete Files="$(BinDir)System.Private.CoreLib.*" />
   </Target>
 
+  <Target Name="RestoreOptData">
+    <Exec Command="$(DnuRestoreCommand) &quot;$(SourceDir).nuget/optdata/project.json&quot; --source https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json" />
+  </Target>
+
   <Target Name="RestoreNETCorePlatforms" AfterTargets="Build" Condition="'$(RestoreDuringBuild)'=='true'">
     <Exec Command="$(DnuRestoreCommand) &quot;$(SourceDir).nuget/init/project.json&quot; --source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
   </Target>

build.sh

@@ -176,6 +176,9 @@ build_coreclr()
             echo $__versionSourceLine > $__versionSourceFile
         fi
 
+        echo "Restoring the OptimizationData package"
+        "$__ProjectRoot/run.sh" sync -optdata
+
 		pushd "$__IntermediatesDir"
         # Regenerate the CMake solution
         __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"

config.json

@@ -48,6 +48,12 @@
       "values": [],
       "defaultValue": ""
     },
+    "RestoreOptData": {
+      "description": "MsBuild target that restores optimization profile data.",
+      "valueType": "target",
+      "values": [],
+      "defaultValue": ""
+    },
     "RestoreDuringBuild": {
       "description": "Enables/disables package restore.",
       "valueType": "property",
@@ -362,6 +368,14 @@
             "RestoreNETCorePlatforms": "default"
           }
         },
+        "optdata": {
+          "description": "Restores optimization profile data for the repository.",
+          "settings": {
+            "Project": "./build.proj",
+            "RestoreDuringBuild": true,
+            "RestoreOptData": "default"
+          }
+        },
         "ab": {
           "description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required",
           "settings": {

extract-from-json.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+import argparse
+import json
+import sys
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="""Extracts information from a json file by navigating the JSON object using a
+            sequence of property accessors and returning the JSON subtree, or the raw data, found
+            at that location."""
+    )
+
+    parser.add_argument(
+        '-f', '--file',
+        metavar='<project.json>',
+        help="Path to project.json file to parse",
+        required=True,
+    )
+
+    parser.add_argument(
+        'property',
+        metavar='property_name',
+        help="""Name of property to extract using object notation.
+            Pass multiple values to drill down into nested objects (in order).""",
+        nargs='*',
+    )
+
+    parser.add_argument(
+        '-r', '--raw',
+        help="""Dumps the raw object found at the requested location.
+            If omitted, returns a JSON formatted object instead.""",
+        action='store_true',
+        default=False
+    )
+
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    with open(args.file) as json_file:
+        selected_property = json.load(json_file)
+
+    for prop in args.property:
+        selected_property = selected_property[prop]
+
+    if args.raw:
+        print(selected_property)
+    else:
+        print(json.dumps(selected_property))
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

pgosupport.cmake

@@ -13,7 +13,7 @@ function(add_pgo TargetName)
     endif(WIN32)
 
     file(TO_NATIVE_PATH
-        "${CLR_CMAKE_PACKAGES_DIR}/Microsoft.DotNet.OptimizationData.Coreclr/${CLR_CMAKE_TARGET_OS}.${CLR_CMAKE_TARGET_ARCH}/${ProfileFileName}"
+        "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
         ProfilePath
     )
 
@@ -37,6 +37,18 @@ function(add_pgo TargetName)
     endforeach(ConfigType)
 endfunction(add_pgo)
 
+set(CLR_CMAKE_OPTDATA_PACKAGEID "optimization.PGO.CoreCLR")
+set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR")
+
+# Parse optdata package version from project.json
+file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/extract-from-json.py" ExtractFromJsonScript)
+file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/src/.nuget/optdata/project.json" OptDataProjectJsonPath)
+execute_process(
+  COMMAND python "${ExtractFromJsonScript}" -rf "${OptDataProjectJsonPath}" dependencies "${CLR_CMAKE_OPTDATA_PACKAGEID}"
+  OUTPUT_VARIABLE CLR_CMAKE_OPTDATA_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
 if(WIN32)
   if(CLR_CMAKE_PGO_INSTRUMENT)
     # Instrumented PGO binaries on Windows introduce an additional runtime dependency, pgort<ver>.dll.

src/.nuget/optdata/project.json

@@ -0,0 +1,11 @@
+{
+    "dependencies": {
+        "optimization.PGO.CoreCLR": "1.1.0-release-20161025"
+    },
+    "frameworks": {
+        "netstandard": {}
+    },
+    "runtimes": {
+        "win7-x64": {}
+    }
+}