Add PGO GENPROFILE support to coreclr and clrjit

(Porting commit 114b588 to release/1.1.0)

Update the cmake build system to enable support for Profile Guided
Optimization (PGO) on Windows, and enable this feature for two target
binaries (coreclr and clrjit).

With this change, toggle between instrumented and profile-optimized
settings for target binaries by passing pgoinstrument argument to the build.cmd
Assume profile-optimized mode by default. Fall back to regular non-PGO
optimized builds if profile data is not available.

Author: dpodder

CMakeLists.txt

@@ -486,6 +486,11 @@ if(CLR_CROSS_COMPONENTS_BUILD)
   include(crosscomponents.cmake)
 endif(CLR_CROSS_COMPONENTS_BUILD)
 
+#-------------------
+# Enable PGO support
+#-------------------
+include(pgosupport.cmake)
+
 #-----------------------------------------
 # Add Projects
 #     - project which require platform header not clr's

build.cmd

@@ -52,6 +52,8 @@ set __BuildTypeChecked=0
 set __BuildTypeRelease=0
 set __BuildJit32="-DBUILD_JIT32=0"
 
+set __PgoInstrument=0
+
 REM __PassThroughArgs is a set of things that will be passed through to nested calls to build.cmd
 REM when using "all".
 set __PassThroughArgs=
@@ -103,6 +105,7 @@ if /i "%1" == "skiptests"           (set __BuildTests=0&set processedArgs=!proce
 if /i "%1" == "skipbuildpackages"   (set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "usenmakemakefiles"   (set __NMakeMakefiles=1&set __ConfigureOnly=1&set __BuildNative=1&set __BuildNativeCoreLib=0&set __BuildCoreLib=0&set __BuildTests=0&set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "buildjit32"          (set __BuildJit32="-DBUILD_JIT32=1"&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "pgoinstrument"       (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "toolset_dir"         (set __ToolsetDir=%2&set __PassThroughArgs=%__PassThroughArgs% %2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 
 if [!processedArgs!]==[] (
@@ -228,7 +231,8 @@ if %__BuildNative% EQU 1 (
     echo %__MsgPrefix%Regenerating the Visual Studio solution
 
     pushd "%__IntermediatesDir%"
-    call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32%
+    set __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
+    call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32% !__ExtraCmakeArgs!
 	@if defined __echo @echo on
     popd
 :SkipConfigure
@@ -281,7 +285,7 @@ if /i "%__DoCrossArchBuild%"=="1" (
     pushd "%__CrossCompIntermediatesDir%"
     set __CMakeBinDir=%__CrossComponentBinDir%
     set "__CMakeBinDir=!__CMakeBinDir:\=/!"
-    set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%"
+    set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
     call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs!
     @if defined __echo @echo on
     popd
@@ -522,6 +526,7 @@ echo     for the specified platform ^(FreeBSD, Linux, NetBSD, OS X or Windows,
 echo     respectively^).
 echo     add nativemscorlib to go further and build the native image for designated mscorlib.
 echo toolset_dir ^<dir^> : set the toolset directory -- Arm64 use only. Required for Arm64 builds.
+echo pgoinstrument: generate instrumented code for profile guided optimization enabled binaries.
 echo configureonly: skip all builds; only run CMake ^(default: CMake and builds are run^)
 echo skipconfigure: skip CMake ^(default: CMake is run^)
 echo skipmscorlib: skip building System.Private.CoreLib ^(default: System.Private.CoreLib is built^).

build.sh

@@ -35,6 +35,7 @@ usage()
     echo "clangx.y - optional argument to build using clang version x.y."
     echo "cross - optional argument to signify cross compilation,"
     echo "      - will use ROOTFS_DIR environment variable if set."
+    echo "pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
     echo "configureonly - do not perform any builds; just configure the build."
     echo "skipconfigure - skip build configuration."
     echo "skipnative - do not build native components."
@@ -177,8 +178,9 @@ build_coreclr()
 
 		pushd "$__IntermediatesDir"
         # Regenerate the CMake solution
-        echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
-        "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
+        __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
+        echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__ExtraCmakeArgs $__cmakeargs"
+        "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__ExtraCmakeArgs" "$__cmakeargs"
         popd
     fi
 
@@ -459,6 +461,7 @@ __RunArgs=
 __MSBCleanBuildArgs=
 __UseNinja=0
 __VerboseBuild=0
+__PgoInstrument=0
 __ConfigureOnly=0
 __SkipConfigure=0
 __SkipRestore=""
@@ -558,6 +561,10 @@ while :; do
             __UseNinja=1
             ;;
 
+        pgoinstrument)
+            __PgoInstrument=1
+            ;;
+
         configureonly)
             __ConfigureOnly=1
             __SkipMSCorLib=1

functions.cmake

@@ -179,6 +179,11 @@ function(install_clr targetName)
     else()  
         install(FILES ${strip_destination_file} DESTINATION .)  
     endif()  
+    if(CLR_CMAKE_PGO_INSTRUMENT)
+        if(WIN32)
+            install(FILES ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${targetName}.pgd DESTINATION PGD OPTIONAL)
+        endif()
+    endif()
   endif()  
 endfunction()
 

pgosupport.cmake

@@ -0,0 +1,68 @@
+function(clr_pgo_unknown_arch)
+    if (WIN32)
+        message(FATAL_ERROR "Only AMD64, ARM and I386 are supported for PGO")
+    else()
+        message(FATAL_ERROR "PGO not currently supported on the current platform")
+    endif()
+endfunction(clr_pgo_unknown_arch)
+
+# Adds Profile Guided Optimization (PGO) flags to the current target
+function(add_pgo TargetName)
+    if(WIN32)
+        set(ProfileFileName "${TargetName}.pgd")
+    endif(WIN32)
+
+    file(TO_NATIVE_PATH
+        "${CLR_CMAKE_PACKAGES_DIR}/Microsoft.DotNet.OptimizationData.Coreclr/${CLR_CMAKE_TARGET_OS}.${CLR_CMAKE_TARGET_ARCH}/${ProfileFileName}"
+        ProfilePath
+    )
+
+    # Enable PGO only for optimized configs
+    set(ConfigTypeList RELEASE RELWITHDEBINFO)
+
+    foreach(ConfigType IN LISTS ConfigTypeList)
+        set(LinkFlagsProperty "LINK_FLAGS_${ConfigType}")
+        if(CLR_CMAKE_PGO_INSTRUMENT)
+            if(WIN32)
+                set_property(TARGET ${TargetName} APPEND_STRING PROPERTY ${LinkFlagsProperty} "/LTCG /GENPROFILE")
+            endif(WIN32)
+        else(CLR_CMAKE_PGO_INSTRUMENT)
+            # If we don't have profile data availble, gracefully fall back to a non-PGO opt build
+            if(EXISTS ${ProfilePath})
+                if(WIN32)
+                    set_property(TARGET ${TargetName} APPEND_STRING PROPERTY ${LinkFlagsProperty} "/LTCG /USEPROFILE:PGD=${ProfilePath}")
+                endif(WIN32)
+            endif(EXISTS ${ProfilePath})
+        endif(CLR_CMAKE_PGO_INSTRUMENT)
+    endforeach(ConfigType)
+endfunction(add_pgo)
+
+if(WIN32)
+  if(CLR_CMAKE_PGO_INSTRUMENT)
+    # Instrumented PGO binaries on Windows introduce an additional runtime dependency, pgort<ver>.dll.
+    # Make sure we copy it next to the installed product to make it easier to redistribute the package.
+
+    string(SUBSTRING ${CMAKE_VS_PLATFORM_TOOLSET} 1 -1 VS_PLATFORM_VERSION_NUMBER)
+    set(PGORT_FILENAME "pgort${VS_PLATFORM_VERSION_NUMBER}.dll")
+
+    get_filename_component(PATH_CXX_ROOTDIR ${CMAKE_CXX_COMPILER} DIRECTORY)
+
+    if(CLR_CMAKE_PLATFORM_ARCH_I386)
+      set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/${PGORT_FILENAME}")
+    elseif(CLR_CMAKE_PLATFORM_ARCH_AMD64)
+      set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/../amd64/${PGORT_FILENAME}")
+    elseif(CLR_CMAKE_PLATFORM_ARCH_ARM)
+      set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/../arm/${PGORT_FILENAME}")
+    else()
+      clr_pgo_unknown_arch()
+    endif()
+
+    if (EXISTS ${PATH_VS_PGORT_DLL})
+      message(STATUS "Found PGO runtime: ${PATH_VS_PGORT_DLL}")
+      install(PROGRAMS ${PATH_VS_PGORT_DLL} DESTINATION .)
+    else()
+      message(FATAL_ERROR "file not found: ${PATH_VS_PGORT_DLL}")
+    endif()
+
+  endif(CLR_CMAKE_PGO_INSTRUMENT)
+endif(WIN32)

src/dlls/mscoree/coreclr/CMakeLists.txt

@@ -174,3 +174,6 @@ endif(WIN32)
 
 # add the install targets
 install_clr(coreclr)
+
+# Enable profile guided optimization
+add_pgo(coreclr)

src/jit/standalone/CMakeLists.txt

@@ -53,3 +53,6 @@ target_link_libraries(${JIT_BASE_NAME}
 
 # add the install targets
 install_clr(${JIT_BASE_NAME})
+
+# Enable profile guided optimization
+add_pgo(${JIT_BASE_NAME})