Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 1602161

Browse files
dpodderdpodder
authored
Merge pull request #7805 from dpodder/pgo-opt
Enable PGO for release/1.1.0 on Windows x64 This ports 114b588 from master into the release branch, and adds some additional changes to enable automated restore and consumption of PGO counts during a release build.
2 parents a4757bc + fa02660 commit 1602161

File tree

11 files changed

+209
-4
lines changed

11 files changed

+209
-4
lines changed

‎CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,11 @@ if(CLR_CROSS_COMPONENTS_BUILD)
486486
include(crosscomponents.cmake)
487487
endif(CLR_CROSS_COMPONENTS_BUILD)
488488

489+
#-------------------
490+
# Enable PGO support
491+
#-------------------
492+
include(pgosupport.cmake)
493+
489494
#-----------------------------------------
490495
# Add Projects
491496
# - project which require platform header not clr's

‎build.cmd‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ set __BuildTypeChecked=0
5252
set __BuildTypeRelease=0
5353
set __BuildJit32="-DBUILD_JIT32=0"
5454

55+
set __PgoInstrument=0
56+
5557
REM __PassThroughArgs is a set of things that will be passed through to nested calls to build.cmd
5658
REM when using "all".
5759
set __PassThroughArgs=
@@ -103,6 +105,7 @@ if /i "%1" == "skiptests" (set __BuildTests=0&set processedArgs=!proce
103105
if /i "%1" == "skipbuildpackages" (set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
104106
if /i "%1" == "usenmakemakefiles" (set __NMakeMakefiles=1&set __ConfigureOnly=1&set __BuildNative=1&set __BuildNativeCoreLib=0&set __BuildCoreLib=0&set __BuildTests=0&set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
105107
if /i "%1" == "buildjit32" (set __BuildJit32="-DBUILD_JIT32=1"&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
108+
if /i "%1" == "pgoinstrument" (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
106109
if /i "%1" == "toolset_dir" (set __ToolsetDir=%2&set __PassThroughArgs=%__PassThroughArgs% %2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
107110

108111
if [!processedArgs!]==[] (
@@ -186,6 +189,15 @@ call "%__VSToolsRoot%\VsDevCmd.bat"
186189

187190
@call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj -generateHeaderWindows -NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h" %__RunArgs% %__UnprocessedBuildArgs%
188191

192+
REM =========================================================================================
193+
REM ===
194+
REM === Restore optimization profile data
195+
REM ===
196+
REM =========================================================================================
197+
198+
echo %__MsgPrefix%Restoring the OptimizationData Package
199+
@call %__ProjectDir%\run.cmd sync -optdata
200+
189201
REM =========================================================================================
190202
REM ===
191203
REM === Build the CLR VM
@@ -228,7 +240,8 @@ if %__BuildNative% EQU 1 (
228240
echo %__MsgPrefix%Regenerating the Visual Studio solution
229241

230242
pushd "%__IntermediatesDir%"
231-
call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32%
243+
set __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
244+
call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32% !__ExtraCmakeArgs!
232245
@if defined __echo @echo on
233246
popd
234247
:SkipConfigure
@@ -281,7 +294,7 @@ if /i "%__DoCrossArchBuild%"=="1" (
281294
pushd "%__CrossCompIntermediatesDir%"
282295
set __CMakeBinDir=%__CrossComponentBinDir%
283296
set "__CMakeBinDir=!__CMakeBinDir:\=/!"
284-
set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%"
297+
set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
285298
call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs!
286299
@if defined __echo @echo on
287300
popd
@@ -522,6 +535,7 @@ echo for the specified platform ^(FreeBSD, Linux, NetBSD, OS X or Windows,
522535
echo respectively^).
523536
echo add nativemscorlib to go further and build the native image for designated mscorlib.
524537
echo toolset_dir ^<dir^> : set the toolset directory -- Arm64 use only. Required for Arm64 builds.
538+
echo pgoinstrument: generate instrumented code for profile guided optimization enabled binaries.
525539
echo configureonly: skip all builds; only run CMake ^(default: CMake and builds are run^)
526540
echo skipconfigure: skip CMake ^(default: CMake is run^)
527541
echo skipmscorlib: skip building System.Private.CoreLib ^(default: System.Private.CoreLib is built^).

‎build.proj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@
2424
<Delete Files="$(BinDir)System.Private.CoreLib.*" />
2525
</Target>
2626

27+
<Target Name="RestoreOptData">
28+
<Exec Command="$(DnuRestoreCommand) &quot;$(SourceDir).nuget/optdata/project.json&quot; --source https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json" />
29+
</Target>
30+
2731
<Target Name="RestoreNETCorePlatforms" AfterTargets="Build" Condition="'$(RestoreDuringBuild)'=='true'">
2832
<Exec Command="$(DnuRestoreCommand) &quot;$(SourceDir).nuget/init/project.json&quot; --source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
2933
</Target>

‎build.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ usage()
3535
echo "clangx.y - optional argument to build using clang version x.y."
3636
echo "cross - optional argument to signify cross compilation,"
3737
echo " - will use ROOTFS_DIR environment variable if set."
38+
echo "pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
3839
echo "configureonly - do not perform any builds; just configure the build."
3940
echo "skipconfigure - skip build configuration."
4041
echo "skipnative - do not build native components."
@@ -175,10 +176,14 @@ build_coreclr()
175176
echo $__versionSourceLine > $__versionSourceFile
176177
fi
177178

179+
echo "Restoring the OptimizationData package"
180+
"$__ProjectRoot/run.sh" sync -optdata
181+
178182
pushd "$__IntermediatesDir"
179183
# Regenerate the CMake solution
180-
echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
181-
"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
184+
__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
185+
echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__ExtraCmakeArgs $__cmakeargs"
186+
"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__ExtraCmakeArgs" "$__cmakeargs"
182187
popd
183188
fi
184189

@@ -459,6 +464,7 @@ __RunArgs=
459464
__MSBCleanBuildArgs=
460465
__UseNinja=0
461466
__VerboseBuild=0
467+
__PgoInstrument=0
462468
__ConfigureOnly=0
463469
__SkipConfigure=0
464470
__SkipRestore=""
@@ -558,6 +564,10 @@ while :; do
558564
__UseNinja=1
559565
;;
560566

567+
pgoinstrument)
568+
__PgoInstrument=1
569+
;;
570+
561571
configureonly)
562572
__ConfigureOnly=1
563573
__SkipMSCorLib=1

‎config.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,12 @@
4848
"values": [],
4949
"defaultValue": ""
5050
},
51+
"RestoreOptData": {
52+
"description": "MsBuild target that restores optimization profile data.",
53+
"valueType": "target",
54+
"values": [],
55+
"defaultValue": ""
56+
},
5157
"RestoreDuringBuild": {
5258
"description": "Enables/disables package restore.",
5359
"valueType": "property",
@@ -362,6 +368,14 @@
362368
"RestoreNETCorePlatforms": "default"
363369
}
364370
},
371+
"optdata": {
372+
"description": "Restores optimization profile data for the repository.",
373+
"settings": {
374+
"Project": "./build.proj",
375+
"RestoreDuringBuild": true,
376+
"RestoreOptData": "default"
377+
}
378+
},
365379
"ab": {
366380
"description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required",
367381
"settings": {

‎extract-from-json.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/python
2+
3+
import argparse
4+
import json
5+
import sys
6+
7+
def parse_args():
8+
parser = argparse.ArgumentParser(
9+
description="""Extracts information from a json file by navigating the JSON object using a
10+
sequence of property accessors and returning the JSON subtree, or the raw data, found
11+
at that location."""
12+
)
13+
14+
parser.add_argument(
15+
'-f', '--file',
16+
metavar='<project.json>',
17+
help="Path to project.json file to parse",
18+
required=True,
19+
)
20+
21+
parser.add_argument(
22+
'property',
23+
metavar='property_name',
24+
help="""Name of property to extract using object notation.
25+
Pass multiple values to drill down into nested objects (in order).""",
26+
nargs='*',
27+
)
28+
29+
parser.add_argument(
30+
'-r', '--raw',
31+
help="""Dumps the raw object found at the requested location.
32+
If omitted, returns a JSON formatted object instead.""",
33+
action='store_true',
34+
default=False
35+
)
36+
37+
return parser.parse_args()
38+
39+
def main():
40+
args = parse_args()
41+
42+
with open(args.file) as json_file:
43+
selected_property = json.load(json_file)
44+
45+
for prop in args.property:
46+
selected_property = selected_property[prop]
47+
48+
if args.raw:
49+
print(selected_property)
50+
else:
51+
print(json.dumps(selected_property))
52+
53+
return 0
54+
55+
if __name__ == "__main__":
56+
sys.exit(main())

‎functions.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,11 @@ function(install_clr targetName)
179179
else()
180180
install(FILES ${strip_destination_file} DESTINATION .)
181181
endif()
182+
if(CLR_CMAKE_PGO_INSTRUMENT)
183+
if(WIN32)
184+
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${targetName}.pgd DESTINATION PGD OPTIONAL)
185+
endif()
186+
endif()
182187
endif()
183188
endfunction()
184189

‎pgosupport.cmake‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
function(clr_pgo_unknown_arch)
2+
if (WIN32)
3+
message(FATAL_ERROR "Only AMD64, ARM and I386 are supported for PGO")
4+
else()
5+
message(FATAL_ERROR "PGO not currently supported on the current platform")
6+
endif()
7+
endfunction(clr_pgo_unknown_arch)
8+
9+
# Adds Profile Guided Optimization (PGO) flags to the current target
10+
function(add_pgo TargetName)
11+
if(WIN32)
12+
set(ProfileFileName "${TargetName}.pgd")
13+
endif(WIN32)
14+
15+
file(TO_NATIVE_PATH
16+
"${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
17+
ProfilePath
18+
)
19+
20+
# Enable PGO only for optimized configs
21+
set(ConfigTypeList RELEASE RELWITHDEBINFO)
22+
23+
foreach(ConfigType IN LISTS ConfigTypeList)
24+
set(LinkFlagsProperty "LINK_FLAGS_${ConfigType}")
25+
if(CLR_CMAKE_PGO_INSTRUMENT)
26+
if(WIN32)
27+
set_property(TARGET ${TargetName} APPEND_STRING PROPERTY ${LinkFlagsProperty} "/LTCG /GENPROFILE")
28+
endif(WIN32)
29+
else(CLR_CMAKE_PGO_INSTRUMENT)
30+
# If we don't have profile data availble, gracefully fall back to a non-PGO opt build
31+
if(EXISTS ${ProfilePath})
32+
if(WIN32)
33+
set_property(TARGET ${TargetName} APPEND_STRING PROPERTY ${LinkFlagsProperty} "/LTCG /USEPROFILE:PGD=${ProfilePath}")
34+
endif(WIN32)
35+
endif(EXISTS ${ProfilePath})
36+
endif(CLR_CMAKE_PGO_INSTRUMENT)
37+
endforeach(ConfigType)
38+
endfunction(add_pgo)
39+
40+
set(CLR_CMAKE_OPTDATA_PACKAGEID "optimization.PGO.CoreCLR")
41+
set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR")
42+
43+
# Parse optdata package version from project.json
44+
file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/extract-from-json.py" ExtractFromJsonScript)
45+
file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/src/.nuget/optdata/project.json" OptDataProjectJsonPath)
46+
execute_process(
47+
COMMAND python "${ExtractFromJsonScript}" -rf "${OptDataProjectJsonPath}" dependencies "${CLR_CMAKE_OPTDATA_PACKAGEID}"
48+
OUTPUT_VARIABLE CLR_CMAKE_OPTDATA_VERSION
49+
OUTPUT_STRIP_TRAILING_WHITESPACE
50+
)
51+
52+
if(WIN32)
53+
if(CLR_CMAKE_PGO_INSTRUMENT)
54+
# Instrumented PGO binaries on Windows introduce an additional runtime dependency, pgort<ver>.dll.
55+
# Make sure we copy it next to the installed product to make it easier to redistribute the package.
56+
57+
string(SUBSTRING ${CMAKE_VS_PLATFORM_TOOLSET} 1 -1 VS_PLATFORM_VERSION_NUMBER)
58+
set(PGORT_FILENAME "pgort${VS_PLATFORM_VERSION_NUMBER}.dll")
59+
60+
get_filename_component(PATH_CXX_ROOTDIR ${CMAKE_CXX_COMPILER} DIRECTORY)
61+
62+
if(CLR_CMAKE_PLATFORM_ARCH_I386)
63+
set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/${PGORT_FILENAME}")
64+
elseif(CLR_CMAKE_PLATFORM_ARCH_AMD64)
65+
set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/../amd64/${PGORT_FILENAME}")
66+
elseif(CLR_CMAKE_PLATFORM_ARCH_ARM)
67+
set(PATH_VS_PGORT_DLL "${PATH_CXX_ROOTDIR}/../arm/${PGORT_FILENAME}")
68+
else()
69+
clr_pgo_unknown_arch()
70+
endif()
71+
72+
if (EXISTS ${PATH_VS_PGORT_DLL})
73+
message(STATUS "Found PGO runtime: ${PATH_VS_PGORT_DLL}")
74+
install(PROGRAMS ${PATH_VS_PGORT_DLL} DESTINATION .)
75+
else()
76+
message(FATAL_ERROR "file not found: ${PATH_VS_PGORT_DLL}")
77+
endif()
78+
79+
endif(CLR_CMAKE_PGO_INSTRUMENT)
80+
endif(WIN32)

‎src/.nuget/optdata/project.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"dependencies": {
3+
"optimization.PGO.CoreCLR": "1.1.0-release-20161025"
4+
},
5+
"frameworks": {
6+
"netstandard": {}
7+
},
8+
"runtimes": {
9+
"win7-x64": {}
10+
}
11+
}

‎src/dlls/mscoree/coreclr/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,3 +174,6 @@ endif(WIN32)
174174

175175
# add the install targets
176176
install_clr(coreclr)
177+
178+
# Enable profile guided optimization
179+
add_pgo(coreclr)

0 commit comments

Comments
 (0)