Skip to content

Commit 6c2d4bf

Browse files
workingjubileeworkingjubilee
authored
Rollup merge of #87918 - mikebenfield:pr-afdo, r=nikic
Enable AutoFDO. This largely involves implementing the options debug-info-for-profiling and profile-sample-use and forwarding them on to LLVM. AutoFDO can be used on x86-64 Linux like this: rustc -O -Clink-arg='Wl,--no-rosegment' -Cdebug-info-for-profiling main.rs -o main perf record -b ./main create_llvm_prof --binary=main --out=code.prof rustc -O -Cprofile-sample-use=code.prof main.rs -o main2 Now `main2` will have feedback directed optimization applied to it. The create_llvm_prof tool can be obtained from this github repository: https://github.com/google/autofdo The option -Clink-arg='Wl,--no-rosegment' is necessary to avoid lld putting an extra RO segment before the executable code, which would make the binary silently incompatible with create_llvm_prof.
2 parents 2ee06e7 + a17193d commit 6c2d4bf

File tree

12 files changed

+120
-9
lines changed

12 files changed

+120
-9
lines changed

‎compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,10 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
263263
attributes::emit_uwtable(llfn, true);
264264
}
265265

266+
if cx.sess().opts.debugging_opts.profile_sample_use.is_some() {
267+
llvm::AddFunctionAttrString(llfn, Function, cstr!("use-sample-profile"));
268+
}
269+
266270
// FIXME: none of these three functions interact with source level attributes.
267271
set_frame_pointer_type(cx, llfn);
268272
set_instrument_function(cx, llfn);

‎compiler/rustc_codegen_llvm/src/back/write.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,13 @@ fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
370370
.map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
371371
}
372372

373+
fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
374+
config
375+
.pgo_sample_use
376+
.as_ref()
377+
.map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
378+
}
379+
373380
pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
374381
// The new pass manager is enabled by default for LLVM >= 13.
375382
// This matches Clang, which also enables it since Clang 13.
@@ -389,6 +396,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
389396
let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
390397
let pgo_gen_path = get_pgo_gen_path(config);
391398
let pgo_use_path = get_pgo_use_path(config);
399+
let pgo_sample_use_path = get_pgo_sample_use_path(config);
392400
let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
393401
// Sanitizer instrumentation is only inserted during the pre-link optimization stage.
394402
let sanitizer_options = if !is_lto {
@@ -439,6 +447,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
439447
pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
440448
config.instrument_coverage,
441449
config.instrument_gcov,
450+
pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
451+
config.debug_info_for_profiling,
442452
llvm_selfprofiler,
443453
selfprofile_before_pass_callback,
444454
selfprofile_after_pass_callback,
@@ -544,6 +554,9 @@ pub(crate) unsafe fn optimize(
544554
if config.instrument_coverage {
545555
llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap());
546556
}
557+
if config.debug_info_for_profiling {
558+
llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap());
559+
}
547560

548561
add_sanitizer_passes(config, &mut extra_passes);
549562

@@ -1001,6 +1014,7 @@ pub unsafe fn with_llvm_pmb(
10011014
let inline_threshold = config.inline_threshold;
10021015
let pgo_gen_path = get_pgo_gen_path(config);
10031016
let pgo_use_path = get_pgo_use_path(config);
1017+
let pgo_sample_use_path = get_pgo_sample_use_path(config);
10041018

10051019
llvm::LLVMRustConfigurePassManagerBuilder(
10061020
builder,
@@ -1011,6 +1025,7 @@ pub unsafe fn with_llvm_pmb(
10111025
prepare_for_thin_lto,
10121026
pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
10131027
pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
1028+
pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
10141029
);
10151030

10161031
llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2176,6 +2176,7 @@ extern "C" {
21762176
PrepareForThinLTO: bool,
21772177
PGOGenPath: *const c_char,
21782178
PGOUsePath: *const c_char,
2179+
PGOSampleUsePath: *const c_char,
21792180
);
21802181
pub fn LLVMRustAddLibraryInfo(
21812182
PM: &PassManager<'a>,
@@ -2210,6 +2211,8 @@ extern "C" {
22102211
PGOUsePath: *const c_char,
22112212
InstrumentCoverage: bool,
22122213
InstrumentGCOV: bool,
2214+
PGOSampleUsePath: *const c_char,
2215+
DebugInfoForProfiling: bool,
22132216
llvm_selfprofiler: *mut c_void,
22142217
begin_callback: SelfProfileBeforePassCallback,
22152218
end_callback: SelfProfileAfterPassCallback,

‎compiler/rustc_codegen_ssa/src/back/linker.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,9 @@ impl<'a> GccLinker<'a> {
286286
config::OptLevel::Aggressive => "O3",
287287
};
288288

289+
if let Some(path) = &self.sess.opts.debugging_opts.profile_sample_use {
290+
self.linker_arg(&format!("-plugin-opt=sample-profile={}", path.display()));
291+
};
289292
self.linker_arg(&format!("-plugin-opt={}", opt_level));
290293
self.linker_arg(&format!("-plugin-opt=mcpu={}", self.target_cpu));
291294
}

‎compiler/rustc_codegen_ssa/src/back/write.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ pub struct ModuleConfig {
8383

8484
pub pgo_gen: SwitchWithOptPath,
8585
pub pgo_use: Option<PathBuf>,
86+
pub pgo_sample_use: Option<PathBuf>,
87+
pub debug_info_for_profiling: bool,
8688
pub instrument_coverage: bool,
8789
pub instrument_gcov: bool,
8890

@@ -176,6 +178,8 @@ impl ModuleConfig {
176178
SwitchWithOptPath::Disabled
177179
),
178180
pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
181+
pgo_sample_use: if_regular!(sess.opts.debugging_opts.profile_sample_use.clone(), None),
182+
debug_info_for_profiling: sess.opts.debugging_opts.debug_info_for_profiling,
179183
instrument_coverage: if_regular!(sess.instrument_coverage(), false),
180184
instrument_gcov: if_regular!(
181185
// compiler_builtins overrides the codegen-units settings,

‎compiler/rustc_interface/src/tests.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,7 @@ fn test_debugging_options_tracking_hash() {
715715
tracked!(chalk, true);
716716
tracked!(codegen_backend, Some("abc".to_string()));
717717
tracked!(crate_attr, vec!["abc".to_string()]);
718+
tracked!(debug_info_for_profiling, true);
718719
tracked!(debug_macros, true);
719720
tracked!(dep_info_omit_d_target, true);
720721
tracked!(dual_proc_macros, true);
@@ -752,6 +753,7 @@ fn test_debugging_options_tracking_hash() {
752753
tracked!(profile, true);
753754
tracked!(profile_emit, Some(PathBuf::from("abc")));
754755
tracked!(profiler_runtime, "abc".to_string());
756+
tracked!(profile_sample_use, Some(PathBuf::from("abc")));
755757
tracked!(relax_elf_relocations, Some(true));
756758
tracked!(relro_level, Some(RelroLevel::Full));
757759
tracked!(remap_cwd_prefix, Some(PathBuf::from("abc")));

‎compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
2626
#include "llvm/Transforms/IPO/AlwaysInliner.h"
2727
#include "llvm/Transforms/IPO/FunctionImport.h"
28+
#include "llvm/Transforms/Utils/AddDiscriminators.h"
2829
#include "llvm/Transforms/Utils/FunctionImportUtils.h"
2930
#include "llvm/LTO/LTO.h"
3031
#include "llvm-c/Transforms/PassManagerBuilder.h"
@@ -39,6 +40,7 @@
3940
#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
4041
#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
4142
#include "llvm/Transforms/Utils/NameAnonGlobals.h"
43+
#include "llvm/Transforms/Utils.h"
4244

4345
using namespace llvm;
4446

@@ -523,21 +525,22 @@ extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
523525
extern "C" void LLVMRustConfigurePassManagerBuilder(
524526
LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
525527
bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
526-
const char* PGOGenPath, const char* PGOUsePath) {
528+
const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath) {
527529
unwrap(PMBR)->MergeFunctions = MergeFunctions;
528530
unwrap(PMBR)->SLPVectorize = SLPVectorize;
529531
unwrap(PMBR)->OptLevel = fromRust(OptLevel);
530532
unwrap(PMBR)->LoopVectorize = LoopVectorize;
531533
unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
532534

533535
if (PGOGenPath) {
534-
assert(!PGOUsePath);
536+
assert(!PGOUsePath && !PGOSampleUsePath);
535537
unwrap(PMBR)->EnablePGOInstrGen = true;
536538
unwrap(PMBR)->PGOInstrGen = PGOGenPath;
537-
}
538-
if (PGOUsePath) {
539-
assert(!PGOGenPath);
539+
} else if (PGOUsePath) {
540+
assert(!PGOSampleUsePath);
540541
unwrap(PMBR)->PGOInstrUse = PGOUsePath;
542+
} else if (PGOSampleUsePath) {
543+
unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
541544
}
542545
}
543546

@@ -759,6 +762,7 @@ LLVMRustOptimizeWithNewPassManager(
759762
LLVMRustSanitizerOptions *SanitizerOptions,
760763
const char *PGOGenPath, const char *PGOUsePath,
761764
bool InstrumentCoverage, bool InstrumentGCOV,
765+
const char *PGOSampleUsePath, bool DebugInfoForProfiling,
762766
void* LlvmSelfProfiler,
763767
LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
764768
LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
@@ -797,11 +801,19 @@ LLVMRustOptimizeWithNewPassManager(
797801

798802
Optional<PGOOptions> PGOOpt;
799803
if (PGOGenPath) {
800-
assert(!PGOUsePath);
801-
PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
804+
assert(!PGOUsePath && !PGOSampleUsePath);
805+
PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
806+
PGOOptions::NoCSAction, DebugInfoForProfiling);
802807
} else if (PGOUsePath) {
803-
assert(!PGOGenPath);
804-
PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
808+
assert(!PGOSampleUsePath);
809+
PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
810+
PGOOptions::NoCSAction, DebugInfoForProfiling);
811+
} else if (PGOSampleUsePath) {
812+
PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
813+
PGOOptions::NoCSAction, DebugInfoForProfiling);
814+
} else if (DebugInfoForProfiling) {
815+
PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
816+
PGOOptions::NoCSAction, DebugInfoForProfiling);
805817
}
806818

807819
#if LLVM_VERSION_GE(12, 0) && !LLVM_VERSION_GE(13,0)

‎compiler/rustc_session/src/config.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2009,6 +2009,15 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
20092009
);
20102010
}
20112011

2012+
if debugging_opts.profile_sample_use.is_some()
2013+
&& (cg.profile_generate.enabled() || cg.profile_use.is_some())
2014+
{
2015+
early_error(
2016+
error_format,
2017+
"option `-Z profile-sample-use` cannot be used with `-C profile-generate` or `-C profile-use`",
2018+
);
2019+
}
2020+
20122021
if debugging_opts.instrument_coverage.is_some()
20132022
&& debugging_opts.instrument_coverage != Some(InstrumentCoverage::Off)
20142023
{

‎compiler/rustc_session/src/options.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,8 @@ options! {
10401040
"combine CGUs into a single one"),
10411041
crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
10421042
"inject the given attribute in the crate"),
1043+
debug_info_for_profiling: bool = (false, parse_bool, [TRACKED],
1044+
"emit discriminators and other data necessary for AutoFDO"),
10431045
debug_macros: bool = (false, parse_bool, [TRACKED],
10441046
"emit line numbers debug info inside macros (default: no)"),
10451047
deduplicate_diagnostics: bool = (true, parse_bool, [UNTRACKED],
@@ -1242,6 +1244,8 @@ options! {
12421244
(default based on relative source path)"),
12431245
profiler_runtime: String = (String::from("profiler_builtins"), parse_string, [TRACKED],
12441246
"name of the profiler runtime crate to automatically inject (default: `profiler_builtins`)"),
1247+
profile_sample_use: Option<PathBuf> = (None, parse_opt_pathbuf, [TRACKED],
1248+
"use the given `.prof` file for sampled profile-guided optimization (also known as AutoFDO)"),
12451249
query_dep_graph: bool = (false, parse_bool, [UNTRACKED],
12461250
"enable queries of the dependency graph for regression testing (default: no)"),
12471251
query_stats: bool = (false, parse_bool, [UNTRACKED],

‎compiler/rustc_session/src/session.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1353,6 +1353,16 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
13531353
}
13541354
}
13551355

1356+
// Do the same for sample profile data.
1357+
if let Some(ref path) = sess.opts.debugging_opts.profile_sample_use {
1358+
if !path.exists() {
1359+
sess.err(&format!(
1360+
"File `{}` passed to `-C profile-sample-use` does not exist.",
1361+
path.display()
1362+
));
1363+
}
1364+
}
1365+
13561366
// Unwind tables cannot be disabled if the target requires them.
13571367
if let Some(include_uwtables) = sess.opts.cg.force_unwind_tables {
13581368
if sess.target.requires_uwtable && !include_uwtables {

0 commit comments

Comments
 (0)