1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10 LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15 BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16 TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::base::wants_wasm_eh;
19use rustc_codegen_ssa::traits::*;
20use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
21use rustc_data_structures::profiling::SelfProfilerRef;
22use rustc_data_structures::small_c_str::SmallCStr;
23use rustc_errors::{DiagCtxtHandle, Level};
24use rustc_fs_util::{link_or_copy, path_to_c_string};
25use rustc_middle::ty::TyCtxt;
26use rustc_session::Session;
27use rustc_session::config::{
28 self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
29};
30use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
31use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
32use tracing::{debug, trace};
33
34use crate::back::command_line_args::quote_command_line_args;
35use crate::back::lto::ThinBuffer;
36use crate::back::owned_target_machine::OwnedTargetMachine;
37use crate::back::profiling::{
38 LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
39};
40use crate::common::AsCCharPtr;
41use crate::errors::{
42 CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
43 WithLlvmError, WriteBytecode,
44};
45use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
46use crate::llvm::{self, DiagnosticInfo};
47use crate::type_::llvm_type_ptr;
48use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
49
50pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
51 match llvm::last_error() {
52 Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)),
53 None => dcx.emit_fatal(err),
54 }
55}
56
57fn write_output_file<'ll>(
58 dcx: DiagCtxtHandle<'_>,
59 target: &'ll llvm::TargetMachine,
60 no_builtins: bool,
61 m: &'ll llvm::Module,
62 output: &Path,
63 dwo_output: Option<&Path>,
64 file_type: llvm::FileType,
65 self_profiler_ref: &SelfProfilerRef,
66 verify_llvm_ir: bool,
67) {
68 debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
69 let output_c = path_to_c_string(output);
70 let dwo_output_c;
71 let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
72 dwo_output_c = path_to_c_string(dwo_output);
73 dwo_output_c.as_ptr()
74 } else {
75 std::ptr::null()
76 };
77 let result = unsafe {
78 let pm = llvm::LLVMCreatePassManager();
79 llvm::LLVMAddAnalysisPasses(target, pm);
80 llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
81 llvm::LLVMRustWriteOutputFile(
82 target,
83 pm,
84 m,
85 output_c.as_ptr(),
86 dwo_output_ptr,
87 file_type,
88 verify_llvm_ir,
89 )
90 };
91
92 if result == llvm::LLVMRustResult::Success {
94 let artifact_kind = match file_type {
95 llvm::FileType::ObjectFile => "object_file",
96 llvm::FileType::AssemblyFile => "assembly_file",
97 };
98 record_artifact_size(self_profiler_ref, artifact_kind, output);
99 if let Some(dwo_file) = dwo_output {
100 record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
101 }
102 }
103
104 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
105}
106
107pub(crate) fn create_informational_target_machine(
108 sess: &Session,
109 only_base_features: bool,
110) -> OwnedTargetMachine {
111 let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
112 let features = llvm_util::global_llvm_features(sess, false, only_base_features);
115 target_machine_factory(sess, config::OptLevel::No, &features)(config)
116 .unwrap_or_else(|err| llvm_err(sess.dcx(), err))
117}
118
119pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
120 let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
121 tcx.output_filenames(()).split_dwarf_path(
122 tcx.sess.split_debuginfo(),
123 tcx.sess.opts.unstable_opts.split_dwarf_kind,
124 mod_name,
125 tcx.sess.invocation_temp.as_deref(),
126 )
127 } else {
128 None
129 };
130
131 let output_obj_file = Some(tcx.output_filenames(()).temp_path_for_cgu(
132 OutputType::Object,
133 mod_name,
134 tcx.sess.invocation_temp.as_deref(),
135 ));
136 let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
137
138 target_machine_factory(
139 tcx.sess,
140 tcx.backend_optimization_level(()),
141 tcx.global_backend_features(()),
142 )(config)
143 .unwrap_or_else(|err| llvm_err(tcx.dcx(), err))
144}
145
146fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
147 use self::config::OptLevel::*;
148 match cfg {
149 No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
150 Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
151 More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
152 Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
153 Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
154 SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
155 }
156}
157
158fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
159 use config::OptLevel::*;
160 match cfg {
161 No => llvm::PassBuilderOptLevel::O0,
162 Less => llvm::PassBuilderOptLevel::O1,
163 More => llvm::PassBuilderOptLevel::O2,
164 Aggressive => llvm::PassBuilderOptLevel::O3,
165 Size => llvm::PassBuilderOptLevel::Os,
166 SizeMin => llvm::PassBuilderOptLevel::Oz,
167 }
168}
169
170fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
171 match relocation_model {
172 RelocModel::Static => llvm::RelocModel::Static,
173 RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
176 RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
177 RelocModel::Ropi => llvm::RelocModel::ROPI,
178 RelocModel::Rwpi => llvm::RelocModel::RWPI,
179 RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
180 }
181}
182
183pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
184 match code_model {
185 Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
186 Some(CodeModel::Small) => llvm::CodeModel::Small,
187 Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
188 Some(CodeModel::Medium) => llvm::CodeModel::Medium,
189 Some(CodeModel::Large) => llvm::CodeModel::Large,
190 None => llvm::CodeModel::None,
191 }
192}
193
194fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
195 match float_abi {
196 None => llvm::FloatAbi::Default,
197 Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
198 Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
199 }
200}
201
202pub(crate) fn target_machine_factory(
203 sess: &Session,
204 optlvl: config::OptLevel,
205 target_features: &[String],
206) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
207 let _prof_timer = sess.prof.generic_activity("target_machine_factory");
209
210 let reloc_model = to_llvm_relocation_model(sess.relocation_model());
211
212 let (opt_level, _) = to_llvm_opt_settings(optlvl);
213 let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
214 llvm::FloatAbi::Soft
215 } else {
216 to_llvm_float_abi(sess.target.llvm_floatabi)
219 };
220
221 let ffunction_sections =
222 sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
223 let fdata_sections = ffunction_sections;
224 let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
225
226 let code_model = to_llvm_code_model(sess.code_model());
227
228 let mut singlethread = sess.target.singlethread;
229
230 if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
234 singlethread = false;
235 }
236
237 let triple = SmallCStr::new(&versioned_llvm_target(sess));
238 let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
239 let features = CString::new(target_features.join(",")).unwrap();
240 let abi = SmallCStr::new(&sess.target.llvm_abiname);
241 let trap_unreachable =
242 sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
243 let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
244
245 let verbose_asm = sess.opts.unstable_opts.verbose_asm;
246 let relax_elf_relocations =
247 sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
248
249 let use_init_array =
250 !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
251
252 let path_mapping = sess.source_map().path_mapping().clone();
253
254 let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
255
256 let argv0 = std::env::current_exe()
260 .unwrap_or_default()
261 .into_os_string()
262 .into_string()
263 .unwrap_or_default();
264 let command_line_args = quote_command_line_args(&sess.expanded_args);
265 sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64);
268
269 let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
270 match sess.opts.debuginfo_compression {
271 rustc_session::config::DebugInfoCompression::Zlib => {
272 if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
273 sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
274 }
275 }
276 rustc_session::config::DebugInfoCompression::Zstd => {
277 if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
278 sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
279 }
280 }
281 rustc_session::config::DebugInfoCompression::None => {}
282 };
283 let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
284
285 let file_name_display_preference =
286 sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
287
288 let use_wasm_eh = wants_wasm_eh(sess);
289
290 let prof = SelfProfilerRef::clone(&sess.prof);
291 Arc::new(move |config: TargetMachineFactoryConfig| {
292 let _prof_timer = prof.generic_activity("target_machine_factory_inner");
294
295 let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
296 let path = path.unwrap_or_default();
297 let path = path_mapping
298 .to_real_filename(path)
299 .to_string_lossy(file_name_display_preference)
300 .into_owned();
301 CString::new(path).unwrap()
302 };
303
304 let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
305 let output_obj_file = path_to_cstring_helper(config.output_obj_file);
306
307 OwnedTargetMachine::new(
308 &triple,
309 &cpu,
310 &features,
311 &abi,
312 code_model,
313 reloc_model,
314 opt_level,
315 float_abi,
316 ffunction_sections,
317 fdata_sections,
318 funique_section_names,
319 trap_unreachable,
320 singlethread,
321 verbose_asm,
322 emit_stack_size_section,
323 relax_elf_relocations,
324 use_init_array,
325 &split_dwarf_file,
326 &output_obj_file,
327 &debuginfo_compression,
328 use_emulated_tls,
329 &argv0,
330 &command_line_args,
331 use_wasm_eh,
332 )
333 })
334}
335
336pub(crate) fn save_temp_bitcode(
337 cgcx: &CodegenContext<LlvmCodegenBackend>,
338 module: &ModuleCodegen<ModuleLlvm>,
339 name: &str,
340) {
341 if !cgcx.save_temps {
342 return;
343 }
344 let ext = format!("{name}.bc");
345 let path = cgcx.output_filenames.temp_path_ext_for_cgu(
346 &ext,
347 &module.name,
348 cgcx.invocation_temp.as_deref(),
349 );
350 write_bitcode_to_file(module, &path)
351}
352
353fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
354 unsafe {
355 let path = path_to_c_string(&path);
356 let llmod = module.module_llvm.llmod();
357 llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
358 }
359}
360
361pub(crate) enum CodegenDiagnosticsStage {
363 Opt,
365 LTO,
367 Codegen,
369}
370
371pub(crate) struct DiagnosticHandlers<'a> {
372 data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
373 llcx: &'a llvm::Context,
374 old_handler: Option<&'a llvm::DiagnosticHandler>,
375}
376
377impl<'a> DiagnosticHandlers<'a> {
378 pub(crate) fn new(
379 cgcx: &'a CodegenContext<LlvmCodegenBackend>,
380 dcx: DiagCtxtHandle<'a>,
381 llcx: &'a llvm::Context,
382 module: &ModuleCodegen<ModuleLlvm>,
383 stage: CodegenDiagnosticsStage,
384 ) -> Self {
385 let remark_passes_all: bool;
386 let remark_passes: Vec<CString>;
387 match &cgcx.remark {
388 Passes::All => {
389 remark_passes_all = true;
390 remark_passes = Vec::new();
391 }
392 Passes::Some(passes) => {
393 remark_passes_all = false;
394 remark_passes =
395 passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
396 }
397 };
398 let remark_passes: Vec<*const c_char> =
399 remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
400 let remark_file = cgcx
401 .remark_dir
402 .as_ref()
403 .map(|dir| {
405 let stage_suffix = match stage {
406 CodegenDiagnosticsStage::Codegen => "codegen",
407 CodegenDiagnosticsStage::Opt => "opt",
408 CodegenDiagnosticsStage::LTO => "lto",
409 };
410 dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
411 })
412 .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
413
414 let pgo_available = cgcx.opts.cg.profile_use.is_some();
415 let data = Box::into_raw(Box::new((cgcx, dcx)));
416 unsafe {
417 let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
418 llvm::LLVMRustContextConfigureDiagnosticHandler(
419 llcx,
420 diagnostic_handler,
421 data.cast(),
422 remark_passes_all,
423 remark_passes.as_ptr(),
424 remark_passes.len(),
425 remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
428 pgo_available,
429 );
430 DiagnosticHandlers { data, llcx, old_handler }
431 }
432 }
433}
434
435impl<'a> Drop for DiagnosticHandlers<'a> {
436 fn drop(&mut self) {
437 unsafe {
438 llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
439 drop(Box::from_raw(self.data));
440 }
441 }
442}
443
444fn report_inline_asm(
445 cgcx: &CodegenContext<LlvmCodegenBackend>,
446 msg: String,
447 level: llvm::DiagnosticLevel,
448 cookie: u64,
449 source: Option<(String, Vec<InnerSpan>)>,
450) {
451 let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
455 SpanData::default()
456 } else {
457 SpanData {
458 lo: BytePos::from_u32(cookie as u32),
459 hi: BytePos::from_u32((cookie >> 32) as u32),
460 ctxt: SyntaxContext::root(),
461 parent: None,
462 }
463 };
464 let level = match level {
465 llvm::DiagnosticLevel::Error => Level::Error,
466 llvm::DiagnosticLevel::Warning => Level::Warning,
467 llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
468 };
469 let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
470 cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
471}
472
473unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
474 if user.is_null() {
475 return;
476 }
477 let (cgcx, dcx) =
478 unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
479
480 match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
481 llvm::diagnostic::InlineAsm(inline) => {
482 report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
483 }
484
485 llvm::diagnostic::Optimization(opt) => {
486 dcx.emit_note(FromLlvmOptimizationDiag {
487 filename: &opt.filename,
488 line: opt.line,
489 column: opt.column,
490 pass_name: &opt.pass_name,
491 kind: match opt.kind {
492 OptimizationRemark => "success",
493 OptimizationMissed | OptimizationFailure => "missed",
494 OptimizationAnalysis
495 | OptimizationAnalysisFPCommute
496 | OptimizationAnalysisAliasing => "analysis",
497 OptimizationRemarkOther => "other",
498 },
499 message: &opt.message,
500 });
501 }
502 llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
503 let message = llvm::build_string(|s| unsafe {
504 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
505 })
506 .expect("non-UTF8 diagnostic");
507 dcx.emit_warn(FromLlvmDiag { message });
508 }
509 llvm::diagnostic::Unsupported(diagnostic_ref) => {
510 let message = llvm::build_string(|s| unsafe {
511 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
512 })
513 .expect("non-UTF8 diagnostic");
514 dcx.emit_err(FromLlvmDiag { message });
515 }
516 llvm::diagnostic::UnknownDiagnostic(..) => {}
517 }
518}
519
520fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
521 match config.pgo_gen {
522 SwitchWithOptPath::Enabled(ref opt_dir_path) => {
523 let path = if let Some(dir_path) = opt_dir_path {
524 dir_path.join("default_%m.profraw")
525 } else {
526 PathBuf::from("default_%m.profraw")
527 };
528
529 Some(CString::new(format!("{}", path.display())).unwrap())
530 }
531 SwitchWithOptPath::Disabled => None,
532 }
533}
534
535fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
536 config
537 .pgo_use
538 .as_ref()
539 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
540}
541
542fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
543 config
544 .pgo_sample_use
545 .as_ref()
546 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
547}
548
549fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
550 config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
551}
552
553#[derive(Debug, Eq, PartialEq)]
557pub(crate) enum AutodiffStage {
558 PreAD,
559 DuringAD,
560 PostAD,
561}
562
563pub(crate) unsafe fn llvm_optimize(
564 cgcx: &CodegenContext<LlvmCodegenBackend>,
565 dcx: DiagCtxtHandle<'_>,
566 module: &ModuleCodegen<ModuleLlvm>,
567 thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
568 config: &ModuleConfig,
569 opt_level: config::OptLevel,
570 opt_stage: llvm::OptStage,
571 autodiff_stage: AutodiffStage,
572) {
573 let consider_ad =
582 cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
583 let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
584 let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
585 let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
586 let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
587 let merge_functions;
588 let unroll_loops;
589 let vectorize_slp;
590 let vectorize_loop;
591
592 if consider_ad && autodiff_stage != AutodiffStage::PostAD {
601 merge_functions = false;
602 unroll_loops = false;
603 vectorize_slp = false;
604 vectorize_loop = false;
605 } else {
606 unroll_loops =
607 opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
608 merge_functions = config.merge_functions;
609 vectorize_slp = config.vectorize_slp;
610 vectorize_loop = config.vectorize_loop;
611 }
612 trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
613 if thin_lto_buffer.is_some() {
614 assert!(
615 matches!(
616 opt_stage,
617 llvm::OptStage::PreLinkNoLTO
618 | llvm::OptStage::PreLinkFatLTO
619 | llvm::OptStage::PreLinkThinLTO
620 ),
621 "the bitcode for LTO can only be obtained at the pre-link stage"
622 );
623 }
624 let pgo_gen_path = get_pgo_gen_path(config);
625 let pgo_use_path = get_pgo_use_path(config);
626 let pgo_sample_use_path = get_pgo_sample_use_path(config);
627 let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
628 let instr_profile_output_path = get_instr_profile_output_path(config);
629 let sanitize_dataflow_abilist: Vec<_> = config
630 .sanitizer_dataflow_abilist
631 .iter()
632 .map(|file| CString::new(file.as_str()).unwrap())
633 .collect();
634 let sanitize_dataflow_abilist_ptrs: Vec<_> =
635 sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
636 let sanitizer_options = if !is_lto {
638 Some(llvm::SanitizerOptions {
639 sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
640 sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
641 sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
642 sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
643 sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
644 sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
645 sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
646 sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
647 sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
648 sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
649 sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
650 sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
651 sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
652 sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
653 sanitize_kernel_address_recover: config
654 .sanitizer_recover
655 .contains(SanitizerSet::KERNELADDRESS),
656 })
657 } else {
658 None
659 };
660
661 let mut llvm_profiler = cgcx
662 .prof
663 .llvm_recording_enabled()
664 .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
665
666 let llvm_selfprofiler =
667 llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
668
669 let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
670
671 let llvm_plugins = config.llvm_plugins.join(",");
672
673 let result = unsafe {
674 llvm::LLVMRustOptimize(
675 module.module_llvm.llmod(),
676 &*module.module_llvm.tm.raw(),
677 to_pass_builder_opt_level(opt_level),
678 opt_stage,
679 cgcx.opts.cg.linker_plugin_lto.enabled(),
680 config.no_prepopulate_passes,
681 config.verify_llvm_ir,
682 config.lint_llvm_ir,
683 thin_lto_buffer,
684 config.emit_thin_lto,
685 config.emit_thin_lto_summary,
686 merge_functions,
687 unroll_loops,
688 vectorize_slp,
689 vectorize_loop,
690 config.no_builtins,
691 config.emit_lifetime_markers,
692 run_enzyme,
693 print_before_enzyme,
694 print_after_enzyme,
695 print_passes,
696 sanitizer_options.as_ref(),
697 pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
698 pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
699 config.instrument_coverage,
700 instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
701 pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
702 config.debug_info_for_profiling,
703 llvm_selfprofiler,
704 selfprofile_before_pass_callback,
705 selfprofile_after_pass_callback,
706 extra_passes.as_c_char_ptr(),
707 extra_passes.len(),
708 llvm_plugins.as_c_char_ptr(),
709 llvm_plugins.len(),
710 )
711 };
712 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
713}
714
715pub(crate) fn optimize(
717 cgcx: &CodegenContext<LlvmCodegenBackend>,
718 dcx: DiagCtxtHandle<'_>,
719 module: &mut ModuleCodegen<ModuleLlvm>,
720 config: &ModuleConfig,
721) {
722 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
723
724 let llcx = &*module.module_llvm.llcx;
725 let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
726
727 if config.emit_no_opt_bc {
728 let out = cgcx.output_filenames.temp_path_ext_for_cgu(
729 "no-opt.bc",
730 &module.name,
731 cgcx.invocation_temp.as_deref(),
732 );
733 write_bitcode_to_file(module, &out)
734 }
735
736 if let Some(opt_level) = config.opt_level {
739 let opt_stage = match cgcx.lto {
740 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
741 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
742 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
743 _ => llvm::OptStage::PreLinkNoLTO,
744 };
745
746 let consider_ad =
749 cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
750 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
751 let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
756 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
757 || config.emit_thin_lto_summary
758 {
759 Some(null_mut())
760 } else {
761 None
762 };
763 unsafe {
764 llvm_optimize(
765 cgcx,
766 dcx,
767 module,
768 thin_lto_buffer.as_mut(),
769 config,
770 opt_level,
771 opt_stage,
772 autodiff_stage,
773 )
774 };
775 if let Some(thin_lto_buffer) = thin_lto_buffer {
776 let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
777 module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
778 let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu(
779 OutputType::ThinLinkBitcode,
780 &module.name,
781 cgcx.invocation_temp.as_deref(),
782 );
783 if config.emit_thin_lto_summary
784 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
785 {
786 let summary_data = thin_lto_buffer.thin_link_data();
787 cgcx.prof.artifact_size(
788 "llvm_bitcode_summary",
789 thin_link_bitcode_filename.to_string_lossy(),
790 summary_data.len() as u64,
791 );
792 let _timer = cgcx.prof.generic_activity_with_arg(
793 "LLVM_module_codegen_emit_bitcode_summary",
794 &*module.name,
795 );
796 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
797 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
798 }
799 }
800 }
801 }
802}
803
804pub(crate) fn codegen(
805 cgcx: &CodegenContext<LlvmCodegenBackend>,
806 module: ModuleCodegen<ModuleLlvm>,
807 config: &ModuleConfig,
808) -> CompiledModule {
809 let dcx = cgcx.create_dcx();
810 let dcx = dcx.handle();
811
812 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
813 {
814 let llmod = module.module_llvm.llmod();
815 let llcx = &*module.module_llvm.llcx;
816 let tm = &*module.module_llvm.tm;
817 let _handlers =
818 DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
819
820 if cgcx.msvc_imps_needed {
821 create_msvc_imps(cgcx, llcx, llmod);
822 }
823
824 let bc_out = cgcx.output_filenames.temp_path_for_cgu(
829 OutputType::Bitcode,
830 &module.name,
831 cgcx.invocation_temp.as_deref(),
832 );
833 let obj_out = cgcx.output_filenames.temp_path_for_cgu(
834 OutputType::Object,
835 &module.name,
836 cgcx.invocation_temp.as_deref(),
837 );
838
839 if config.bitcode_needed() {
840 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
841 let thin = {
842 let _timer = cgcx.prof.generic_activity_with_arg(
843 "LLVM_module_codegen_make_bitcode",
844 &*module.name,
845 );
846 ThinBuffer::new(llmod, config.emit_thin_lto)
847 };
848 let data = thin.data();
849 let _timer = cgcx
850 .prof
851 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
852 if let Some(bitcode_filename) = bc_out.file_name() {
853 cgcx.prof.artifact_size(
854 "llvm_bitcode",
855 bitcode_filename.to_string_lossy(),
856 data.len() as u64,
857 );
858 }
859 if let Err(err) = fs::write(&bc_out, data) {
860 dcx.emit_err(WriteBytecode { path: &bc_out, err });
861 }
862 }
863
864 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
865 let _timer = cgcx
866 .prof
867 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
868 let thin_bc =
869 module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
870 embed_bitcode(cgcx, llcx, llmod, &thin_bc);
871 }
872 }
873
874 if config.emit_ir {
875 let _timer =
876 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
877 let out = cgcx.output_filenames.temp_path_for_cgu(
878 OutputType::LlvmAssembly,
879 &module.name,
880 cgcx.invocation_temp.as_deref(),
881 );
882 let out_c = path_to_c_string(&out);
883
884 extern "C" fn demangle_callback(
885 input_ptr: *const c_char,
886 input_len: size_t,
887 output_ptr: *mut c_char,
888 output_len: size_t,
889 ) -> size_t {
890 let input =
891 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
892
893 let Ok(input) = str::from_utf8(input) else { return 0 };
894
895 let output = unsafe {
896 slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
897 };
898 let mut cursor = io::Cursor::new(output);
899
900 let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
901
902 if write!(cursor, "{demangled:#}").is_err() {
903 return 0;
905 }
906
907 cursor.position() as size_t
908 }
909
910 let result =
911 unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
912
913 if result == llvm::LLVMRustResult::Success {
914 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
915 }
916
917 result
918 .into_result()
919 .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }));
920 }
921
922 if config.emit_asm {
923 let _timer =
924 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
925 let path = cgcx.output_filenames.temp_path_for_cgu(
926 OutputType::Assembly,
927 &module.name,
928 cgcx.invocation_temp.as_deref(),
929 );
930
931 let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
936 llvm::LLVMCloneModule(llmod)
937 } else {
938 llmod
939 };
940 write_output_file(
941 dcx,
942 tm.raw(),
943 config.no_builtins,
944 llmod,
945 &path,
946 None,
947 llvm::FileType::AssemblyFile,
948 &cgcx.prof,
949 config.verify_llvm_ir,
950 );
951 }
952
953 match config.emit_obj {
954 EmitObj::ObjectCode(_) => {
955 let _timer = cgcx
956 .prof
957 .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
958
959 let dwo_out = cgcx
960 .output_filenames
961 .temp_path_dwo_for_cgu(&module.name, cgcx.invocation_temp.as_deref());
962 let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
963 (SplitDebuginfo::Off, _) => None,
965 _ if !cgcx.target_can_use_split_dwarf => None,
968 (_, SplitDwarfKind::Single) => None,
971 (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
974 };
975
976 write_output_file(
977 dcx,
978 tm.raw(),
979 config.no_builtins,
980 llmod,
981 &obj_out,
982 dwo_out,
983 llvm::FileType::ObjectFile,
984 &cgcx.prof,
985 config.verify_llvm_ir,
986 );
987 }
988
989 EmitObj::Bitcode => {
990 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
991 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
992 dcx.emit_err(CopyBitcode { err });
993 }
994
995 if !config.emit_bc {
996 debug!("removing_bitcode {:?}", bc_out);
997 ensure_removed(dcx, &bc_out);
998 }
999 }
1000
1001 EmitObj::None => {}
1002 }
1003
1004 record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
1005 }
1006
1007 let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
1016 && cgcx.target_can_use_split_dwarf
1017 && cgcx.split_debuginfo != SplitDebuginfo::Off
1018 && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1019 module.into_compiled_module(
1020 config.emit_obj != EmitObj::None,
1021 dwarf_object_emitted,
1022 config.emit_bc,
1023 config.emit_asm,
1024 config.emit_ir,
1025 &cgcx.output_filenames,
1026 cgcx.invocation_temp.as_deref(),
1027 )
1028}
1029
1030fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1031 let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1032 asm.extend_from_slice(b".ascii \"");
1033 asm.reserve(data.len());
1034 for &byte in data {
1035 if byte == b'\\' || byte == b'"' {
1036 asm.push(b'\\');
1037 asm.push(byte);
1038 } else if byte < 0x20 || byte >= 0x80 {
1039 asm.push(b'\\');
1042 asm.push(b'0' + ((byte >> 6) & 0x7));
1043 asm.push(b'0' + ((byte >> 3) & 0x7));
1044 asm.push(b'0' + ((byte >> 0) & 0x7));
1045 } else {
1046 asm.push(byte);
1047 }
1048 }
1049 asm.extend_from_slice(b"\"\n");
1050 asm
1051}
1052
1053pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1054 if cgcx.target_is_like_darwin {
1055 c"__LLVM,__bitcode"
1056 } else if cgcx.target_is_like_aix {
1057 c".ipa"
1058 } else {
1059 c".llvmbc"
1060 }
1061}
1062
1063fn embed_bitcode(
1065 cgcx: &CodegenContext<LlvmCodegenBackend>,
1066 llcx: &llvm::Context,
1067 llmod: &llvm::Module,
1068 bitcode: &[u8],
1069) {
1070 if cgcx.target_is_like_darwin
1109 || cgcx.target_is_like_aix
1110 || cgcx.target_arch == "wasm32"
1111 || cgcx.target_arch == "wasm64"
1112 {
1113 let llconst = common::bytes_in_context(llcx, bitcode);
1115 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1116 llvm::set_initializer(llglobal, llconst);
1117
1118 llvm::set_section(llglobal, bitcode_section_name(cgcx));
1119 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1120 llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE);
1121
1122 let llconst = common::bytes_in_context(llcx, &[]);
1123 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1124 llvm::set_initializer(llglobal, llconst);
1125 let section = if cgcx.target_is_like_darwin {
1126 c"__LLVM,__cmdline"
1127 } else if cgcx.target_is_like_aix {
1128 c".info"
1129 } else {
1130 c".llvmcmd"
1131 };
1132 llvm::set_section(llglobal, section);
1133 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1134 } else {
1135 let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1137 let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1138 llvm::append_module_inline_asm(llmod, &asm);
1139 let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
1140 llvm::append_module_inline_asm(llmod, &asm);
1141 }
1142}
1143
1144fn create_msvc_imps(
1150 cgcx: &CodegenContext<LlvmCodegenBackend>,
1151 llcx: &llvm::Context,
1152 llmod: &llvm::Module,
1153) {
1154 if !cgcx.msvc_imps_needed {
1155 return;
1156 }
1157 let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1162
1163 let ptr_ty = llvm_type_ptr(llcx);
1164 let globals = base::iter_globals(llmod)
1165 .filter(|&val| {
1166 llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1167 })
1168 .filter_map(|val| {
1169 let name = llvm::get_value_name(val);
1171 if ignored(&name) { None } else { Some((val, name)) }
1172 })
1173 .map(move |(val, name)| {
1174 let mut imp_name = prefix.as_bytes().to_vec();
1175 imp_name.extend(name);
1176 let imp_name = CString::new(imp_name).unwrap();
1177 (imp_name, val)
1178 })
1179 .collect::<Vec<_>>();
1180
1181 for (imp_name, val) in globals {
1182 let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1183
1184 llvm::set_initializer(imp, val);
1185 llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1186 }
1187
1188 fn ignored(symbol_name: &[u8]) -> bool {
1190 symbol_name.starts_with(b"__llvm_profile_")
1192 }
1193}
1194
1195fn record_artifact_size(
1196 self_profiler_ref: &SelfProfilerRef,
1197 artifact_kind: &'static str,
1198 path: &Path,
1199) {
1200 if !self_profiler_ref.enabled() {
1202 return;
1203 }
1204
1205 if let Some(artifact_name) = path.file_name() {
1206 let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1207 self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1208 }
1209}
1210
1211fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1212 if !prof.enabled() {
1213 return;
1214 }
1215
1216 let raw_stats =
1217 llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1218 .expect("cannot get module instruction stats");
1219
1220 #[derive(serde::Deserialize)]
1221 struct InstructionsStats {
1222 module: String,
1223 total: u64,
1224 }
1225
1226 let InstructionsStats { module, total } =
1227 serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1228 prof.artifact_size("cgu_instructions", module, total);
1229}