1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10 LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15 BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16 TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::base::wants_wasm_eh;
19use rustc_codegen_ssa::traits::*;
20use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
21use rustc_data_structures::profiling::SelfProfilerRef;
22use rustc_data_structures::small_c_str::SmallCStr;
23use rustc_errors::{DiagCtxtHandle, Level};
24use rustc_fs_util::{link_or_copy, path_to_c_string};
25use rustc_middle::ty::TyCtxt;
26use rustc_session::Session;
27use rustc_session::config::{
28 self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
29};
30use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
31use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
32use tracing::{debug, trace};
33
34use crate::back::lto::ThinBuffer;
35use crate::back::owned_target_machine::OwnedTargetMachine;
36use crate::back::profiling::{
37 LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
38};
39use crate::common::AsCCharPtr;
40use crate::errors::{
41 CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
42 WithLlvmError, WriteBytecode,
43};
44use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
45use crate::llvm::{self, DiagnosticInfo};
46use crate::type_::llvm_type_ptr;
47use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
48
49pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
50 match llvm::last_error() {
51 Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)),
52 None => dcx.emit_fatal(err),
53 }
54}
55
56fn write_output_file<'ll>(
57 dcx: DiagCtxtHandle<'_>,
58 target: &'ll llvm::TargetMachine,
59 no_builtins: bool,
60 m: &'ll llvm::Module,
61 output: &Path,
62 dwo_output: Option<&Path>,
63 file_type: llvm::FileType,
64 self_profiler_ref: &SelfProfilerRef,
65 verify_llvm_ir: bool,
66) {
67 debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
68 let output_c = path_to_c_string(output);
69 let dwo_output_c;
70 let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
71 dwo_output_c = path_to_c_string(dwo_output);
72 dwo_output_c.as_ptr()
73 } else {
74 std::ptr::null()
75 };
76 let result = unsafe {
77 let pm = llvm::LLVMCreatePassManager();
78 llvm::LLVMAddAnalysisPasses(target, pm);
79 llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
80 llvm::LLVMRustWriteOutputFile(
81 target,
82 pm,
83 m,
84 output_c.as_ptr(),
85 dwo_output_ptr,
86 file_type,
87 verify_llvm_ir,
88 )
89 };
90
91 if result == llvm::LLVMRustResult::Success {
93 let artifact_kind = match file_type {
94 llvm::FileType::ObjectFile => "object_file",
95 llvm::FileType::AssemblyFile => "assembly_file",
96 };
97 record_artifact_size(self_profiler_ref, artifact_kind, output);
98 if let Some(dwo_file) = dwo_output {
99 record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
100 }
101 }
102
103 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
104}
105
106pub(crate) fn create_informational_target_machine(
107 sess: &Session,
108 only_base_features: bool,
109) -> OwnedTargetMachine {
110 let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
111 let features = llvm_util::global_llvm_features(sess, false, only_base_features);
114 target_machine_factory(sess, config::OptLevel::No, &features)(config)
115 .unwrap_or_else(|err| llvm_err(sess.dcx(), err))
116}
117
118pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
119 let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
120 tcx.output_filenames(()).split_dwarf_path(
121 tcx.sess.split_debuginfo(),
122 tcx.sess.opts.unstable_opts.split_dwarf_kind,
123 mod_name,
124 tcx.sess.invocation_temp.as_deref(),
125 )
126 } else {
127 None
128 };
129
130 let output_obj_file = Some(tcx.output_filenames(()).temp_path_for_cgu(
131 OutputType::Object,
132 mod_name,
133 tcx.sess.invocation_temp.as_deref(),
134 ));
135 let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
136
137 target_machine_factory(
138 tcx.sess,
139 tcx.backend_optimization_level(()),
140 tcx.global_backend_features(()),
141 )(config)
142 .unwrap_or_else(|err| llvm_err(tcx.dcx(), err))
143}
144
145fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
146 use self::config::OptLevel::*;
147 match cfg {
148 No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
149 Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
150 More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
151 Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
152 Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
153 SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
154 }
155}
156
157fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
158 use config::OptLevel::*;
159 match cfg {
160 No => llvm::PassBuilderOptLevel::O0,
161 Less => llvm::PassBuilderOptLevel::O1,
162 More => llvm::PassBuilderOptLevel::O2,
163 Aggressive => llvm::PassBuilderOptLevel::O3,
164 Size => llvm::PassBuilderOptLevel::Os,
165 SizeMin => llvm::PassBuilderOptLevel::Oz,
166 }
167}
168
169fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
170 match relocation_model {
171 RelocModel::Static => llvm::RelocModel::Static,
172 RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
175 RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
176 RelocModel::Ropi => llvm::RelocModel::ROPI,
177 RelocModel::Rwpi => llvm::RelocModel::RWPI,
178 RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
179 }
180}
181
182pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
183 match code_model {
184 Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
185 Some(CodeModel::Small) => llvm::CodeModel::Small,
186 Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
187 Some(CodeModel::Medium) => llvm::CodeModel::Medium,
188 Some(CodeModel::Large) => llvm::CodeModel::Large,
189 None => llvm::CodeModel::None,
190 }
191}
192
193fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
194 match float_abi {
195 None => llvm::FloatAbi::Default,
196 Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
197 Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
198 }
199}
200
201pub(crate) fn target_machine_factory(
202 sess: &Session,
203 optlvl: config::OptLevel,
204 target_features: &[String],
205) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
206 let _prof_timer = sess.prof.generic_activity("target_machine_factory");
208
209 let reloc_model = to_llvm_relocation_model(sess.relocation_model());
210
211 let (opt_level, _) = to_llvm_opt_settings(optlvl);
212 let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
213 llvm::FloatAbi::Soft
214 } else {
215 to_llvm_float_abi(sess.target.llvm_floatabi)
218 };
219
220 let ffunction_sections =
221 sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
222 let fdata_sections = ffunction_sections;
223 let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
224
225 let code_model = to_llvm_code_model(sess.code_model());
226
227 let mut singlethread = sess.target.singlethread;
228
229 if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
233 singlethread = false;
234 }
235
236 let triple = SmallCStr::new(&versioned_llvm_target(sess));
237 let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
238 let features = CString::new(target_features.join(",")).unwrap();
239 let abi = SmallCStr::new(&sess.target.llvm_abiname);
240 let trap_unreachable =
241 sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
242 let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
243
244 let verbose_asm = sess.opts.unstable_opts.verbose_asm;
245 let relax_elf_relocations =
246 sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
247
248 let use_init_array =
249 !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
250
251 let path_mapping = sess.source_map().path_mapping().clone();
252
253 let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
254
255 let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
256 match sess.opts.debuginfo_compression {
257 rustc_session::config::DebugInfoCompression::Zlib => {
258 if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
259 sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
260 }
261 }
262 rustc_session::config::DebugInfoCompression::Zstd => {
263 if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
264 sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
265 }
266 }
267 rustc_session::config::DebugInfoCompression::None => {}
268 };
269 let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
270
271 let file_name_display_preference =
272 sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
273
274 let use_wasm_eh = wants_wasm_eh(sess);
275
276 let prof = SelfProfilerRef::clone(&sess.prof);
277 Arc::new(move |config: TargetMachineFactoryConfig| {
278 let _prof_timer = prof.generic_activity("target_machine_factory_inner");
280
281 let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
282 let path = path.unwrap_or_default();
283 let path = path_mapping
284 .to_real_filename(path)
285 .to_string_lossy(file_name_display_preference)
286 .into_owned();
287 CString::new(path).unwrap()
288 };
289
290 let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
291 let output_obj_file = path_to_cstring_helper(config.output_obj_file);
292
293 OwnedTargetMachine::new(
294 &triple,
295 &cpu,
296 &features,
297 &abi,
298 code_model,
299 reloc_model,
300 opt_level,
301 float_abi,
302 ffunction_sections,
303 fdata_sections,
304 funique_section_names,
305 trap_unreachable,
306 singlethread,
307 verbose_asm,
308 emit_stack_size_section,
309 relax_elf_relocations,
310 use_init_array,
311 &split_dwarf_file,
312 &output_obj_file,
313 &debuginfo_compression,
314 use_emulated_tls,
315 use_wasm_eh,
316 )
317 })
318}
319
320pub(crate) fn save_temp_bitcode(
321 cgcx: &CodegenContext<LlvmCodegenBackend>,
322 module: &ModuleCodegen<ModuleLlvm>,
323 name: &str,
324) {
325 if !cgcx.save_temps {
326 return;
327 }
328 let ext = format!("{name}.bc");
329 let path = cgcx.output_filenames.temp_path_ext_for_cgu(
330 &ext,
331 &module.name,
332 cgcx.invocation_temp.as_deref(),
333 );
334 write_bitcode_to_file(module, &path)
335}
336
337fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
338 unsafe {
339 let path = path_to_c_string(&path);
340 let llmod = module.module_llvm.llmod();
341 llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
342 }
343}
344
345pub(crate) enum CodegenDiagnosticsStage {
347 Opt,
349 LTO,
351 Codegen,
353}
354
355pub(crate) struct DiagnosticHandlers<'a> {
356 data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
357 llcx: &'a llvm::Context,
358 old_handler: Option<&'a llvm::DiagnosticHandler>,
359}
360
361impl<'a> DiagnosticHandlers<'a> {
362 pub(crate) fn new(
363 cgcx: &'a CodegenContext<LlvmCodegenBackend>,
364 dcx: DiagCtxtHandle<'a>,
365 llcx: &'a llvm::Context,
366 module: &ModuleCodegen<ModuleLlvm>,
367 stage: CodegenDiagnosticsStage,
368 ) -> Self {
369 let remark_passes_all: bool;
370 let remark_passes: Vec<CString>;
371 match &cgcx.remark {
372 Passes::All => {
373 remark_passes_all = true;
374 remark_passes = Vec::new();
375 }
376 Passes::Some(passes) => {
377 remark_passes_all = false;
378 remark_passes =
379 passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
380 }
381 };
382 let remark_passes: Vec<*const c_char> =
383 remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
384 let remark_file = cgcx
385 .remark_dir
386 .as_ref()
387 .map(|dir| {
389 let stage_suffix = match stage {
390 CodegenDiagnosticsStage::Codegen => "codegen",
391 CodegenDiagnosticsStage::Opt => "opt",
392 CodegenDiagnosticsStage::LTO => "lto",
393 };
394 dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
395 })
396 .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
397
398 let pgo_available = cgcx.opts.cg.profile_use.is_some();
399 let data = Box::into_raw(Box::new((cgcx, dcx)));
400 unsafe {
401 let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
402 llvm::LLVMRustContextConfigureDiagnosticHandler(
403 llcx,
404 diagnostic_handler,
405 data.cast(),
406 remark_passes_all,
407 remark_passes.as_ptr(),
408 remark_passes.len(),
409 remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
412 pgo_available,
413 );
414 DiagnosticHandlers { data, llcx, old_handler }
415 }
416 }
417}
418
419impl<'a> Drop for DiagnosticHandlers<'a> {
420 fn drop(&mut self) {
421 unsafe {
422 llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
423 drop(Box::from_raw(self.data));
424 }
425 }
426}
427
428fn report_inline_asm(
429 cgcx: &CodegenContext<LlvmCodegenBackend>,
430 msg: String,
431 level: llvm::DiagnosticLevel,
432 cookie: u64,
433 source: Option<(String, Vec<InnerSpan>)>,
434) {
435 let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
439 SpanData::default()
440 } else {
441 SpanData {
442 lo: BytePos::from_u32(cookie as u32),
443 hi: BytePos::from_u32((cookie >> 32) as u32),
444 ctxt: SyntaxContext::root(),
445 parent: None,
446 }
447 };
448 let level = match level {
449 llvm::DiagnosticLevel::Error => Level::Error,
450 llvm::DiagnosticLevel::Warning => Level::Warning,
451 llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
452 };
453 let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
454 cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
455}
456
457unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
458 if user.is_null() {
459 return;
460 }
461 let (cgcx, dcx) =
462 unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
463
464 match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
465 llvm::diagnostic::InlineAsm(inline) => {
466 report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
467 }
468
469 llvm::diagnostic::Optimization(opt) => {
470 dcx.emit_note(FromLlvmOptimizationDiag {
471 filename: &opt.filename,
472 line: opt.line,
473 column: opt.column,
474 pass_name: &opt.pass_name,
475 kind: match opt.kind {
476 OptimizationRemark => "success",
477 OptimizationMissed | OptimizationFailure => "missed",
478 OptimizationAnalysis
479 | OptimizationAnalysisFPCommute
480 | OptimizationAnalysisAliasing => "analysis",
481 OptimizationRemarkOther => "other",
482 },
483 message: &opt.message,
484 });
485 }
486 llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
487 let message = llvm::build_string(|s| unsafe {
488 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
489 })
490 .expect("non-UTF8 diagnostic");
491 dcx.emit_warn(FromLlvmDiag { message });
492 }
493 llvm::diagnostic::Unsupported(diagnostic_ref) => {
494 let message = llvm::build_string(|s| unsafe {
495 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
496 })
497 .expect("non-UTF8 diagnostic");
498 dcx.emit_err(FromLlvmDiag { message });
499 }
500 llvm::diagnostic::UnknownDiagnostic(..) => {}
501 }
502}
503
504fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
505 match config.pgo_gen {
506 SwitchWithOptPath::Enabled(ref opt_dir_path) => {
507 let path = if let Some(dir_path) = opt_dir_path {
508 dir_path.join("default_%m.profraw")
509 } else {
510 PathBuf::from("default_%m.profraw")
511 };
512
513 Some(CString::new(format!("{}", path.display())).unwrap())
514 }
515 SwitchWithOptPath::Disabled => None,
516 }
517}
518
519fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
520 config
521 .pgo_use
522 .as_ref()
523 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
524}
525
526fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
527 config
528 .pgo_sample_use
529 .as_ref()
530 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
531}
532
533fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
534 config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
535}
536
537#[derive(Debug, Eq, PartialEq)]
541pub(crate) enum AutodiffStage {
542 PreAD,
543 DuringAD,
544 PostAD,
545}
546
547pub(crate) unsafe fn llvm_optimize(
548 cgcx: &CodegenContext<LlvmCodegenBackend>,
549 dcx: DiagCtxtHandle<'_>,
550 module: &ModuleCodegen<ModuleLlvm>,
551 thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
552 config: &ModuleConfig,
553 opt_level: config::OptLevel,
554 opt_stage: llvm::OptStage,
555 autodiff_stage: AutodiffStage,
556) {
557 let consider_ad =
566 cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
567 let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
568 let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
569 let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
570 let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
571 let merge_functions;
572 let unroll_loops;
573 let vectorize_slp;
574 let vectorize_loop;
575
576 if consider_ad && autodiff_stage != AutodiffStage::PostAD {
585 merge_functions = false;
586 unroll_loops = false;
587 vectorize_slp = false;
588 vectorize_loop = false;
589 } else {
590 unroll_loops =
591 opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
592 merge_functions = config.merge_functions;
593 vectorize_slp = config.vectorize_slp;
594 vectorize_loop = config.vectorize_loop;
595 }
596 trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
597 if thin_lto_buffer.is_some() {
598 assert!(
599 matches!(
600 opt_stage,
601 llvm::OptStage::PreLinkNoLTO
602 | llvm::OptStage::PreLinkFatLTO
603 | llvm::OptStage::PreLinkThinLTO
604 ),
605 "the bitcode for LTO can only be obtained at the pre-link stage"
606 );
607 }
608 let pgo_gen_path = get_pgo_gen_path(config);
609 let pgo_use_path = get_pgo_use_path(config);
610 let pgo_sample_use_path = get_pgo_sample_use_path(config);
611 let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
612 let instr_profile_output_path = get_instr_profile_output_path(config);
613 let sanitize_dataflow_abilist: Vec<_> = config
614 .sanitizer_dataflow_abilist
615 .iter()
616 .map(|file| CString::new(file.as_str()).unwrap())
617 .collect();
618 let sanitize_dataflow_abilist_ptrs: Vec<_> =
619 sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
620 let sanitizer_options = if !is_lto {
622 Some(llvm::SanitizerOptions {
623 sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
624 sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
625 sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
626 sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
627 sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
628 sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
629 sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
630 sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
631 sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
632 sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
633 sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
634 sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
635 sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
636 sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
637 sanitize_kernel_address_recover: config
638 .sanitizer_recover
639 .contains(SanitizerSet::KERNELADDRESS),
640 })
641 } else {
642 None
643 };
644
645 let mut llvm_profiler = cgcx
646 .prof
647 .llvm_recording_enabled()
648 .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
649
650 let llvm_selfprofiler =
651 llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
652
653 let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
654
655 let llvm_plugins = config.llvm_plugins.join(",");
656
657 let result = unsafe {
658 llvm::LLVMRustOptimize(
659 module.module_llvm.llmod(),
660 &*module.module_llvm.tm.raw(),
661 to_pass_builder_opt_level(opt_level),
662 opt_stage,
663 cgcx.opts.cg.linker_plugin_lto.enabled(),
664 config.no_prepopulate_passes,
665 config.verify_llvm_ir,
666 config.lint_llvm_ir,
667 thin_lto_buffer,
668 config.emit_thin_lto,
669 config.emit_thin_lto_summary,
670 merge_functions,
671 unroll_loops,
672 vectorize_slp,
673 vectorize_loop,
674 config.no_builtins,
675 config.emit_lifetime_markers,
676 run_enzyme,
677 print_before_enzyme,
678 print_after_enzyme,
679 print_passes,
680 sanitizer_options.as_ref(),
681 pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
682 pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
683 config.instrument_coverage,
684 instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
685 pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
686 config.debug_info_for_profiling,
687 llvm_selfprofiler,
688 selfprofile_before_pass_callback,
689 selfprofile_after_pass_callback,
690 extra_passes.as_c_char_ptr(),
691 extra_passes.len(),
692 llvm_plugins.as_c_char_ptr(),
693 llvm_plugins.len(),
694 )
695 };
696 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
697}
698
699pub(crate) fn optimize(
701 cgcx: &CodegenContext<LlvmCodegenBackend>,
702 dcx: DiagCtxtHandle<'_>,
703 module: &mut ModuleCodegen<ModuleLlvm>,
704 config: &ModuleConfig,
705) {
706 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
707
708 let llcx = &*module.module_llvm.llcx;
709 let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
710
711 if config.emit_no_opt_bc {
712 let out = cgcx.output_filenames.temp_path_ext_for_cgu(
713 "no-opt.bc",
714 &module.name,
715 cgcx.invocation_temp.as_deref(),
716 );
717 write_bitcode_to_file(module, &out)
718 }
719
720 if let Some(opt_level) = config.opt_level {
723 let opt_stage = match cgcx.lto {
724 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
725 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
726 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
727 _ => llvm::OptStage::PreLinkNoLTO,
728 };
729
730 let consider_ad =
733 cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
734 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
735 let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
740 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
741 || config.emit_thin_lto_summary
742 {
743 Some(null_mut())
744 } else {
745 None
746 };
747 unsafe {
748 llvm_optimize(
749 cgcx,
750 dcx,
751 module,
752 thin_lto_buffer.as_mut(),
753 config,
754 opt_level,
755 opt_stage,
756 autodiff_stage,
757 )
758 };
759 if let Some(thin_lto_buffer) = thin_lto_buffer {
760 let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
761 module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
762 let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu(
763 OutputType::ThinLinkBitcode,
764 &module.name,
765 cgcx.invocation_temp.as_deref(),
766 );
767 if config.emit_thin_lto_summary
768 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
769 {
770 let summary_data = thin_lto_buffer.thin_link_data();
771 cgcx.prof.artifact_size(
772 "llvm_bitcode_summary",
773 thin_link_bitcode_filename.to_string_lossy(),
774 summary_data.len() as u64,
775 );
776 let _timer = cgcx.prof.generic_activity_with_arg(
777 "LLVM_module_codegen_emit_bitcode_summary",
778 &*module.name,
779 );
780 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
781 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
782 }
783 }
784 }
785 }
786}
787
788pub(crate) fn codegen(
789 cgcx: &CodegenContext<LlvmCodegenBackend>,
790 module: ModuleCodegen<ModuleLlvm>,
791 config: &ModuleConfig,
792) -> CompiledModule {
793 let dcx = cgcx.create_dcx();
794 let dcx = dcx.handle();
795
796 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
797 {
798 let llmod = module.module_llvm.llmod();
799 let llcx = &*module.module_llvm.llcx;
800 let tm = &*module.module_llvm.tm;
801 let _handlers =
802 DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
803
804 if cgcx.msvc_imps_needed {
805 create_msvc_imps(cgcx, llcx, llmod);
806 }
807
808 let bc_out = cgcx.output_filenames.temp_path_for_cgu(
813 OutputType::Bitcode,
814 &module.name,
815 cgcx.invocation_temp.as_deref(),
816 );
817 let obj_out = cgcx.output_filenames.temp_path_for_cgu(
818 OutputType::Object,
819 &module.name,
820 cgcx.invocation_temp.as_deref(),
821 );
822
823 if config.bitcode_needed() {
824 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
825 let thin = {
826 let _timer = cgcx.prof.generic_activity_with_arg(
827 "LLVM_module_codegen_make_bitcode",
828 &*module.name,
829 );
830 ThinBuffer::new(llmod, config.emit_thin_lto)
831 };
832 let data = thin.data();
833 let _timer = cgcx
834 .prof
835 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
836 if let Some(bitcode_filename) = bc_out.file_name() {
837 cgcx.prof.artifact_size(
838 "llvm_bitcode",
839 bitcode_filename.to_string_lossy(),
840 data.len() as u64,
841 );
842 }
843 if let Err(err) = fs::write(&bc_out, data) {
844 dcx.emit_err(WriteBytecode { path: &bc_out, err });
845 }
846 }
847
848 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
849 let _timer = cgcx
850 .prof
851 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
852 let thin_bc =
853 module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
854 embed_bitcode(cgcx, llcx, llmod, &thin_bc);
855 }
856 }
857
858 if config.emit_ir {
859 let _timer =
860 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
861 let out = cgcx.output_filenames.temp_path_for_cgu(
862 OutputType::LlvmAssembly,
863 &module.name,
864 cgcx.invocation_temp.as_deref(),
865 );
866 let out_c = path_to_c_string(&out);
867
868 extern "C" fn demangle_callback(
869 input_ptr: *const c_char,
870 input_len: size_t,
871 output_ptr: *mut c_char,
872 output_len: size_t,
873 ) -> size_t {
874 let input =
875 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
876
877 let Ok(input) = str::from_utf8(input) else { return 0 };
878
879 let output = unsafe {
880 slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
881 };
882 let mut cursor = io::Cursor::new(output);
883
884 let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
885
886 if write!(cursor, "{demangled:#}").is_err() {
887 return 0;
889 }
890
891 cursor.position() as size_t
892 }
893
894 let result =
895 unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
896
897 if result == llvm::LLVMRustResult::Success {
898 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
899 }
900
901 result
902 .into_result()
903 .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }));
904 }
905
906 if config.emit_asm {
907 let _timer =
908 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
909 let path = cgcx.output_filenames.temp_path_for_cgu(
910 OutputType::Assembly,
911 &module.name,
912 cgcx.invocation_temp.as_deref(),
913 );
914
915 let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
920 llvm::LLVMCloneModule(llmod)
921 } else {
922 llmod
923 };
924 write_output_file(
925 dcx,
926 tm.raw(),
927 config.no_builtins,
928 llmod,
929 &path,
930 None,
931 llvm::FileType::AssemblyFile,
932 &cgcx.prof,
933 config.verify_llvm_ir,
934 );
935 }
936
937 match config.emit_obj {
938 EmitObj::ObjectCode(_) => {
939 let _timer = cgcx
940 .prof
941 .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
942
943 let dwo_out = cgcx
944 .output_filenames
945 .temp_path_dwo_for_cgu(&module.name, cgcx.invocation_temp.as_deref());
946 let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
947 (SplitDebuginfo::Off, _) => None,
949 _ if !cgcx.target_can_use_split_dwarf => None,
952 (_, SplitDwarfKind::Single) => None,
955 (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
958 };
959
960 write_output_file(
961 dcx,
962 tm.raw(),
963 config.no_builtins,
964 llmod,
965 &obj_out,
966 dwo_out,
967 llvm::FileType::ObjectFile,
968 &cgcx.prof,
969 config.verify_llvm_ir,
970 );
971 }
972
973 EmitObj::Bitcode => {
974 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
975 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
976 dcx.emit_err(CopyBitcode { err });
977 }
978
979 if !config.emit_bc {
980 debug!("removing_bitcode {:?}", bc_out);
981 ensure_removed(dcx, &bc_out);
982 }
983 }
984
985 EmitObj::None => {}
986 }
987
988 record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
989 }
990
991 let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
1000 && cgcx.target_can_use_split_dwarf
1001 && cgcx.split_debuginfo != SplitDebuginfo::Off
1002 && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1003 module.into_compiled_module(
1004 config.emit_obj != EmitObj::None,
1005 dwarf_object_emitted,
1006 config.emit_bc,
1007 config.emit_asm,
1008 config.emit_ir,
1009 &cgcx.output_filenames,
1010 cgcx.invocation_temp.as_deref(),
1011 )
1012}
1013
1014fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1015 let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1016 asm.extend_from_slice(b".ascii \"");
1017 asm.reserve(data.len());
1018 for &byte in data {
1019 if byte == b'\\' || byte == b'"' {
1020 asm.push(b'\\');
1021 asm.push(byte);
1022 } else if byte < 0x20 || byte >= 0x80 {
1023 asm.push(b'\\');
1026 asm.push(b'0' + ((byte >> 6) & 0x7));
1027 asm.push(b'0' + ((byte >> 3) & 0x7));
1028 asm.push(b'0' + ((byte >> 0) & 0x7));
1029 } else {
1030 asm.push(byte);
1031 }
1032 }
1033 asm.extend_from_slice(b"\"\n");
1034 asm
1035}
1036
1037pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1038 if cgcx.target_is_like_darwin {
1039 c"__LLVM,__bitcode"
1040 } else if cgcx.target_is_like_aix {
1041 c".ipa"
1042 } else {
1043 c".llvmbc"
1044 }
1045}
1046
1047fn embed_bitcode(
1049 cgcx: &CodegenContext<LlvmCodegenBackend>,
1050 llcx: &llvm::Context,
1051 llmod: &llvm::Module,
1052 bitcode: &[u8],
1053) {
1054 if cgcx.target_is_like_darwin
1093 || cgcx.target_is_like_aix
1094 || cgcx.target_arch == "wasm32"
1095 || cgcx.target_arch == "wasm64"
1096 {
1097 let llconst = common::bytes_in_context(llcx, bitcode);
1099 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1100 llvm::set_initializer(llglobal, llconst);
1101
1102 llvm::set_section(llglobal, bitcode_section_name(cgcx));
1103 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1104 llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE);
1105
1106 let llconst = common::bytes_in_context(llcx, &[]);
1107 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1108 llvm::set_initializer(llglobal, llconst);
1109 let section = if cgcx.target_is_like_darwin {
1110 c"__LLVM,__cmdline"
1111 } else if cgcx.target_is_like_aix {
1112 c".info"
1113 } else {
1114 c".llvmcmd"
1115 };
1116 llvm::set_section(llglobal, section);
1117 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1118 } else {
1119 let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1121 let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1122 llvm::append_module_inline_asm(llmod, &asm);
1123 let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
1124 llvm::append_module_inline_asm(llmod, &asm);
1125 }
1126}
1127
1128fn create_msvc_imps(
1134 cgcx: &CodegenContext<LlvmCodegenBackend>,
1135 llcx: &llvm::Context,
1136 llmod: &llvm::Module,
1137) {
1138 if !cgcx.msvc_imps_needed {
1139 return;
1140 }
1141 let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1146
1147 let ptr_ty = llvm_type_ptr(llcx);
1148 let globals = base::iter_globals(llmod)
1149 .filter(|&val| {
1150 llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1151 })
1152 .filter_map(|val| {
1153 let name = llvm::get_value_name(val);
1155 if ignored(&name) { None } else { Some((val, name)) }
1156 })
1157 .map(move |(val, name)| {
1158 let mut imp_name = prefix.as_bytes().to_vec();
1159 imp_name.extend(name);
1160 let imp_name = CString::new(imp_name).unwrap();
1161 (imp_name, val)
1162 })
1163 .collect::<Vec<_>>();
1164
1165 for (imp_name, val) in globals {
1166 let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1167
1168 llvm::set_initializer(imp, val);
1169 llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1170 }
1171
1172 fn ignored(symbol_name: &[u8]) -> bool {
1174 symbol_name.starts_with(b"__llvm_profile_")
1176 }
1177}
1178
1179fn record_artifact_size(
1180 self_profiler_ref: &SelfProfilerRef,
1181 artifact_kind: &'static str,
1182 path: &Path,
1183) {
1184 if !self_profiler_ref.enabled() {
1186 return;
1187 }
1188
1189 if let Some(artifact_name) = path.file_name() {
1190 let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1191 self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1192 }
1193}
1194
1195fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1196 if !prof.enabled() {
1197 return;
1198 }
1199
1200 let raw_stats =
1201 llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1202 .expect("cannot get module instruction stats");
1203
1204 #[derive(serde::Deserialize)]
1205 struct InstructionsStats {
1206 module: String,
1207 total: u64,
1208 }
1209
1210 let InstructionsStats { module, total } =
1211 serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1212 prof.artifact_size("cgu_instructions", module, total);
1213}