1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10 LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15 BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16 TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::base::wants_wasm_eh;
19use rustc_codegen_ssa::traits::*;
20use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
21use rustc_data_structures::profiling::SelfProfilerRef;
22use rustc_data_structures::small_c_str::SmallCStr;
23use rustc_errors::{DiagCtxtHandle, FatalError, Level};
24use rustc_fs_util::{link_or_copy, path_to_c_string};
25use rustc_middle::ty::TyCtxt;
26use rustc_session::Session;
27use rustc_session::config::{
28 self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
29};
30use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
31use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
32use tracing::{debug, trace};
33
34use crate::back::lto::ThinBuffer;
35use crate::back::owned_target_machine::OwnedTargetMachine;
36use crate::back::profiling::{
37 LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
38};
39use crate::common::AsCCharPtr;
40use crate::errors::{
41 CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
42 WithLlvmError, WriteBytecode,
43};
44use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
45use crate::llvm::{self, DiagnosticInfo};
46use crate::type_::Type;
47use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
48
49pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError {
50 match llvm::last_error() {
51 Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)),
52 None => dcx.emit_almost_fatal(err),
53 }
54}
55
56fn write_output_file<'ll>(
57 dcx: DiagCtxtHandle<'_>,
58 target: &'ll llvm::TargetMachine,
59 no_builtins: bool,
60 m: &'ll llvm::Module,
61 output: &Path,
62 dwo_output: Option<&Path>,
63 file_type: llvm::FileType,
64 self_profiler_ref: &SelfProfilerRef,
65 verify_llvm_ir: bool,
66) -> Result<(), FatalError> {
67 debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
68 let output_c = path_to_c_string(output);
69 let dwo_output_c;
70 let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
71 dwo_output_c = path_to_c_string(dwo_output);
72 dwo_output_c.as_ptr()
73 } else {
74 std::ptr::null()
75 };
76 let result = unsafe {
77 let pm = llvm::LLVMCreatePassManager();
78 llvm::LLVMAddAnalysisPasses(target, pm);
79 llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
80 llvm::LLVMRustWriteOutputFile(
81 target,
82 pm,
83 m,
84 output_c.as_ptr(),
85 dwo_output_ptr,
86 file_type,
87 verify_llvm_ir,
88 )
89 };
90
91 if result == llvm::LLVMRustResult::Success {
93 let artifact_kind = match file_type {
94 llvm::FileType::ObjectFile => "object_file",
95 llvm::FileType::AssemblyFile => "assembly_file",
96 };
97 record_artifact_size(self_profiler_ref, artifact_kind, output);
98 if let Some(dwo_file) = dwo_output {
99 record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
100 }
101 }
102
103 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
104}
105
106pub(crate) fn create_informational_target_machine(
107 sess: &Session,
108 only_base_features: bool,
109) -> OwnedTargetMachine {
110 let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
111 let features = llvm_util::global_llvm_features(sess, false, only_base_features);
114 target_machine_factory(sess, config::OptLevel::No, &features)(config)
115 .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise())
116}
117
118pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
119 let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
120 tcx.output_filenames(()).split_dwarf_path(
121 tcx.sess.split_debuginfo(),
122 tcx.sess.opts.unstable_opts.split_dwarf_kind,
123 mod_name,
124 tcx.sess.invocation_temp.as_deref(),
125 )
126 } else {
127 None
128 };
129
130 let output_obj_file = Some(tcx.output_filenames(()).temp_path_for_cgu(
131 OutputType::Object,
132 mod_name,
133 tcx.sess.invocation_temp.as_deref(),
134 ));
135 let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
136
137 target_machine_factory(
138 tcx.sess,
139 tcx.backend_optimization_level(()),
140 tcx.global_backend_features(()),
141 )(config)
142 .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise())
143}
144
145fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
146 use self::config::OptLevel::*;
147 match cfg {
148 No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
149 Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
150 More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
151 Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
152 Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
153 SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
154 }
155}
156
157fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
158 use config::OptLevel::*;
159 match cfg {
160 No => llvm::PassBuilderOptLevel::O0,
161 Less => llvm::PassBuilderOptLevel::O1,
162 More => llvm::PassBuilderOptLevel::O2,
163 Aggressive => llvm::PassBuilderOptLevel::O3,
164 Size => llvm::PassBuilderOptLevel::Os,
165 SizeMin => llvm::PassBuilderOptLevel::Oz,
166 }
167}
168
169fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
170 match relocation_model {
171 RelocModel::Static => llvm::RelocModel::Static,
172 RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
175 RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
176 RelocModel::Ropi => llvm::RelocModel::ROPI,
177 RelocModel::Rwpi => llvm::RelocModel::RWPI,
178 RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
179 }
180}
181
182pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
183 match code_model {
184 Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
185 Some(CodeModel::Small) => llvm::CodeModel::Small,
186 Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
187 Some(CodeModel::Medium) => llvm::CodeModel::Medium,
188 Some(CodeModel::Large) => llvm::CodeModel::Large,
189 None => llvm::CodeModel::None,
190 }
191}
192
193fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
194 match float_abi {
195 None => llvm::FloatAbi::Default,
196 Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
197 Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
198 }
199}
200
201pub(crate) fn target_machine_factory(
202 sess: &Session,
203 optlvl: config::OptLevel,
204 target_features: &[String],
205) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
206 let reloc_model = to_llvm_relocation_model(sess.relocation_model());
207
208 let (opt_level, _) = to_llvm_opt_settings(optlvl);
209 let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
210 llvm::FloatAbi::Soft
211 } else {
212 to_llvm_float_abi(sess.target.llvm_floatabi)
215 };
216
217 let ffunction_sections =
218 sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
219 let fdata_sections = ffunction_sections;
220 let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
221
222 let code_model = to_llvm_code_model(sess.code_model());
223
224 let mut singlethread = sess.target.singlethread;
225
226 if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
230 singlethread = false;
231 }
232
233 let triple = SmallCStr::new(&versioned_llvm_target(sess));
234 let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
235 let features = CString::new(target_features.join(",")).unwrap();
236 let abi = SmallCStr::new(&sess.target.llvm_abiname);
237 let trap_unreachable =
238 sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
239 let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
240
241 let verbose_asm = sess.opts.unstable_opts.verbose_asm;
242 let relax_elf_relocations =
243 sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
244
245 let use_init_array =
246 !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
247
248 let path_mapping = sess.source_map().path_mapping().clone();
249
250 let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
251
252 let args_cstr_buff = {
255 let mut args_cstr_buff: Vec<u8> = Vec::new();
256 let exe_path = std::env::current_exe().unwrap_or_default();
257 let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default();
258
259 args_cstr_buff.extend_from_slice(exe_path_str.as_bytes());
260 args_cstr_buff.push(0);
261
262 for arg in sess.expanded_args.iter() {
263 args_cstr_buff.extend_from_slice(arg.as_bytes());
264 args_cstr_buff.push(0);
265 }
266
267 args_cstr_buff
268 };
269
270 let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
271 match sess.opts.debuginfo_compression {
272 rustc_session::config::DebugInfoCompression::Zlib => {
273 if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
274 sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
275 }
276 }
277 rustc_session::config::DebugInfoCompression::Zstd => {
278 if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
279 sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
280 }
281 }
282 rustc_session::config::DebugInfoCompression::None => {}
283 };
284 let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
285
286 let file_name_display_preference =
287 sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
288
289 let use_wasm_eh = wants_wasm_eh(sess);
290
291 Arc::new(move |config: TargetMachineFactoryConfig| {
292 let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
293 let path = path.unwrap_or_default();
294 let path = path_mapping
295 .to_real_filename(path)
296 .to_string_lossy(file_name_display_preference)
297 .into_owned();
298 CString::new(path).unwrap()
299 };
300
301 let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
302 let output_obj_file = path_to_cstring_helper(config.output_obj_file);
303
304 OwnedTargetMachine::new(
305 &triple,
306 &cpu,
307 &features,
308 &abi,
309 code_model,
310 reloc_model,
311 opt_level,
312 float_abi,
313 ffunction_sections,
314 fdata_sections,
315 funique_section_names,
316 trap_unreachable,
317 singlethread,
318 verbose_asm,
319 emit_stack_size_section,
320 relax_elf_relocations,
321 use_init_array,
322 &split_dwarf_file,
323 &output_obj_file,
324 &debuginfo_compression,
325 use_emulated_tls,
326 &args_cstr_buff,
327 use_wasm_eh,
328 )
329 })
330}
331
332pub(crate) fn save_temp_bitcode(
333 cgcx: &CodegenContext<LlvmCodegenBackend>,
334 module: &ModuleCodegen<ModuleLlvm>,
335 name: &str,
336) {
337 if !cgcx.save_temps {
338 return;
339 }
340 let ext = format!("{name}.bc");
341 let path = cgcx.output_filenames.temp_path_ext_for_cgu(
342 &ext,
343 &module.name,
344 cgcx.invocation_temp.as_deref(),
345 );
346 write_bitcode_to_file(module, &path)
347}
348
349fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
350 unsafe {
351 let path = path_to_c_string(&path);
352 let llmod = module.module_llvm.llmod();
353 llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
354 }
355}
356
357pub(crate) enum CodegenDiagnosticsStage {
359 Opt,
361 LTO,
363 Codegen,
365}
366
367pub(crate) struct DiagnosticHandlers<'a> {
368 data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
369 llcx: &'a llvm::Context,
370 old_handler: Option<&'a llvm::DiagnosticHandler>,
371}
372
373impl<'a> DiagnosticHandlers<'a> {
374 pub(crate) fn new(
375 cgcx: &'a CodegenContext<LlvmCodegenBackend>,
376 dcx: DiagCtxtHandle<'a>,
377 llcx: &'a llvm::Context,
378 module: &ModuleCodegen<ModuleLlvm>,
379 stage: CodegenDiagnosticsStage,
380 ) -> Self {
381 let remark_passes_all: bool;
382 let remark_passes: Vec<CString>;
383 match &cgcx.remark {
384 Passes::All => {
385 remark_passes_all = true;
386 remark_passes = Vec::new();
387 }
388 Passes::Some(passes) => {
389 remark_passes_all = false;
390 remark_passes =
391 passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
392 }
393 };
394 let remark_passes: Vec<*const c_char> =
395 remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
396 let remark_file = cgcx
397 .remark_dir
398 .as_ref()
399 .map(|dir| {
401 let stage_suffix = match stage {
402 CodegenDiagnosticsStage::Codegen => "codegen",
403 CodegenDiagnosticsStage::Opt => "opt",
404 CodegenDiagnosticsStage::LTO => "lto",
405 };
406 dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
407 })
408 .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
409
410 let pgo_available = cgcx.opts.cg.profile_use.is_some();
411 let data = Box::into_raw(Box::new((cgcx, dcx)));
412 unsafe {
413 let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
414 llvm::LLVMRustContextConfigureDiagnosticHandler(
415 llcx,
416 diagnostic_handler,
417 data.cast(),
418 remark_passes_all,
419 remark_passes.as_ptr(),
420 remark_passes.len(),
421 remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
424 pgo_available,
425 );
426 DiagnosticHandlers { data, llcx, old_handler }
427 }
428 }
429}
430
431impl<'a> Drop for DiagnosticHandlers<'a> {
432 fn drop(&mut self) {
433 unsafe {
434 llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
435 drop(Box::from_raw(self.data));
436 }
437 }
438}
439
440fn report_inline_asm(
441 cgcx: &CodegenContext<LlvmCodegenBackend>,
442 msg: String,
443 level: llvm::DiagnosticLevel,
444 cookie: u64,
445 source: Option<(String, Vec<InnerSpan>)>,
446) {
447 let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
451 SpanData::default()
452 } else {
453 SpanData {
454 lo: BytePos::from_u32(cookie as u32),
455 hi: BytePos::from_u32((cookie >> 32) as u32),
456 ctxt: SyntaxContext::root(),
457 parent: None,
458 }
459 };
460 let level = match level {
461 llvm::DiagnosticLevel::Error => Level::Error,
462 llvm::DiagnosticLevel::Warning => Level::Warning,
463 llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
464 };
465 let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
466 cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
467}
468
469unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
470 if user.is_null() {
471 return;
472 }
473 let (cgcx, dcx) =
474 unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
475
476 match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
477 llvm::diagnostic::InlineAsm(inline) => {
478 report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
479 }
480
481 llvm::diagnostic::Optimization(opt) => {
482 dcx.emit_note(FromLlvmOptimizationDiag {
483 filename: &opt.filename,
484 line: opt.line,
485 column: opt.column,
486 pass_name: &opt.pass_name,
487 kind: match opt.kind {
488 OptimizationRemark => "success",
489 OptimizationMissed | OptimizationFailure => "missed",
490 OptimizationAnalysis
491 | OptimizationAnalysisFPCommute
492 | OptimizationAnalysisAliasing => "analysis",
493 OptimizationRemarkOther => "other",
494 },
495 message: &opt.message,
496 });
497 }
498 llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
499 let message = llvm::build_string(|s| unsafe {
500 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
501 })
502 .expect("non-UTF8 diagnostic");
503 dcx.emit_warn(FromLlvmDiag { message });
504 }
505 llvm::diagnostic::Unsupported(diagnostic_ref) => {
506 let message = llvm::build_string(|s| unsafe {
507 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
508 })
509 .expect("non-UTF8 diagnostic");
510 dcx.emit_err(FromLlvmDiag { message });
511 }
512 llvm::diagnostic::UnknownDiagnostic(..) => {}
513 }
514}
515
516fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
517 match config.pgo_gen {
518 SwitchWithOptPath::Enabled(ref opt_dir_path) => {
519 let path = if let Some(dir_path) = opt_dir_path {
520 dir_path.join("default_%m.profraw")
521 } else {
522 PathBuf::from("default_%m.profraw")
523 };
524
525 Some(CString::new(format!("{}", path.display())).unwrap())
526 }
527 SwitchWithOptPath::Disabled => None,
528 }
529}
530
531fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
532 config
533 .pgo_use
534 .as_ref()
535 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
536}
537
538fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
539 config
540 .pgo_sample_use
541 .as_ref()
542 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
543}
544
545fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
546 config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
547}
548
549#[derive(Debug, Eq, PartialEq)]
553pub(crate) enum AutodiffStage {
554 PreAD,
555 DuringAD,
556 PostAD,
557}
558
559pub(crate) unsafe fn llvm_optimize(
560 cgcx: &CodegenContext<LlvmCodegenBackend>,
561 dcx: DiagCtxtHandle<'_>,
562 module: &ModuleCodegen<ModuleLlvm>,
563 thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
564 config: &ModuleConfig,
565 opt_level: config::OptLevel,
566 opt_stage: llvm::OptStage,
567 autodiff_stage: AutodiffStage,
568) -> Result<(), FatalError> {
569 let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
578 let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
579 let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
580 let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
581 let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
582 let merge_functions;
583 let unroll_loops;
584 let vectorize_slp;
585 let vectorize_loop;
586
587 if consider_ad && autodiff_stage != AutodiffStage::PostAD {
596 merge_functions = false;
597 unroll_loops = false;
598 vectorize_slp = false;
599 vectorize_loop = false;
600 } else {
601 unroll_loops =
602 opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
603 merge_functions = config.merge_functions;
604 vectorize_slp = config.vectorize_slp;
605 vectorize_loop = config.vectorize_loop;
606 }
607 trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
608 if thin_lto_buffer.is_some() {
609 assert!(
610 matches!(
611 opt_stage,
612 llvm::OptStage::PreLinkNoLTO
613 | llvm::OptStage::PreLinkFatLTO
614 | llvm::OptStage::PreLinkThinLTO
615 ),
616 "the bitcode for LTO can only be obtained at the pre-link stage"
617 );
618 }
619 let pgo_gen_path = get_pgo_gen_path(config);
620 let pgo_use_path = get_pgo_use_path(config);
621 let pgo_sample_use_path = get_pgo_sample_use_path(config);
622 let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
623 let instr_profile_output_path = get_instr_profile_output_path(config);
624 let sanitize_dataflow_abilist: Vec<_> = config
625 .sanitizer_dataflow_abilist
626 .iter()
627 .map(|file| CString::new(file.as_str()).unwrap())
628 .collect();
629 let sanitize_dataflow_abilist_ptrs: Vec<_> =
630 sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
631 let sanitizer_options = if !is_lto {
633 Some(llvm::SanitizerOptions {
634 sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
635 sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
636 sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
637 sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
638 sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
639 sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
640 sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
641 sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
642 sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
643 sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
644 sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
645 sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
646 sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
647 sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
648 sanitize_kernel_address_recover: config
649 .sanitizer_recover
650 .contains(SanitizerSet::KERNELADDRESS),
651 })
652 } else {
653 None
654 };
655
656 let mut llvm_profiler = cgcx
657 .prof
658 .llvm_recording_enabled()
659 .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
660
661 let llvm_selfprofiler =
662 llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
663
664 let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
665
666 let llvm_plugins = config.llvm_plugins.join(",");
667
668 let result = unsafe {
669 llvm::LLVMRustOptimize(
670 module.module_llvm.llmod(),
671 &*module.module_llvm.tm.raw(),
672 to_pass_builder_opt_level(opt_level),
673 opt_stage,
674 cgcx.opts.cg.linker_plugin_lto.enabled(),
675 config.no_prepopulate_passes,
676 config.verify_llvm_ir,
677 config.lint_llvm_ir,
678 thin_lto_buffer,
679 config.emit_thin_lto,
680 config.emit_thin_lto_summary,
681 merge_functions,
682 unroll_loops,
683 vectorize_slp,
684 vectorize_loop,
685 config.no_builtins,
686 config.emit_lifetime_markers,
687 run_enzyme,
688 print_before_enzyme,
689 print_after_enzyme,
690 print_passes,
691 sanitizer_options.as_ref(),
692 pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
693 pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
694 config.instrument_coverage,
695 instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
696 pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
697 config.debug_info_for_profiling,
698 llvm_selfprofiler,
699 selfprofile_before_pass_callback,
700 selfprofile_after_pass_callback,
701 extra_passes.as_c_char_ptr(),
702 extra_passes.len(),
703 llvm_plugins.as_c_char_ptr(),
704 llvm_plugins.len(),
705 )
706 };
707 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
708}
709
710pub(crate) fn optimize(
712 cgcx: &CodegenContext<LlvmCodegenBackend>,
713 dcx: DiagCtxtHandle<'_>,
714 module: &mut ModuleCodegen<ModuleLlvm>,
715 config: &ModuleConfig,
716) -> Result<(), FatalError> {
717 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
718
719 let llcx = &*module.module_llvm.llcx;
720 let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
721
722 if config.emit_no_opt_bc {
723 let out = cgcx.output_filenames.temp_path_ext_for_cgu(
724 "no-opt.bc",
725 &module.name,
726 cgcx.invocation_temp.as_deref(),
727 );
728 write_bitcode_to_file(module, &out)
729 }
730
731 if let Some(opt_level) = config.opt_level {
734 let opt_stage = match cgcx.lto {
735 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
736 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
737 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
738 _ => llvm::OptStage::PreLinkNoLTO,
739 };
740
741 let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
744 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
745 let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
750 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
751 || config.emit_thin_lto_summary
752 {
753 Some(null_mut())
754 } else {
755 None
756 };
757 unsafe {
758 llvm_optimize(
759 cgcx,
760 dcx,
761 module,
762 thin_lto_buffer.as_mut(),
763 config,
764 opt_level,
765 opt_stage,
766 autodiff_stage,
767 )
768 }?;
769 if let Some(thin_lto_buffer) = thin_lto_buffer {
770 let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
771 module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
772 let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu(
773 OutputType::ThinLinkBitcode,
774 &module.name,
775 cgcx.invocation_temp.as_deref(),
776 );
777 if config.emit_thin_lto_summary
778 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
779 {
780 let summary_data = thin_lto_buffer.thin_link_data();
781 cgcx.prof.artifact_size(
782 "llvm_bitcode_summary",
783 thin_link_bitcode_filename.to_string_lossy(),
784 summary_data.len() as u64,
785 );
786 let _timer = cgcx.prof.generic_activity_with_arg(
787 "LLVM_module_codegen_emit_bitcode_summary",
788 &*module.name,
789 );
790 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
791 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
792 }
793 }
794 }
795 }
796 Ok(())
797}
798
799pub(crate) fn codegen(
800 cgcx: &CodegenContext<LlvmCodegenBackend>,
801 module: ModuleCodegen<ModuleLlvm>,
802 config: &ModuleConfig,
803) -> Result<CompiledModule, FatalError> {
804 let dcx = cgcx.create_dcx();
805 let dcx = dcx.handle();
806
807 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
808 {
809 let llmod = module.module_llvm.llmod();
810 let llcx = &*module.module_llvm.llcx;
811 let tm = &*module.module_llvm.tm;
812 let _handlers =
813 DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
814
815 if cgcx.msvc_imps_needed {
816 create_msvc_imps(cgcx, llcx, llmod);
817 }
818
819 let bc_out = cgcx.output_filenames.temp_path_for_cgu(
824 OutputType::Bitcode,
825 &module.name,
826 cgcx.invocation_temp.as_deref(),
827 );
828 let obj_out = cgcx.output_filenames.temp_path_for_cgu(
829 OutputType::Object,
830 &module.name,
831 cgcx.invocation_temp.as_deref(),
832 );
833
834 if config.bitcode_needed() {
835 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
836 let thin = {
837 let _timer = cgcx.prof.generic_activity_with_arg(
838 "LLVM_module_codegen_make_bitcode",
839 &*module.name,
840 );
841 ThinBuffer::new(llmod, config.emit_thin_lto, false)
842 };
843 let data = thin.data();
844 let _timer = cgcx
845 .prof
846 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
847 if let Some(bitcode_filename) = bc_out.file_name() {
848 cgcx.prof.artifact_size(
849 "llvm_bitcode",
850 bitcode_filename.to_string_lossy(),
851 data.len() as u64,
852 );
853 }
854 if let Err(err) = fs::write(&bc_out, data) {
855 dcx.emit_err(WriteBytecode { path: &bc_out, err });
856 }
857 }
858
859 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
860 let _timer = cgcx
861 .prof
862 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
863 let thin_bc =
864 module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
865 embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc);
866 }
867 }
868
869 if config.emit_ir {
870 let _timer =
871 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
872 let out = cgcx.output_filenames.temp_path_for_cgu(
873 OutputType::LlvmAssembly,
874 &module.name,
875 cgcx.invocation_temp.as_deref(),
876 );
877 let out_c = path_to_c_string(&out);
878
879 extern "C" fn demangle_callback(
880 input_ptr: *const c_char,
881 input_len: size_t,
882 output_ptr: *mut c_char,
883 output_len: size_t,
884 ) -> size_t {
885 let input =
886 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
887
888 let Ok(input) = str::from_utf8(input) else { return 0 };
889
890 let output = unsafe {
891 slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
892 };
893 let mut cursor = io::Cursor::new(output);
894
895 let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
896
897 if write!(cursor, "{demangled:#}").is_err() {
898 return 0;
900 }
901
902 cursor.position() as size_t
903 }
904
905 let result =
906 unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
907
908 if result == llvm::LLVMRustResult::Success {
909 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
910 }
911
912 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?;
913 }
914
915 if config.emit_asm {
916 let _timer =
917 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
918 let path = cgcx.output_filenames.temp_path_for_cgu(
919 OutputType::Assembly,
920 &module.name,
921 cgcx.invocation_temp.as_deref(),
922 );
923
924 let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
929 llvm::LLVMCloneModule(llmod)
930 } else {
931 llmod
932 };
933 write_output_file(
934 dcx,
935 tm.raw(),
936 config.no_builtins,
937 llmod,
938 &path,
939 None,
940 llvm::FileType::AssemblyFile,
941 &cgcx.prof,
942 config.verify_llvm_ir,
943 )?;
944 }
945
946 match config.emit_obj {
947 EmitObj::ObjectCode(_) => {
948 let _timer = cgcx
949 .prof
950 .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
951
952 let dwo_out = cgcx
953 .output_filenames
954 .temp_path_dwo_for_cgu(&module.name, cgcx.invocation_temp.as_deref());
955 let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
956 (SplitDebuginfo::Off, _) => None,
958 _ if !cgcx.target_can_use_split_dwarf => None,
961 (_, SplitDwarfKind::Single) => None,
964 (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
967 };
968
969 write_output_file(
970 dcx,
971 tm.raw(),
972 config.no_builtins,
973 llmod,
974 &obj_out,
975 dwo_out,
976 llvm::FileType::ObjectFile,
977 &cgcx.prof,
978 config.verify_llvm_ir,
979 )?;
980 }
981
982 EmitObj::Bitcode => {
983 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
984 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
985 dcx.emit_err(CopyBitcode { err });
986 }
987
988 if !config.emit_bc {
989 debug!("removing_bitcode {:?}", bc_out);
990 ensure_removed(dcx, &bc_out);
991 }
992 }
993
994 EmitObj::None => {}
995 }
996
997 record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
998 }
999
1000 let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
1009 && cgcx.target_can_use_split_dwarf
1010 && cgcx.split_debuginfo != SplitDebuginfo::Off
1011 && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1012 Ok(module.into_compiled_module(
1013 config.emit_obj != EmitObj::None,
1014 dwarf_object_emitted,
1015 config.emit_bc,
1016 config.emit_asm,
1017 config.emit_ir,
1018 &cgcx.output_filenames,
1019 cgcx.invocation_temp.as_deref(),
1020 ))
1021}
1022
1023fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1024 let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1025 asm.extend_from_slice(b".ascii \"");
1026 asm.reserve(data.len());
1027 for &byte in data {
1028 if byte == b'\\' || byte == b'"' {
1029 asm.push(b'\\');
1030 asm.push(byte);
1031 } else if byte < 0x20 || byte >= 0x80 {
1032 asm.push(b'\\');
1035 asm.push(b'0' + ((byte >> 6) & 0x7));
1036 asm.push(b'0' + ((byte >> 3) & 0x7));
1037 asm.push(b'0' + ((byte >> 0) & 0x7));
1038 } else {
1039 asm.push(byte);
1040 }
1041 }
1042 asm.extend_from_slice(b"\"\n");
1043 asm
1044}
1045
1046pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1047 if cgcx.target_is_like_darwin {
1048 c"__LLVM,__bitcode"
1049 } else if cgcx.target_is_like_aix {
1050 c".ipa"
1051 } else {
1052 c".llvmbc"
1053 }
1054}
1055
1056fn embed_bitcode(
1058 cgcx: &CodegenContext<LlvmCodegenBackend>,
1059 llcx: &llvm::Context,
1060 llmod: &llvm::Module,
1061 cmdline: &str,
1062 bitcode: &[u8],
1063) {
1064 if cgcx.target_is_like_darwin
1101 || cgcx.target_is_like_aix
1102 || cgcx.target_arch == "wasm32"
1103 || cgcx.target_arch == "wasm64"
1104 {
1105 let llconst = common::bytes_in_context(llcx, bitcode);
1107 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1108 llvm::set_initializer(llglobal, llconst);
1109
1110 llvm::set_section(llglobal, bitcode_section_name(cgcx));
1111 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1112 llvm::LLVMSetGlobalConstant(llglobal, llvm::True);
1113
1114 let llconst = common::bytes_in_context(llcx, cmdline.as_bytes());
1115 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1116 llvm::set_initializer(llglobal, llconst);
1117 let section = if cgcx.target_is_like_darwin {
1118 c"__LLVM,__cmdline"
1119 } else if cgcx.target_is_like_aix {
1120 c".info"
1121 } else {
1122 c".llvmcmd"
1123 };
1124 llvm::set_section(llglobal, section);
1125 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1126 } else {
1127 let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1129 let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1130 llvm::append_module_inline_asm(llmod, &asm);
1131 let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes());
1132 llvm::append_module_inline_asm(llmod, &asm);
1133 }
1134}
1135
1136fn create_msvc_imps(
1142 cgcx: &CodegenContext<LlvmCodegenBackend>,
1143 llcx: &llvm::Context,
1144 llmod: &llvm::Module,
1145) {
1146 if !cgcx.msvc_imps_needed {
1147 return;
1148 }
1149 let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1154
1155 let ptr_ty = Type::ptr_llcx(llcx);
1156 let globals = base::iter_globals(llmod)
1157 .filter(|&val| {
1158 llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1159 })
1160 .filter_map(|val| {
1161 let name = llvm::get_value_name(val);
1163 if ignored(&name) { None } else { Some((val, name)) }
1164 })
1165 .map(move |(val, name)| {
1166 let mut imp_name = prefix.as_bytes().to_vec();
1167 imp_name.extend(name);
1168 let imp_name = CString::new(imp_name).unwrap();
1169 (imp_name, val)
1170 })
1171 .collect::<Vec<_>>();
1172
1173 for (imp_name, val) in globals {
1174 let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1175
1176 llvm::set_initializer(imp, val);
1177 llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1178 }
1179
1180 fn ignored(symbol_name: &[u8]) -> bool {
1182 symbol_name.starts_with(b"__llvm_profile_")
1184 }
1185}
1186
1187fn record_artifact_size(
1188 self_profiler_ref: &SelfProfilerRef,
1189 artifact_kind: &'static str,
1190 path: &Path,
1191) {
1192 if !self_profiler_ref.enabled() {
1194 return;
1195 }
1196
1197 if let Some(artifact_name) = path.file_name() {
1198 let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1199 self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1200 }
1201}
1202
1203fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1204 if !prof.enabled() {
1205 return;
1206 }
1207
1208 let raw_stats =
1209 llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1210 .expect("cannot get module instruction stats");
1211
1212 #[derive(serde::Deserialize)]
1213 struct InstructionsStats {
1214 module: String,
1215 total: u64,
1216 }
1217
1218 let InstructionsStats { module, total } =
1219 serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1220 prof.artifact_size("cgu_instructions", module, total);
1221}