1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10 LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15 BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16 TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::traits::*;
19use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
20use rustc_data_structures::profiling::SelfProfilerRef;
21use rustc_data_structures::small_c_str::SmallCStr;
22use rustc_errors::{DiagCtxtHandle, FatalError, Level};
23use rustc_fs_util::{link_or_copy, path_to_c_string};
24use rustc_middle::ty::TyCtxt;
25use rustc_session::Session;
26use rustc_session::config::{
27 self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
28};
29use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
30use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
31use tracing::{debug, trace};
32
33use crate::back::lto::ThinBuffer;
34use crate::back::owned_target_machine::OwnedTargetMachine;
35use crate::back::profiling::{
36 LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
37};
38use crate::common::AsCCharPtr;
39use crate::errors::{
40 CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
41 WithLlvmError, WriteBytecode,
42};
43use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
44use crate::llvm::{self, DiagnosticInfo};
45use crate::type_::Type;
46use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
47
48pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError {
49 match llvm::last_error() {
50 Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)),
51 None => dcx.emit_almost_fatal(err),
52 }
53}
54
55fn write_output_file<'ll>(
56 dcx: DiagCtxtHandle<'_>,
57 target: &'ll llvm::TargetMachine,
58 no_builtins: bool,
59 m: &'ll llvm::Module,
60 output: &Path,
61 dwo_output: Option<&Path>,
62 file_type: llvm::FileType,
63 self_profiler_ref: &SelfProfilerRef,
64 verify_llvm_ir: bool,
65) -> Result<(), FatalError> {
66 debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
67 let output_c = path_to_c_string(output);
68 let dwo_output_c;
69 let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
70 dwo_output_c = path_to_c_string(dwo_output);
71 dwo_output_c.as_ptr()
72 } else {
73 std::ptr::null()
74 };
75 let result = unsafe {
76 let pm = llvm::LLVMCreatePassManager();
77 llvm::LLVMAddAnalysisPasses(target, pm);
78 llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
79 llvm::LLVMRustWriteOutputFile(
80 target,
81 pm,
82 m,
83 output_c.as_ptr(),
84 dwo_output_ptr,
85 file_type,
86 verify_llvm_ir,
87 )
88 };
89
90 if result == llvm::LLVMRustResult::Success {
92 let artifact_kind = match file_type {
93 llvm::FileType::ObjectFile => "object_file",
94 llvm::FileType::AssemblyFile => "assembly_file",
95 };
96 record_artifact_size(self_profiler_ref, artifact_kind, output);
97 if let Some(dwo_file) = dwo_output {
98 record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
99 }
100 }
101
102 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
103}
104
105pub(crate) fn create_informational_target_machine(
106 sess: &Session,
107 only_base_features: bool,
108) -> OwnedTargetMachine {
109 let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
110 let features = llvm_util::global_llvm_features(sess, false, only_base_features);
113 target_machine_factory(sess, config::OptLevel::No, &features)(config)
114 .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise())
115}
116
117pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
118 let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
119 tcx.output_filenames(()).split_dwarf_path(
120 tcx.sess.split_debuginfo(),
121 tcx.sess.opts.unstable_opts.split_dwarf_kind,
122 Some(mod_name),
123 )
124 } else {
125 None
126 };
127
128 let output_obj_file =
129 Some(tcx.output_filenames(()).temp_path(OutputType::Object, Some(mod_name)));
130 let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
131
132 target_machine_factory(
133 tcx.sess,
134 tcx.backend_optimization_level(()),
135 tcx.global_backend_features(()),
136 )(config)
137 .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise())
138}
139
140fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
141 use self::config::OptLevel::*;
142 match cfg {
143 No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
144 Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
145 More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
146 Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
147 Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
148 SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
149 }
150}
151
152fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
153 use config::OptLevel::*;
154 match cfg {
155 No => llvm::PassBuilderOptLevel::O0,
156 Less => llvm::PassBuilderOptLevel::O1,
157 More => llvm::PassBuilderOptLevel::O2,
158 Aggressive => llvm::PassBuilderOptLevel::O3,
159 Size => llvm::PassBuilderOptLevel::Os,
160 SizeMin => llvm::PassBuilderOptLevel::Oz,
161 }
162}
163
164fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
165 match relocation_model {
166 RelocModel::Static => llvm::RelocModel::Static,
167 RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
170 RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
171 RelocModel::Ropi => llvm::RelocModel::ROPI,
172 RelocModel::Rwpi => llvm::RelocModel::RWPI,
173 RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
174 }
175}
176
177pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
178 match code_model {
179 Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
180 Some(CodeModel::Small) => llvm::CodeModel::Small,
181 Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
182 Some(CodeModel::Medium) => llvm::CodeModel::Medium,
183 Some(CodeModel::Large) => llvm::CodeModel::Large,
184 None => llvm::CodeModel::None,
185 }
186}
187
188fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
189 match float_abi {
190 None => llvm::FloatAbi::Default,
191 Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
192 Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
193 }
194}
195
196pub(crate) fn target_machine_factory(
197 sess: &Session,
198 optlvl: config::OptLevel,
199 target_features: &[String],
200) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
201 let reloc_model = to_llvm_relocation_model(sess.relocation_model());
202
203 let (opt_level, _) = to_llvm_opt_settings(optlvl);
204 let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
205 llvm::FloatAbi::Soft
206 } else {
207 to_llvm_float_abi(sess.target.llvm_floatabi)
210 };
211
212 let ffunction_sections =
213 sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
214 let fdata_sections = ffunction_sections;
215 let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
216
217 let code_model = to_llvm_code_model(sess.code_model());
218
219 let mut singlethread = sess.target.singlethread;
220
221 if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
225 singlethread = false;
226 }
227
228 let triple = SmallCStr::new(&versioned_llvm_target(sess));
229 let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
230 let features = CString::new(target_features.join(",")).unwrap();
231 let abi = SmallCStr::new(&sess.target.llvm_abiname);
232 let trap_unreachable =
233 sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
234 let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
235
236 let verbose_asm = sess.opts.unstable_opts.verbose_asm;
237 let relax_elf_relocations =
238 sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
239
240 let use_init_array =
241 !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
242
243 let path_mapping = sess.source_map().path_mapping().clone();
244
245 let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
246
247 let args_cstr_buff = {
250 let mut args_cstr_buff: Vec<u8> = Vec::new();
251 let exe_path = std::env::current_exe().unwrap_or_default();
252 let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default();
253
254 args_cstr_buff.extend_from_slice(exe_path_str.as_bytes());
255 args_cstr_buff.push(0);
256
257 for arg in sess.expanded_args.iter() {
258 args_cstr_buff.extend_from_slice(arg.as_bytes());
259 args_cstr_buff.push(0);
260 }
261
262 args_cstr_buff
263 };
264
265 let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
266 match sess.opts.debuginfo_compression {
267 rustc_session::config::DebugInfoCompression::Zlib => {
268 if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
269 sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
270 }
271 }
272 rustc_session::config::DebugInfoCompression::Zstd => {
273 if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
274 sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
275 }
276 }
277 rustc_session::config::DebugInfoCompression::None => {}
278 };
279 let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
280
281 let file_name_display_preference =
282 sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
283
284 Arc::new(move |config: TargetMachineFactoryConfig| {
285 let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
286 let path = path.unwrap_or_default();
287 let path = path_mapping
288 .to_real_filename(path)
289 .to_string_lossy(file_name_display_preference)
290 .into_owned();
291 CString::new(path).unwrap()
292 };
293
294 let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
295 let output_obj_file = path_to_cstring_helper(config.output_obj_file);
296
297 OwnedTargetMachine::new(
298 &triple,
299 &cpu,
300 &features,
301 &abi,
302 code_model,
303 reloc_model,
304 opt_level,
305 float_abi,
306 ffunction_sections,
307 fdata_sections,
308 funique_section_names,
309 trap_unreachable,
310 singlethread,
311 verbose_asm,
312 emit_stack_size_section,
313 relax_elf_relocations,
314 use_init_array,
315 &split_dwarf_file,
316 &output_obj_file,
317 &debuginfo_compression,
318 use_emulated_tls,
319 &args_cstr_buff,
320 )
321 })
322}
323
324pub(crate) fn save_temp_bitcode(
325 cgcx: &CodegenContext<LlvmCodegenBackend>,
326 module: &ModuleCodegen<ModuleLlvm>,
327 name: &str,
328) {
329 if !cgcx.save_temps {
330 return;
331 }
332 let ext = format!("{name}.bc");
333 let cgu = Some(&module.name[..]);
334 let path = cgcx.output_filenames.temp_path_ext(&ext, cgu);
335 write_bitcode_to_file(module, &path)
336}
337
338fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
339 unsafe {
340 let path = path_to_c_string(&path);
341 let llmod = module.module_llvm.llmod();
342 llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
343 }
344}
345
346pub(crate) enum CodegenDiagnosticsStage {
348 Opt,
350 LTO,
352 Codegen,
354}
355
356pub(crate) struct DiagnosticHandlers<'a> {
357 data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
358 llcx: &'a llvm::Context,
359 old_handler: Option<&'a llvm::DiagnosticHandler>,
360}
361
362impl<'a> DiagnosticHandlers<'a> {
363 pub(crate) fn new(
364 cgcx: &'a CodegenContext<LlvmCodegenBackend>,
365 dcx: DiagCtxtHandle<'a>,
366 llcx: &'a llvm::Context,
367 module: &ModuleCodegen<ModuleLlvm>,
368 stage: CodegenDiagnosticsStage,
369 ) -> Self {
370 let remark_passes_all: bool;
371 let remark_passes: Vec<CString>;
372 match &cgcx.remark {
373 Passes::All => {
374 remark_passes_all = true;
375 remark_passes = Vec::new();
376 }
377 Passes::Some(passes) => {
378 remark_passes_all = false;
379 remark_passes =
380 passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
381 }
382 };
383 let remark_passes: Vec<*const c_char> =
384 remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
385 let remark_file = cgcx
386 .remark_dir
387 .as_ref()
388 .map(|dir| {
390 let stage_suffix = match stage {
391 CodegenDiagnosticsStage::Codegen => "codegen",
392 CodegenDiagnosticsStage::Opt => "opt",
393 CodegenDiagnosticsStage::LTO => "lto",
394 };
395 dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
396 })
397 .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
398
399 let pgo_available = cgcx.opts.cg.profile_use.is_some();
400 let data = Box::into_raw(Box::new((cgcx, dcx)));
401 unsafe {
402 let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
403 llvm::LLVMRustContextConfigureDiagnosticHandler(
404 llcx,
405 diagnostic_handler,
406 data.cast(),
407 remark_passes_all,
408 remark_passes.as_ptr(),
409 remark_passes.len(),
410 remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
413 pgo_available,
414 );
415 DiagnosticHandlers { data, llcx, old_handler }
416 }
417 }
418}
419
420impl<'a> Drop for DiagnosticHandlers<'a> {
421 fn drop(&mut self) {
422 unsafe {
423 llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
424 drop(Box::from_raw(self.data));
425 }
426 }
427}
428
429fn report_inline_asm(
430 cgcx: &CodegenContext<LlvmCodegenBackend>,
431 msg: String,
432 level: llvm::DiagnosticLevel,
433 cookie: u64,
434 source: Option<(String, Vec<InnerSpan>)>,
435) {
436 let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
440 SpanData::default()
441 } else {
442 let lo = BytePos::from_u32(cookie as u32);
443 let hi = BytePos::from_u32((cookie >> 32) as u32);
444 SpanData {
445 lo,
446 hi: if hi.to_u32() != 0 { hi } else { lo },
448 ctxt: SyntaxContext::root(),
449 parent: None,
450 }
451 };
452 let level = match level {
453 llvm::DiagnosticLevel::Error => Level::Error,
454 llvm::DiagnosticLevel::Warning => Level::Warning,
455 llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
456 };
457 let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
458 cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
459}
460
461unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
462 if user.is_null() {
463 return;
464 }
465 let (cgcx, dcx) =
466 unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
467
468 match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
469 llvm::diagnostic::InlineAsm(inline) => {
470 report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
471 }
472
473 llvm::diagnostic::Optimization(opt) => {
474 dcx.emit_note(FromLlvmOptimizationDiag {
475 filename: &opt.filename,
476 line: opt.line,
477 column: opt.column,
478 pass_name: &opt.pass_name,
479 kind: match opt.kind {
480 OptimizationRemark => "success",
481 OptimizationMissed | OptimizationFailure => "missed",
482 OptimizationAnalysis
483 | OptimizationAnalysisFPCommute
484 | OptimizationAnalysisAliasing => "analysis",
485 OptimizationRemarkOther => "other",
486 },
487 message: &opt.message,
488 });
489 }
490 llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
491 let message = llvm::build_string(|s| unsafe {
492 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
493 })
494 .expect("non-UTF8 diagnostic");
495 dcx.emit_warn(FromLlvmDiag { message });
496 }
497 llvm::diagnostic::Unsupported(diagnostic_ref) => {
498 let message = llvm::build_string(|s| unsafe {
499 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
500 })
501 .expect("non-UTF8 diagnostic");
502 dcx.emit_err(FromLlvmDiag { message });
503 }
504 llvm::diagnostic::UnknownDiagnostic(..) => {}
505 }
506}
507
508fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
509 match config.pgo_gen {
510 SwitchWithOptPath::Enabled(ref opt_dir_path) => {
511 let path = if let Some(dir_path) = opt_dir_path {
512 dir_path.join("default_%m.profraw")
513 } else {
514 PathBuf::from("default_%m.profraw")
515 };
516
517 Some(CString::new(format!("{}", path.display())).unwrap())
518 }
519 SwitchWithOptPath::Disabled => None,
520 }
521}
522
523fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
524 config
525 .pgo_use
526 .as_ref()
527 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
528}
529
530fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
531 config
532 .pgo_sample_use
533 .as_ref()
534 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
535}
536
537fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
538 config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
539}
540
541#[derive(Debug, Eq, PartialEq)]
545pub(crate) enum AutodiffStage {
546 PreAD,
547 DuringAD,
548 PostAD,
549}
550
551pub(crate) unsafe fn llvm_optimize(
552 cgcx: &CodegenContext<LlvmCodegenBackend>,
553 dcx: DiagCtxtHandle<'_>,
554 module: &ModuleCodegen<ModuleLlvm>,
555 thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
556 config: &ModuleConfig,
557 opt_level: config::OptLevel,
558 opt_stage: llvm::OptStage,
559 autodiff_stage: AutodiffStage,
560) -> Result<(), FatalError> {
561 let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
570 let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
571 let unroll_loops;
572 let vectorize_slp;
573 let vectorize_loop;
574
575 if consider_ad && autodiff_stage != AutodiffStage::PostAD {
579 unroll_loops = false;
580 vectorize_slp = false;
581 vectorize_loop = false;
582 } else {
583 unroll_loops =
584 opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
585 vectorize_slp = config.vectorize_slp;
586 vectorize_loop = config.vectorize_loop;
587 }
588 trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
589 if thin_lto_buffer.is_some() {
590 assert!(
591 matches!(
592 opt_stage,
593 llvm::OptStage::PreLinkNoLTO
594 | llvm::OptStage::PreLinkFatLTO
595 | llvm::OptStage::PreLinkThinLTO
596 ),
597 "the bitcode for LTO can only be obtained at the pre-link stage"
598 );
599 }
600 let pgo_gen_path = get_pgo_gen_path(config);
601 let pgo_use_path = get_pgo_use_path(config);
602 let pgo_sample_use_path = get_pgo_sample_use_path(config);
603 let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
604 let instr_profile_output_path = get_instr_profile_output_path(config);
605 let sanitize_dataflow_abilist: Vec<_> = config
606 .sanitizer_dataflow_abilist
607 .iter()
608 .map(|file| CString::new(file.as_str()).unwrap())
609 .collect();
610 let sanitize_dataflow_abilist_ptrs: Vec<_> =
611 sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
612 let sanitizer_options = if !is_lto {
614 Some(llvm::SanitizerOptions {
615 sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
616 sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
617 sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
618 sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
619 sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
620 sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
621 sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
622 sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
623 sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
624 sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
625 sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
626 sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
627 sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
628 sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
629 sanitize_kernel_address_recover: config
630 .sanitizer_recover
631 .contains(SanitizerSet::KERNELADDRESS),
632 })
633 } else {
634 None
635 };
636
637 let mut llvm_profiler = cgcx
638 .prof
639 .llvm_recording_enabled()
640 .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
641
642 let llvm_selfprofiler =
643 llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
644
645 let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
646
647 let llvm_plugins = config.llvm_plugins.join(",");
648
649 let result = unsafe {
650 llvm::LLVMRustOptimize(
651 module.module_llvm.llmod(),
652 &*module.module_llvm.tm.raw(),
653 to_pass_builder_opt_level(opt_level),
654 opt_stage,
655 cgcx.opts.cg.linker_plugin_lto.enabled(),
656 config.no_prepopulate_passes,
657 config.verify_llvm_ir,
658 config.lint_llvm_ir,
659 thin_lto_buffer,
660 config.emit_thin_lto,
661 config.emit_thin_lto_summary,
662 config.merge_functions,
663 unroll_loops,
664 vectorize_slp,
665 vectorize_loop,
666 config.no_builtins,
667 config.emit_lifetime_markers,
668 run_enzyme,
669 sanitizer_options.as_ref(),
670 pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
671 pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
672 config.instrument_coverage,
673 instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
674 pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
675 config.debug_info_for_profiling,
676 llvm_selfprofiler,
677 selfprofile_before_pass_callback,
678 selfprofile_after_pass_callback,
679 extra_passes.as_c_char_ptr(),
680 extra_passes.len(),
681 llvm_plugins.as_c_char_ptr(),
682 llvm_plugins.len(),
683 )
684 };
685 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
686}
687
688pub(crate) unsafe fn optimize(
690 cgcx: &CodegenContext<LlvmCodegenBackend>,
691 dcx: DiagCtxtHandle<'_>,
692 module: &mut ModuleCodegen<ModuleLlvm>,
693 config: &ModuleConfig,
694) -> Result<(), FatalError> {
695 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
696
697 let llcx = &*module.module_llvm.llcx;
698 let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
699
700 let module_name = module.name.clone();
701 let module_name = Some(&module_name[..]);
702
703 if config.emit_no_opt_bc {
704 let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name);
705 write_bitcode_to_file(module, &out)
706 }
707
708 if let Some(opt_level) = config.opt_level {
711 let opt_stage = match cgcx.lto {
712 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
713 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
714 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
715 _ => llvm::OptStage::PreLinkNoLTO,
716 };
717
718 let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
721 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
722 let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
727 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
728 || config.emit_thin_lto_summary
729 {
730 Some(null_mut())
731 } else {
732 None
733 };
734 unsafe {
735 llvm_optimize(
736 cgcx,
737 dcx,
738 module,
739 thin_lto_buffer.as_mut(),
740 config,
741 opt_level,
742 opt_stage,
743 autodiff_stage,
744 )
745 }?;
746 if let Some(thin_lto_buffer) = thin_lto_buffer {
747 let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
748 module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
749 let bc_summary_out =
750 cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name);
751 if config.emit_thin_lto_summary
752 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
753 {
754 let summary_data = thin_lto_buffer.thin_link_data();
755 cgcx.prof.artifact_size(
756 "llvm_bitcode_summary",
757 thin_link_bitcode_filename.to_string_lossy(),
758 summary_data.len() as u64,
759 );
760 let _timer = cgcx.prof.generic_activity_with_arg(
761 "LLVM_module_codegen_emit_bitcode_summary",
762 &*module.name,
763 );
764 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
765 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
766 }
767 }
768 }
769 }
770 Ok(())
771}
772
773pub(crate) fn link(
774 cgcx: &CodegenContext<LlvmCodegenBackend>,
775 dcx: DiagCtxtHandle<'_>,
776 mut modules: Vec<ModuleCodegen<ModuleLlvm>>,
777) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
778 use super::lto::{Linker, ModuleBuffer};
779 modules.sort_by(|a, b| a.name.cmp(&b.name));
781 let (first, elements) =
782 modules.split_first().expect("Bug! modules must contain at least one module.");
783
784 let mut linker = Linker::new(first.module_llvm.llmod());
785 for module in elements {
786 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name);
787 let buffer = ModuleBuffer::new(module.module_llvm.llmod());
788 linker
789 .add(buffer.data())
790 .map_err(|()| llvm_err(dcx, LlvmError::SerializeModule { name: &module.name }))?;
791 }
792 drop(linker);
793 Ok(modules.remove(0))
794}
795
796pub(crate) unsafe fn codegen(
797 cgcx: &CodegenContext<LlvmCodegenBackend>,
798 dcx: DiagCtxtHandle<'_>,
799 module: ModuleCodegen<ModuleLlvm>,
800 config: &ModuleConfig,
801) -> Result<CompiledModule, FatalError> {
802 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
803 {
804 let llmod = module.module_llvm.llmod();
805 let llcx = &*module.module_llvm.llcx;
806 let tm = &*module.module_llvm.tm;
807 let module_name = module.name.clone();
808 let module_name = Some(&module_name[..]);
809 let _handlers =
810 DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
811
812 if cgcx.msvc_imps_needed {
813 create_msvc_imps(cgcx, llcx, llmod);
814 }
815
816 let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
821 let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name);
822
823 if config.bitcode_needed() {
824 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
825 let thin = {
826 let _timer = cgcx.prof.generic_activity_with_arg(
827 "LLVM_module_codegen_make_bitcode",
828 &*module.name,
829 );
830 ThinBuffer::new(llmod, config.emit_thin_lto, false)
831 };
832 let data = thin.data();
833 let _timer = cgcx
834 .prof
835 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
836 if let Some(bitcode_filename) = bc_out.file_name() {
837 cgcx.prof.artifact_size(
838 "llvm_bitcode",
839 bitcode_filename.to_string_lossy(),
840 data.len() as u64,
841 );
842 }
843 if let Err(err) = fs::write(&bc_out, data) {
844 dcx.emit_err(WriteBytecode { path: &bc_out, err });
845 }
846 }
847
848 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
849 let _timer = cgcx
850 .prof
851 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
852 let thin_bc =
853 module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
854 unsafe {
855 embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc);
856 }
857 }
858 }
859
860 if config.emit_ir {
861 let _timer =
862 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
863 let out = cgcx.output_filenames.temp_path(OutputType::LlvmAssembly, module_name);
864 let out_c = path_to_c_string(&out);
865
866 extern "C" fn demangle_callback(
867 input_ptr: *const c_char,
868 input_len: size_t,
869 output_ptr: *mut c_char,
870 output_len: size_t,
871 ) -> size_t {
872 let input =
873 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
874
875 let Ok(input) = str::from_utf8(input) else { return 0 };
876
877 let output = unsafe {
878 slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
879 };
880 let mut cursor = io::Cursor::new(output);
881
882 let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
883
884 if write!(cursor, "{demangled:#}").is_err() {
885 return 0;
887 }
888
889 cursor.position() as size_t
890 }
891
892 let result =
893 unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
894
895 if result == llvm::LLVMRustResult::Success {
896 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
897 }
898
899 result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?;
900 }
901
902 if config.emit_asm {
903 let _timer =
904 cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
905 let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name);
906
907 let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
912 unsafe { llvm::LLVMCloneModule(llmod) }
913 } else {
914 llmod
915 };
916 write_output_file(
917 dcx,
918 tm.raw(),
919 config.no_builtins,
920 llmod,
921 &path,
922 None,
923 llvm::FileType::AssemblyFile,
924 &cgcx.prof,
925 config.verify_llvm_ir,
926 )?;
927 }
928
929 match config.emit_obj {
930 EmitObj::ObjectCode(_) => {
931 let _timer = cgcx
932 .prof
933 .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
934
935 let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name);
936 let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
937 (SplitDebuginfo::Off, _) => None,
939 _ if !cgcx.target_can_use_split_dwarf => None,
942 (_, SplitDwarfKind::Single) => None,
945 (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
948 };
949
950 write_output_file(
951 dcx,
952 tm.raw(),
953 config.no_builtins,
954 llmod,
955 &obj_out,
956 dwo_out,
957 llvm::FileType::ObjectFile,
958 &cgcx.prof,
959 config.verify_llvm_ir,
960 )?;
961 }
962
963 EmitObj::Bitcode => {
964 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
965 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
966 dcx.emit_err(CopyBitcode { err });
967 }
968
969 if !config.emit_bc {
970 debug!("removing_bitcode {:?}", bc_out);
971 ensure_removed(dcx, &bc_out);
972 }
973 }
974
975 EmitObj::None => {}
976 }
977
978 record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
979 }
980
981 let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
990 && cgcx.target_can_use_split_dwarf
991 && cgcx.split_debuginfo != SplitDebuginfo::Off
992 && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
993 Ok(module.into_compiled_module(
994 config.emit_obj != EmitObj::None,
995 dwarf_object_emitted,
996 config.emit_bc,
997 config.emit_asm,
998 config.emit_ir,
999 &cgcx.output_filenames,
1000 ))
1001}
1002
1003fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1004 let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1005 asm.extend_from_slice(b".ascii \"");
1006 asm.reserve(data.len());
1007 for &byte in data {
1008 if byte == b'\\' || byte == b'"' {
1009 asm.push(b'\\');
1010 asm.push(byte);
1011 } else if byte < 0x20 || byte >= 0x80 {
1012 asm.push(b'\\');
1015 asm.push(b'0' + ((byte >> 6) & 0x7));
1016 asm.push(b'0' + ((byte >> 3) & 0x7));
1017 asm.push(b'0' + ((byte >> 0) & 0x7));
1018 } else {
1019 asm.push(byte);
1020 }
1021 }
1022 asm.extend_from_slice(b"\"\n");
1023 asm
1024}
1025
1026pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1027 if cgcx.target_is_like_osx {
1028 c"__LLVM,__bitcode"
1029 } else if cgcx.target_is_like_aix {
1030 c".ipa"
1031 } else {
1032 c".llvmbc"
1033 }
1034}
1035
1036unsafe fn embed_bitcode(
1038 cgcx: &CodegenContext<LlvmCodegenBackend>,
1039 llcx: &llvm::Context,
1040 llmod: &llvm::Module,
1041 cmdline: &str,
1042 bitcode: &[u8],
1043) {
1044 unsafe {
1080 if cgcx.target_is_like_osx
1081 || cgcx.target_is_like_aix
1082 || cgcx.target_arch == "wasm32"
1083 || cgcx.target_arch == "wasm64"
1084 {
1085 let llconst = common::bytes_in_context(llcx, bitcode);
1087 let llglobal =
1088 llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1089 llvm::set_initializer(llglobal, llconst);
1090
1091 llvm::set_section(llglobal, bitcode_section_name(cgcx));
1092 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1093 llvm::LLVMSetGlobalConstant(llglobal, llvm::True);
1094
1095 let llconst = common::bytes_in_context(llcx, cmdline.as_bytes());
1096 let llglobal =
1097 llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1098 llvm::set_initializer(llglobal, llconst);
1099 let section = if cgcx.target_is_like_osx {
1100 c"__LLVM,__cmdline"
1101 } else if cgcx.target_is_like_aix {
1102 c".info"
1103 } else {
1104 c".llvmcmd"
1105 };
1106 llvm::set_section(llglobal, section);
1107 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1108 } else {
1109 let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1111 let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1112 llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len());
1113 let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes());
1114 llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len());
1115 }
1116 }
1117}
1118
1119fn create_msvc_imps(
1125 cgcx: &CodegenContext<LlvmCodegenBackend>,
1126 llcx: &llvm::Context,
1127 llmod: &llvm::Module,
1128) {
1129 if !cgcx.msvc_imps_needed {
1130 return;
1131 }
1132 let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1137
1138 let ptr_ty = Type::ptr_llcx(llcx);
1139 let globals = base::iter_globals(llmod)
1140 .filter(|&val| {
1141 llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1142 })
1143 .filter_map(|val| {
1144 let name = llvm::get_value_name(val);
1146 if ignored(name) { None } else { Some((val, name)) }
1147 })
1148 .map(move |(val, name)| {
1149 let mut imp_name = prefix.as_bytes().to_vec();
1150 imp_name.extend(name);
1151 let imp_name = CString::new(imp_name).unwrap();
1152 (imp_name, val)
1153 })
1154 .collect::<Vec<_>>();
1155
1156 for (imp_name, val) in globals {
1157 let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1158
1159 llvm::set_initializer(imp, val);
1160 llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1161 }
1162
1163 fn ignored(symbol_name: &[u8]) -> bool {
1165 symbol_name.starts_with(b"__llvm_profile_")
1167 }
1168}
1169
1170fn record_artifact_size(
1171 self_profiler_ref: &SelfProfilerRef,
1172 artifact_kind: &'static str,
1173 path: &Path,
1174) {
1175 if !self_profiler_ref.enabled() {
1177 return;
1178 }
1179
1180 if let Some(artifact_name) = path.file_name() {
1181 let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1182 self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1183 }
1184}
1185
1186fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1187 if !prof.enabled() {
1188 return;
1189 }
1190
1191 let raw_stats =
1192 llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1193 .expect("cannot get module instruction stats");
1194
1195 #[derive(serde::Deserialize)]
1196 struct InstructionsStats {
1197 module: String,
1198 total: u64,
1199 }
1200
1201 let InstructionsStats { module, total } =
1202 serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1203 prof.artifact_size("cgu_instructions", module, total);
1204}