rustc_codegen_llvm/back/
write.rs

1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10    LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15    BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16    TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::base::wants_wasm_eh;
19use rustc_codegen_ssa::traits::*;
20use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
21use rustc_data_structures::profiling::SelfProfilerRef;
22use rustc_data_structures::small_c_str::SmallCStr;
23use rustc_errors::{DiagCtxtHandle, FatalError, Level};
24use rustc_fs_util::{link_or_copy, path_to_c_string};
25use rustc_middle::ty::TyCtxt;
26use rustc_session::Session;
27use rustc_session::config::{
28    self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
29};
30use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
31use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
32use tracing::{debug, trace};
33
34use crate::back::lto::ThinBuffer;
35use crate::back::owned_target_machine::OwnedTargetMachine;
36use crate::back::profiling::{
37    LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
38};
39use crate::common::AsCCharPtr;
40use crate::errors::{
41    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
42    WithLlvmError, WriteBytecode,
43};
44use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
45use crate::llvm::{self, DiagnosticInfo};
46use crate::type_::Type;
47use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
48
49pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError {
50    match llvm::last_error() {
51        Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)),
52        None => dcx.emit_almost_fatal(err),
53    }
54}
55
56fn write_output_file<'ll>(
57    dcx: DiagCtxtHandle<'_>,
58    target: &'ll llvm::TargetMachine,
59    no_builtins: bool,
60    m: &'ll llvm::Module,
61    output: &Path,
62    dwo_output: Option<&Path>,
63    file_type: llvm::FileType,
64    self_profiler_ref: &SelfProfilerRef,
65    verify_llvm_ir: bool,
66) -> Result<(), FatalError> {
67    debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
68    let output_c = path_to_c_string(output);
69    let dwo_output_c;
70    let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
71        dwo_output_c = path_to_c_string(dwo_output);
72        dwo_output_c.as_ptr()
73    } else {
74        std::ptr::null()
75    };
76    let result = unsafe {
77        let pm = llvm::LLVMCreatePassManager();
78        llvm::LLVMAddAnalysisPasses(target, pm);
79        llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
80        llvm::LLVMRustWriteOutputFile(
81            target,
82            pm,
83            m,
84            output_c.as_ptr(),
85            dwo_output_ptr,
86            file_type,
87            verify_llvm_ir,
88        )
89    };
90
91    // Record artifact sizes for self-profiling
92    if result == llvm::LLVMRustResult::Success {
93        let artifact_kind = match file_type {
94            llvm::FileType::ObjectFile => "object_file",
95            llvm::FileType::AssemblyFile => "assembly_file",
96        };
97        record_artifact_size(self_profiler_ref, artifact_kind, output);
98        if let Some(dwo_file) = dwo_output {
99            record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
100        }
101    }
102
103    result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
104}
105
106pub(crate) fn create_informational_target_machine(
107    sess: &Session,
108    only_base_features: bool,
109) -> OwnedTargetMachine {
110    let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
111    // Can't use query system here quite yet because this function is invoked before the query
112    // system/tcx is set up.
113    let features = llvm_util::global_llvm_features(sess, false, only_base_features);
114    target_machine_factory(sess, config::OptLevel::No, &features)(config)
115        .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise())
116}
117
118pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
119    let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
120        tcx.output_filenames(()).split_dwarf_path(
121            tcx.sess.split_debuginfo(),
122            tcx.sess.opts.unstable_opts.split_dwarf_kind,
123            mod_name,
124            tcx.sess.invocation_temp.as_deref(),
125        )
126    } else {
127        None
128    };
129
130    let output_obj_file = Some(tcx.output_filenames(()).temp_path_for_cgu(
131        OutputType::Object,
132        mod_name,
133        tcx.sess.invocation_temp.as_deref(),
134    ));
135    let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
136
137    target_machine_factory(
138        tcx.sess,
139        tcx.backend_optimization_level(()),
140        tcx.global_backend_features(()),
141    )(config)
142    .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise())
143}
144
145fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
146    use self::config::OptLevel::*;
147    match cfg {
148        No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
149        Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
150        More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
151        Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
152        Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
153        SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
154    }
155}
156
157fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
158    use config::OptLevel::*;
159    match cfg {
160        No => llvm::PassBuilderOptLevel::O0,
161        Less => llvm::PassBuilderOptLevel::O1,
162        More => llvm::PassBuilderOptLevel::O2,
163        Aggressive => llvm::PassBuilderOptLevel::O3,
164        Size => llvm::PassBuilderOptLevel::Os,
165        SizeMin => llvm::PassBuilderOptLevel::Oz,
166    }
167}
168
169fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
170    match relocation_model {
171        RelocModel::Static => llvm::RelocModel::Static,
172        // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra
173        // attribute.
174        RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
175        RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
176        RelocModel::Ropi => llvm::RelocModel::ROPI,
177        RelocModel::Rwpi => llvm::RelocModel::RWPI,
178        RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
179    }
180}
181
182pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
183    match code_model {
184        Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
185        Some(CodeModel::Small) => llvm::CodeModel::Small,
186        Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
187        Some(CodeModel::Medium) => llvm::CodeModel::Medium,
188        Some(CodeModel::Large) => llvm::CodeModel::Large,
189        None => llvm::CodeModel::None,
190    }
191}
192
193fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
194    match float_abi {
195        None => llvm::FloatAbi::Default,
196        Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
197        Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
198    }
199}
200
201pub(crate) fn target_machine_factory(
202    sess: &Session,
203    optlvl: config::OptLevel,
204    target_features: &[String],
205) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
206    let reloc_model = to_llvm_relocation_model(sess.relocation_model());
207
208    let (opt_level, _) = to_llvm_opt_settings(optlvl);
209    let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
210        llvm::FloatAbi::Soft
211    } else {
212        // `validate_commandline_args_with_session_available` has already warned about this being
213        // ignored. Let's make sure LLVM doesn't suddenly start using this flag on more targets.
214        to_llvm_float_abi(sess.target.llvm_floatabi)
215    };
216
217    let ffunction_sections =
218        sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
219    let fdata_sections = ffunction_sections;
220    let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
221
222    let code_model = to_llvm_code_model(sess.code_model());
223
224    let mut singlethread = sess.target.singlethread;
225
226    // On the wasm target once the `atomics` feature is enabled that means that
227    // we're no longer single-threaded, or otherwise we don't want LLVM to
228    // lower atomic operations to single-threaded operations.
229    if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
230        singlethread = false;
231    }
232
233    let triple = SmallCStr::new(&versioned_llvm_target(sess));
234    let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
235    let features = CString::new(target_features.join(",")).unwrap();
236    let abi = SmallCStr::new(&sess.target.llvm_abiname);
237    let trap_unreachable =
238        sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
239    let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
240
241    let verbose_asm = sess.opts.unstable_opts.verbose_asm;
242    let relax_elf_relocations =
243        sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
244
245    let use_init_array =
246        !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
247
248    let path_mapping = sess.source_map().path_mapping().clone();
249
250    let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
251
252    // copy the exe path, followed by path all into one buffer
253    // null terminating them so we can use them as null terminated strings
254    let args_cstr_buff = {
255        let mut args_cstr_buff: Vec<u8> = Vec::new();
256        let exe_path = std::env::current_exe().unwrap_or_default();
257        let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default();
258
259        args_cstr_buff.extend_from_slice(exe_path_str.as_bytes());
260        args_cstr_buff.push(0);
261
262        for arg in sess.expanded_args.iter() {
263            args_cstr_buff.extend_from_slice(arg.as_bytes());
264            args_cstr_buff.push(0);
265        }
266
267        args_cstr_buff
268    };
269
270    let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
271    match sess.opts.debuginfo_compression {
272        rustc_session::config::DebugInfoCompression::Zlib => {
273            if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
274                sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
275            }
276        }
277        rustc_session::config::DebugInfoCompression::Zstd => {
278            if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
279                sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
280            }
281        }
282        rustc_session::config::DebugInfoCompression::None => {}
283    };
284    let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
285
286    let file_name_display_preference =
287        sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
288
289    let use_wasm_eh = wants_wasm_eh(sess);
290
291    Arc::new(move |config: TargetMachineFactoryConfig| {
292        let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
293            let path = path.unwrap_or_default();
294            let path = path_mapping
295                .to_real_filename(path)
296                .to_string_lossy(file_name_display_preference)
297                .into_owned();
298            CString::new(path).unwrap()
299        };
300
301        let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
302        let output_obj_file = path_to_cstring_helper(config.output_obj_file);
303
304        OwnedTargetMachine::new(
305            &triple,
306            &cpu,
307            &features,
308            &abi,
309            code_model,
310            reloc_model,
311            opt_level,
312            float_abi,
313            ffunction_sections,
314            fdata_sections,
315            funique_section_names,
316            trap_unreachable,
317            singlethread,
318            verbose_asm,
319            emit_stack_size_section,
320            relax_elf_relocations,
321            use_init_array,
322            &split_dwarf_file,
323            &output_obj_file,
324            &debuginfo_compression,
325            use_emulated_tls,
326            &args_cstr_buff,
327            use_wasm_eh,
328        )
329    })
330}
331
332pub(crate) fn save_temp_bitcode(
333    cgcx: &CodegenContext<LlvmCodegenBackend>,
334    module: &ModuleCodegen<ModuleLlvm>,
335    name: &str,
336) {
337    if !cgcx.save_temps {
338        return;
339    }
340    let ext = format!("{name}.bc");
341    let path = cgcx.output_filenames.temp_path_ext_for_cgu(
342        &ext,
343        &module.name,
344        cgcx.invocation_temp.as_deref(),
345    );
346    write_bitcode_to_file(module, &path)
347}
348
349fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
350    unsafe {
351        let path = path_to_c_string(&path);
352        let llmod = module.module_llvm.llmod();
353        llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
354    }
355}
356
357/// In what context is a dignostic handler being attached to a codegen unit?
358pub(crate) enum CodegenDiagnosticsStage {
359    /// Prelink optimization stage.
360    Opt,
361    /// LTO/ThinLTO postlink optimization stage.
362    LTO,
363    /// Code generation.
364    Codegen,
365}
366
367pub(crate) struct DiagnosticHandlers<'a> {
368    data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
369    llcx: &'a llvm::Context,
370    old_handler: Option<&'a llvm::DiagnosticHandler>,
371}
372
373impl<'a> DiagnosticHandlers<'a> {
374    pub(crate) fn new(
375        cgcx: &'a CodegenContext<LlvmCodegenBackend>,
376        dcx: DiagCtxtHandle<'a>,
377        llcx: &'a llvm::Context,
378        module: &ModuleCodegen<ModuleLlvm>,
379        stage: CodegenDiagnosticsStage,
380    ) -> Self {
381        let remark_passes_all: bool;
382        let remark_passes: Vec<CString>;
383        match &cgcx.remark {
384            Passes::All => {
385                remark_passes_all = true;
386                remark_passes = Vec::new();
387            }
388            Passes::Some(passes) => {
389                remark_passes_all = false;
390                remark_passes =
391                    passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
392            }
393        };
394        let remark_passes: Vec<*const c_char> =
395            remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
396        let remark_file = cgcx
397            .remark_dir
398            .as_ref()
399            // Use the .opt.yaml file suffix, which is supported by LLVM's opt-viewer.
400            .map(|dir| {
401                let stage_suffix = match stage {
402                    CodegenDiagnosticsStage::Codegen => "codegen",
403                    CodegenDiagnosticsStage::Opt => "opt",
404                    CodegenDiagnosticsStage::LTO => "lto",
405                };
406                dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
407            })
408            .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
409
410        let pgo_available = cgcx.opts.cg.profile_use.is_some();
411        let data = Box::into_raw(Box::new((cgcx, dcx)));
412        unsafe {
413            let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
414            llvm::LLVMRustContextConfigureDiagnosticHandler(
415                llcx,
416                diagnostic_handler,
417                data.cast(),
418                remark_passes_all,
419                remark_passes.as_ptr(),
420                remark_passes.len(),
421                // The `as_ref()` is important here, otherwise the `CString` will be dropped
422                // too soon!
423                remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
424                pgo_available,
425            );
426            DiagnosticHandlers { data, llcx, old_handler }
427        }
428    }
429}
430
431impl<'a> Drop for DiagnosticHandlers<'a> {
432    fn drop(&mut self) {
433        unsafe {
434            llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
435            drop(Box::from_raw(self.data));
436        }
437    }
438}
439
440fn report_inline_asm(
441    cgcx: &CodegenContext<LlvmCodegenBackend>,
442    msg: String,
443    level: llvm::DiagnosticLevel,
444    cookie: u64,
445    source: Option<(String, Vec<InnerSpan>)>,
446) {
447    // In LTO build we may get srcloc values from other crates which are invalid
448    // since they use a different source map. To be safe we just suppress these
449    // in LTO builds.
450    let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
451        SpanData::default()
452    } else {
453        SpanData {
454            lo: BytePos::from_u32(cookie as u32),
455            hi: BytePos::from_u32((cookie >> 32) as u32),
456            ctxt: SyntaxContext::root(),
457            parent: None,
458        }
459    };
460    let level = match level {
461        llvm::DiagnosticLevel::Error => Level::Error,
462        llvm::DiagnosticLevel::Warning => Level::Warning,
463        llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
464    };
465    let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
466    cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
467}
468
469unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
470    if user.is_null() {
471        return;
472    }
473    let (cgcx, dcx) =
474        unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
475
476    match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
477        llvm::diagnostic::InlineAsm(inline) => {
478            report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
479        }
480
481        llvm::diagnostic::Optimization(opt) => {
482            dcx.emit_note(FromLlvmOptimizationDiag {
483                filename: &opt.filename,
484                line: opt.line,
485                column: opt.column,
486                pass_name: &opt.pass_name,
487                kind: match opt.kind {
488                    OptimizationRemark => "success",
489                    OptimizationMissed | OptimizationFailure => "missed",
490                    OptimizationAnalysis
491                    | OptimizationAnalysisFPCommute
492                    | OptimizationAnalysisAliasing => "analysis",
493                    OptimizationRemarkOther => "other",
494                },
495                message: &opt.message,
496            });
497        }
498        llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
499            let message = llvm::build_string(|s| unsafe {
500                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
501            })
502            .expect("non-UTF8 diagnostic");
503            dcx.emit_warn(FromLlvmDiag { message });
504        }
505        llvm::diagnostic::Unsupported(diagnostic_ref) => {
506            let message = llvm::build_string(|s| unsafe {
507                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
508            })
509            .expect("non-UTF8 diagnostic");
510            dcx.emit_err(FromLlvmDiag { message });
511        }
512        llvm::diagnostic::UnknownDiagnostic(..) => {}
513    }
514}
515
516fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
517    match config.pgo_gen {
518        SwitchWithOptPath::Enabled(ref opt_dir_path) => {
519            let path = if let Some(dir_path) = opt_dir_path {
520                dir_path.join("default_%m.profraw")
521            } else {
522                PathBuf::from("default_%m.profraw")
523            };
524
525            Some(CString::new(format!("{}", path.display())).unwrap())
526        }
527        SwitchWithOptPath::Disabled => None,
528    }
529}
530
531fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
532    config
533        .pgo_use
534        .as_ref()
535        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
536}
537
538fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
539    config
540        .pgo_sample_use
541        .as_ref()
542        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
543}
544
545fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
546    config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
547}
548
549// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling)
550// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes.
551// PostAD will run all opts, including size increasing opts.
552#[derive(Debug, Eq, PartialEq)]
553pub(crate) enum AutodiffStage {
554    PreAD,
555    DuringAD,
556    PostAD,
557}
558
559pub(crate) unsafe fn llvm_optimize(
560    cgcx: &CodegenContext<LlvmCodegenBackend>,
561    dcx: DiagCtxtHandle<'_>,
562    module: &ModuleCodegen<ModuleLlvm>,
563    thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
564    config: &ModuleConfig,
565    opt_level: config::OptLevel,
566    opt_stage: llvm::OptStage,
567    autodiff_stage: AutodiffStage,
568) -> Result<(), FatalError> {
569    // Enzyme:
570    // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized
571    // source code. However, benchmarks show that optimizations increasing the code size
572    // tend to reduce AD performance. Therefore deactivate them before AD, then differentiate the code
573    // and finally re-optimize the module, now with all optimizations available.
574    // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting
575    // differentiated.
576
577    let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
578    let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
579    let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
580    let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
581    let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
582    let merge_functions;
583    let unroll_loops;
584    let vectorize_slp;
585    let vectorize_loop;
586
587    // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing
588    // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt).
589    // We therefore have two calls to llvm_optimize, if autodiff is used.
590    //
591    // We also must disable merge_functions, since autodiff placeholder/dummy bodies tend to be
592    // identical. We run opts before AD, so there is a chance that LLVM will merge our dummies.
593    // In that case, we lack some dummy bodies and can't replace them with the real AD code anymore.
594    // We then would need to abort compilation. This was especially common in test cases.
595    if consider_ad && autodiff_stage != AutodiffStage::PostAD {
596        merge_functions = false;
597        unroll_loops = false;
598        vectorize_slp = false;
599        vectorize_loop = false;
600    } else {
601        unroll_loops =
602            opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
603        merge_functions = config.merge_functions;
604        vectorize_slp = config.vectorize_slp;
605        vectorize_loop = config.vectorize_loop;
606    }
607    trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
608    if thin_lto_buffer.is_some() {
609        assert!(
610            matches!(
611                opt_stage,
612                llvm::OptStage::PreLinkNoLTO
613                    | llvm::OptStage::PreLinkFatLTO
614                    | llvm::OptStage::PreLinkThinLTO
615            ),
616            "the bitcode for LTO can only be obtained at the pre-link stage"
617        );
618    }
619    let pgo_gen_path = get_pgo_gen_path(config);
620    let pgo_use_path = get_pgo_use_path(config);
621    let pgo_sample_use_path = get_pgo_sample_use_path(config);
622    let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
623    let instr_profile_output_path = get_instr_profile_output_path(config);
624    let sanitize_dataflow_abilist: Vec<_> = config
625        .sanitizer_dataflow_abilist
626        .iter()
627        .map(|file| CString::new(file.as_str()).unwrap())
628        .collect();
629    let sanitize_dataflow_abilist_ptrs: Vec<_> =
630        sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
631    // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
632    let sanitizer_options = if !is_lto {
633        Some(llvm::SanitizerOptions {
634            sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
635            sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
636            sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
637            sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
638            sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
639            sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
640            sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
641            sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
642            sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
643            sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
644            sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
645            sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
646            sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
647            sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
648            sanitize_kernel_address_recover: config
649                .sanitizer_recover
650                .contains(SanitizerSet::KERNELADDRESS),
651        })
652    } else {
653        None
654    };
655
656    let mut llvm_profiler = cgcx
657        .prof
658        .llvm_recording_enabled()
659        .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
660
661    let llvm_selfprofiler =
662        llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
663
664    let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
665
666    let llvm_plugins = config.llvm_plugins.join(",");
667
668    let result = unsafe {
669        llvm::LLVMRustOptimize(
670            module.module_llvm.llmod(),
671            &*module.module_llvm.tm.raw(),
672            to_pass_builder_opt_level(opt_level),
673            opt_stage,
674            cgcx.opts.cg.linker_plugin_lto.enabled(),
675            config.no_prepopulate_passes,
676            config.verify_llvm_ir,
677            config.lint_llvm_ir,
678            thin_lto_buffer,
679            config.emit_thin_lto,
680            config.emit_thin_lto_summary,
681            merge_functions,
682            unroll_loops,
683            vectorize_slp,
684            vectorize_loop,
685            config.no_builtins,
686            config.emit_lifetime_markers,
687            run_enzyme,
688            print_before_enzyme,
689            print_after_enzyme,
690            print_passes,
691            sanitizer_options.as_ref(),
692            pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
693            pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
694            config.instrument_coverage,
695            instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
696            pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
697            config.debug_info_for_profiling,
698            llvm_selfprofiler,
699            selfprofile_before_pass_callback,
700            selfprofile_after_pass_callback,
701            extra_passes.as_c_char_ptr(),
702            extra_passes.len(),
703            llvm_plugins.as_c_char_ptr(),
704            llvm_plugins.len(),
705        )
706    };
707    result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
708}
709
710// Unsafe due to LLVM calls.
711pub(crate) fn optimize(
712    cgcx: &CodegenContext<LlvmCodegenBackend>,
713    dcx: DiagCtxtHandle<'_>,
714    module: &mut ModuleCodegen<ModuleLlvm>,
715    config: &ModuleConfig,
716) -> Result<(), FatalError> {
717    let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
718
719    let llcx = &*module.module_llvm.llcx;
720    let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
721
722    if config.emit_no_opt_bc {
723        let out = cgcx.output_filenames.temp_path_ext_for_cgu(
724            "no-opt.bc",
725            &module.name,
726            cgcx.invocation_temp.as_deref(),
727        );
728        write_bitcode_to_file(module, &out)
729    }
730
731    // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts
732
733    if let Some(opt_level) = config.opt_level {
734        let opt_stage = match cgcx.lto {
735            Lto::Fat => llvm::OptStage::PreLinkFatLTO,
736            Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
737            _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
738            _ => llvm::OptStage::PreLinkNoLTO,
739        };
740
741        // If we know that we will later run AD, then we disable vectorization and loop unrolling.
742        // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
743        let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
744        let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
745        // The embedded bitcode is used to run LTO/ThinLTO.
746        // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO.
747        // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at
748        // this point.
749        let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
750            && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
751            || config.emit_thin_lto_summary
752        {
753            Some(null_mut())
754        } else {
755            None
756        };
757        unsafe {
758            llvm_optimize(
759                cgcx,
760                dcx,
761                module,
762                thin_lto_buffer.as_mut(),
763                config,
764                opt_level,
765                opt_stage,
766                autodiff_stage,
767            )
768        }?;
769        if let Some(thin_lto_buffer) = thin_lto_buffer {
770            let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
771            module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
772            let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu(
773                OutputType::ThinLinkBitcode,
774                &module.name,
775                cgcx.invocation_temp.as_deref(),
776            );
777            if config.emit_thin_lto_summary
778                && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
779            {
780                let summary_data = thin_lto_buffer.thin_link_data();
781                cgcx.prof.artifact_size(
782                    "llvm_bitcode_summary",
783                    thin_link_bitcode_filename.to_string_lossy(),
784                    summary_data.len() as u64,
785                );
786                let _timer = cgcx.prof.generic_activity_with_arg(
787                    "LLVM_module_codegen_emit_bitcode_summary",
788                    &*module.name,
789                );
790                if let Err(err) = fs::write(&bc_summary_out, summary_data) {
791                    dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
792                }
793            }
794        }
795    }
796    Ok(())
797}
798
799pub(crate) fn codegen(
800    cgcx: &CodegenContext<LlvmCodegenBackend>,
801    module: ModuleCodegen<ModuleLlvm>,
802    config: &ModuleConfig,
803) -> Result<CompiledModule, FatalError> {
804    let dcx = cgcx.create_dcx();
805    let dcx = dcx.handle();
806
807    let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
808    {
809        let llmod = module.module_llvm.llmod();
810        let llcx = &*module.module_llvm.llcx;
811        let tm = &*module.module_llvm.tm;
812        let _handlers =
813            DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
814
815        if cgcx.msvc_imps_needed {
816            create_msvc_imps(cgcx, llcx, llmod);
817        }
818
819        // Note that if object files are just LLVM bitcode we write bitcode,
820        // copy it to the .o file, and delete the bitcode if it wasn't
821        // otherwise requested.
822
823        let bc_out = cgcx.output_filenames.temp_path_for_cgu(
824            OutputType::Bitcode,
825            &module.name,
826            cgcx.invocation_temp.as_deref(),
827        );
828        let obj_out = cgcx.output_filenames.temp_path_for_cgu(
829            OutputType::Object,
830            &module.name,
831            cgcx.invocation_temp.as_deref(),
832        );
833
834        if config.bitcode_needed() {
835            if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
836                let thin = {
837                    let _timer = cgcx.prof.generic_activity_with_arg(
838                        "LLVM_module_codegen_make_bitcode",
839                        &*module.name,
840                    );
841                    ThinBuffer::new(llmod, config.emit_thin_lto, false)
842                };
843                let data = thin.data();
844                let _timer = cgcx
845                    .prof
846                    .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
847                if let Some(bitcode_filename) = bc_out.file_name() {
848                    cgcx.prof.artifact_size(
849                        "llvm_bitcode",
850                        bitcode_filename.to_string_lossy(),
851                        data.len() as u64,
852                    );
853                }
854                if let Err(err) = fs::write(&bc_out, data) {
855                    dcx.emit_err(WriteBytecode { path: &bc_out, err });
856                }
857            }
858
859            if config.embed_bitcode() && module.kind == ModuleKind::Regular {
860                let _timer = cgcx
861                    .prof
862                    .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
863                let thin_bc =
864                    module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
865                embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc);
866            }
867        }
868
869        if config.emit_ir {
870            let _timer =
871                cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
872            let out = cgcx.output_filenames.temp_path_for_cgu(
873                OutputType::LlvmAssembly,
874                &module.name,
875                cgcx.invocation_temp.as_deref(),
876            );
877            let out_c = path_to_c_string(&out);
878
879            extern "C" fn demangle_callback(
880                input_ptr: *const c_char,
881                input_len: size_t,
882                output_ptr: *mut c_char,
883                output_len: size_t,
884            ) -> size_t {
885                let input =
886                    unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
887
888                let Ok(input) = str::from_utf8(input) else { return 0 };
889
890                let output = unsafe {
891                    slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
892                };
893                let mut cursor = io::Cursor::new(output);
894
895                let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
896
897                if write!(cursor, "{demangled:#}").is_err() {
898                    // Possible only if provided buffer is not big enough
899                    return 0;
900                }
901
902                cursor.position() as size_t
903            }
904
905            let result =
906                unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
907
908            if result == llvm::LLVMRustResult::Success {
909                record_artifact_size(&cgcx.prof, "llvm_ir", &out);
910            }
911
912            result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?;
913        }
914
915        if config.emit_asm {
916            let _timer =
917                cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
918            let path = cgcx.output_filenames.temp_path_for_cgu(
919                OutputType::Assembly,
920                &module.name,
921                cgcx.invocation_temp.as_deref(),
922            );
923
924            // We can't use the same module for asm and object code output,
925            // because that triggers various errors like invalid IR or broken
926            // binaries. So we must clone the module to produce the asm output
927            // if we are also producing object code.
928            let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
929                llvm::LLVMCloneModule(llmod)
930            } else {
931                llmod
932            };
933            write_output_file(
934                dcx,
935                tm.raw(),
936                config.no_builtins,
937                llmod,
938                &path,
939                None,
940                llvm::FileType::AssemblyFile,
941                &cgcx.prof,
942                config.verify_llvm_ir,
943            )?;
944        }
945
946        match config.emit_obj {
947            EmitObj::ObjectCode(_) => {
948                let _timer = cgcx
949                    .prof
950                    .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
951
952                let dwo_out = cgcx
953                    .output_filenames
954                    .temp_path_dwo_for_cgu(&module.name, cgcx.invocation_temp.as_deref());
955                let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
956                    // Don't change how DWARF is emitted when disabled.
957                    (SplitDebuginfo::Off, _) => None,
958                    // Don't provide a DWARF object path if split debuginfo is enabled but this is
959                    // a platform that doesn't support Split DWARF.
960                    _ if !cgcx.target_can_use_split_dwarf => None,
961                    // Don't provide a DWARF object path in single mode, sections will be written
962                    // into the object as normal but ignored by linker.
963                    (_, SplitDwarfKind::Single) => None,
964                    // Emit (a subset of the) DWARF into a separate dwarf object file in split
965                    // mode.
966                    (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
967                };
968
969                write_output_file(
970                    dcx,
971                    tm.raw(),
972                    config.no_builtins,
973                    llmod,
974                    &obj_out,
975                    dwo_out,
976                    llvm::FileType::ObjectFile,
977                    &cgcx.prof,
978                    config.verify_llvm_ir,
979                )?;
980            }
981
982            EmitObj::Bitcode => {
983                debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
984                if let Err(err) = link_or_copy(&bc_out, &obj_out) {
985                    dcx.emit_err(CopyBitcode { err });
986                }
987
988                if !config.emit_bc {
989                    debug!("removing_bitcode {:?}", bc_out);
990                    ensure_removed(dcx, &bc_out);
991                }
992            }
993
994            EmitObj::None => {}
995        }
996
997        record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
998    }
999
1000    // `.dwo` files are only emitted if:
1001    //
1002    // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not
1003    //   produce dwarf objects, even if otherwise enabled)
1004    // - Target supports Split DWARF
1005    // - Split debuginfo is enabled
1006    // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different
1007    //   sections in the `.o` files).
1008    let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
1009        && cgcx.target_can_use_split_dwarf
1010        && cgcx.split_debuginfo != SplitDebuginfo::Off
1011        && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1012    Ok(module.into_compiled_module(
1013        config.emit_obj != EmitObj::None,
1014        dwarf_object_emitted,
1015        config.emit_bc,
1016        config.emit_asm,
1017        config.emit_ir,
1018        &cgcx.output_filenames,
1019        cgcx.invocation_temp.as_deref(),
1020    ))
1021}
1022
1023fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1024    let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1025    asm.extend_from_slice(b".ascii \"");
1026    asm.reserve(data.len());
1027    for &byte in data {
1028        if byte == b'\\' || byte == b'"' {
1029            asm.push(b'\\');
1030            asm.push(byte);
1031        } else if byte < 0x20 || byte >= 0x80 {
1032            // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed
1033            // width, while hex escapes will consume following characters.
1034            asm.push(b'\\');
1035            asm.push(b'0' + ((byte >> 6) & 0x7));
1036            asm.push(b'0' + ((byte >> 3) & 0x7));
1037            asm.push(b'0' + ((byte >> 0) & 0x7));
1038        } else {
1039            asm.push(byte);
1040        }
1041    }
1042    asm.extend_from_slice(b"\"\n");
1043    asm
1044}
1045
1046pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1047    if cgcx.target_is_like_darwin {
1048        c"__LLVM,__bitcode"
1049    } else if cgcx.target_is_like_aix {
1050        c".ipa"
1051    } else {
1052        c".llvmbc"
1053    }
1054}
1055
1056/// Embed the bitcode of an LLVM module for LTO in the LLVM module itself.
1057fn embed_bitcode(
1058    cgcx: &CodegenContext<LlvmCodegenBackend>,
1059    llcx: &llvm::Context,
1060    llmod: &llvm::Module,
1061    cmdline: &str,
1062    bitcode: &[u8],
1063) {
1064    // We're adding custom sections to the output object file, but we definitely
1065    // do not want these custom sections to make their way into the final linked
1066    // executable. The purpose of these custom sections is for tooling
1067    // surrounding object files to work with the LLVM IR, if necessary. For
1068    // example rustc's own LTO will look for LLVM IR inside of the object file
1069    // in these sections by default.
1070    //
1071    // To handle this is a bit different depending on the object file format
1072    // used by the backend, broken down into a few different categories:
1073    //
1074    // * Mach-O - this is for macOS. Inspecting the source code for the native
1075    //   linker here shows that the `.llvmbc` and `.llvmcmd` sections are
1076    //   automatically skipped by the linker. In that case there's nothing extra
1077    //   that we need to do here.
1078    //
1079    // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and
1080    //   `.llvmcmd` sections, so there's nothing extra we need to do.
1081    //
1082    // * COFF - if we don't do anything the linker will by default copy all
1083    //   these sections to the output artifact, not what we want! To subvert
1084    //   this we want to flag the sections we inserted here as
1085    //   `IMAGE_SCN_LNK_REMOVE`.
1086    //
1087    // * ELF - this is very similar to COFF above. One difference is that these
1088    //   sections are removed from the output linked artifact when
1089    //   `--gc-sections` is passed, which we pass by default. If that flag isn't
1090    //   passed though then these sections will show up in the final output.
1091    //   Additionally the flag that we need to set here is `SHF_EXCLUDE`.
1092    //
1093    // * XCOFF - AIX linker ignores content in .ipa and .info if no auxiliary
1094    //   symbol associated with these sections.
1095    //
1096    // Unfortunately, LLVM provides no way to set custom section flags. For ELF
1097    // and COFF we emit the sections using module level inline assembly for that
1098    // reason (see issue #90326 for historical background).
1099
1100    if cgcx.target_is_like_darwin
1101        || cgcx.target_is_like_aix
1102        || cgcx.target_arch == "wasm32"
1103        || cgcx.target_arch == "wasm64"
1104    {
1105        // We don't need custom section flags, create LLVM globals.
1106        let llconst = common::bytes_in_context(llcx, bitcode);
1107        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1108        llvm::set_initializer(llglobal, llconst);
1109
1110        llvm::set_section(llglobal, bitcode_section_name(cgcx));
1111        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1112        llvm::LLVMSetGlobalConstant(llglobal, llvm::True);
1113
1114        let llconst = common::bytes_in_context(llcx, cmdline.as_bytes());
1115        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1116        llvm::set_initializer(llglobal, llconst);
1117        let section = if cgcx.target_is_like_darwin {
1118            c"__LLVM,__cmdline"
1119        } else if cgcx.target_is_like_aix {
1120            c".info"
1121        } else {
1122            c".llvmcmd"
1123        };
1124        llvm::set_section(llglobal, section);
1125        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1126    } else {
1127        // We need custom section flags, so emit module-level inline assembly.
1128        let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1129        let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1130        llvm::append_module_inline_asm(llmod, &asm);
1131        let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes());
1132        llvm::append_module_inline_asm(llmod, &asm);
1133    }
1134}
1135
1136// Create a `__imp_<symbol> = &symbol` global for every public static `symbol`.
1137// This is required to satisfy `dllimport` references to static data in .rlibs
1138// when using MSVC linker. We do this only for data, as linker can fix up
1139// code references on its own.
1140// See #26591, #27438
1141fn create_msvc_imps(
1142    cgcx: &CodegenContext<LlvmCodegenBackend>,
1143    llcx: &llvm::Context,
1144    llmod: &llvm::Module,
1145) {
1146    if !cgcx.msvc_imps_needed {
1147        return;
1148    }
1149    // The x86 ABI seems to require that leading underscores are added to symbol
1150    // names, so we need an extra underscore on x86. There's also a leading
1151    // '\x01' here which disables LLVM's symbol mangling (e.g., no extra
1152    // underscores added in front).
1153    let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1154
1155    let ptr_ty = Type::ptr_llcx(llcx);
1156    let globals = base::iter_globals(llmod)
1157        .filter(|&val| {
1158            llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1159        })
1160        .filter_map(|val| {
1161            // Exclude some symbols that we know are not Rust symbols.
1162            let name = llvm::get_value_name(val);
1163            if ignored(&name) { None } else { Some((val, name)) }
1164        })
1165        .map(move |(val, name)| {
1166            let mut imp_name = prefix.as_bytes().to_vec();
1167            imp_name.extend(name);
1168            let imp_name = CString::new(imp_name).unwrap();
1169            (imp_name, val)
1170        })
1171        .collect::<Vec<_>>();
1172
1173    for (imp_name, val) in globals {
1174        let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1175
1176        llvm::set_initializer(imp, val);
1177        llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1178    }
1179
1180    // Use this function to exclude certain symbols from `__imp` generation.
1181    fn ignored(symbol_name: &[u8]) -> bool {
1182        // These are symbols generated by LLVM's profiling instrumentation
1183        symbol_name.starts_with(b"__llvm_profile_")
1184    }
1185}
1186
1187fn record_artifact_size(
1188    self_profiler_ref: &SelfProfilerRef,
1189    artifact_kind: &'static str,
1190    path: &Path,
1191) {
1192    // Don't stat the file if we are not going to record its size.
1193    if !self_profiler_ref.enabled() {
1194        return;
1195    }
1196
1197    if let Some(artifact_name) = path.file_name() {
1198        let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1199        self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1200    }
1201}
1202
1203fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1204    if !prof.enabled() {
1205        return;
1206    }
1207
1208    let raw_stats =
1209        llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1210            .expect("cannot get module instruction stats");
1211
1212    #[derive(serde::Deserialize)]
1213    struct InstructionsStats {
1214        module: String,
1215        total: u64,
1216    }
1217
1218    let InstructionsStats { module, total } =
1219        serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1220    prof.artifact_size("cgu_instructions", module, total);
1221}