rustc_codegen_llvm/back/
write.rs

1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::ptr::null_mut;
5use std::sync::Arc;
6use std::{fs, slice, str};
7
8use libc::{c_char, c_int, c_void, size_t};
9use llvm::{
10    LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
11};
12use rustc_codegen_ssa::back::link::ensure_removed;
13use rustc_codegen_ssa::back::versioned_llvm_target;
14use rustc_codegen_ssa::back::write::{
15    BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
16    TargetMachineFactoryFn,
17};
18use rustc_codegen_ssa::base::wants_wasm_eh;
19use rustc_codegen_ssa::traits::*;
20use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
21use rustc_data_structures::profiling::SelfProfilerRef;
22use rustc_data_structures::small_c_str::SmallCStr;
23use rustc_errors::{DiagCtxtHandle, Level};
24use rustc_fs_util::{link_or_copy, path_to_c_string};
25use rustc_middle::ty::TyCtxt;
26use rustc_session::Session;
27use rustc_session::config::{
28    self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
29};
30use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym};
31use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
32use tracing::{debug, trace};
33
34use crate::back::command_line_args::quote_command_line_args;
35use crate::back::lto::ThinBuffer;
36use crate::back::owned_target_machine::OwnedTargetMachine;
37use crate::back::profiling::{
38    LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
39};
40use crate::common::AsCCharPtr;
41use crate::errors::{
42    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
43    WithLlvmError, WriteBytecode,
44};
45use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
46use crate::llvm::{self, DiagnosticInfo};
47use crate::type_::llvm_type_ptr;
48use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
49
50pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
51    match llvm::last_error() {
52        Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)),
53        None => dcx.emit_fatal(err),
54    }
55}
56
57fn write_output_file<'ll>(
58    dcx: DiagCtxtHandle<'_>,
59    target: &'ll llvm::TargetMachine,
60    no_builtins: bool,
61    m: &'ll llvm::Module,
62    output: &Path,
63    dwo_output: Option<&Path>,
64    file_type: llvm::FileType,
65    self_profiler_ref: &SelfProfilerRef,
66    verify_llvm_ir: bool,
67) {
68    debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
69    let output_c = path_to_c_string(output);
70    let dwo_output_c;
71    let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
72        dwo_output_c = path_to_c_string(dwo_output);
73        dwo_output_c.as_ptr()
74    } else {
75        std::ptr::null()
76    };
77    let result = unsafe {
78        let pm = llvm::LLVMCreatePassManager();
79        llvm::LLVMAddAnalysisPasses(target, pm);
80        llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins);
81        llvm::LLVMRustWriteOutputFile(
82            target,
83            pm,
84            m,
85            output_c.as_ptr(),
86            dwo_output_ptr,
87            file_type,
88            verify_llvm_ir,
89        )
90    };
91
92    // Record artifact sizes for self-profiling
93    if result == llvm::LLVMRustResult::Success {
94        let artifact_kind = match file_type {
95            llvm::FileType::ObjectFile => "object_file",
96            llvm::FileType::AssemblyFile => "assembly_file",
97        };
98        record_artifact_size(self_profiler_ref, artifact_kind, output);
99        if let Some(dwo_file) = dwo_output {
100            record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
101        }
102    }
103
104    result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
105}
106
107pub(crate) fn create_informational_target_machine(
108    sess: &Session,
109    only_base_features: bool,
110) -> OwnedTargetMachine {
111    let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
112    // Can't use query system here quite yet because this function is invoked before the query
113    // system/tcx is set up.
114    let features = llvm_util::global_llvm_features(sess, false, only_base_features);
115    target_machine_factory(sess, config::OptLevel::No, &features)(config)
116        .unwrap_or_else(|err| llvm_err(sess.dcx(), err))
117}
118
119pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
120    let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
121        tcx.output_filenames(()).split_dwarf_path(
122            tcx.sess.split_debuginfo(),
123            tcx.sess.opts.unstable_opts.split_dwarf_kind,
124            mod_name,
125            tcx.sess.invocation_temp.as_deref(),
126        )
127    } else {
128        None
129    };
130
131    let output_obj_file = Some(tcx.output_filenames(()).temp_path_for_cgu(
132        OutputType::Object,
133        mod_name,
134        tcx.sess.invocation_temp.as_deref(),
135    ));
136    let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
137
138    target_machine_factory(
139        tcx.sess,
140        tcx.backend_optimization_level(()),
141        tcx.global_backend_features(()),
142    )(config)
143    .unwrap_or_else(|err| llvm_err(tcx.dcx(), err))
144}
145
146fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
147    use self::config::OptLevel::*;
148    match cfg {
149        No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
150        Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
151        More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
152        Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
153        Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
154        SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
155    }
156}
157
158fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
159    use config::OptLevel::*;
160    match cfg {
161        No => llvm::PassBuilderOptLevel::O0,
162        Less => llvm::PassBuilderOptLevel::O1,
163        More => llvm::PassBuilderOptLevel::O2,
164        Aggressive => llvm::PassBuilderOptLevel::O3,
165        Size => llvm::PassBuilderOptLevel::Os,
166        SizeMin => llvm::PassBuilderOptLevel::Oz,
167    }
168}
169
170fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
171    match relocation_model {
172        RelocModel::Static => llvm::RelocModel::Static,
173        // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra
174        // attribute.
175        RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
176        RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
177        RelocModel::Ropi => llvm::RelocModel::ROPI,
178        RelocModel::Rwpi => llvm::RelocModel::RWPI,
179        RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
180    }
181}
182
183pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
184    match code_model {
185        Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
186        Some(CodeModel::Small) => llvm::CodeModel::Small,
187        Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
188        Some(CodeModel::Medium) => llvm::CodeModel::Medium,
189        Some(CodeModel::Large) => llvm::CodeModel::Large,
190        None => llvm::CodeModel::None,
191    }
192}
193
194fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
195    match float_abi {
196        None => llvm::FloatAbi::Default,
197        Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
198        Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
199    }
200}
201
202pub(crate) fn target_machine_factory(
203    sess: &Session,
204    optlvl: config::OptLevel,
205    target_features: &[String],
206) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
207    // Self-profile timer for creating a _factory_.
208    let _prof_timer = sess.prof.generic_activity("target_machine_factory");
209
210    let reloc_model = to_llvm_relocation_model(sess.relocation_model());
211
212    let (opt_level, _) = to_llvm_opt_settings(optlvl);
213    let float_abi = if sess.target.arch == "arm" && sess.opts.cg.soft_float {
214        llvm::FloatAbi::Soft
215    } else {
216        // `validate_commandline_args_with_session_available` has already warned about this being
217        // ignored. Let's make sure LLVM doesn't suddenly start using this flag on more targets.
218        to_llvm_float_abi(sess.target.llvm_floatabi)
219    };
220
221    let ffunction_sections =
222        sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
223    let fdata_sections = ffunction_sections;
224    let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
225
226    let code_model = to_llvm_code_model(sess.code_model());
227
228    let mut singlethread = sess.target.singlethread;
229
230    // On the wasm target once the `atomics` feature is enabled that means that
231    // we're no longer single-threaded, or otherwise we don't want LLVM to
232    // lower atomic operations to single-threaded operations.
233    if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
234        singlethread = false;
235    }
236
237    let triple = SmallCStr::new(&versioned_llvm_target(sess));
238    let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
239    let features = CString::new(target_features.join(",")).unwrap();
240    let abi = SmallCStr::new(&sess.target.llvm_abiname);
241    let trap_unreachable =
242        sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
243    let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
244
245    let verbose_asm = sess.opts.unstable_opts.verbose_asm;
246    let relax_elf_relocations =
247        sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
248
249    let use_init_array =
250        !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
251
252    let path_mapping = sess.source_map().path_mapping().clone();
253
254    let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
255
256    // Command-line information to be included in the target machine.
257    // This seems to only be used for embedding in PDB debuginfo files.
258    // FIXME(Zalathar): Maybe skip this for non-PDB targets?
259    let argv0 = std::env::current_exe()
260        .unwrap_or_default()
261        .into_os_string()
262        .into_string()
263        .unwrap_or_default();
264    let command_line_args = quote_command_line_args(&sess.expanded_args);
265    // Self-profile counter for the number of bytes produced by command-line quoting.
266    // Values are summed, so the summary result is cumulative across all TM factories.
267    sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64);
268
269    let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
270    match sess.opts.debuginfo_compression {
271        rustc_session::config::DebugInfoCompression::Zlib => {
272            if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
273                sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" });
274            }
275        }
276        rustc_session::config::DebugInfoCompression::Zstd => {
277            if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
278                sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" });
279            }
280        }
281        rustc_session::config::DebugInfoCompression::None => {}
282    };
283    let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
284
285    let file_name_display_preference =
286        sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO);
287
288    let use_wasm_eh = wants_wasm_eh(sess);
289
290    let prof = SelfProfilerRef::clone(&sess.prof);
291    Arc::new(move |config: TargetMachineFactoryConfig| {
292        // Self-profile timer for invoking a factory to create a target machine.
293        let _prof_timer = prof.generic_activity("target_machine_factory_inner");
294
295        let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
296            let path = path.unwrap_or_default();
297            let path = path_mapping
298                .to_real_filename(path)
299                .to_string_lossy(file_name_display_preference)
300                .into_owned();
301            CString::new(path).unwrap()
302        };
303
304        let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
305        let output_obj_file = path_to_cstring_helper(config.output_obj_file);
306
307        OwnedTargetMachine::new(
308            &triple,
309            &cpu,
310            &features,
311            &abi,
312            code_model,
313            reloc_model,
314            opt_level,
315            float_abi,
316            ffunction_sections,
317            fdata_sections,
318            funique_section_names,
319            trap_unreachable,
320            singlethread,
321            verbose_asm,
322            emit_stack_size_section,
323            relax_elf_relocations,
324            use_init_array,
325            &split_dwarf_file,
326            &output_obj_file,
327            &debuginfo_compression,
328            use_emulated_tls,
329            &argv0,
330            &command_line_args,
331            use_wasm_eh,
332        )
333    })
334}
335
336pub(crate) fn save_temp_bitcode(
337    cgcx: &CodegenContext<LlvmCodegenBackend>,
338    module: &ModuleCodegen<ModuleLlvm>,
339    name: &str,
340) {
341    if !cgcx.save_temps {
342        return;
343    }
344    let ext = format!("{name}.bc");
345    let path = cgcx.output_filenames.temp_path_ext_for_cgu(
346        &ext,
347        &module.name,
348        cgcx.invocation_temp.as_deref(),
349    );
350    write_bitcode_to_file(module, &path)
351}
352
353fn write_bitcode_to_file(module: &ModuleCodegen<ModuleLlvm>, path: &Path) {
354    unsafe {
355        let path = path_to_c_string(&path);
356        let llmod = module.module_llvm.llmod();
357        llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr());
358    }
359}
360
361/// In what context is a dignostic handler being attached to a codegen unit?
362pub(crate) enum CodegenDiagnosticsStage {
363    /// Prelink optimization stage.
364    Opt,
365    /// LTO/ThinLTO postlink optimization stage.
366    LTO,
367    /// Code generation.
368    Codegen,
369}
370
371pub(crate) struct DiagnosticHandlers<'a> {
372    data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>),
373    llcx: &'a llvm::Context,
374    old_handler: Option<&'a llvm::DiagnosticHandler>,
375}
376
377impl<'a> DiagnosticHandlers<'a> {
378    pub(crate) fn new(
379        cgcx: &'a CodegenContext<LlvmCodegenBackend>,
380        dcx: DiagCtxtHandle<'a>,
381        llcx: &'a llvm::Context,
382        module: &ModuleCodegen<ModuleLlvm>,
383        stage: CodegenDiagnosticsStage,
384    ) -> Self {
385        let remark_passes_all: bool;
386        let remark_passes: Vec<CString>;
387        match &cgcx.remark {
388            Passes::All => {
389                remark_passes_all = true;
390                remark_passes = Vec::new();
391            }
392            Passes::Some(passes) => {
393                remark_passes_all = false;
394                remark_passes =
395                    passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
396            }
397        };
398        let remark_passes: Vec<*const c_char> =
399            remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
400        let remark_file = cgcx
401            .remark_dir
402            .as_ref()
403            // Use the .opt.yaml file suffix, which is supported by LLVM's opt-viewer.
404            .map(|dir| {
405                let stage_suffix = match stage {
406                    CodegenDiagnosticsStage::Codegen => "codegen",
407                    CodegenDiagnosticsStage::Opt => "opt",
408                    CodegenDiagnosticsStage::LTO => "lto",
409                };
410                dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name))
411            })
412            .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
413
414        let pgo_available = cgcx.opts.cg.profile_use.is_some();
415        let data = Box::into_raw(Box::new((cgcx, dcx)));
416        unsafe {
417            let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
418            llvm::LLVMRustContextConfigureDiagnosticHandler(
419                llcx,
420                diagnostic_handler,
421                data.cast(),
422                remark_passes_all,
423                remark_passes.as_ptr(),
424                remark_passes.len(),
425                // The `as_ref()` is important here, otherwise the `CString` will be dropped
426                // too soon!
427                remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
428                pgo_available,
429            );
430            DiagnosticHandlers { data, llcx, old_handler }
431        }
432    }
433}
434
435impl<'a> Drop for DiagnosticHandlers<'a> {
436    fn drop(&mut self) {
437        unsafe {
438            llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
439            drop(Box::from_raw(self.data));
440        }
441    }
442}
443
444fn report_inline_asm(
445    cgcx: &CodegenContext<LlvmCodegenBackend>,
446    msg: String,
447    level: llvm::DiagnosticLevel,
448    cookie: u64,
449    source: Option<(String, Vec<InnerSpan>)>,
450) {
451    // In LTO build we may get srcloc values from other crates which are invalid
452    // since they use a different source map. To be safe we just suppress these
453    // in LTO builds.
454    let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
455        SpanData::default()
456    } else {
457        SpanData {
458            lo: BytePos::from_u32(cookie as u32),
459            hi: BytePos::from_u32((cookie >> 32) as u32),
460            ctxt: SyntaxContext::root(),
461            parent: None,
462        }
463    };
464    let level = match level {
465        llvm::DiagnosticLevel::Error => Level::Error,
466        llvm::DiagnosticLevel::Warning => Level::Warning,
467        llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note,
468    };
469    let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
470    cgcx.diag_emitter.inline_asm_error(span, msg, level, source);
471}
472
473unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
474    if user.is_null() {
475        return;
476    }
477    let (cgcx, dcx) =
478        unsafe { *(user as *const (&CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'_>)) };
479
480    match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
481        llvm::diagnostic::InlineAsm(inline) => {
482            report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source);
483        }
484
485        llvm::diagnostic::Optimization(opt) => {
486            dcx.emit_note(FromLlvmOptimizationDiag {
487                filename: &opt.filename,
488                line: opt.line,
489                column: opt.column,
490                pass_name: &opt.pass_name,
491                kind: match opt.kind {
492                    OptimizationRemark => "success",
493                    OptimizationMissed | OptimizationFailure => "missed",
494                    OptimizationAnalysis
495                    | OptimizationAnalysisFPCommute
496                    | OptimizationAnalysisAliasing => "analysis",
497                    OptimizationRemarkOther => "other",
498                },
499                message: &opt.message,
500            });
501        }
502        llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
503            let message = llvm::build_string(|s| unsafe {
504                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
505            })
506            .expect("non-UTF8 diagnostic");
507            dcx.emit_warn(FromLlvmDiag { message });
508        }
509        llvm::diagnostic::Unsupported(diagnostic_ref) => {
510            let message = llvm::build_string(|s| unsafe {
511                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
512            })
513            .expect("non-UTF8 diagnostic");
514            dcx.emit_err(FromLlvmDiag { message });
515        }
516        llvm::diagnostic::UnknownDiagnostic(..) => {}
517    }
518}
519
520fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
521    match config.pgo_gen {
522        SwitchWithOptPath::Enabled(ref opt_dir_path) => {
523            let path = if let Some(dir_path) = opt_dir_path {
524                dir_path.join("default_%m.profraw")
525            } else {
526                PathBuf::from("default_%m.profraw")
527            };
528
529            Some(CString::new(format!("{}", path.display())).unwrap())
530        }
531        SwitchWithOptPath::Disabled => None,
532    }
533}
534
535fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
536    config
537        .pgo_use
538        .as_ref()
539        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
540}
541
542fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
543    config
544        .pgo_sample_use
545        .as_ref()
546        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
547}
548
549fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
550    config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
551}
552
553// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling)
554// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes.
555// PostAD will run all opts, including size increasing opts.
556#[derive(Debug, Eq, PartialEq)]
557pub(crate) enum AutodiffStage {
558    PreAD,
559    DuringAD,
560    PostAD,
561}
562
563pub(crate) unsafe fn llvm_optimize(
564    cgcx: &CodegenContext<LlvmCodegenBackend>,
565    dcx: DiagCtxtHandle<'_>,
566    module: &ModuleCodegen<ModuleLlvm>,
567    thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>,
568    config: &ModuleConfig,
569    opt_level: config::OptLevel,
570    opt_stage: llvm::OptStage,
571    autodiff_stage: AutodiffStage,
572) {
573    // Enzyme:
574    // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized
575    // source code. However, benchmarks show that optimizations increasing the code size
576    // tend to reduce AD performance. Therefore deactivate them before AD, then differentiate the code
577    // and finally re-optimize the module, now with all optimizations available.
578    // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting
579    // differentiated.
580
581    let consider_ad =
582        cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
583    let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
584    let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
585    let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
586    let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
587    let merge_functions;
588    let unroll_loops;
589    let vectorize_slp;
590    let vectorize_loop;
591
592    // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing
593    // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt).
594    // We therefore have two calls to llvm_optimize, if autodiff is used.
595    //
596    // We also must disable merge_functions, since autodiff placeholder/dummy bodies tend to be
597    // identical. We run opts before AD, so there is a chance that LLVM will merge our dummies.
598    // In that case, we lack some dummy bodies and can't replace them with the real AD code anymore.
599    // We then would need to abort compilation. This was especially common in test cases.
600    if consider_ad && autodiff_stage != AutodiffStage::PostAD {
601        merge_functions = false;
602        unroll_loops = false;
603        vectorize_slp = false;
604        vectorize_loop = false;
605    } else {
606        unroll_loops =
607            opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
608        merge_functions = config.merge_functions;
609        vectorize_slp = config.vectorize_slp;
610        vectorize_loop = config.vectorize_loop;
611    }
612    trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
613    if thin_lto_buffer.is_some() {
614        assert!(
615            matches!(
616                opt_stage,
617                llvm::OptStage::PreLinkNoLTO
618                    | llvm::OptStage::PreLinkFatLTO
619                    | llvm::OptStage::PreLinkThinLTO
620            ),
621            "the bitcode for LTO can only be obtained at the pre-link stage"
622        );
623    }
624    let pgo_gen_path = get_pgo_gen_path(config);
625    let pgo_use_path = get_pgo_use_path(config);
626    let pgo_sample_use_path = get_pgo_sample_use_path(config);
627    let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
628    let instr_profile_output_path = get_instr_profile_output_path(config);
629    let sanitize_dataflow_abilist: Vec<_> = config
630        .sanitizer_dataflow_abilist
631        .iter()
632        .map(|file| CString::new(file.as_str()).unwrap())
633        .collect();
634    let sanitize_dataflow_abilist_ptrs: Vec<_> =
635        sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
636    // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
637    let sanitizer_options = if !is_lto {
638        Some(llvm::SanitizerOptions {
639            sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
640            sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
641            sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
642            sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
643            sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
644            sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
645            sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
646            sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
647            sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
648            sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
649            sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
650            sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
651            sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
652            sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
653            sanitize_kernel_address_recover: config
654                .sanitizer_recover
655                .contains(SanitizerSet::KERNELADDRESS),
656        })
657    } else {
658        None
659    };
660
661    let mut llvm_profiler = cgcx
662        .prof
663        .llvm_recording_enabled()
664        .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
665
666    let llvm_selfprofiler =
667        llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
668
669    let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
670
671    let llvm_plugins = config.llvm_plugins.join(",");
672
673    let result = unsafe {
674        llvm::LLVMRustOptimize(
675            module.module_llvm.llmod(),
676            &*module.module_llvm.tm.raw(),
677            to_pass_builder_opt_level(opt_level),
678            opt_stage,
679            cgcx.opts.cg.linker_plugin_lto.enabled(),
680            config.no_prepopulate_passes,
681            config.verify_llvm_ir,
682            config.lint_llvm_ir,
683            thin_lto_buffer,
684            config.emit_thin_lto,
685            config.emit_thin_lto_summary,
686            merge_functions,
687            unroll_loops,
688            vectorize_slp,
689            vectorize_loop,
690            config.no_builtins,
691            config.emit_lifetime_markers,
692            run_enzyme,
693            print_before_enzyme,
694            print_after_enzyme,
695            print_passes,
696            sanitizer_options.as_ref(),
697            pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
698            pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
699            config.instrument_coverage,
700            instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
701            pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
702            config.debug_info_for_profiling,
703            llvm_selfprofiler,
704            selfprofile_before_pass_callback,
705            selfprofile_after_pass_callback,
706            extra_passes.as_c_char_ptr(),
707            extra_passes.len(),
708            llvm_plugins.as_c_char_ptr(),
709            llvm_plugins.len(),
710        )
711    };
712    result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
713}
714
715// Unsafe due to LLVM calls.
716pub(crate) fn optimize(
717    cgcx: &CodegenContext<LlvmCodegenBackend>,
718    dcx: DiagCtxtHandle<'_>,
719    module: &mut ModuleCodegen<ModuleLlvm>,
720    config: &ModuleConfig,
721) {
722    let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
723
724    let llcx = &*module.module_llvm.llcx;
725    let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
726
727    if config.emit_no_opt_bc {
728        let out = cgcx.output_filenames.temp_path_ext_for_cgu(
729            "no-opt.bc",
730            &module.name,
731            cgcx.invocation_temp.as_deref(),
732        );
733        write_bitcode_to_file(module, &out)
734    }
735
736    // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts
737
738    if let Some(opt_level) = config.opt_level {
739        let opt_stage = match cgcx.lto {
740            Lto::Fat => llvm::OptStage::PreLinkFatLTO,
741            Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
742            _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
743            _ => llvm::OptStage::PreLinkNoLTO,
744        };
745
746        // If we know that we will later run AD, then we disable vectorization and loop unrolling.
747        // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
748        let consider_ad =
749            cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable);
750        let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
751        // The embedded bitcode is used to run LTO/ThinLTO.
752        // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO.
753        // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at
754        // this point.
755        let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular
756            && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
757            || config.emit_thin_lto_summary
758        {
759            Some(null_mut())
760        } else {
761            None
762        };
763        unsafe {
764            llvm_optimize(
765                cgcx,
766                dcx,
767                module,
768                thin_lto_buffer.as_mut(),
769                config,
770                opt_level,
771                opt_stage,
772                autodiff_stage,
773            )
774        };
775        if let Some(thin_lto_buffer) = thin_lto_buffer {
776            let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) };
777            module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec());
778            let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu(
779                OutputType::ThinLinkBitcode,
780                &module.name,
781                cgcx.invocation_temp.as_deref(),
782            );
783            if config.emit_thin_lto_summary
784                && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
785            {
786                let summary_data = thin_lto_buffer.thin_link_data();
787                cgcx.prof.artifact_size(
788                    "llvm_bitcode_summary",
789                    thin_link_bitcode_filename.to_string_lossy(),
790                    summary_data.len() as u64,
791                );
792                let _timer = cgcx.prof.generic_activity_with_arg(
793                    "LLVM_module_codegen_emit_bitcode_summary",
794                    &*module.name,
795                );
796                if let Err(err) = fs::write(&bc_summary_out, summary_data) {
797                    dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
798                }
799            }
800        }
801    }
802}
803
804pub(crate) fn codegen(
805    cgcx: &CodegenContext<LlvmCodegenBackend>,
806    module: ModuleCodegen<ModuleLlvm>,
807    config: &ModuleConfig,
808) -> CompiledModule {
809    let dcx = cgcx.create_dcx();
810    let dcx = dcx.handle();
811
812    let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
813    {
814        let llmod = module.module_llvm.llmod();
815        let llcx = &*module.module_llvm.llcx;
816        let tm = &*module.module_llvm.tm;
817        let _handlers =
818            DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
819
820        if cgcx.msvc_imps_needed {
821            create_msvc_imps(cgcx, llcx, llmod);
822        }
823
824        // Note that if object files are just LLVM bitcode we write bitcode,
825        // copy it to the .o file, and delete the bitcode if it wasn't
826        // otherwise requested.
827
828        let bc_out = cgcx.output_filenames.temp_path_for_cgu(
829            OutputType::Bitcode,
830            &module.name,
831            cgcx.invocation_temp.as_deref(),
832        );
833        let obj_out = cgcx.output_filenames.temp_path_for_cgu(
834            OutputType::Object,
835            &module.name,
836            cgcx.invocation_temp.as_deref(),
837        );
838
839        if config.bitcode_needed() {
840            if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
841                let thin = {
842                    let _timer = cgcx.prof.generic_activity_with_arg(
843                        "LLVM_module_codegen_make_bitcode",
844                        &*module.name,
845                    );
846                    ThinBuffer::new(llmod, config.emit_thin_lto)
847                };
848                let data = thin.data();
849                let _timer = cgcx
850                    .prof
851                    .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
852                if let Some(bitcode_filename) = bc_out.file_name() {
853                    cgcx.prof.artifact_size(
854                        "llvm_bitcode",
855                        bitcode_filename.to_string_lossy(),
856                        data.len() as u64,
857                    );
858                }
859                if let Err(err) = fs::write(&bc_out, data) {
860                    dcx.emit_err(WriteBytecode { path: &bc_out, err });
861                }
862            }
863
864            if config.embed_bitcode() && module.kind == ModuleKind::Regular {
865                let _timer = cgcx
866                    .prof
867                    .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
868                let thin_bc =
869                    module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode");
870                embed_bitcode(cgcx, llcx, llmod, &thin_bc);
871            }
872        }
873
874        if config.emit_ir {
875            let _timer =
876                cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
877            let out = cgcx.output_filenames.temp_path_for_cgu(
878                OutputType::LlvmAssembly,
879                &module.name,
880                cgcx.invocation_temp.as_deref(),
881            );
882            let out_c = path_to_c_string(&out);
883
884            extern "C" fn demangle_callback(
885                input_ptr: *const c_char,
886                input_len: size_t,
887                output_ptr: *mut c_char,
888                output_len: size_t,
889            ) -> size_t {
890                let input =
891                    unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
892
893                let Ok(input) = str::from_utf8(input) else { return 0 };
894
895                let output = unsafe {
896                    slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
897                };
898                let mut cursor = io::Cursor::new(output);
899
900                let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
901
902                if write!(cursor, "{demangled:#}").is_err() {
903                    // Possible only if provided buffer is not big enough
904                    return 0;
905                }
906
907                cursor.position() as size_t
908            }
909
910            let result =
911                unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
912
913            if result == llvm::LLVMRustResult::Success {
914                record_artifact_size(&cgcx.prof, "llvm_ir", &out);
915            }
916
917            result
918                .into_result()
919                .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }));
920        }
921
922        if config.emit_asm {
923            let _timer =
924                cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
925            let path = cgcx.output_filenames.temp_path_for_cgu(
926                OutputType::Assembly,
927                &module.name,
928                cgcx.invocation_temp.as_deref(),
929            );
930
931            // We can't use the same module for asm and object code output,
932            // because that triggers various errors like invalid IR or broken
933            // binaries. So we must clone the module to produce the asm output
934            // if we are also producing object code.
935            let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
936                llvm::LLVMCloneModule(llmod)
937            } else {
938                llmod
939            };
940            write_output_file(
941                dcx,
942                tm.raw(),
943                config.no_builtins,
944                llmod,
945                &path,
946                None,
947                llvm::FileType::AssemblyFile,
948                &cgcx.prof,
949                config.verify_llvm_ir,
950            );
951        }
952
953        match config.emit_obj {
954            EmitObj::ObjectCode(_) => {
955                let _timer = cgcx
956                    .prof
957                    .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
958
959                let dwo_out = cgcx
960                    .output_filenames
961                    .temp_path_dwo_for_cgu(&module.name, cgcx.invocation_temp.as_deref());
962                let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
963                    // Don't change how DWARF is emitted when disabled.
964                    (SplitDebuginfo::Off, _) => None,
965                    // Don't provide a DWARF object path if split debuginfo is enabled but this is
966                    // a platform that doesn't support Split DWARF.
967                    _ if !cgcx.target_can_use_split_dwarf => None,
968                    // Don't provide a DWARF object path in single mode, sections will be written
969                    // into the object as normal but ignored by linker.
970                    (_, SplitDwarfKind::Single) => None,
971                    // Emit (a subset of the) DWARF into a separate dwarf object file in split
972                    // mode.
973                    (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
974                };
975
976                write_output_file(
977                    dcx,
978                    tm.raw(),
979                    config.no_builtins,
980                    llmod,
981                    &obj_out,
982                    dwo_out,
983                    llvm::FileType::ObjectFile,
984                    &cgcx.prof,
985                    config.verify_llvm_ir,
986                );
987            }
988
989            EmitObj::Bitcode => {
990                debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
991                if let Err(err) = link_or_copy(&bc_out, &obj_out) {
992                    dcx.emit_err(CopyBitcode { err });
993                }
994
995                if !config.emit_bc {
996                    debug!("removing_bitcode {:?}", bc_out);
997                    ensure_removed(dcx, &bc_out);
998                }
999            }
1000
1001            EmitObj::None => {}
1002        }
1003
1004        record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
1005    }
1006
1007    // `.dwo` files are only emitted if:
1008    //
1009    // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not
1010    //   produce dwarf objects, even if otherwise enabled)
1011    // - Target supports Split DWARF
1012    // - Split debuginfo is enabled
1013    // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different
1014    //   sections in the `.o` files).
1015    let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
1016        && cgcx.target_can_use_split_dwarf
1017        && cgcx.split_debuginfo != SplitDebuginfo::Off
1018        && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1019    module.into_compiled_module(
1020        config.emit_obj != EmitObj::None,
1021        dwarf_object_emitted,
1022        config.emit_bc,
1023        config.emit_asm,
1024        config.emit_ir,
1025        &cgcx.output_filenames,
1026        cgcx.invocation_temp.as_deref(),
1027    )
1028}
1029
1030fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1031    let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1032    asm.extend_from_slice(b".ascii \"");
1033    asm.reserve(data.len());
1034    for &byte in data {
1035        if byte == b'\\' || byte == b'"' {
1036            asm.push(b'\\');
1037            asm.push(byte);
1038        } else if byte < 0x20 || byte >= 0x80 {
1039            // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed
1040            // width, while hex escapes will consume following characters.
1041            asm.push(b'\\');
1042            asm.push(b'0' + ((byte >> 6) & 0x7));
1043            asm.push(b'0' + ((byte >> 3) & 0x7));
1044            asm.push(b'0' + ((byte >> 0) & 0x7));
1045        } else {
1046            asm.push(byte);
1047        }
1048    }
1049    asm.extend_from_slice(b"\"\n");
1050    asm
1051}
1052
1053pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr {
1054    if cgcx.target_is_like_darwin {
1055        c"__LLVM,__bitcode"
1056    } else if cgcx.target_is_like_aix {
1057        c".ipa"
1058    } else {
1059        c".llvmbc"
1060    }
1061}
1062
1063/// Embed the bitcode of an LLVM module for LTO in the LLVM module itself.
1064fn embed_bitcode(
1065    cgcx: &CodegenContext<LlvmCodegenBackend>,
1066    llcx: &llvm::Context,
1067    llmod: &llvm::Module,
1068    bitcode: &[u8],
1069) {
1070    // We're adding custom sections to the output object file, but we definitely
1071    // do not want these custom sections to make their way into the final linked
1072    // executable. The purpose of these custom sections is for tooling
1073    // surrounding object files to work with the LLVM IR, if necessary. For
1074    // example rustc's own LTO will look for LLVM IR inside of the object file
1075    // in these sections by default.
1076    //
1077    // To handle this is a bit different depending on the object file format
1078    // used by the backend, broken down into a few different categories:
1079    //
1080    // * Mach-O - this is for macOS. Inspecting the source code for the native
1081    //   linker here shows that the `.llvmbc` and `.llvmcmd` sections are
1082    //   automatically skipped by the linker. In that case there's nothing extra
1083    //   that we need to do here. We do need to make sure that the
1084    //   `__LLVM,__cmdline` section exists even though it is empty as otherwise
1085    //   ld64 rejects the object file.
1086    //
1087    // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and
1088    //   `.llvmcmd` sections, so there's nothing extra we need to do.
1089    //
1090    // * COFF - if we don't do anything the linker will by default copy all
1091    //   these sections to the output artifact, not what we want! To subvert
1092    //   this we want to flag the sections we inserted here as
1093    //   `IMAGE_SCN_LNK_REMOVE`.
1094    //
1095    // * ELF - this is very similar to COFF above. One difference is that these
1096    //   sections are removed from the output linked artifact when
1097    //   `--gc-sections` is passed, which we pass by default. If that flag isn't
1098    //   passed though then these sections will show up in the final output.
1099    //   Additionally the flag that we need to set here is `SHF_EXCLUDE`.
1100    //
1101    // * XCOFF - AIX linker ignores content in .ipa and .info if no auxiliary
1102    //   symbol associated with these sections.
1103    //
1104    // Unfortunately, LLVM provides no way to set custom section flags. For ELF
1105    // and COFF we emit the sections using module level inline assembly for that
1106    // reason (see issue #90326 for historical background).
1107
1108    if cgcx.target_is_like_darwin
1109        || cgcx.target_is_like_aix
1110        || cgcx.target_arch == "wasm32"
1111        || cgcx.target_arch == "wasm64"
1112    {
1113        // We don't need custom section flags, create LLVM globals.
1114        let llconst = common::bytes_in_context(llcx, bitcode);
1115        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1116        llvm::set_initializer(llglobal, llconst);
1117
1118        llvm::set_section(llglobal, bitcode_section_name(cgcx));
1119        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1120        llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE);
1121
1122        let llconst = common::bytes_in_context(llcx, &[]);
1123        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1124        llvm::set_initializer(llglobal, llconst);
1125        let section = if cgcx.target_is_like_darwin {
1126            c"__LLVM,__cmdline"
1127        } else if cgcx.target_is_like_aix {
1128            c".info"
1129        } else {
1130            c".llvmcmd"
1131        };
1132        llvm::set_section(llglobal, section);
1133        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1134    } else {
1135        // We need custom section flags, so emit module-level inline assembly.
1136        let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1137        let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1138        llvm::append_module_inline_asm(llmod, &asm);
1139        let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
1140        llvm::append_module_inline_asm(llmod, &asm);
1141    }
1142}
1143
1144// Create a `__imp_<symbol> = &symbol` global for every public static `symbol`.
1145// This is required to satisfy `dllimport` references to static data in .rlibs
1146// when using MSVC linker. We do this only for data, as linker can fix up
1147// code references on its own.
1148// See #26591, #27438
1149fn create_msvc_imps(
1150    cgcx: &CodegenContext<LlvmCodegenBackend>,
1151    llcx: &llvm::Context,
1152    llmod: &llvm::Module,
1153) {
1154    if !cgcx.msvc_imps_needed {
1155        return;
1156    }
1157    // The x86 ABI seems to require that leading underscores are added to symbol
1158    // names, so we need an extra underscore on x86. There's also a leading
1159    // '\x01' here which disables LLVM's symbol mangling (e.g., no extra
1160    // underscores added in front).
1161    let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
1162
1163    let ptr_ty = llvm_type_ptr(llcx);
1164    let globals = base::iter_globals(llmod)
1165        .filter(|&val| {
1166            llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)
1167        })
1168        .filter_map(|val| {
1169            // Exclude some symbols that we know are not Rust symbols.
1170            let name = llvm::get_value_name(val);
1171            if ignored(&name) { None } else { Some((val, name)) }
1172        })
1173        .map(move |(val, name)| {
1174            let mut imp_name = prefix.as_bytes().to_vec();
1175            imp_name.extend(name);
1176            let imp_name = CString::new(imp_name).unwrap();
1177            (imp_name, val)
1178        })
1179        .collect::<Vec<_>>();
1180
1181    for (imp_name, val) in globals {
1182        let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1183
1184        llvm::set_initializer(imp, val);
1185        llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage);
1186    }
1187
1188    // Use this function to exclude certain symbols from `__imp` generation.
1189    fn ignored(symbol_name: &[u8]) -> bool {
1190        // These are symbols generated by LLVM's profiling instrumentation
1191        symbol_name.starts_with(b"__llvm_profile_")
1192    }
1193}
1194
1195fn record_artifact_size(
1196    self_profiler_ref: &SelfProfilerRef,
1197    artifact_kind: &'static str,
1198    path: &Path,
1199) {
1200    // Don't stat the file if we are not going to record its size.
1201    if !self_profiler_ref.enabled() {
1202        return;
1203    }
1204
1205    if let Some(artifact_name) = path.file_name() {
1206        let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1207        self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1208    }
1209}
1210
1211fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
1212    if !prof.enabled() {
1213        return;
1214    }
1215
1216    let raw_stats =
1217        llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
1218            .expect("cannot get module instruction stats");
1219
1220    #[derive(serde::Deserialize)]
1221    struct InstructionsStats {
1222        module: String,
1223        total: u64,
1224    }
1225
1226    let InstructionsStats { module, total } =
1227        serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
1228    prof.artifact_size("cgu_instructions", module, total);
1229}