Skip to main content

rustc_codegen_llvm/back/
write.rs

1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::{fs, ptr, slice, str};
6
7use libc::{c_char, c_int, c_void, size_t};
8use rustc_codegen_ssa::back::link::ensure_removed;
9use rustc_codegen_ssa::back::write::{
10    BitcodeSection, CodegenContext, EmitObj, InlineAsmError, ModuleConfig, SharedEmitter,
11    TargetMachineFactoryConfig, TargetMachineFactoryFn,
12};
13use rustc_codegen_ssa::base::wants_wasm_eh;
14use rustc_codegen_ssa::traits::*;
15use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
16use rustc_data_structures::profiling::SelfProfilerRef;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_errors::{DiagCtxt, DiagCtxtHandle, Level};
19use rustc_fs_util::{link_or_copy, path_to_c_string};
20use rustc_middle::ty::TyCtxt;
21use rustc_session::Session;
22use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath};
23use rustc_span::{BytePos, DUMMY_SP, InnerSpan, Pos, RemapPathScopeComponents};
24use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
25use tracing::{debug, trace};
26
27use crate::back::llvm_backend::ModuleRefs;
28use crate::back::lto::{Buffer, ModuleBuffer};
29use crate::back::owned_target_machine::OwnedTargetMachine;
30use crate::back::profiling::{
31    LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
32};
33use crate::builder::SBuilder;
34use crate::builder::gpu_offload::scalar_width;
35use crate::context::SimpleCx;
36use crate::diagnostics::{
37    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, ParseTargetMachineConfig,
38    UnsupportedCompression, WithLlvmError, WriteBytecode,
39};
40use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
41use crate::llvm::{self, DiagnosticInfo, TypeKind};
42use crate::type_::llvm_type_ptr;
43use crate::{LlvmCodegenBackend, ModuleLlvm, attributes, base, common, llvm_util};
44
45pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
46    match llvm::last_error() {
47        Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)),
48        None => dcx.emit_fatal(err),
49    }
50}
51
52fn write_output_file<'ll>(
53    dcx: DiagCtxtHandle<'_>,
54    target: &'ll llvm::TargetMachine,
55    no_builtins: bool,
56    m: llvm::Module<'ll>,
57    output: &Path,
58    dwo_output: Option<&Path>,
59    file_type: llvm::FileType,
60    self_profiler_ref: &SelfProfilerRef,
61    verify_llvm_ir: bool,
62) {
63    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:63",
                        "rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
                        ::tracing_core::__macro_support::Option::Some(63u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("write_output_file output={0:?} dwo_output={1:?}",
                                                    output, dwo_output) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
64    let output_c = path_to_c_string(output);
65    let dwo_output_c;
66    let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
67        dwo_output_c = path_to_c_string(dwo_output);
68        dwo_output_c.as_ptr()
69    } else {
70        std::ptr::null()
71    };
72    let result = unsafe {
73        llvm::LLVMRustWriteOutputFile(
74            target,
75            m,
76            output_c.as_ptr(),
77            dwo_output_ptr,
78            file_type,
79            verify_llvm_ir,
80            no_builtins,
81        )
82    };
83
84    // Record artifact sizes for self-profiling
85    if result == llvm::LLVMRustResult::Success {
86        let artifact_kind = match file_type {
87            llvm::FileType::ObjectFile => "object_file",
88            llvm::FileType::AssemblyFile => "assembly_file",
89        };
90        record_artifact_size(self_profiler_ref, artifact_kind, output);
91        if let Some(dwo_file) = dwo_output {
92            record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
93        }
94    }
95
96    result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
97}
98
99pub(crate) fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine {
100    let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
101    target_machine_factory(sess, config::OptLevel::No)(sess.dcx(), config)
102}
103
104pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
105    let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
106        tcx.output_filenames(()).split_dwarf_path(
107            tcx.sess.split_debuginfo(),
108            tcx.sess.opts.unstable_opts.split_dwarf_kind,
109            mod_name,
110        )
111    } else {
112        None
113    };
114
115    let output_obj_file =
116        Some(tcx.output_filenames(()).temp_path_for_cgu(OutputType::Object, mod_name));
117    let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
118
119    target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(tcx.dcx(), config)
120}
121
122fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
123    use self::config::OptLevel::*;
124    match cfg {
125        No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
126        Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
127        More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
128        Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
129        Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
130        SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
131    }
132}
133
134fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
135    use config::OptLevel::*;
136    match cfg {
137        No => llvm::PassBuilderOptLevel::O0,
138        Less => llvm::PassBuilderOptLevel::O1,
139        More => llvm::PassBuilderOptLevel::O2,
140        Aggressive => llvm::PassBuilderOptLevel::O3,
141        Size => llvm::PassBuilderOptLevel::Os,
142        SizeMin => llvm::PassBuilderOptLevel::Oz,
143    }
144}
145
146fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
147    match relocation_model {
148        RelocModel::Static => llvm::RelocModel::Static,
149        // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra
150        // attribute.
151        RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
152        RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
153        RelocModel::Ropi => llvm::RelocModel::ROPI,
154        RelocModel::Rwpi => llvm::RelocModel::RWPI,
155        RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
156    }
157}
158
159pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
160    match code_model {
161        Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
162        Some(CodeModel::Small) => llvm::CodeModel::Small,
163        Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
164        Some(CodeModel::Medium) => llvm::CodeModel::Medium,
165        Some(CodeModel::Large) => llvm::CodeModel::Large,
166        None => llvm::CodeModel::None,
167    }
168}
169
170fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
171    match float_abi {
172        None => llvm::FloatAbi::Default,
173        Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
174        Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
175    }
176}
177
178pub(crate) fn target_machine_factory(
179    sess: &Session,
180    optlvl: config::OptLevel,
181) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
182    // Self-profile timer for creating a _factory_.
183    let _prof_timer = sess.prof.generic_activity("target_machine_factory");
184
185    let reloc_model = to_llvm_relocation_model(sess.relocation_model());
186
187    let (opt_level, _) = to_llvm_opt_settings(optlvl);
188    let float_abi = to_llvm_float_abi(sess.target.llvm_floatabi);
189
190    let ffunction_sections =
191        sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
192    let fdata_sections = ffunction_sections;
193    let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
194
195    let code_model = to_llvm_code_model(sess.code_model());
196
197    let singlethread = sess.target.singlethread(&sess.internal_target_features);
198
199    let triple = SmallCStr::new(&llvm_util::llvm_target(sess));
200    let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
201    let features = CString::new(sess.global_backend_features.join(",")).unwrap();
202    let abi = SmallCStr::new(sess.target.llvm_abiname.desc());
203    let trap_unreachable =
204        sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
205    let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
206
207    let verbose_asm = sess.opts.unstable_opts.verbose_asm;
208    let relax_x86_elf_relocations =
209        sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
210
211    let use_init_array =
212        !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
213
214    let path_mapping = sess.source_map().path_mapping().clone();
215    let working_dir = sess.source_map().working_dir().clone();
216
217    let use_emulated_tls = #[allow(non_exhaustive_omitted_patterns)] match sess.tls_model() {
    TlsModel::Emulated => true,
    _ => false,
}matches!(sess.tls_model(), TlsModel::Emulated);
218
219    let debuginfo_compression = match sess.opts.unstable_opts.debuginfo_compression {
220        config::DebugInfoCompression::None => llvm::CompressionKind::None,
221        config::DebugInfoCompression::Zlib => {
222            if llvm::LLVMRustLLVMHasZlibCompression() {
223                llvm::CompressionKind::Zlib
224            } else {
225                sess.dcx().emit_warn(UnsupportedCompression { algorithm: "zlib" });
226                llvm::CompressionKind::None
227            }
228        }
229        config::DebugInfoCompression::Zstd => {
230            if llvm::LLVMRustLLVMHasZstdCompression() {
231                llvm::CompressionKind::Zstd
232            } else {
233                sess.dcx().emit_warn(UnsupportedCompression { algorithm: "zstd" });
234                llvm::CompressionKind::None
235            }
236        }
237    };
238
239    let use_wasm_eh = wants_wasm_eh(&sess.target);
240
241    let large_data_threshold = sess.opts.unstable_opts.large_data_threshold.unwrap_or(0);
242
243    let prof = SelfProfilerRef::clone(&sess.prof);
244    Arc::new(move |dcx: DiagCtxtHandle<'_>, config: TargetMachineFactoryConfig| {
245        // Self-profile timer for invoking a factory to create a target machine.
246        let _prof_timer = prof.generic_activity("target_machine_factory_inner");
247
248        let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
249            let path = path.unwrap_or_default();
250            let path = path_mapping
251                .to_real_filename(&working_dir, path)
252                .path(RemapPathScopeComponents::DEBUGINFO)
253                .to_string_lossy()
254                .into_owned();
255            CString::new(path).unwrap()
256        };
257
258        let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
259        let output_obj_file = path_to_cstring_helper(config.output_obj_file);
260
261        OwnedTargetMachine::new(
262            &triple,
263            &cpu,
264            &features,
265            &abi,
266            code_model,
267            reloc_model,
268            opt_level,
269            float_abi,
270            ffunction_sections,
271            fdata_sections,
272            funique_section_names,
273            trap_unreachable,
274            singlethread,
275            verbose_asm,
276            emit_stack_size_section,
277            relax_x86_elf_relocations,
278            use_init_array,
279            &split_dwarf_file,
280            &output_obj_file,
281            debuginfo_compression,
282            use_emulated_tls,
283            use_wasm_eh,
284            large_data_threshold,
285        )
286        .unwrap_or_else(|err| dcx.emit_fatal(ParseTargetMachineConfig(err)))
287    })
288}
289
290pub(crate) fn save_temp_bitcode(
291    cgcx: &CodegenContext,
292    module: &ModuleCodegen<ModuleLlvm>,
293    name: &str,
294) {
295    if !cgcx.save_temps {
296        return;
297    }
298    let ext = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}.bc", name))
    })format!("{name}.bc");
299    let path = cgcx.output_filenames.temp_path_ext_for_cgu(&ext, &module.name);
300    write_bitcode_to_file(&module.module_llvm, &path)
301}
302
303fn write_bitcode_to_file(module: &ModuleLlvm, path: &Path) {
304    let path = path_to_c_string(&path);
305    module.with_module_refs(|ModuleRefs { llmod, .. }| {
306        unsafe { llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr()) };
307    });
308}
309
310/// In what context is a diagnostic handler being attached to a codegen unit?
311pub(crate) enum CodegenDiagnosticsStage {
312    /// Prelink optimization stage.
313    Opt,
314    /// LTO/ThinLTO postlink optimization stage.
315    LTO,
316    /// Code generation.
317    Codegen,
318}
319
320/// Context data that is passed to [`diagnostic_handler_callback`].
321#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl<'a>
    ::core::clone::TrivialClone for DiagnosticHandlerCallbackData<'a> {
}
#[automatically_derived]impl<'a> ::core::clone::Clone for
    DiagnosticHandlerCallbackData<'a> {
    #[inline]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<&'a CodegenContext>;
        let _: ::core::clone::AssertParamIsClone<&'a SharedEmitter>;
        *self
    }
}Clone, #[automatically_derived]impl<'a> ::core::marker::Copy for
    DiagnosticHandlerCallbackData<'a> {
}Copy)]
322struct DiagnosticHandlerCallbackData<'a> {
323    cgcx: &'a CodegenContext,
324    shared_emitter: &'a SharedEmitter,
325}
326
327pub(crate) struct DiagnosticHandlers<'a, 'll> {
328    llcx: llvm::Context<'ll>,
329    old_handler: Option<ptr::NonNull<llvm::DiagnosticHandler>>,
330
331    /// Ownership of the box-allocated data that was installed (as a pointer)
332    /// alongside [`diagnostic_handler_callback`].
333    ///
334    /// To be freed via [`Box::from_raw`] after the handler is removed.
335    diagnostic_handler_callback_data: *mut DiagnosticHandlerCallbackData<'a>,
336}
337
338impl<'a, 'll> DiagnosticHandlers<'a, 'll> {
339    pub(crate) fn new(
340        cgcx: &'a CodegenContext,
341        shared_emitter: &'a SharedEmitter,
342        llcx: llvm::Context<'ll>,
343        module: &ModuleCodegen<ModuleLlvm>,
344        stage: CodegenDiagnosticsStage,
345    ) -> Self {
346        let remark_passes_all: bool;
347        let remark_passes: Vec<CString>;
348        match &cgcx.remark {
349            Passes::All => {
350                remark_passes_all = true;
351                remark_passes = Vec::new();
352            }
353            Passes::Some(passes) => {
354                remark_passes_all = false;
355                remark_passes =
356                    passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
357            }
358        };
359        let remark_passes: Vec<*const c_char> =
360            remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
361        let remark_file = cgcx
362            .remark_dir
363            .as_ref()
364            // Use the .opt.yaml file suffix, which is supported by LLVM's opt-viewer.
365            .map(|dir| {
366                let stage_suffix = match stage {
367                    CodegenDiagnosticsStage::Codegen => "codegen",
368                    CodegenDiagnosticsStage::Opt => "opt",
369                    CodegenDiagnosticsStage::LTO => "lto",
370                };
371                dir.join(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}.{1}.opt.yaml", module.name,
                stage_suffix))
    })format!("{}.{stage_suffix}.opt.yaml", module.name))
372            })
373            .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
374
375        let pgo_available = cgcx.module_config.pgo_use.is_some();
376        let diagnostic_handler_callback_data =
377            Box::into_raw(Box::new(DiagnosticHandlerCallbackData { cgcx, shared_emitter }));
378        unsafe {
379            let old_handler = llvm::LLVMRustContextTakeDiagnosticHandler(llcx);
380            llvm::LLVMRustContextConfigureDiagnosticHandler(
381                llcx,
382                diagnostic_handler_callback,
383                diagnostic_handler_callback_data.cast::<c_void>(),
384                remark_passes_all,
385                remark_passes.as_ptr(),
386                remark_passes.len(),
387                // The `as_ref()` is important here, otherwise the `CString` will be dropped
388                // too soon!
389                remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
390                pgo_available,
391            );
392            DiagnosticHandlers { llcx, old_handler, diagnostic_handler_callback_data }
393        }
394    }
395}
396
397impl<'a, 'll> Drop for DiagnosticHandlers<'a, 'll> {
398    fn drop(&mut self) {
399        unsafe {
400            llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
401            drop(Box::from_raw(self.diagnostic_handler_callback_data));
402        }
403    }
404}
405
406fn report_inline_asm(
407    cgcx: &CodegenContext,
408    msg: String,
409    severity: llvm::DiagnosticSeverity,
410    cookie: u64,
411    source: Option<(String, Vec<InnerSpan>)>,
412) -> InlineAsmError {
413    // In LTO build we may get srcloc values from other crates which are invalid
414    // since they use a different source map. To be safe we just suppress these
415    // in LTO builds.
416    let (lo, hi) = if cookie == 0 || #[allow(non_exhaustive_omitted_patterns)] match cgcx.lto {
    Lto::Fat | Lto::Thin => true,
    _ => false,
}matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
417        (DUMMY_SP.lo(), DUMMY_SP.hi())
418    } else {
419        let lo = BytePos::from_u32(cookie as u32);
420        let hi = BytePos::from_u32((cookie >> 32) as u32);
421        (lo, hi)
422    };
423    let level = match severity {
424        llvm::DiagnosticSeverity::Error => Level::Error,
425        llvm::DiagnosticSeverity::Warning => Level::Warning(None),
426        llvm::DiagnosticSeverity::Note | llvm::DiagnosticSeverity::Remark => Level::Note,
427    };
428    let msg = msg.trim_prefix("error: ").to_string();
429    InlineAsmError { lo, hi, msg, level, source }
430}
431
432unsafe extern "C" fn diagnostic_handler_callback(info: &DiagnosticInfo, data: *mut c_void) {
433    if data.is_null() {
434        return;
435    }
436    // After casting the data pointer, pass it into an inner function to limit its lifetimes.
437    diagnostic_handler_callback_inner(info, unsafe {
438        &*data.cast::<DiagnosticHandlerCallbackData<'_>>()
439    });
440}
441
442fn diagnostic_handler_callback_inner(
443    info: &DiagnosticInfo,
444    data: &DiagnosticHandlerCallbackData<'_>,
445) {
446    let &DiagnosticHandlerCallbackData { cgcx, shared_emitter } = data;
447
448    let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
449    let dcx = dcx.handle();
450
451    match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
452        llvm::diagnostic::InlineAsm(inline) => {
453            // FIXME use dcx
454            shared_emitter.inline_asm_error(report_inline_asm(
455                cgcx,
456                inline.message,
457                inline.severity,
458                inline.cookie,
459                inline.source,
460            ));
461        }
462
463        llvm::diagnostic::Optimization(opt) => {
464            dcx.emit_note(FromLlvmOptimizationDiag {
465                filename: &opt.filename,
466                line: opt.line,
467                column: opt.column,
468                pass_name: &opt.pass_name,
469                kind: match opt.kind {
470                    OptimizationRemark => "success",
471                    OptimizationMissed | OptimizationFailure => "missed",
472                    OptimizationAnalysis
473                    | OptimizationAnalysisFPCommute
474                    | OptimizationAnalysisAliasing => "analysis",
475                    OptimizationRemarkOther => "other",
476                },
477                message: &opt.message,
478            });
479        }
480
481        // FIXME(#164677): Don't ignore stack size warnings.
482        llvm::diagnostic::StackSize => {}
483
484        llvm::diagnostic::Other(diagnostic_ref) => {
485            let message = llvm::build_string(|s| unsafe {
486                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
487            })
488            .expect("non-UTF8 diagnostic");
489            match diagnostic_ref.severity() {
490                llvm::DiagnosticSeverity::Error => {
491                    dcx.emit_err(FromLlvmDiag { message });
492                }
493                llvm::DiagnosticSeverity::Warning => {
494                    dcx.emit_warn(FromLlvmDiag { message });
495                }
496                _ => {}
497            };
498        }
499    }
500}
501
502fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
503    match config.pgo_gen {
504        SwitchWithOptPath::Enabled(ref opt_dir_path) => {
505            let path = if let Some(dir_path) = opt_dir_path {
506                dir_path.join("default_%m.profraw")
507            } else {
508                PathBuf::from("default_%m.profraw")
509            };
510
511            Some(CString::new(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}", path.display()))
    })format!("{}", path.display())).unwrap())
512        }
513        SwitchWithOptPath::Disabled => None,
514    }
515}
516
517fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
518    config
519        .pgo_use
520        .as_ref()
521        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
522}
523
524fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
525    config
526        .pgo_sample_use
527        .as_ref()
528        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
529}
530
531fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
532    config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
533}
534
535// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling)
536// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes.
537// PostAD will run all opts, including size increasing opts.
538#[derive(#[automatically_derived]impl ::core::fmt::Debug for AutodiffStage {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                AutodiffStage::PreAD => "PreAD",
                AutodiffStage::DuringAD => "DuringAD",
                AutodiffStage::PostAD => "PostAD",
            })
    }
}Debug, #[automatically_derived]impl ::core::cmp::Eq for AutodiffStage { }Eq, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    AutodiffStage {
}
#[automatically_derived]impl ::core::cmp::PartialEq for AutodiffStage {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq)]
539pub(crate) enum AutodiffStage {
540    PreAD,
541    DuringAD,
542    PostAD,
543}
544
545pub(crate) unsafe fn llvm_optimize<'ll>(
546    cgcx: &CodegenContext,
547    prof: &SelfProfilerRef,
548    dcx: DiagCtxtHandle<'_>,
549    module_refs: ModuleRefs<'ll>,
550    thin_lto_buffer: Option<&mut Option<Buffer>>,
551    thin_lto_summary_buffer: Option<&mut Option<Buffer>>,
552    config: &ModuleConfig,
553    opt_level: config::OptLevel,
554    opt_stage: llvm::OptStage,
555    autodiff_stage: AutodiffStage,
556) {
557    let ModuleRefs { llcx, llmod, tm, .. } = module_refs;
558
559    // Enzyme:
560    // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized
561    // source code. However, benchmarks show that optimizations increasing the code size
562    // tend to reduce AD performance. Therefore deactivate them before AD, then differentiate the code
563    // and finally re-optimize the module, now with all optimizations available.
564    // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting
565    // differentiated.
566
567    let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable);
568    let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
569    let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
570    let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
571    let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
572    let passes_after_enzyme = if autodiff_stage == AutodiffStage::PostAD {
573        config.autodiff_post_passes.as_deref()
574    } else {
575        None
576    };
577
578    let merge_functions;
579    let unroll_loops;
580    let vectorize_slp;
581    let vectorize_loop;
582
583    // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing
584    // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt).
585    // We therefore have two calls to llvm_optimize, if autodiff is used.
586    //
587    // We also must disable merge_functions, since autodiff placeholder/dummy bodies tend to be
588    // identical. We run opts before AD, so there is a chance that LLVM will merge our dummies.
589    // In that case, we lack some dummy bodies and can't replace them with the real AD code anymore.
590    // We then would need to abort compilation. This was especially common in test cases.
591    if consider_ad && autodiff_stage != AutodiffStage::PostAD {
592        merge_functions = false;
593        unroll_loops = false;
594        vectorize_slp = false;
595        vectorize_loop = false;
596    } else {
597        unroll_loops =
598            opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
599        merge_functions = config.merge_functions;
600        vectorize_slp = config.vectorize_slp;
601        vectorize_loop = config.vectorize_loop;
602    }
603    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:603",
                        "rustc_codegen_llvm::back::write", ::tracing::Level::TRACE,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
                        ::tracing_core::__macro_support::Option::Some(603u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
                        ::tracing_core::field::FieldSet::new(&[{
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("unroll_loops")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("unroll_loops");
                                            NAME.as_str()
                                        },
                                        {
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("vectorize_slp")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("vectorize_slp");
                                            NAME.as_str()
                                        },
                                        {
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("vectorize_loop")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("vectorize_loop");
                                            NAME.as_str()
                                        },
                                        {
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("run_enzyme")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("run_enzyme");
                                            NAME.as_str()
                                        }], ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::TRACE <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&unroll_loops)
                                            as &dyn ::tracing::field::Value)),
                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&vectorize_slp)
                                            as &dyn ::tracing::field::Value)),
                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&vectorize_loop)
                                            as &dyn ::tracing::field::Value)),
                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&run_enzyme)
                                            as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
604    if thin_lto_buffer.is_some() {
605        if !#[allow(non_exhaustive_omitted_patterns)] match opt_stage {
            llvm::OptStage::PreLinkNoLTO | llvm::OptStage::PreLinkFatLTO |
                llvm::OptStage::PreLinkThinLTO => true,
            _ => false,
        } {
    {
        ::core::panicking::panic_fmt(format_args!("the bitcode for LTO can only be obtained at the pre-link stage"));
    }
};assert!(
606            matches!(
607                opt_stage,
608                llvm::OptStage::PreLinkNoLTO
609                    | llvm::OptStage::PreLinkFatLTO
610                    | llvm::OptStage::PreLinkThinLTO
611            ),
612            "the bitcode for LTO can only be obtained at the pre-link stage"
613        );
614    }
615    let pgo_gen_path = get_pgo_gen_path(config);
616    let pgo_use_path = get_pgo_use_path(config);
617    let pgo_sample_use_path = get_pgo_sample_use_path(config);
618    let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
619    let is_final_stage =
620        !#[allow(non_exhaustive_omitted_patterns)] match opt_stage {
    llvm::OptStage::PreLinkFatLTO | llvm::OptStage::PreLinkThinLTO => true,
    _ => false,
}matches!(opt_stage, llvm::OptStage::PreLinkFatLTO | llvm::OptStage::PreLinkThinLTO);
621    let instr_profile_output_path = get_instr_profile_output_path(config);
622    let sanitize_dataflow_abilist: Vec<_> = config
623        .sanitizer_dataflow_abilist
624        .iter()
625        .map(|file| CString::new(file.as_str()).unwrap())
626        .collect();
627    let sanitize_dataflow_abilist_ptrs: Vec<_> =
628        sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
629    // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
630    let sanitizer_options = if !is_lto {
631        Some(llvm::SanitizerOptions {
632            sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
633            sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
634            sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
635            sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
636            sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
637            sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
638            sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
639            sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
640            sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
641            sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
642            sanitize_realtime: config.sanitizer.contains(SanitizerSet::REALTIME),
643            sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
644            sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
645            sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
646            sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
647            sanitize_kernel_address_recover: config
648                .sanitizer_recover
649                .contains(SanitizerSet::KERNELADDRESS),
650            sanitize_kernel_hwaddress: config.sanitizer.contains(SanitizerSet::KERNELHWADDRESS),
651            sanitize_kernel_hwaddress_recover: config
652                .sanitizer_recover
653                .contains(SanitizerSet::KERNELHWADDRESS),
654        })
655    } else {
656        None
657    };
658
659    // Wraps `old_fn` into a new kernel that takes the implicit `dyn_ptr` as its first argument.
660    // Returns the new function, or `None` if nothing was wrapped
661    fn handle_offload<'ll>(
662        cx: &SimpleCx<'ll>,
663        old_fn: &'ll llvm::Value,
664    ) -> Option<&'ll llvm::Value> {
665        let old_fn_ty = cx.get_type_of_global(old_fn);
666        let old_param_types = cx.func_params_types(old_fn_ty);
667        let old_param_count = old_param_types.len();
668        if old_param_count == 0 {
669            return None;
670        }
671
672        // Create the new parameter list, with ptr as the first argument
673        let mut new_param_types = Vec::with_capacity(old_param_count as usize + 1);
674        new_param_types.push(cx.type_ptr());
675
676        // This relies on undocumented LLVM knowledge that scalars must be passed as i64
677        for &old_ty in &old_param_types {
678            let new_ty = match cx.type_kind(old_ty) {
679                TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::Integer => {
680                    cx.type_i64()
681                }
682                _ => old_ty,
683            };
684            new_param_types.push(new_ty);
685        }
686
687        // Create the new function type
688        let ret_ty = unsafe { llvm::LLVMGetReturnType(old_fn_ty) };
689        let new_fn_ty = cx.type_func(&new_param_types, ret_ty);
690
691        // Create the new function, with a temporary .offload name to avoid a name collision.
692        let old_fn_name = String::from_utf8(llvm::get_value_name(old_fn)).unwrap();
693        let new_fn_name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}.offload", &old_fn_name))
    })format!("{}.offload", &old_fn_name);
694        let new_fn = cx.add_func(&new_fn_name, new_fn_ty);
695        let a0 = llvm::get_param(new_fn, 0);
696        llvm::set_value_name(a0, CString::new("dyn_ptr").unwrap().as_bytes());
697
698        let bb = SBuilder::append_block(cx, new_fn, "entry");
699        let mut builder = SBuilder::build(cx, bb);
700
701        let mut old_args_rebuilt = Vec::with_capacity(old_param_types.len());
702
703        for (i, &old_ty) in old_param_types.iter().enumerate() {
704            let new_arg = llvm::get_param(new_fn, (i + 1) as u32);
705
706            let rebuilt = match cx.type_kind(old_ty) {
707                TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::Integer => {
708                    let num_bits = scalar_width(cx, old_ty);
709
710                    let trunc = builder.trunc(new_arg, cx.type_ix(num_bits));
711                    builder.bitcast(trunc, old_ty)
712                }
713                _ => new_arg,
714            };
715
716            old_args_rebuilt.push(rebuilt);
717        }
718
719        builder.ret_void();
720
721        // Here we map the old arguments to the new arguments, with an offset of 1 to make sure
722        // that we don't use the newly added `%dyn_ptr`.
723        unsafe {
724            llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_wrapper(
725                old_fn,
726                new_fn,
727                old_args_rebuilt.as_slice(),
728            );
729        }
730
731        llvm::set_linkage(new_fn, llvm::get_linkage(old_fn));
732        llvm::set_visibility(new_fn, llvm::get_visibility(old_fn));
733
734        // Replace all uses of old_fn with new_fn (RAUW)
735        unsafe {
736            llvm::LLVMReplaceAllUsesWith(old_fn, new_fn);
737        }
738        let name = llvm::get_value_name(old_fn);
739        unsafe {
740            llvm::LLVMDeleteFunction(old_fn);
741        }
742        // Now we can re-use the old name, without name collision.
743        llvm::set_value_name(new_fn, &name);
744        Some(new_fn)
745    }
746
747    if cgcx.target_is_like_gpu
748        && config.offload.iter().any(|o| #[allow(non_exhaustive_omitted_patterns)] match o {
    config::Offload::Device(_) => true,
    _ => false,
}matches!(o, config::Offload::Device(_)))
749    {
750        let cx = SimpleCx::new(llmod, llcx, cgcx.pointer_size);
751        for func in cx.get_functions() {
752            let offload_kernel = "offload-kernel";
753            if !attributes::has_string_attr(func, offload_kernel) {
754                continue;
755            }
756            match handle_offload(&cx, func) {
757                Some(new_fn) => attributes::remove_string_attr_from_llfn(new_fn, offload_kernel),
758                None => attributes::remove_string_attr_from_llfn(func, offload_kernel),
759            }
760        }
761    }
762
763    let mut llvm_profiler = prof
764        .llvm_recording_enabled()
765        .then(|| LlvmSelfProfiler::new(prof.get_self_profiler().unwrap()));
766
767    let llvm_selfprofiler =
768        llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
769
770    let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
771
772    let llvm_plugins = config.llvm_plugins.join(",");
773
774    let enzyme_fn = if consider_ad {
775        let wrapper = llvm::EnzymeWrapper::get_instance();
776        wrapper.registerEnzymeAndPassPipeline
777    } else {
778        std::ptr::null()
779    };
780
781    let result = unsafe {
782        llvm::LLVMRustOptimize(
783            llmod,
784            tm,
785            to_pass_builder_opt_level(opt_level),
786            opt_stage,
787            cgcx.use_linker_plugin_lto,
788            config.no_prepopulate_passes,
789            config.verify_llvm_ir,
790            config.lint_llvm_ir,
791            thin_lto_buffer,
792            thin_lto_summary_buffer,
793            merge_functions,
794            unroll_loops,
795            vectorize_slp,
796            vectorize_loop,
797            config.no_builtins,
798            config.emit_lifetime_markers,
799            enzyme_fn,
800            print_before_enzyme,
801            print_after_enzyme,
802            print_passes,
803            sanitizer_options.as_ref(),
804            pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
805            pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
806            config.instrument_coverage,
807            instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
808            pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
809            config.debug_info_for_profiling,
810            llvm_selfprofiler,
811            selfprofile_before_pass_callback,
812            selfprofile_after_pass_callback,
813            passes_after_enzyme.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()),
814            passes_after_enzyme.map(|s| s.len()).unwrap_or(0),
815            extra_passes.as_ptr(),
816            extra_passes.len(),
817            llvm_plugins.as_ptr(),
818            llvm_plugins.len(),
819        )
820    };
821
822    // This assumes that we previously compiled our kernels for a gpu target, which created a
823    // `<crate>.ptx` (NVPTX) or `<crate>.elf` (AMDGPU) artifact.
824    // The user is supposed to provide us with a path to this artifact,
825    // we don't need any other artifacts from the previous run.
826    if !cgcx.target_is_like_gpu && is_final_stage {
827        if let Some(device_path) = config
828            .offload
829            .iter()
830            .find_map(|o| if let config::Offload::Host(path) = o { Some(path) } else { None })
831        {
832            let device_pathbuf = PathBuf::from(device_path);
833            if device_pathbuf.is_relative() {
834                dcx.emit_err(crate::diagnostics::OffloadWithoutAbsPath);
835            } else if !#[allow(non_exhaustive_omitted_patterns)] match device_pathbuf.extension().and_then(|e|
            e.to_str()) {
    Some("ptx" | "elf") => true,
    _ => false,
}matches!(
836                device_pathbuf.extension().and_then(|e| e.to_str()),
837                Some("ptx" | "elf")
838            ) {
839                dcx.emit_err(crate::diagnostics::OffloadWrongFileName);
840            } else if !device_pathbuf.exists() {
841                dcx.emit_err(crate::diagnostics::OffloadNonexistingPath);
842            }
843            let device_image_c = path_to_c_string(device_pathbuf.as_path());
844            let ok = unsafe {
845                llvm::RustOffloadWrapper::get_instance()
846                    .llvm_rust_offload_wrap_images(llmod, device_image_c.as_c_str())
847            };
848            if !ok {
849                dcx.emit_err(crate::diagnostics::OffloadWrapImagesFailed);
850            }
851        }
852    }
853    result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
854}
855
856pub(crate) fn optimize(
857    cgcx: &CodegenContext,
858    prof: &SelfProfilerRef,
859    shared_emitter: &SharedEmitter,
860    module: &ModuleCodegen<ModuleLlvm>,
861    config: &ModuleConfig,
862) {
863    // Pass `module_refs` to an inner function to avoid having to indent the whole thing.
864    //
865    // It feels a bit odd to be passing both `module` and `module_refs` here, but it's
866    // unclear whether trying to avoid that would actually be an improvement.
867    module.module_llvm.with_module_refs(|module_refs| {
868        optimize_inner(cgcx, prof, shared_emitter, module, module_refs, config);
869    });
870}
871
872fn optimize_inner<'ll>(
873    cgcx: &CodegenContext,
874    prof: &SelfProfilerRef,
875    shared_emitter: &SharedEmitter,
876    module: &ModuleCodegen<ModuleLlvm>,
877    module_refs: ModuleRefs<'ll>, // Should be the refs from `module.module_llvm`
878    config: &ModuleConfig,
879) {
880    let _timer = prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
881
882    let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
883    let dcx = dcx.handle();
884
885    let _handlers = DiagnosticHandlers::new(
886        cgcx,
887        shared_emitter,
888        module_refs.llcx,
889        module,
890        CodegenDiagnosticsStage::Opt,
891    );
892
893    if module.kind == ModuleKind::Regular {
894        save_temp_bitcode(cgcx, module, "no-opt");
895    }
896
897    // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts
898
899    if let Some(opt_level) = config.opt_level {
900        let opt_stage = match cgcx.lto {
901            Lto::Fat => llvm::OptStage::PreLinkFatLTO,
902            Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
903            _ if cgcx.use_linker_plugin_lto => llvm::OptStage::PreLinkThinLTO,
904            _ => llvm::OptStage::PreLinkNoLTO,
905        };
906
907        // If we know that we will later run AD, then we disable vectorization and loop unrolling.
908        // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
909        let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable);
910        let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
911        // The embedded bitcode is used to run LTO/ThinLTO.
912        // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO.
913        // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at
914        // this point.
915        let (mut thin_lto_buffer, mut thin_lto_summary_buffer) = if (module.kind
916            == ModuleKind::Regular
917            && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
918            || config.emit_thin_lto_summary
919        {
920            (Some(None), config.emit_thin_lto_summary.then_some(None))
921        } else {
922            (None, None)
923        };
924        unsafe {
925            llvm_optimize(
926                cgcx,
927                prof,
928                dcx,
929                module_refs,
930                thin_lto_buffer.as_mut(),
931                thin_lto_summary_buffer.as_mut(),
932                config,
933                opt_level,
934                opt_stage,
935                autodiff_stage,
936            )
937        };
938        if let Some(thin_lto_buffer) = thin_lto_buffer {
939            let thin_lto_buffer = thin_lto_buffer.unwrap();
940            module
941                .module_llvm
942                .thin_lto_buffer
943                .set(thin_lto_buffer.data().to_vec())
944                .expect("thin_lto_buffer should be unset");
945            let bc_summary_out =
946                cgcx.output_filenames.temp_path_for_cgu(OutputType::ThinLinkBitcode, &module.name);
947            if let Some(thin_lto_summary_buffer) = thin_lto_summary_buffer
948                && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
949            {
950                let thin_lto_summary_buffer = thin_lto_summary_buffer.unwrap();
951                let summary_data = thin_lto_summary_buffer.data();
952                prof.artifact_size(
953                    "llvm_bitcode_summary",
954                    thin_link_bitcode_filename.to_string_lossy(),
955                    summary_data.len() as u64,
956                );
957                let _timer = prof.generic_activity_with_arg(
958                    "LLVM_module_codegen_emit_bitcode_summary",
959                    &*module.name,
960                );
961                if let Err(err) = fs::write(&bc_summary_out, summary_data) {
962                    dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
963                }
964            }
965        }
966    }
967}
968
969pub(crate) fn codegen(
970    cgcx: &CodegenContext,
971    prof: &SelfProfilerRef,
972    shared_emitter: &SharedEmitter,
973    module: ModuleCodegen<ModuleLlvm>,
974    config: &ModuleConfig,
975) -> CompiledModule {
976    let _timer = prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
977
978    let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
979    let dcx = dcx.handle();
980
981    module.module_llvm.with_module_refs(|ModuleRefs { llcx, llmod, tm, .. }| {
982        let _handlers = DiagnosticHandlers::new(
983            cgcx,
984            shared_emitter,
985            llcx,
986            &module,
987            CodegenDiagnosticsStage::Codegen,
988        );
989
990        if cgcx.msvc_imps_needed {
991            create_msvc_imps(cgcx, llcx, llmod);
992        }
993
994        // Note that if object files are just LLVM bitcode we write bitcode,
995        // copy it to the .o file, and delete the bitcode if it wasn't
996        // otherwise requested.
997
998        let bc_out = cgcx.output_filenames.temp_path_for_cgu(OutputType::Bitcode, &module.name);
999        let obj_out = cgcx.output_filenames.temp_path_for_cgu(OutputType::Object, &module.name);
1000
1001        if config.bitcode_needed() {
1002            if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
1003                let thin = {
1004                    let _timer = prof.generic_activity_with_arg(
1005                        "LLVM_module_codegen_make_bitcode",
1006                        &*module.name,
1007                    );
1008                    ModuleBuffer::new(llmod, cgcx.lto != Lto::Fat)
1009                };
1010                let data = thin.data();
1011                let _timer = prof
1012                    .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
1013                if let Some(bitcode_filename) = bc_out.file_name() {
1014                    prof.artifact_size(
1015                        "llvm_bitcode",
1016                        bitcode_filename.to_string_lossy(),
1017                        data.len() as u64,
1018                    );
1019                }
1020                if let Err(err) = fs::write(&bc_out, data) {
1021                    dcx.emit_err(WriteBytecode { path: &bc_out, err });
1022                }
1023            }
1024
1025            if config.embed_bitcode() && module.kind == ModuleKind::Regular {
1026                let _timer = prof
1027                    .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
1028                let thin_bc =
1029                    module.module_llvm.thin_lto_buffer.get().expect("cannot find embedded bitcode");
1030                embed_bitcode(cgcx, llcx, llmod, &thin_bc);
1031            }
1032        }
1033
1034        if config.emit_ir {
1035            let _timer =
1036                prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
1037            let out =
1038                cgcx.output_filenames.temp_path_for_cgu(OutputType::LlvmAssembly, &module.name);
1039            let out_c = path_to_c_string(&out);
1040
1041            extern "C" fn demangle_callback(
1042                input_ptr: *const c_char,
1043                input_len: size_t,
1044                output_ptr: *mut c_char,
1045                output_len: size_t,
1046            ) -> size_t {
1047                let input =
1048                    unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
1049
1050                let Ok(input) = str::from_utf8(input) else { return 0 };
1051
1052                let output = unsafe {
1053                    slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
1054                };
1055                let mut cursor = io::Cursor::new(output);
1056
1057                let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
1058
1059                if cursor.write_fmt(format_args!("{0:#}", demangled))write!(cursor, "{demangled:#}").is_err() {
1060                    // Possible only if provided buffer is not big enough
1061                    return 0;
1062                }
1063
1064                cursor.position() as size_t
1065            }
1066
1067            let result =
1068                unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
1069
1070            if result == llvm::LLVMRustResult::Success {
1071                record_artifact_size(prof, "llvm_ir", &out);
1072            }
1073
1074            result
1075                .into_result()
1076                .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }));
1077        }
1078
1079        if config.emit_asm {
1080            let _timer =
1081                prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
1082            let path = cgcx.output_filenames.temp_path_for_cgu(OutputType::Assembly, &module.name);
1083
1084            // We can't use the same module for asm and object code output,
1085            // because that triggers various errors like invalid IR or broken
1086            // binaries. So we must clone the module to produce the asm output
1087            // if we are also producing object code.
1088            let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
1089                llvm::LLVMCloneModule(llmod)
1090            } else {
1091                llmod
1092            };
1093            write_output_file(
1094                dcx,
1095                tm,
1096                config.no_builtins,
1097                llmod,
1098                &path,
1099                None,
1100                llvm::FileType::AssemblyFile,
1101                prof,
1102                config.verify_llvm_ir,
1103            );
1104        }
1105
1106        match config.emit_obj {
1107            EmitObj::ObjectCode(_) => {
1108                let _timer =
1109                    prof.generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
1110
1111                let dwo_out = cgcx.output_filenames.temp_path_dwo_for_cgu(&module.name);
1112                let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
1113                    // Don't change how DWARF is emitted when disabled.
1114                    (SplitDebuginfo::Off, _) => None,
1115                    // Don't provide a DWARF object path if split debuginfo is enabled but this is
1116                    // a platform that doesn't support Split DWARF.
1117                    _ if !cgcx.target_can_use_split_dwarf => None,
1118                    // Don't provide a DWARF object path in single mode, sections will be written
1119                    // into the object as normal but ignored by linker.
1120                    (_, SplitDwarfKind::Single) => None,
1121                    // Emit (a subset of the) DWARF into a separate dwarf object file in split
1122                    // mode.
1123                    (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
1124                };
1125
1126                write_output_file(
1127                    dcx,
1128                    tm,
1129                    config.no_builtins,
1130                    llmod,
1131                    &obj_out,
1132                    dwo_out,
1133                    llvm::FileType::ObjectFile,
1134                    prof,
1135                    config.verify_llvm_ir,
1136                );
1137            }
1138
1139            EmitObj::Bitcode => {
1140                {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:1140",
                        "rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
                        ::tracing_core::__macro_support::Option::Some(1140u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("copying bitcode {0:?} to obj {1:?}",
                                                    bc_out, obj_out) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
1141                if let Err(err) = link_or_copy(&bc_out, &obj_out) {
1142                    dcx.emit_err(CopyBitcode { err });
1143                }
1144
1145                if !config.emit_bc {
1146                    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:1146",
                        "rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
                        ::tracing_core::__macro_support::Option::Some(1146u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing_bitcode {0:?}",
                                                    bc_out) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("removing_bitcode {:?}", bc_out);
1147                    ensure_removed(dcx, &bc_out);
1148                }
1149            }
1150
1151            EmitObj::None => {}
1152        }
1153
1154        record_llvm_cgu_instructions_stats(prof, &module.name, llmod);
1155    });
1156
1157    // `.dwo` files are only emitted if:
1158    //
1159    // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not
1160    //   produce dwarf objects, even if otherwise enabled)
1161    // - Target supports Split DWARF
1162    // - Split debuginfo is enabled
1163    // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different
1164    //   sections in the `.o` files).
1165    let dwarf_object_emitted = #[allow(non_exhaustive_omitted_patterns)] match config.emit_obj {
    EmitObj::ObjectCode(_) => true,
    _ => false,
}matches!(config.emit_obj, EmitObj::ObjectCode(_))
1166        && cgcx.target_can_use_split_dwarf
1167        && cgcx.split_debuginfo != SplitDebuginfo::Off
1168        && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1169    module.into_compiled_module(
1170        config.emit_obj != EmitObj::None,
1171        dwarf_object_emitted,
1172        config.emit_bc,
1173        config.emit_asm,
1174        config.emit_ir,
1175        &cgcx.output_filenames,
1176    )
1177}
1178
1179fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1180    let mut asm = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".section {0},\"{1}\"\n",
                section_name, section_flags))
    })format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1181    asm.extend_from_slice(b".ascii \"");
1182    asm.reserve(data.len());
1183    for &byte in data {
1184        if byte == b'\\' || byte == b'"' {
1185            asm.push(b'\\');
1186            asm.push(byte);
1187        } else if byte < 0x20 || byte >= 0x80 {
1188            // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed
1189            // width, while hex escapes will consume following characters.
1190            asm.push(b'\\');
1191            asm.push(b'0' + ((byte >> 6) & 0x7));
1192            asm.push(b'0' + ((byte >> 3) & 0x7));
1193            asm.push(b'0' + ((byte >> 0) & 0x7));
1194        } else {
1195            asm.push(byte);
1196        }
1197    }
1198    asm.extend_from_slice(b"\"\n");
1199    asm
1200}
1201
1202pub(crate) fn bitcode_section_name(cgcx: &CodegenContext) -> &'static CStr {
1203    if cgcx.target_is_like_darwin {
1204        c"__LLVM,__bitcode"
1205    } else if cgcx.target_is_like_aix {
1206        c".ipa"
1207    } else {
1208        c".llvmbc"
1209    }
1210}
1211
1212/// Embed the bitcode of an LLVM module for LTO in the LLVM module itself.
1213fn embed_bitcode<'ll>(
1214    cgcx: &CodegenContext,
1215    llcx: llvm::Context<'ll>,
1216    llmod: llvm::Module<'ll>,
1217    bitcode: &[u8],
1218) {
1219    // We're adding custom sections to the output object file, but we definitely
1220    // do not want these custom sections to make their way into the final linked
1221    // executable. The purpose of these custom sections is for tooling
1222    // surrounding object files to work with the LLVM IR, if necessary. For
1223    // example rustc's own LTO will look for LLVM IR inside of the object file
1224    // in these sections by default.
1225    //
1226    // To handle this is a bit different depending on the object file format
1227    // used by the backend, broken down into a few different categories:
1228    //
1229    // * Mach-O - this is for macOS. Inspecting the source code for the native
1230    //   linker here shows that the `.llvmbc` and `.llvmcmd` sections are
1231    //   automatically skipped by the linker. In that case there's nothing extra
1232    //   that we need to do here. We do need to make sure that the
1233    //   `__LLVM,__cmdline` section exists even though it is empty as otherwise
1234    //   ld64 rejects the object file.
1235    //
1236    // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and
1237    //   `.llvmcmd` sections, so there's nothing extra we need to do.
1238    //
1239    // * COFF - if we don't do anything the linker will by default copy all
1240    //   these sections to the output artifact, not what we want! To subvert
1241    //   this we want to flag the sections we inserted here as
1242    //   `IMAGE_SCN_LNK_REMOVE`.
1243    //
1244    // * ELF - this is very similar to COFF above. One difference is that these
1245    //   sections are removed from the output linked artifact when
1246    //   `--gc-sections` is passed, which we pass by default. If that flag isn't
1247    //   passed though then these sections will show up in the final output.
1248    //   Additionally the flag that we need to set here is `SHF_EXCLUDE`.
1249    //
1250    // * XCOFF - AIX linker ignores content in .ipa and .info if no auxiliary
1251    //   symbol associated with these sections.
1252    //
1253    // Unfortunately, LLVM provides no way to set custom section flags. For ELF
1254    // and COFF we emit the sections using module level inline assembly for that
1255    // reason (see issue #90326 for historical background).
1256
1257    if cgcx.target_is_like_darwin
1258        || cgcx.target_is_like_aix
1259        || cgcx.target_arch == "wasm32"
1260        || cgcx.target_arch == "wasm64"
1261    {
1262        // We don't need custom section flags, create LLVM globals.
1263        let llconst = common::bytes_in_context(llcx, bitcode);
1264        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1265        llvm::set_initializer(llglobal, llconst);
1266
1267        llvm::set_section(llglobal, bitcode_section_name(cgcx));
1268        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1269        llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE);
1270
1271        let llconst = common::bytes_in_context(llcx, &[]);
1272        let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1273        llvm::set_initializer(llglobal, llconst);
1274        let section = if cgcx.target_is_like_darwin {
1275            c"__LLVM,__cmdline"
1276        } else if cgcx.target_is_like_aix {
1277            c".info"
1278        } else {
1279            c".llvmcmd"
1280        };
1281        llvm::set_section(llglobal, section);
1282        llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1283    } else {
1284        // We need custom section flags, so emit module-level inline assembly.
1285        let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1286        let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1287        llvm::append_module_inline_asm(llmod, &asm, "", "");
1288        let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
1289        llvm::append_module_inline_asm(llmod, &asm, "", "");
1290    }
1291}
1292
1293// Create a `__imp_<symbol> = &symbol` global for each externally visible
1294// static data symbol, including aliases to static data.
1295// This is required to satisfy `dllimport` references to static data in .rlibs
1296// when using MSVC linker. We do this only for data, as linker can fix up
1297// code references on its own.
1298// See #26591, #27438
1299fn create_msvc_imps<'ll>(
1300    cgcx: &CodegenContext,
1301    llcx: llvm::Context<'ll>,
1302    llmod: llvm::Module<'ll>,
1303) {
1304    if !cgcx.msvc_imps_needed {
1305        return;
1306    }
1307    // The x86 ABI seems to require that leading underscores are added to symbol
1308    // names, so we need an extra underscore on x86. There's also a leading
1309    // '\x01' here which disables LLVM's symbol mangling (e.g., no extra
1310    // underscores added in front).
1311    let prefix: &[u8] = if cgcx.target_arch == "x86" { b"\x01__imp__" } else { b"\x01__imp_" };
1312
1313    let ptr_ty = llvm_type_ptr(llcx);
1314    let symbols = std::iter::chain(
1315        base::iter_globals(llmod),
1316        base::iter_global_aliases(llmod).filter(|&val| {
1317            llvm::LLVMGetTypeKind(unsafe { llvm::LLVMGlobalGetValueType(val) }).to_rust()
1318                != llvm::TypeKind::Function
1319        }),
1320    )
1321    .map(|val| (val, llvm::get_linkage(val)))
1322    .filter(|&(val, linkage)| {
1323        #[allow(non_exhaustive_omitted_patterns)] match linkage {
    llvm::Linkage::ExternalLinkage | llvm::Linkage::WeakAnyLinkage => true,
    _ => false,
}matches!(linkage, llvm::Linkage::ExternalLinkage | llvm::Linkage::WeakAnyLinkage)
1324            && !llvm::is_declaration(val)
1325    })
1326    .collect::<Vec<_>>();
1327
1328    for (val, linkage) in symbols {
1329        let name = llvm::get_value_name(val);
1330        // Exclude some symbols that we know are not Rust symbols.
1331        if ignored(&name) {
1332            continue;
1333        }
1334
1335        let mut imp_name = prefix.to_vec();
1336        imp_name.extend(name);
1337        let imp_name = CString::new(imp_name).unwrap();
1338
1339        let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1340
1341        llvm::set_initializer(imp, val);
1342        llvm::set_linkage(imp, linkage);
1343    }
1344
1345    // Use this function to exclude certain symbols from `__imp` generation.
1346    fn ignored(symbol_name: &[u8]) -> bool {
1347        // These are symbols generated by LLVM's profiling instrumentation
1348        symbol_name.starts_with(b"__llvm_profile_")
1349    }
1350}
1351
1352fn record_artifact_size(
1353    self_profiler_ref: &SelfProfilerRef,
1354    artifact_kind: &'static str,
1355    path: &Path,
1356) {
1357    // Don't stat the file if we are not going to record its size.
1358    if !self_profiler_ref.enabled() {
1359        return;
1360    }
1361
1362    if let Some(artifact_name) = path.file_name() {
1363        let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1364        self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1365    }
1366}
1367
1368fn record_llvm_cgu_instructions_stats<'ll>(
1369    prof: &SelfProfilerRef,
1370    name: &str,
1371    llmod: llvm::Module<'ll>,
1372) {
1373    if !prof.enabled() {
1374        return;
1375    }
1376
1377    let total = unsafe { llvm::LLVMRustModuleInstructionStats(llmod) };
1378    prof.artifact_size("cgu_instructions", name, total);
1379}