Skip to main content

rustc_codegen_llvm/
llvm_util.rs

1use std::borrow::Cow;
2use std::collections::VecDeque;
3use std::ffi::{CStr, CString};
4use std::fmt::Write;
5use std::path::Path;
6use std::sync::Once;
7use std::{ptr, slice, str};
8
9use libc::c_int;
10use rustc_abi::Endian;
11use rustc_codegen_ssa::back::versioned_llvm_target;
12use rustc_codegen_ssa::base::wants_wasm_eh;
13use rustc_codegen_ssa::target_features::internal_target_features;
14use rustc_codegen_ssa::{TargetConfig, target_features};
15use rustc_data_structures::fx::FxHashSet;
16use rustc_data_structures::small_c_str::SmallCStr;
17use rustc_fs_util::path_to_c_string;
18use rustc_session::config::{NATIVE_CPU, PrintKind, PrintRequest};
19use rustc_session::{EarlySession, Session};
20use rustc_span::{bug, sym};
21use rustc_target::spec::{
22    Arch, CfgAbi, Env, MergeFunctions, Os, PanicStrategy, SmallDataThresholdSupport, Target,
23};
24use smallvec::{SmallVec, smallvec};
25
26use crate::back::owned_mc_subtarget_info::OwnedMCSubtargetInfo;
27use crate::back::write::{create_informational_target_machine, llvm_err};
28use crate::{diagnostics, llvm};
29
30static INIT: Once = Once::new();
31
32pub(crate) fn init(sess: &EarlySession) {
33    unsafe {
34        // Before we touch LLVM, make sure that multithreading is enabled.
35        if !llvm::LLVMIsMultithreaded().is_true() {
36            ::rustc_span::macros::bug_impl(None,
    format_args!("LLVM compiled without support for threads"),
    Location::caller());bug!("LLVM compiled without support for threads");
37        }
38        INIT.call_once(|| {
39            configure_llvm(sess);
40        });
41    }
42}
43
44fn require_inited() {
45    if !INIT.is_completed() {
46        ::rustc_span::macros::bug_impl(None, format_args!("LLVM is not initialized"),
    Location::caller());bug!("LLVM is not initialized");
47    }
48}
49
50unsafe fn configure_llvm(sess: &EarlySession) {
51    let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len();
52    let mut llvm_c_strs = Vec::with_capacity(n_args + 1);
53    let mut llvm_args = Vec::with_capacity(n_args + 1);
54
55    // Check to ensure we're running against the correct LLVM version.
56    unsafe {
57        let (llvm_major, llvm_minor, llvm_patch) = get_version();
58        let expected_version = llvm::LLVMRustVersionMajor();
59        if llvm_major != expected_version {
60            sess.dcx().emit_fatal(diagnostics::LlvmVersionMismatch {
61                expected_version,
62                llvm_major,
63                llvm_minor,
64                llvm_patch,
65                dll_loc: &match rustc_session::filesearch::dll_path(llvm::LLVMGetVersion as *mut _)
66                {
67                    Ok(path) => ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(" at {0}", path.display()))
    })format!(" at {}", path.display()),
68                    Err(_) => String::new(),
69                },
70            })
71        }
72    }
73
74    unsafe {
75        llvm::LLVMRustInstallErrorHandlers();
76    }
77    // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog
78    // box for the purpose of launching a debugger. However, on CI this will
79    // cause it to hang until it times out, which can take several hours.
80    if std::env::var_os("CI").is_some() {
81        unsafe {
82            llvm::LLVMRustDisableSystemDialogsOnCrash();
83        }
84    }
85
86    fn llvm_arg_to_arg_name(full_arg: &str) -> &str {
87        full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("")
88    }
89
90    let cg_opts = sess.opts.cg.llvm_args.iter().map(AsRef::as_ref);
91    let tg_opts = sess.target.llvm_args.iter().map(AsRef::as_ref);
92    // Target-spec args are passed to LLVM before user `-Cllvm-args`. LLVM's
93    // `cl::opt` parser is last-wins, so this lets `-Cllvm-args=...` override
94    // a value already set in the target spec (e.g. `-wasm-use-legacy-eh`).
95    let sess_args = tg_opts.chain(cg_opts);
96
97    let user_specified_args: FxHashSet<_> =
98        sess_args.clone().map(|s| llvm_arg_to_arg_name(s)).filter(|s| !s.is_empty()).collect();
99
100    {
101        // This adds the given argument to LLVM. Unless `force` is true
102        // user specified arguments are *not* overridden.
103        let mut add = |arg: &str, force: bool| {
104            if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) {
105                let s = CString::new(arg).unwrap();
106                llvm_args.push(s.as_ptr());
107                llvm_c_strs.push(s);
108            }
109        };
110        // Set the llvm "program name" to make usage and invalid argument messages more clear.
111        add("rustc -Cllvm-args=\"...\" with", true);
112        if sess.opts.unstable_opts.time_llvm_passes {
113            add("-time-passes", false);
114        }
115        if sess.opts.unstable_opts.print_llvm_passes {
116            add("-debug-pass=Structure", false);
117        }
118        if sess.target.generate_arange_section
119            && !sess.opts.unstable_opts.no_generate_arange_section
120        {
121            add("-generate-arange-section", false);
122        }
123
124        match sess.merge_functions() {
125            MergeFunctions::Disabled | MergeFunctions::Trampolines => {}
126            MergeFunctions::Aliases => {
127                add("-mergefunc-use-aliases", false);
128            }
129        }
130
131        if get_version() < (24, 0, 0) && wants_wasm_eh(&sess.target) {
132            add("-wasm-enable-eh", false);
133        }
134
135        // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes
136        // during inlining. Unfortunately these may block other optimizations.
137        add("-preserve-alignment-assumptions-during-inlining=false", false);
138
139        // Use non-zero `import-instr-limit` multiplier for cold callsites.
140        add("-import-cold-multiplier=0.1", false);
141
142        if sess.print_llvm_stats() || sess.print_llvm_stats_json().is_some() {
143            add("-stats", false);
144        }
145
146        for arg in sess_args {
147            add(&(*arg), true);
148        }
149
150        match (
151            sess.opts.unstable_opts.small_data_threshold,
152            sess.target.small_data_threshold_support(),
153        ) {
154            // Set up the small-data optimization limit for architectures that use
155            // an LLVM argument to control this.
156            (Some(threshold), SmallDataThresholdSupport::LlvmArg(arg)) => {
157                add(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("--{0}={1}", arg, threshold))
    })format!("--{arg}={threshold}"), false)
158            }
159            _ => (),
160        };
161    }
162
163    if sess.opts.unstable_opts.llvm_time_trace {
164        unsafe { llvm::LLVMRustTimeTraceProfilerInitialize() };
165    }
166
167    rustc_llvm::initialize_available_targets();
168
169    unsafe { llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()) };
170}
171
172pub(crate) fn time_trace_profiler_finish(file_name: &Path) {
173    unsafe {
174        let file_name = path_to_c_string(file_name);
175        llvm::LLVMRustTimeTraceProfilerFinish(file_name.as_ptr());
176    }
177}
178
179enum TargetFeatureFoldStrength<'a> {
180    // The feature is only tied when enabling the feature, disabling
181    // this feature shouldn't disable the tied feature.
182    EnableOnly(&'a str),
183    // The feature is tied for both enabling and disabling this feature.
184    Both(&'a str),
185}
186
187impl<'a> TargetFeatureFoldStrength<'a> {
188    fn as_str(&self) -> &'a str {
189        match self {
190            TargetFeatureFoldStrength::EnableOnly(feat) => feat,
191            TargetFeatureFoldStrength::Both(feat) => feat,
192        }
193    }
194}
195
196pub(crate) struct LLVMFeature<'a> {
197    llvm_feature_name: &'a str,
198    dependencies: SmallVec<[TargetFeatureFoldStrength<'a>; 1]>,
199}
200
201impl<'a> LLVMFeature<'a> {
202    fn new(llvm_feature_name: &'a str) -> Self {
203        Self { llvm_feature_name, dependencies: SmallVec::new() }
204    }
205
206    fn with_dependencies(
207        llvm_feature_name: &'a str,
208        dependencies: SmallVec<[TargetFeatureFoldStrength<'a>; 1]>,
209    ) -> Self {
210        Self { llvm_feature_name, dependencies }
211    }
212}
213
214impl<'a> IntoIterator for LLVMFeature<'a> {
215    type Item = &'a str;
216    type IntoIter = impl Iterator<Item = &'a str>;
217
218    fn into_iter(self) -> Self::IntoIter {
219        let dependencies = self.dependencies.into_iter().map(|feat| feat.as_str());
220        std::iter::once(self.llvm_feature_name).chain(dependencies)
221    }
222}
223
224/// Convert a Rust feature name to an LLVM feature name. Returning `None` means the
225/// feature should be skipped, usually because it is not supported by the current
226/// LLVM version.
227///
228/// WARNING: the features after applying `to_llvm_features` must be known
229/// to LLVM or the feature detection code will walk past the end of the feature
230/// array, leading to crashes.
231///
232/// To find a list of LLVM's names, see llvm-project/llvm/lib/Target/{ARCH}/*.td
233/// where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`.
234///
235/// Check the current rustc fork of LLVM in the repo at
236/// <https://github.com/rust-lang/llvm-project/>. The commit in use can be found via the
237/// `llvm-project` submodule in <https://github.com/rust-lang/rust/tree/HEAD/src> Though note that
238/// Rust can also be build with an external precompiled version of LLVM which might lead to failures
239/// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics.
240pub(crate) fn to_llvm_features<'a>(target: &Target, s: &'a str) -> Option<LLVMFeature<'a>> {
241    let (major, _, _) = get_version();
242    match target.arch {
243        Arch::AArch64 | Arch::Arm64EC => {
244            match s {
245                "rcpc2" => Some(LLVMFeature::new("rcpc-immo")),
246                "dpb" => Some(LLVMFeature::new("ccpp")),
247                "dpb2" => Some(LLVMFeature::new("ccdp")),
248                "frintts" => Some(LLVMFeature::new("fptoint")),
249                "fcma" => Some(LLVMFeature::new("complxnum")),
250                "pmuv3" => Some(LLVMFeature::new("perfmon")),
251                "paca" => Some(LLVMFeature::new("pauth")),
252                "pacg" => Some(LLVMFeature::new("pauth")),
253                "flagm2" => Some(LLVMFeature::new("altnzcv")),
254                // Rust ties fp and neon together.
255                "neon" => Some(LLVMFeature::with_dependencies(
256                    "neon",
257                    {
    let count = 0usize + 1usize;
    let mut vec = ::smallvec::SmallVec::new();
    if count <= vec.inline_size() {
        vec.push(TargetFeatureFoldStrength::Both("fp-armv8"));
        vec
    } else {
        ::smallvec::SmallVec::from_vec(::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
                    [TargetFeatureFoldStrength::Both("fp-armv8")])))
    }
}smallvec![TargetFeatureFoldStrength::Both("fp-armv8")],
258                )),
259                // In LLVM neon implicitly enables fp, but we manually enable
260                // neon when a feature only implicitly enables fp
261                "fhm" => Some(LLVMFeature::new("fp16fml")),
262                "fp16" => Some(LLVMFeature::new("fullfp16")),
263                // Filter out features that are not supported by the current LLVM version
264                "fpmr" => None, // only existed in 18
265                // Withdrawn by ARM; removed from LLVM in 22
266                "tme" => None,
267                s => Some(LLVMFeature::new(s)),
268            }
269        }
270        Arch::Arm => match s {
271            "fp16" => Some(LLVMFeature::new("fullfp16")),
272            s => Some(LLVMFeature::new(s)),
273        },
274        Arch::Nvptx64 => match s {
275            "sm_101" if major >= 24 => Some(LLVMFeature::new("sm_110")),
276            "sm_101a" if major >= 24 => Some(LLVMFeature::new("sm_110a")),
277            "sm_101f" if major >= 24 => Some(LLVMFeature::new("sm_110f")),
278            s => Some(LLVMFeature::new(s)),
279        },
280        // Filter out features that are not supported by the current LLVM version
281        Arch::PowerPC | Arch::PowerPC64 => match s {
282            "power8-crypto" => Some(LLVMFeature::new("crypto")),
283            s => Some(LLVMFeature::new(s)),
284        },
285        Arch::RiscV32 | Arch::RiscV64 => match s {
286            // Filter out Rust-specific *virtual* target feature
287            "zkne_or_zknd" => None,
288            s => Some(LLVMFeature::new(s)),
289        },
290        Arch::Sparc | Arch::Sparc64 => match s {
291            "leoncasa" => Some(LLVMFeature::new("hasleoncasa")),
292            s => Some(LLVMFeature::new(s)),
293        },
294        Arch::X86 | Arch::X86_64 => match s {
295            "sse4.2" => Some(LLVMFeature::with_dependencies(
296                "sse4.2",
297                {
    let count = 0usize + 1usize;
    let mut vec = ::smallvec::SmallVec::new();
    if count <= vec.inline_size() {
        vec.push(TargetFeatureFoldStrength::EnableOnly("crc32"));
        vec
    } else {
        ::smallvec::SmallVec::from_vec(::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
                    [TargetFeatureFoldStrength::EnableOnly("crc32")])))
    }
}smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")],
298            )),
299            "pclmulqdq" => Some(LLVMFeature::new("pclmul")),
300            "rdrand" => Some(LLVMFeature::new("rdrnd")),
301            "bmi1" => Some(LLVMFeature::new("bmi")),
302            "cmpxchg16b" => Some(LLVMFeature::new("cx16")),
303            "lahfsahf" => Some(LLVMFeature::new("sahf")),
304            "apxf" => Some(LLVMFeature::with_dependencies(
305                "egpr",
306                {
    let count =
        0usize + 1usize + 1usize + 1usize + 1usize + 1usize + 1usize + 1usize;
    let mut vec = ::smallvec::SmallVec::new();
    if count <= vec.inline_size() {
        vec.push(TargetFeatureFoldStrength::Both("push2pop2"));
        vec.push(TargetFeatureFoldStrength::Both("ppx"));
        vec.push(TargetFeatureFoldStrength::Both("ndd"));
        vec.push(TargetFeatureFoldStrength::Both("ccmp"));
        vec.push(TargetFeatureFoldStrength::Both("cf"));
        vec.push(TargetFeatureFoldStrength::Both("nf"));
        vec.push(TargetFeatureFoldStrength::Both("zu"));
        vec
    } else {
        ::smallvec::SmallVec::from_vec(::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
                    [TargetFeatureFoldStrength::Both("push2pop2"),
                            TargetFeatureFoldStrength::Both("ppx"),
                            TargetFeatureFoldStrength::Both("ndd"),
                            TargetFeatureFoldStrength::Both("ccmp"),
                            TargetFeatureFoldStrength::Both("cf"),
                            TargetFeatureFoldStrength::Both("nf"),
                            TargetFeatureFoldStrength::Both("zu")])))
    }
}smallvec![
307                    TargetFeatureFoldStrength::Both("push2pop2"),
308                    TargetFeatureFoldStrength::Both("ppx"),
309                    TargetFeatureFoldStrength::Both("ndd"),
310                    TargetFeatureFoldStrength::Both("ccmp"),
311                    TargetFeatureFoldStrength::Both("cf"),
312                    TargetFeatureFoldStrength::Both("nf"),
313                    TargetFeatureFoldStrength::Both("zu"),
314                ],
315            )),
316            s => Some(LLVMFeature::new(s)),
317        },
318        _ => Some(LLVMFeature::new(s)),
319    }
320}
321
322/// Used to generate cfg variables and apply features.
323/// Must express features in the way Rust understands them.
324///
325/// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled outside codegen.
326pub(crate) fn target_config(sess: &EarlySession) -> TargetConfig {
327    require_inited();
328    let target_features = global_llvm_features(sess, /* for_cfg */ true);
329
330    let triple = SmallCStr::new(&llvm_target(sess));
331    let cpu = SmallCStr::new(target_cpu(sess));
332    let features = CString::new(target_features.join(",")).unwrap();
333    let mc_subtarget_info = OwnedMCSubtargetInfo::new(&triple, &cpu, &features)
334        .unwrap_or_else(|err| llvm_err(sess.dcx(), err));
335
336    let internal_target_features = internal_target_features(
337        sess,
338        |feature| {
339            to_llvm_features(&sess.target, feature)
340                .map(|f| SmallVec::<[&str; 2]>::from_iter(f.into_iter()))
341                .unwrap_or_default()
342        },
343        |feature| {
344            // This closure determines whether the target CPU has the feature according to LLVM. We
345            // do *not* consider the `-Ctarget-feature`s here (that's why we passed `for_cfg: true`
346            // to `global_llvm_features` above) because that will be handled later in
347            // `internal_target_features`.
348            if let Some(feat) = to_llvm_features(&sess.target, feature)
349                && mc_subtarget_info.has_features(feat)
350            {
351                true
352            } else {
353                false
354            }
355        },
356    );
357
358    let mut cfg = TargetConfig {
359        internal_target_features,
360        has_reliable_f16: true,
361        has_reliable_f16_math: true,
362        has_reliable_f16b: true,
363        has_reliable_f128: true,
364        has_reliable_f128_math: true,
365    };
366
367    update_target_reliable_float_cfg(&sess.target, &mut cfg);
368    cfg
369}
370
371/// Determine whether or not experimental float types are reliable based on known bugs.
372fn update_target_reliable_float_cfg(target: &Target, cfg: &mut TargetConfig) {
373    let target_arch = &target.arch;
374    let target_os = &target.options.os;
375    let target_endian = &target.options.endian;
376    let target_env = &target.options.env;
377    let target_abi = &target.options.cfg_abi;
378    let target_pointer_width = target.pointer_width;
379    let version = get_version();
380    let (major, _, _) = version;
381
382    cfg.has_reliable_f16 = match (target_arch, target_os) {
383        // MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054> resolved in GCC 16
384        // but our toolchain hasn't been updated.
385        (Arch::X86_64, Os::Windows) if *target_env == Env::Gnu && *target_abi != CfgAbi::Llvm => {
386            false
387        }
388        // `f16` support only requires that symbols converting to and from `f32` are available. We
389        // provide these in `compiler-builtins`, so `f16` should be available on all platforms that
390        // do not have other ABI issues or LLVM crashes.
391        _ => true,
392    };
393
394    // The heuristic for evaluating to true is twofold, namely;
395    //
396    // 1. Can LLVM compile an IR snippet containing `fpext bfloat %<var> to float`
397    // 2. Does the documentation indicate `bf16` support, can be seen in the
398    //    tracking issue; <https://github.com/rust-lang/rust/issues/160630>
399    cfg.has_reliable_f16b = match (target_arch, target_os) {
400        // This is similar to <https://github.com/llvm/llvm-project/issues/94434>, however
401        // does not work until LLVM 23 on Windows.
402        (Arch::Arm64EC, _) => major >= 23,
403        (Arch::AArch64 | Arch::Arm, _) => true,
404        // FIXME(f16b) until <https://github.com/llvm/llvm-project/issues/97896>
405        // is resolved the below do not have a reliable `f16b`, on a widening
406        // path a call to `__truncsfbf2` is emitted. Or when using architectural
407        // extensions a non-portable narrowing instruction is emitted.
408        (Arch::X86_64 | Arch::RiscV64 | Arch::LoongArch64, _) => false,
409        _ => false,
410    };
411
412    cfg.has_reliable_f128 = match (target_arch, target_os) {
413        // Unsupported https://github.com/llvm/llvm-project/issues/121122
414        (Arch::AmdGpu, _) => false,
415        (Arch::Arm64EC, _) if major < 23 => false, // (fixed in llvm23)
416        // Selection bug <https://github.com/llvm/llvm-project/issues/95471>. This issue is closed
417        // but basic math still does not work.
418        (Arch::Nvptx64, _) => false,
419        // ABI/LLVM bugs:
420        // - with +vsx <https://github.com/llvm/llvm-project/pull/216613>
421        // - without +vsx <https://github.com/rust-lang/rust/issues/125109>
422        (Arch::PowerPC, _) => false,
423        // AIX does not support f128.
424        (Arch::PowerPC64, Os::Aix) => false,
425        // ABI bugs on BE without +vsx <https://github.com/rust-lang/rust/issues/125109>.
426        (Arch::PowerPC64, _) => cfg.internal_target_features.contains(&sym::vsx),
427        // MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054> (fixed in llvm23)
428        (Arch::X86_64, Os::Windows)
429            if *target_env == Env::Gnu && *target_abi != CfgAbi::Llvm && major < 23 =>
430        {
431            false
432        }
433        // There are no known problems on other platforms, so the only requirement is that symbols
434        // are available. `compiler-builtins` provides all symbols required for core `f128`
435        // support, so this should work for everything else.
436        _ => true,
437    };
438
439    // Assume that working `f16` means working `f16` math for most platforms, since
440    // operations just go through `f32`.
441    cfg.has_reliable_f16_math = cfg.has_reliable_f16;
442
443    cfg.has_reliable_f128_math = match (target_arch, target_os) {
444        // LLVM lowers `fp128` math to `long double` symbols even on platforms where
445        // `long double` is not IEEE binary128. See
446        // <https://github.com/llvm/llvm-project/issues/44744>.
447        //
448        // This rules out anything that doesn't have `long double` = `binary128`: <= 32 bits
449        // (`long double` is `f64`) and anything other than Linux (Windows and MacOS use `f64`).
450        // x86-64 would also be a problem, except there's a special fix for linux-gnu.
451        // We have to reject musl anyway because it doesn't have the symbols for f128 math.
452        _ if target_pointer_width <= 32 => false,
453        _ if *target_os != Os::Linux => false,
454        _ if *target_env != Env::Gnu => false,
455        // On big-endian powerpc the symbol selection is correct, despite __ibmf128 being
456        // long double on the target, but the f128 symbols are not defined.
457        (Arch::PowerPC | Arch::PowerPC64, _) if *target_endian == Endian::Big => false,
458        // After applying all these exceptions, the rest should work.
459        _ => true,
460    } && cfg.has_reliable_f128;
461}
462
463pub(crate) fn print_version() {
464    let (major, minor, patch) = get_version();
465    {
    ::std::io::_print(format_args!("LLVM version: {0}.{1}.{2}\n", major,
            minor, patch));
};println!("LLVM version: {major}.{minor}.{patch}");
466}
467
468/// Returns the version of LLVM that we are actually linked to at runtime.
469pub(crate) fn get_version() -> (u32, u32, u32) {
470    let mut llvm_major = 0;
471    let mut llvm_minor = 0;
472    let mut llvm_patch = 0;
473    llvm::LLVMGetVersion(&mut llvm_major, &mut llvm_minor, &mut llvm_patch);
474    (llvm_major, llvm_minor, llvm_patch)
475}
476
477/// The LLVM target triple to use for the session's target.
478///
479/// The UEFI targets are lowered through LLVM's `*-unknown-uefi` targets with
480/// LLVM 24 or later, and through the Windows targets with older LLVMs.
481pub(crate) fn llvm_target(sess: &EarlySession) -> Cow<'_, str> {
482    let triple = versioned_llvm_target(sess);
483    if sess.target.os == Os::Uefi && get_version() < (24, 0, 0) {
484        return triple.replace("-unknown-uefi", "-unknown-windows").into();
485    }
486    triple
487}
488
489pub(crate) fn print_passes() {
490    // Can be called without initializing LLVM
491    unsafe {
492        llvm::LLVMRustPrintPasses();
493    }
494}
495
496fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> {
497    let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) };
498    let mut ret = Vec::with_capacity(len);
499    for i in 0..len {
500        unsafe {
501            let mut feature = ptr::null();
502            let mut desc = ptr::null();
503            llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc);
504            if feature.is_null() || desc.is_null() {
505                ::rustc_span::macros::bug_impl(None,
    format_args!("LLVM returned a `null` target feature string"),
    Location::caller());bug!("LLVM returned a `null` target feature string");
506            }
507            let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| {
508                ::rustc_span::macros::bug_impl(None,
    format_args!("LLVM returned a non-utf8 feature string: {0}", e),
    Location::caller());bug!("LLVM returned a non-utf8 feature string: {}", e);
509            });
510            let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| {
511                ::rustc_span::macros::bug_impl(None,
    format_args!("LLVM returned a non-utf8 feature string: {0}", e),
    Location::caller());bug!("LLVM returned a non-utf8 feature string: {}", e);
512            });
513            ret.push((feature, desc));
514        }
515    }
516    ret
517}
518
519pub(crate) fn print(req: &PrintRequest, out: &mut String, sess: &Session) {
520    require_inited();
521    let tm = create_informational_target_machine(sess);
522    match req.kind {
523        PrintKind::TargetCPUs => print_target_cpus(sess, tm.raw(), out),
524        PrintKind::TargetFeatures => print_target_features(sess, tm.raw(), out),
525        _ => ::rustc_span::macros::bug_impl(None,
    format_args!("rustc_codegen_llvm can\'t handle print request: {0:?}",
        req), Location::caller())bug!("rustc_codegen_llvm can't handle print request: {:?}", req),
526    }
527}
528
529fn print_target_cpus(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) {
530    let cpu_names = llvm::build_string(|s| unsafe {
531        llvm::LLVMRustPrintTargetCPUs(&tm, s);
532    })
533    .unwrap();
534
535    struct Cpu<'a> {
536        cpu_name: &'a str,
537        remark: String,
538    }
539    // Compare CPU against current target to label the default. Do not print it if
540    // `need_explicit_cpu` is set, because in that case the concept of default makes less sense.
541    let target_cpu = handle_native(&sess.target.cpu);
542    let make_remark = |cpu_name| {
543        if cpu_name == target_cpu && !sess.target.need_explicit_cpu {
544            // FIXME(#132514): This prints the LLVM target string, which can be
545            // different from the Rust target string. Is that intended?
546            let target = &sess.target.llvm_target;
547            ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(" - This is the default target CPU for the current build target (currently {0}).",
                target))
    })format!(
548                " - This is the default target CPU for the current build target (currently {target})."
549            )
550        } else {
551            "".to_owned()
552        }
553    };
554    let mut cpus = cpu_names
555        .lines()
556        .filter(|cpu_name| {
557            !sess.target.unsupported_cpus.contains(&std::borrow::Cow::Borrowed(*cpu_name))
558        })
559        .map(|cpu_name| Cpu { cpu_name, remark: make_remark(cpu_name) })
560        .collect::<VecDeque<_>>();
561
562    // Only print the "native" entry when host and target are the same arch,
563    // since otherwise it could be wrong or misleading.
564    // Also do not print it if `requires_consistent_cpu` is set, because in this case
565    // "native" would be rejected.
566    if sess.host.arch == sess.target.arch && !sess.target.requires_consistent_cpu {
567        let host = get_host_cpu_name();
568        cpus.push_front(Cpu {
569            cpu_name: NATIVE_CPU,
570            remark: ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(" - Select the CPU of the current host (currently {0}).",
                host))
    })format!(" - Select the CPU of the current host (currently {host})."),
571        });
572    }
573
574    let max_name_width = cpus.iter().map(|cpu| cpu.cpu_name.len()).max().unwrap_or(0);
575    out.write_fmt(format_args!("Available CPUs for this target:\n"))writeln!(out, "Available CPUs for this target:").unwrap();
576    for Cpu { cpu_name, remark } in cpus {
577        // Only pad the CPU name if there's a remark to print after it.
578        let width = if remark.is_empty() { 0 } else { max_name_width };
579        out.write_fmt(format_args!("    {0:<1$}{2}\n", cpu_name, width, remark))writeln!(out, "    {cpu_name:<width$}{remark}").unwrap();
580    }
581}
582
583fn print_target_features(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) {
584    let mut llvm_target_features = llvm_target_features(tm);
585    let mut known_llvm_target_features = FxHashSet::<&'static str>::default();
586    let mut rustc_target_features = sess
587        .target
588        .rust_target_features()
589        .iter()
590        .filter_map(|(feature, gate, _implied)| {
591            if !gate.in_cfg() {
592                // Only list (experimentally) supported features.
593                return None;
594            }
595            // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these
596            // strings.
597            let llvm_feature = to_llvm_features(&sess.target, *feature)?.llvm_feature_name;
598            let desc =
599                match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() {
600                    Some(index) => {
601                        known_llvm_target_features.insert(llvm_feature);
602                        llvm_target_features[index].1
603                    }
604                    None => "",
605                };
606
607            Some((*feature, desc))
608        })
609        .collect::<Vec<_>>();
610
611    // Since we add this at the end ...
612    rustc_target_features.extend_from_slice(&[(
613        "crt-static",
614        "Enables C Run-time Libraries to be statically linked",
615    )]);
616    // ... we need to sort the list again.
617    rustc_target_features.sort();
618
619    llvm_target_features.retain(|(f, _d)| !known_llvm_target_features.contains(f));
620
621    let max_feature_len = llvm_target_features
622        .iter()
623        .chain(rustc_target_features.iter())
624        .map(|(feature, _desc)| feature.len())
625        .max()
626        .unwrap_or(0);
627
628    out.write_fmt(format_args!("Features supported by rustc for this target:\n"))writeln!(out, "Features supported by rustc for this target:").unwrap();
629    for (feature, desc) in &rustc_target_features {
630        out.write_fmt(format_args!("    {0:1$} - {2}.\n", feature, max_feature_len,
        desc))writeln!(out, "    {feature:max_feature_len$} - {desc}.").unwrap();
631    }
632    out.write_fmt(format_args!("\nCode-generation features supported by LLVM for this target:\n"))writeln!(out, "\nCode-generation features supported by LLVM for this target:").unwrap();
633    for (feature, desc) in &llvm_target_features {
634        out.write_fmt(format_args!("    {0:1$} - {2}.\n", feature, max_feature_len,
        desc))writeln!(out, "    {feature:max_feature_len$} - {desc}.").unwrap();
635    }
636    if llvm_target_features.is_empty() {
637        out.write_fmt(format_args!("    Target features listing is not supported by this LLVM version.\n"))writeln!(out, "    Target features listing is not supported by this LLVM version.")
638            .unwrap();
639    }
640    out.write_fmt(format_args!("\nUse +feature to enable a feature, or -feature to disable it.\n"))writeln!(out, "\nUse +feature to enable a feature, or -feature to disable it.").unwrap();
641    out.write_fmt(format_args!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n\n"))writeln!(out, "For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n")
642        .unwrap();
643    out.write_fmt(format_args!("Code-generation features cannot be used in cfg or #[target_feature],\n"))writeln!(out, "Code-generation features cannot be used in cfg or #[target_feature],").unwrap();
644    out.write_fmt(format_args!("and may be renamed or removed in a future version of LLVM or rustc.\n\n"))writeln!(out, "and may be renamed or removed in a future version of LLVM or rustc.\n").unwrap();
645}
646
647/// Returns the host CPU name, according to LLVM.
648fn get_host_cpu_name() -> &'static str {
649    let mut len = 0;
650    // SAFETY: The underlying C++ global function returns a `StringRef` that
651    // isn't tied to any particular backing buffer, so it must be 'static.
652    let slice: &'static [u8] = unsafe {
653        let ptr = llvm::LLVMRustGetHostCPUName(&mut len);
654        if !!ptr.is_null() {
    ::core::panicking::panic("assertion failed: !ptr.is_null()")
};assert!(!ptr.is_null());
655        slice::from_raw_parts(ptr, len)
656    };
657    str::from_utf8(slice).expect("host CPU name should be UTF-8")
658}
659
660/// If the given string is `"native"`, returns the host CPU name according to
661/// LLVM. Otherwise, the string is returned as-is.
662fn handle_native(cpu_name: &str) -> &str {
663    match cpu_name {
664        NATIVE_CPU => get_host_cpu_name(),
665        _ => cpu_name,
666    }
667}
668
669pub(crate) fn target_cpu(sess: &EarlySession) -> &str {
670    let cpu_name = sess.opts.cg.target_cpu.as_deref().unwrap_or_else(|| &sess.target.cpu);
671    handle_native(cpu_name)
672}
673
674/// The target features for compiler flags other than `-Ctarget-features`.
675fn llvm_features_by_flags(sess: &EarlySession, features: &mut Vec<String>) {
676    if wants_wasm_eh(&sess.target) && sess.panic_strategy() == PanicStrategy::Unwind {
677        features.push("+exception-handling".into());
678    }
679
680    target_features::retpoline_features_by_flags(sess, features);
681    target_features::sanitizer_features_by_flags(sess, features);
682
683    // -Zfixed-x18
684    if sess.opts.unstable_opts.fixed_x18 {
685        if sess.target.arch != Arch::AArch64 {
686            sess.dcx()
687                .emit_fatal(diagnostics::FixedX18InvalidArch { arch: sess.target.arch.desc() });
688        } else {
689            features.push("+reserve-x18".into());
690        }
691    }
692}
693
694/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`,
695/// `--target` and similar).
696///
697/// If `for_cfg` is `true` then we are assembling the feature list for the purpose of populating
698/// [`rustc_codegen_ssa::TargetConfig`] based on what LLVM actually enables in this configuration.
699/// `-Ctarget-feature` should be ignored in that case since it is already processed separately.
700pub(crate) fn global_llvm_features(sess: &EarlySession, for_cfg: bool) -> Vec<String> {
701    // Features that come earlier are overridden by conflicting features later in the string.
702    // Typically we'll want more explicit settings to override the implicit ones, so:
703    //
704    // * Features from -Ctarget-cpu=*; are overridden by [^1]
705    // * Features implied by --target; are overridden by
706    // * Features from -Ctarget-feature; are overridden by
707    // * function specific features.
708    //
709    // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly
710    // through LLVM TargetMachine implementation.
711    //
712    // FIXME(nagisa): it isn't clear what's the best interaction between features implied by
713    // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always
714    // override anything that's implicit, so e.g. when there's no `--target` flag, features implied
715    // the host target are overridden by `-Ctarget-cpu=*`. On the other hand, what about when both
716    // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both
717    // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence
718    // should be taken in cases like these.
719    let mut features = ::alloc::vec::Vec::new()vec![];
720
721    // -Ctarget-cpu=native
722    match sess.opts.cg.target_cpu {
723        Some(ref s) if s == NATIVE_CPU => {
724            // We have already figured out the actual CPU name with `LLVMRustGetHostCPUName` and set
725            // that for LLVM, so the features implied by that CPU name will be available everywhere.
726            // However, that is not sufficient: e.g. `skylake` alone is not sufficient to tell if
727            // some of the instructions are available or not. So we have to also explicitly ask for
728            // the exact set of features available on the host, and enable all of them.
729            let features_string = unsafe {
730                let ptr = llvm::LLVMGetHostCPUFeatures();
731                let features_string = if !ptr.is_null() {
732                    CStr::from_ptr(ptr)
733                        .to_str()
734                        .unwrap_or_else(|e| {
735                            ::rustc_span::macros::bug_impl(None,
    format_args!("LLVM returned a non-utf8 features string: {0}", e),
    Location::caller());bug!("LLVM returned a non-utf8 features string: {}", e);
736                        })
737                        .to_owned()
738                } else {
739                    ::rustc_span::macros::bug_impl(None,
    format_args!("could not allocate host CPU features, LLVM returned a `null` string"),
    Location::caller());bug!("could not allocate host CPU features, LLVM returned a `null` string");
740                };
741
742                llvm::LLVMDisposeMessage(ptr);
743
744                features_string
745            };
746            if !features_string.is_empty() {
747                features.extend(features_string.split(',').map(String::from));
748            }
749        }
750        Some(_) | None => {}
751    };
752
753    let mut extend_backend_features = |feature: &str, enable: bool| {
754        let enable_disable = if enable { '+' } else { '-' };
755        // We run through `to_llvm_features` when
756        // passing requests down to LLVM. This means that all in-language
757        // features also work on the command line instead of having two
758        // different names when the LLVM name and the Rust name differ.
759        let Some(llvm_feature) = to_llvm_features(&sess.target, feature) else { return };
760
761        features.extend(
762            std::iter::once(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", enable_disable,
                llvm_feature.llvm_feature_name))
    })format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)).chain(
763                llvm_feature.dependencies.into_iter().filter_map(move |feat| {
764                    match (enable, feat) {
765                        (_, TargetFeatureFoldStrength::Both(f))
766                        | (true, TargetFeatureFoldStrength::EnableOnly(f)) => {
767                            Some(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", enable_disable, f))
    })format!("{enable_disable}{f}"))
768                        }
769                        _ => None,
770                    }
771                }),
772            ),
773        );
774    };
775
776    // Features implied by an implicit or explicit `--target`.
777    target_features::target_spec_to_backend_features(sess, &mut extend_backend_features);
778
779    // -Ctarget-features. Skipped for `cfg` as there we parse -Ctarget-features directly instead of
780    // going via an LLVM target machine (which avoids accidentally picking up LLVM-level target
781    // feature implications that we do not want).
782    if !for_cfg {
783        target_features::flag_to_backend_features(sess, extend_backend_features);
784    }
785
786    // `-C` flags that map to LLVM target features.
787    // We need to include them even with `only_base_features` as this is used to populate
788    // `sess.internal_target_features` where we very much want them to be present (e.g. the inline
789    // asm logic uses that to check which registers may be used).
790    llvm_features_by_flags(sess, &mut features);
791
792    // `-Zllvm-target-features`, all the way at the end to overwrite everything.
793    // Should be picked up by `cfg` (e.g. if someone enables AVX this way).
794    for feature in sess.opts.unstable_opts.llvm_target_feature.split(',') {
795        if feature.is_empty() {
796            continue;
797        }
798        if feature.starts_with('+') || feature.starts_with('-') {
799            features.push(feature.to_owned());
800        } else {
801            // LLVM seems to silently ignore entries without leading `+`/`-`. Let's emit a warning
802            // to avoid confusion. But only emit this warning once, under `for_cfg`.
803            if for_cfg {
804                sess.dcx().emit_warn(diagnostics::UnknownLlvmTargetFeaturePrefix { feature });
805            }
806        }
807    }
808
809    features
810}
811
812pub(crate) fn tune_cpu(sess: &Session) -> Option<&str> {
813    let name = sess.opts.unstable_opts.tune_cpu.as_ref()?;
814    Some(handle_native(name))
815}
816
817pub(crate) fn target_has_mnemonic(sess: &Session, mnemonic: &str) -> bool {
818    require_inited();
819    let tm = create_informational_target_machine(sess);
820    let cstr = SmallCStr::new(mnemonic);
821    unsafe { llvm::LLVMRustTargetHasMnemonic(tm.raw(), cstr.as_ptr()) }
822}