rustc_codegen_llvm/
asm.rs

1use std::assert_matches;
2
3use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
4use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
5use rustc_codegen_ssa::mir::operand::OperandValue;
6use rustc_codegen_ssa::traits::*;
7use rustc_data_structures::fx::FxHashMap;
8use rustc_middle::ty::Instance;
9use rustc_middle::ty::layout::TyAndLayout;
10use rustc_middle::{bug, span_bug};
11use rustc_span::{Pos, Span, Symbol, sym};
12use rustc_target::asm::*;
13use smallvec::SmallVec;
14use tracing::debug;
15
16use crate::attributes;
17use crate::builder::Builder;
18use crate::common::Funclet;
19use crate::context::CodegenCx;
20use crate::llvm::{self, ToLlvmBool, Type, Value};
21use crate::type_of::LayoutLlvmExt;
22
23impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
24    fn codegen_inline_asm(
25        &mut self,
26        template: &[InlineAsmTemplatePiece],
27        operands: &[InlineAsmOperandRef<'tcx, Self>],
28        options: InlineAsmOptions,
29        line_spans: &[Span],
30        instance: Instance<'_>,
31        dest: Option<Self::BasicBlock>,
32        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
33    ) {
34        let asm_arch = self.tcx.sess.asm_arch.unwrap();
35
36        // Collect the types of output operands
37        let mut constraints = ::alloc::vec::Vec::new()vec![];
38        let mut clobbers = ::alloc::vec::Vec::new()vec![];
39        let mut output_types = ::alloc::vec::Vec::new()vec![];
40        let mut op_idx = FxHashMap::default();
41        let mut clobbered_x87 = false;
42        for (idx, op) in operands.iter().enumerate() {
43            match *op {
44                InlineAsmOperandRef::Out { reg, late, place } => {
45                    let is_target_supported = |reg_class: InlineAsmRegClass| {
46                        for &(_, feature) in reg_class.supported_types(asm_arch, true) {
47                            if let Some(feature) = feature {
48                                if self
49                                    .tcx
50                                    .asm_target_features(instance.def_id())
51                                    .contains(&feature)
52                                {
53                                    return true;
54                                }
55                            } else {
56                                // Register class is unconditionally supported
57                                return true;
58                            }
59                        }
60                        false
61                    };
62
63                    let mut layout = None;
64                    let ty = if let Some(ref place) = place {
65                        layout = Some(&place.layout);
66                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
67                    } else if #[allow(non_exhaustive_omitted_patterns)] match reg.reg_class() {
    InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg |
        X86InlineAsmRegClass::x87_reg) => true,
    _ => false,
}matches!(
68                        reg.reg_class(),
69                        InlineAsmRegClass::X86(
70                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
71                        )
72                    ) {
73                        // Special handling for x87/mmx registers: we always
74                        // clobber the whole set if one register is marked as
75                        // clobbered. This is due to the way LLVM handles the
76                        // FP stack in inline assembly.
77                        if !clobbered_x87 {
78                            clobbered_x87 = true;
79                            clobbers.push("~{st}".to_string());
80                            for i in 1..=7 {
81                                clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{{st({0})}}", i))
    })format!("~{{st({})}}", i));
82                            }
83                        }
84                        continue;
85                    } else if !is_target_supported(reg.reg_class())
86                        || reg.reg_class().is_clobber_only(asm_arch, true)
87                    {
88                        // We turn discarded outputs into clobber constraints
89                        // if the target feature needed by the register class is
90                        // disabled. This is necessary otherwise LLVM will try
91                        // to actually allocate a register for the dummy output.
92                        match reg {
    InlineAsmRegOrRegClass::Reg(_) => {}
    ref left_val => {
        ::core::panicking::assert_matches_failed(left_val,
            "InlineAsmRegOrRegClass::Reg(_)", ::core::option::Option::None);
    }
};assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
93                        clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{0}", reg_to_llvm(reg, None)))
    })format!("~{}", reg_to_llvm(reg, None)));
94                        continue;
95                    } else {
96                        // If the output is discarded, we don't really care what
97                        // type is used. We're just using this to tell LLVM to
98                        // reserve the register.
99                        dummy_output_type(self.cx, reg.reg_class())
100                    };
101                    output_types.push(ty);
102                    op_idx.insert(idx, constraints.len());
103                    let prefix = if late { "=" } else { "=&" };
104                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, layout)))
    })format!("{}{}", prefix, reg_to_llvm(reg, layout)));
105                }
106                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
107                    let layout = if let Some(ref out_place) = out_place {
108                        &out_place.layout
109                    } else {
110                        // LLVM required tied operands to have the same type,
111                        // so we just use the type of the input.
112                        &in_value.layout
113                    };
114                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
115                    output_types.push(ty);
116                    op_idx.insert(idx, constraints.len());
117                    let prefix = if late { "=" } else { "=&" };
118                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, Some(layout))))
    })format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
119                }
120                _ => {}
121            }
122        }
123
124        // Collect input operands
125        let mut inputs = ::alloc::vec::Vec::new()vec![];
126        for (idx, op) in operands.iter().enumerate() {
127            match *op {
128                InlineAsmOperandRef::In { reg, value } => {
129                    let llval = llvm_fixup_input(
130                        self,
131                        value.immediate(),
132                        reg.reg_class(),
133                        &value.layout,
134                        instance,
135                    );
136                    inputs.push(llval);
137                    op_idx.insert(idx, constraints.len());
138                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
139                }
140                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
141                    let value = llvm_fixup_input(
142                        self,
143                        in_value.immediate(),
144                        reg.reg_class(),
145                        &in_value.layout,
146                        instance,
147                    );
148                    inputs.push(value);
149
150                    // In the case of fixed registers, we have the choice of
151                    // either using a tied operand or duplicating the constraint.
152                    // We prefer the latter because it matches the behavior of
153                    // Clang.
154                    if late && #[allow(non_exhaustive_omitted_patterns)] match reg {
    InlineAsmRegOrRegClass::Reg(_) => true,
    _ => false,
}matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
155                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
156                    } else {
157                        constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}", op_idx[&idx]))
    })format!("{}", op_idx[&idx]));
158                    }
159                }
160                InlineAsmOperandRef::SymFn { instance } => {
161                    inputs.push(self.cx.get_fn(instance));
162                    op_idx.insert(idx, constraints.len());
163                    constraints.push("s".to_string());
164                }
165                InlineAsmOperandRef::SymStatic { def_id } => {
166                    inputs.push(self.cx.get_static(def_id));
167                    op_idx.insert(idx, constraints.len());
168                    constraints.push("s".to_string());
169                }
170                _ => {}
171            }
172        }
173
174        // Build the template string
175        let mut labels = ::alloc::vec::Vec::new()vec![];
176        let mut template_str = String::new();
177        for piece in template {
178            match *piece {
179                InlineAsmTemplatePiece::String(ref s) => {
180                    if s.contains('$') {
181                        for c in s.chars() {
182                            if c == '$' {
183                                template_str.push_str("$$");
184                            } else {
185                                template_str.push(c);
186                            }
187                        }
188                    } else {
189                        template_str.push_str(s)
190                    }
191                }
192                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
193                    match operands[operand_idx] {
194                        InlineAsmOperandRef::In { reg, .. }
195                        | InlineAsmOperandRef::Out { reg, .. }
196                        | InlineAsmOperandRef::InOut { reg, .. } => {
197                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
198                            if let Some(modifier) = modifier {
199                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:{1}}}",
                op_idx[&operand_idx], modifier))
    })format!(
200                                    "${{{}:{}}}",
201                                    op_idx[&operand_idx], modifier
202                                ));
203                            } else {
204                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}}}", op_idx[&operand_idx]))
    })format!("${{{}}}", op_idx[&operand_idx]));
205                            }
206                        }
207                        InlineAsmOperandRef::Const { ref string } => {
208                            // Const operands get injected directly into the template
209                            template_str.push_str(string);
210                        }
211                        InlineAsmOperandRef::SymFn { .. }
212                        | InlineAsmOperandRef::SymStatic { .. } => {
213                            // Only emit the raw symbol name
214                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
215                        }
216                        InlineAsmOperandRef::Label { label } => {
217                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:l}}", constraints.len()))
    })format!("${{{}:l}}", constraints.len()));
218                            constraints.push("!i".to_owned());
219                            labels.push(label);
220                        }
221                    }
222                }
223            }
224        }
225
226        constraints.append(&mut clobbers);
227        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
228            match asm_arch {
229                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
230                    constraints.push("~{cc}".to_string());
231                }
232                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
233                    constraints.extend_from_slice(&[
234                        "~{dirflag}".to_string(),
235                        "~{fpsr}".to_string(),
236                        "~{flags}".to_string(),
237                    ]);
238                }
239                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
240                    constraints.extend_from_slice(&[
241                        "~{fflags}".to_string(),
242                        "~{vtype}".to_string(),
243                        "~{vl}".to_string(),
244                        "~{vxsat}".to_string(),
245                        "~{vxrm}".to_string(),
246                    ]);
247                }
248                InlineAsmArch::Avr => {
249                    constraints.push("~{sreg}".to_string());
250                }
251                InlineAsmArch::Nvptx64 => {}
252                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
253                InlineAsmArch::Hexagon => {}
254                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
255                    constraints.extend_from_slice(&[
256                        "~{$fcc0}".to_string(),
257                        "~{$fcc1}".to_string(),
258                        "~{$fcc2}".to_string(),
259                        "~{$fcc3}".to_string(),
260                        "~{$fcc4}".to_string(),
261                        "~{$fcc5}".to_string(),
262                        "~{$fcc6}".to_string(),
263                        "~{$fcc7}".to_string(),
264                    ]);
265                }
266                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
267                InlineAsmArch::S390x => {
268                    constraints.push("~{cc}".to_string());
269                }
270                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
271                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
272                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
273                    constraints.push("~{icc}".to_string());
274                    constraints.push("~{fcc0}".to_string());
275                    constraints.push("~{fcc1}".to_string());
276                    constraints.push("~{fcc2}".to_string());
277                    constraints.push("~{fcc3}".to_string());
278                }
279                InlineAsmArch::SpirV => {}
280                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
281                InlineAsmArch::Bpf => {}
282                InlineAsmArch::Msp430 => {
283                    constraints.push("~{sr}".to_string());
284                }
285                InlineAsmArch::M68k => {
286                    constraints.push("~{ccr}".to_string());
287                }
288                InlineAsmArch::CSKY => {
289                    constraints.push("~{psr}".to_string());
290                }
291            }
292        }
293        if !options.contains(InlineAsmOptions::NOMEM) {
294            // This is actually ignored by LLVM, but it's probably best to keep
295            // it just in case. LLVM instead uses the ReadOnly/ReadNone
296            // attributes on the call instruction to optimize.
297            constraints.push("~{memory}".to_string());
298        }
299        let volatile = !options.contains(InlineAsmOptions::PURE);
300        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
301        let output_type = match &output_types[..] {
302            [] => self.type_void(),
303            [ty] => ty,
304            tys => self.type_struct(tys, false),
305        };
306        let dialect = match asm_arch {
307            InlineAsmArch::X86 | InlineAsmArch::X86_64
308                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
309            {
310                llvm::AsmDialect::Intel
311            }
312            _ => llvm::AsmDialect::Att,
313        };
314        let result = inline_asm_call(
315            self,
316            &template_str,
317            &constraints.join(","),
318            &inputs,
319            output_type,
320            &labels,
321            volatile,
322            alignstack,
323            dialect,
324            line_spans,
325            options.contains(InlineAsmOptions::MAY_UNWIND),
326            dest,
327            catch_funclet,
328        )
329        .unwrap_or_else(|| ::rustc_middle::util::bug::span_bug_fmt(line_spans[0],
    format_args!("LLVM asm constraint validation failed"))span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
330
331        let mut attrs = SmallVec::<[_; 2]>::new();
332        if options.contains(InlineAsmOptions::PURE) {
333            if options.contains(InlineAsmOptions::NOMEM) {
334                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
335            } else if options.contains(InlineAsmOptions::READONLY) {
336                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
337            }
338            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
339        } else if options.contains(InlineAsmOptions::NOMEM) {
340            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
341        } else if options.contains(InlineAsmOptions::READONLY) {
342            attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
343        }
344        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
345
346        // Write results to outputs. We need to do this for all possible control flow.
347        //
348        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
349        // is used (because we need to codegen callbr which always needs a destination), so
350        // here we use the NORETURN option to determine if `dest` should be used.
351        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
352            .into_iter()
353            .chain(labels.iter().copied().map(Some))
354        {
355            if let Some(block) = block {
356                self.switch_to_block(block);
357            }
358
359            for (idx, op) in operands.iter().enumerate() {
360                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
361                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
362                {
363                    let value = if output_types.len() == 1 {
364                        result
365                    } else {
366                        self.extract_value(result, op_idx[&idx] as u64)
367                    };
368                    let value =
369                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
370                    OperandValue::Immediate(value).store(self, place);
371                }
372            }
373        }
374    }
375}
376
377impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
378    fn codegen_global_asm(
379        &mut self,
380        template: &[InlineAsmTemplatePiece],
381        operands: &[GlobalAsmOperandRef<'tcx>],
382        options: InlineAsmOptions,
383        _line_spans: &[Span],
384    ) {
385        let asm_arch = self.tcx.sess.asm_arch.unwrap();
386
387        // Build the template string
388        let mut template_str = String::new();
389
390        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
391        // but AT&T can be specified explicitly.
392        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
393            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
394                template_str.push_str(".att_syntax\n")
395            } else {
396                template_str.push_str(".intel_syntax\n")
397            }
398        }
399
400        for piece in template {
401            match *piece {
402                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
403                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => {
404                    use rustc_codegen_ssa::back::symbol_export::escape_symbol_name;
405                    match operands[operand_idx] {
406                        GlobalAsmOperandRef::Const { ref string } => {
407                            // Const operands get injected directly into the
408                            // template. Note that we don't need to escape $
409                            // here unlike normal inline assembly.
410                            template_str.push_str(string);
411                        }
412                        GlobalAsmOperandRef::SymFn { instance } => {
413                            let llval = self.get_fn(instance);
414                            self.add_compiler_used_global(llval);
415                            let symbol = llvm::build_string(|s| unsafe {
416                                llvm::LLVMRustGetMangledName(llval, s);
417                            })
418                            .expect("symbol is not valid UTF-8");
419                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
420                        }
421                        GlobalAsmOperandRef::SymStatic { def_id } => {
422                            let llval = self
423                                .renamed_statics
424                                .borrow()
425                                .get(&def_id)
426                                .copied()
427                                .unwrap_or_else(|| self.get_static(def_id));
428                            self.add_compiler_used_global(llval);
429                            let symbol = llvm::build_string(|s| unsafe {
430                                llvm::LLVMRustGetMangledName(llval, s);
431                            })
432                            .expect("symbol is not valid UTF-8");
433                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
434                        }
435                    }
436                }
437            }
438        }
439
440        // Just to play it safe, if intel was used, reset the assembly syntax to att.
441        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
442            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
443        {
444            template_str.push_str("\n.att_syntax\n");
445        }
446
447        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
448    }
449
450    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
451        let llval = self.get_fn(instance);
452        llvm::build_string(|s| unsafe {
453            llvm::LLVMRustGetMangledName(llval, s);
454        })
455        .expect("symbol is not valid UTF-8")
456    }
457}
458
459pub(crate) fn inline_asm_call<'ll>(
460    bx: &mut Builder<'_, 'll, '_>,
461    asm: &str,
462    cons: &str,
463    inputs: &[&'ll Value],
464    output: &'ll llvm::Type,
465    labels: &[&'ll llvm::BasicBlock],
466    volatile: bool,
467    alignstack: bool,
468    dia: llvm::AsmDialect,
469    line_spans: &[Span],
470    unwind: bool,
471    dest: Option<&'ll llvm::BasicBlock>,
472    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
473) -> Option<&'ll Value> {
474    let argtys = inputs
475        .iter()
476        .map(|v| {
477            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:477",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(477u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("Asm Input Type: {0:?}",
                                                    *v) as &dyn Value))])
            });
    } else { ; }
};debug!("Asm Input Type: {:?}", *v);
478            bx.cx.val_ty(*v)
479        })
480        .collect::<Vec<_>>();
481
482    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:482",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(482u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("Asm Output Type: {0:?}",
                                                    output) as &dyn Value))])
            });
    } else { ; }
};debug!("Asm Output Type: {:?}", output);
483    let fty = bx.cx.type_func(&argtys, output);
484
485    // Ask LLVM to verify that the constraints are well-formed.
486    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
487    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:487",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(487u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("constraint verification result: {0:?}",
                                                    constraints_ok) as &dyn Value))])
            });
    } else { ; }
};debug!("constraint verification result: {:?}", constraints_ok);
488    if !constraints_ok {
489        // LLVM has detected an issue with our constraints, so bail out.
490        return None;
491    }
492
493    let v = unsafe {
494        llvm::LLVMGetInlineAsm(
495            fty,
496            asm.as_ptr(),
497            asm.len(),
498            cons.as_ptr(),
499            cons.len(),
500            volatile.to_llvm_bool(),
501            alignstack.to_llvm_bool(),
502            dia,
503            unwind.to_llvm_bool(),
504        )
505    };
506
507    let call = if !labels.is_empty() {
508        if !catch_funclet.is_none() {
    ::core::panicking::panic("assertion failed: catch_funclet.is_none()")
};assert!(catch_funclet.is_none());
509        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
510    } else if let Some((catch, funclet)) = catch_funclet {
511        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
512    } else {
513        bx.call(fty, None, None, v, inputs, None, None)
514    };
515
516    // Store mark in a metadata node so we can map LLVM errors
517    // back to source locations. See #17552.
518    let key = "srcloc";
519    let kind = bx.get_md_kind_id(key);
520
521    // `srcloc` contains one 64-bit integer for each line of assembly code,
522    // where the lower 32 bits hold the lo byte position and the upper 32 bits
523    // hold the hi byte position.
524    let mut srcloc = ::alloc::vec::Vec::new()vec![];
525    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
526        // LLVM inserts an extra line to add the ".intel_syntax", so add
527        // a dummy srcloc entry for it.
528        //
529        // Don't do this if we only have 1 line span since that may be
530        // due to the asm template string coming from a macro. LLVM will
531        // default to the first srcloc for lines that don't have an
532        // associated srcloc.
533        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
534    }
535    srcloc.extend(line_spans.iter().map(|span| {
536        llvm::LLVMValueAsMetadata(
537            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
538        )
539    }));
540    bx.cx.set_metadata_node(call, kind, &srcloc);
541
542    Some(call)
543}
544
545/// If the register is an xmm/ymm/zmm register then return its index.
546fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
547    use X86InlineAsmReg::*;
548    match reg {
549        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
550            Some(reg as u32 - xmm0 as u32)
551        }
552        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
553            Some(reg as u32 - ymm0 as u32)
554        }
555        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
556            Some(reg as u32 - zmm0 as u32)
557        }
558        _ => None,
559    }
560}
561
562/// If the register is an AArch64 integer register then return its index.
563fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
564    match reg {
565        InlineAsmReg::AArch64(r) => r.reg_index(),
566        _ => None,
567    }
568}
569
570/// If the register is an AArch64 vector register then return its index.
571fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
572    match reg {
573        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
574        _ => None,
575    }
576}
577
578/// Converts a register class to an LLVM constraint code.
579fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
580    use InlineAsmRegClass::*;
581    match reg {
582        // For vector registers LLVM wants the register name to match the type size.
583        InlineAsmRegOrRegClass::Reg(reg) => {
584            if let Some(idx) = xmm_reg_index(reg) {
585                let class = if let Some(layout) = layout {
586                    match layout.size.bytes() {
587                        64 => 'z',
588                        32 => 'y',
589                        _ => 'x',
590                    }
591                } else {
592                    // We use f32 as the type for discarded outputs
593                    'x'
594                };
595                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}mm{1}}}", class, idx))
    })format!("{{{}mm{}}}", class, idx)
596            } else if let Some(idx) = a64_reg_index(reg) {
597                let class = if let Some(layout) = layout {
598                    match layout.size.bytes() {
599                        8 => 'x',
600                        _ => 'w',
601                    }
602                } else {
603                    // We use i32 as the type for discarded outputs
604                    'w'
605                };
606                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
607                    // LLVM doesn't recognize x30. use lr instead.
608                    "{lr}".to_string()
609                } else {
610                    ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
611                }
612            } else if let Some(idx) = a64_vreg_index(reg) {
613                let class = if let Some(layout) = layout {
614                    match layout.size.bytes() {
615                        16 => 'q',
616                        8 => 'd',
617                        4 => 's',
618                        2 => 'h',
619                        1 => 'd', // We fixup i8 to i8x8
620                        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
621                    }
622                } else {
623                    // We use i64x2 as the type for discarded outputs
624                    'q'
625                };
626                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
627            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
628                // LLVM doesn't recognize r14
629                "{lr}".to_string()
630            } else {
631                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}}}", reg.name()))
    })format!("{{{}}}", reg.name())
632            }
633        }
634        // The constraints can be retrieved from
635        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
636        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
637            AArch64(AArch64InlineAsmRegClass::reg) => "r",
638            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
639            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
640            AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
641            Arm(ArmInlineAsmRegClass::reg) => "r",
642            Arm(ArmInlineAsmRegClass::sreg)
643            | Arm(ArmInlineAsmRegClass::dreg_low16)
644            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
645            Arm(ArmInlineAsmRegClass::sreg_low16)
646            | Arm(ArmInlineAsmRegClass::dreg_low8)
647            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
648            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
649            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
650            Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
651            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
652            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
653            Mips(MipsInlineAsmRegClass::reg) => "r",
654            Mips(MipsInlineAsmRegClass::freg) => "f",
655            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
656            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
657            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
658            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
659            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
660            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
661            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
662            PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa",
663            PowerPC(
664                PowerPCInlineAsmRegClass::cr
665                | PowerPCInlineAsmRegClass::ctr
666                | PowerPCInlineAsmRegClass::lr
667                | PowerPCInlineAsmRegClass::xer
668                | PowerPCInlineAsmRegClass::spe_acc,
669            ) => {
670                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
671            }
672            RiscV(RiscVInlineAsmRegClass::reg) => "r",
673            RiscV(RiscVInlineAsmRegClass::freg) => "f",
674            RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
675            X86(X86InlineAsmRegClass::reg) => "r",
676            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
677            X86(X86InlineAsmRegClass::reg_byte) => "q",
678            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
679            X86(X86InlineAsmRegClass::zmm_reg) => "v",
680            X86(X86InlineAsmRegClass::kreg) => "^Yk",
681            X86(
682                X86InlineAsmRegClass::x87_reg
683                | X86InlineAsmRegClass::mmx_reg
684                | X86InlineAsmRegClass::kreg0
685                | X86InlineAsmRegClass::tmm_reg,
686            ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
687            Wasm(WasmInlineAsmRegClass::local) => "r",
688            Bpf(BpfInlineAsmRegClass::reg) => "r",
689            Bpf(BpfInlineAsmRegClass::wreg) => "w",
690            Avr(AvrInlineAsmRegClass::reg) => "r",
691            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
692            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
693            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
694            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
695            S390x(S390xInlineAsmRegClass::reg) => "r",
696            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
697            S390x(S390xInlineAsmRegClass::freg) => "f",
698            S390x(S390xInlineAsmRegClass::vreg) => "v",
699            S390x(S390xInlineAsmRegClass::areg) => {
700                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
701            }
702            Sparc(SparcInlineAsmRegClass::reg) => "r",
703            Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
704            Msp430(Msp430InlineAsmRegClass::reg) => "r",
705            M68k(M68kInlineAsmRegClass::reg) => "r",
706            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
707            M68k(M68kInlineAsmRegClass::reg_data) => "d",
708            CSKY(CSKYInlineAsmRegClass::reg) => "r",
709            CSKY(CSKYInlineAsmRegClass::freg) => "f",
710            SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
711            Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
712        }
713        .to_string(),
714    }
715}
716
717/// Converts a modifier into LLVM's equivalent modifier.
718fn modifier_to_llvm(
719    arch: InlineAsmArch,
720    reg: InlineAsmRegClass,
721    modifier: Option<char>,
722) -> Option<char> {
723    use InlineAsmRegClass::*;
724    // The modifiers can be retrieved from
725    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
726    match reg {
727        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
728        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
729            if modifier == Some('v') {
730                None
731            } else {
732                modifier
733            }
734        }
735        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
736        Arm(ArmInlineAsmRegClass::reg) => None,
737        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
738        Arm(ArmInlineAsmRegClass::dreg)
739        | Arm(ArmInlineAsmRegClass::dreg_low16)
740        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
741        Arm(ArmInlineAsmRegClass::qreg)
742        | Arm(ArmInlineAsmRegClass::qreg_low8)
743        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
744            if modifier.is_none() {
745                Some('q')
746            } else {
747                modifier
748            }
749        }
750        Hexagon(_) => None,
751        LoongArch(_) => None,
752        Mips(_) => None,
753        Nvptx(_) => None,
754        PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
755            // The documentation for the 'x' modifier is missing for llvm, and the gcc
756            // documentation is simply "use this for any vsx argument". It is needed
757            // to ensure the correct vsx register number is used.
758            if modifier.is_none() { Some('x') } else { modifier }
759        }
760        PowerPC(_) => None,
761        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
762        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
763        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
764            None if arch == InlineAsmArch::X86_64 => Some('q'),
765            None => Some('k'),
766            Some('l') => Some('b'),
767            Some('h') => Some('h'),
768            Some('x') => Some('w'),
769            Some('e') => Some('k'),
770            Some('r') => Some('q'),
771            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
772        },
773        X86(X86InlineAsmRegClass::reg_byte) => None,
774        X86(reg @ X86InlineAsmRegClass::xmm_reg)
775        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
776        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
777            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
778            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
779            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
780            (_, Some('x')) => Some('x'),
781            (_, Some('y')) => Some('t'),
782            (_, Some('z')) => Some('g'),
783            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
784        },
785        X86(X86InlineAsmRegClass::kreg) => None,
786        X86(
787            X86InlineAsmRegClass::x87_reg
788            | X86InlineAsmRegClass::mmx_reg
789            | X86InlineAsmRegClass::kreg0
790            | X86InlineAsmRegClass::tmm_reg,
791        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
792        Wasm(WasmInlineAsmRegClass::local) => None,
793        Bpf(_) => None,
794        Avr(AvrInlineAsmRegClass::reg_pair)
795        | Avr(AvrInlineAsmRegClass::reg_iw)
796        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
797            Some('h') => Some('B'),
798            Some('l') => Some('A'),
799            _ => None,
800        },
801        Avr(_) => None,
802        S390x(_) => None,
803        Sparc(_) => None,
804        Msp430(_) => None,
805        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
806        M68k(_) => None,
807        CSKY(_) => None,
808        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
809    }
810}
811
812/// Type to use for outputs that are discarded. It doesn't really matter what
813/// the type is, as long as it is valid for the constraint code.
814fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
815    use InlineAsmRegClass::*;
816    match reg {
817        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
818        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
819            cx.type_vector(cx.type_i64(), 2)
820        }
821        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
822        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
823        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
824        Arm(ArmInlineAsmRegClass::dreg)
825        | Arm(ArmInlineAsmRegClass::dreg_low16)
826        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
827        Arm(ArmInlineAsmRegClass::qreg)
828        | Arm(ArmInlineAsmRegClass::qreg_low8)
829        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
830        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
831        Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
832        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
833        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
834        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
835        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
836        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
837        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
838        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
839        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
840        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
841        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
842        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
843        PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4),
844        PowerPC(
845            PowerPCInlineAsmRegClass::cr
846            | PowerPCInlineAsmRegClass::ctr
847            | PowerPCInlineAsmRegClass::lr
848            | PowerPCInlineAsmRegClass::xer
849            | PowerPCInlineAsmRegClass::spe_acc,
850        ) => {
851            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
852        }
853        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
854        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
855        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
856        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
857        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
858        X86(X86InlineAsmRegClass::xmm_reg)
859        | X86(X86InlineAsmRegClass::ymm_reg)
860        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
861        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
862        X86(
863            X86InlineAsmRegClass::x87_reg
864            | X86InlineAsmRegClass::mmx_reg
865            | X86InlineAsmRegClass::kreg0
866            | X86InlineAsmRegClass::tmm_reg,
867        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
868        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
869        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
870        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
871        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
872        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
873        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
874        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
875        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
876        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
877        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
878        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
879        S390x(S390xInlineAsmRegClass::areg) => {
880            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
881        }
882        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
883        Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
884        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
885        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
886        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
887        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
888        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
889        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
890        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
891        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
892    }
893}
894
895/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
896/// the equivalent integer type.
897fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
898    let dl = &cx.tcx.data_layout;
899    match scalar.primitive() {
900        Primitive::Int(Integer::I8, _) => cx.type_i8(),
901        Primitive::Int(Integer::I16, _) => cx.type_i16(),
902        Primitive::Int(Integer::I32, _) => cx.type_i32(),
903        Primitive::Int(Integer::I64, _) => cx.type_i64(),
904        Primitive::Float(Float::F16) => cx.type_f16(),
905        Primitive::Float(Float::F32) => cx.type_f32(),
906        Primitive::Float(Float::F64) => cx.type_f64(),
907        Primitive::Float(Float::F128) => cx.type_f128(),
908        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
909        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
910        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
911    }
912}
913
914fn any_target_feature_enabled(
915    cx: &CodegenCx<'_, '_>,
916    instance: Instance<'_>,
917    features: &[Symbol],
918) -> bool {
919    let enabled = cx.tcx.asm_target_features(instance.def_id());
920    features.iter().any(|feat| enabled.contains(feat))
921}
922
923/// Fix up an input value to work around LLVM bugs.
924fn llvm_fixup_input<'ll, 'tcx>(
925    bx: &mut Builder<'_, 'll, 'tcx>,
926    mut value: &'ll Value,
927    reg: InlineAsmRegClass,
928    layout: &TyAndLayout<'tcx>,
929    instance: Instance<'_>,
930) -> &'ll Value {
931    use InlineAsmRegClass::*;
932    let dl = &bx.tcx.data_layout;
933    match (reg, layout.backend_repr) {
934        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
935            if let Primitive::Int(Integer::I8, _) = s.primitive() {
936                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
937                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
938            } else {
939                value
940            }
941        }
942        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
943            if s.primitive() != Primitive::Float(Float::F128) =>
944        {
945            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
946            let count = 16 / layout.size.bytes();
947            let vec_ty = bx.cx.type_vector(elem_ty, count);
948            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
949            if let Primitive::Pointer(_) = s.primitive() {
950                let t = bx.type_from_integer(dl.ptr_sized_integer());
951                value = bx.ptrtoint(value, t);
952            }
953            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
954        }
955        (
956            AArch64(AArch64InlineAsmRegClass::vreg_low16),
957            BackendRepr::SimdVector { element, count },
958        ) if layout.size.bytes() == 8 => {
959            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
960            let vec_ty = bx.cx.type_vector(elem_ty, count);
961            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
962            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
963        }
964        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
965            if s.primitive() == Primitive::Float(Float::F64) =>
966        {
967            bx.bitcast(value, bx.cx.type_i64())
968        }
969        (
970            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
971            BackendRepr::SimdVector { .. },
972        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
973        (
974            X86(
975                X86InlineAsmRegClass::xmm_reg
976                | X86InlineAsmRegClass::ymm_reg
977                | X86InlineAsmRegClass::zmm_reg,
978            ),
979            BackendRepr::Scalar(s),
980        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
981            && s.primitive() == Primitive::Float(Float::F128) =>
982        {
983            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
984        }
985        (
986            X86(
987                X86InlineAsmRegClass::xmm_reg
988                | X86InlineAsmRegClass::ymm_reg
989                | X86InlineAsmRegClass::zmm_reg,
990            ),
991            BackendRepr::Scalar(s),
992        ) if s.primitive() == Primitive::Float(Float::F16) => {
993            let value = bx.insert_element(
994                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
995                value,
996                bx.const_usize(0),
997            );
998            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
999        }
1000        (
1001            X86(
1002                X86InlineAsmRegClass::xmm_reg
1003                | X86InlineAsmRegClass::ymm_reg
1004                | X86InlineAsmRegClass::zmm_reg,
1005            ),
1006            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1007        ) if element.primitive() == Primitive::Float(Float::F16) => {
1008            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1009        }
1010        (
1011            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1012            BackendRepr::Scalar(s),
1013        ) => {
1014            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1015                bx.bitcast(value, bx.cx.type_f32())
1016            } else {
1017                value
1018            }
1019        }
1020        (
1021            Arm(
1022                ArmInlineAsmRegClass::dreg
1023                | ArmInlineAsmRegClass::dreg_low8
1024                | ArmInlineAsmRegClass::dreg_low16,
1025            ),
1026            BackendRepr::Scalar(s),
1027        ) => {
1028            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1029                bx.bitcast(value, bx.cx.type_f64())
1030            } else {
1031                value
1032            }
1033        }
1034        (
1035            Arm(
1036                ArmInlineAsmRegClass::dreg
1037                | ArmInlineAsmRegClass::dreg_low8
1038                | ArmInlineAsmRegClass::dreg_low16
1039                | ArmInlineAsmRegClass::qreg
1040                | ArmInlineAsmRegClass::qreg_low4
1041                | ArmInlineAsmRegClass::qreg_low8,
1042            ),
1043            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1044        ) if element.primitive() == Primitive::Float(Float::F16) => {
1045            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1046        }
1047        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1048            if s.primitive() == Primitive::Float(Float::F16) =>
1049        {
1050            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1051            let value = bx.bitcast(value, bx.type_i16());
1052            let value = bx.zext(value, bx.type_i32());
1053            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1054            bx.bitcast(value, bx.type_f32())
1055        }
1056        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1057            match s.primitive() {
1058                // MIPS only supports register-length arithmetics.
1059                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1060                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1061                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1062                _ => value,
1063            }
1064        }
1065        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1066            if s.primitive() == Primitive::Float(Float::F16)
1067                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1068        {
1069            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1070            let value = bx.bitcast(value, bx.type_i16());
1071            let value = bx.zext(value, bx.type_i32());
1072            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1073            bx.bitcast(value, bx.type_f32())
1074        }
1075        (
1076            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1077            BackendRepr::Scalar(s),
1078        ) if s.primitive() == Primitive::Float(Float::F32) => {
1079            let value = bx.insert_element(
1080                bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
1081                value,
1082                bx.const_usize(0),
1083            );
1084            bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
1085        }
1086        (
1087            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1088            BackendRepr::Scalar(s),
1089        ) if s.primitive() == Primitive::Float(Float::F64) => {
1090            let value = bx.insert_element(
1091                bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
1092                value,
1093                bx.const_usize(0),
1094            );
1095            bx.bitcast(value, bx.type_vector(bx.type_f64(), 2))
1096        }
1097        _ => value,
1098    }
1099}
1100
1101/// Fix up an output value to work around LLVM bugs.
1102fn llvm_fixup_output<'ll, 'tcx>(
1103    bx: &mut Builder<'_, 'll, 'tcx>,
1104    mut value: &'ll Value,
1105    reg: InlineAsmRegClass,
1106    layout: &TyAndLayout<'tcx>,
1107    instance: Instance<'_>,
1108) -> &'ll Value {
1109    use InlineAsmRegClass::*;
1110    match (reg, layout.backend_repr) {
1111        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1112            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1113                bx.extract_element(value, bx.const_i32(0))
1114            } else {
1115                value
1116            }
1117        }
1118        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1119            if s.primitive() != Primitive::Float(Float::F128) =>
1120        {
1121            value = bx.extract_element(value, bx.const_i32(0));
1122            if let Primitive::Pointer(_) = s.primitive() {
1123                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1124            }
1125            value
1126        }
1127        (
1128            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1129            BackendRepr::SimdVector { element, count },
1130        ) if layout.size.bytes() == 8 => {
1131            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1132            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1133            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1134            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1135        }
1136        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1137            if s.primitive() == Primitive::Float(Float::F64) =>
1138        {
1139            bx.bitcast(value, bx.cx.type_f64())
1140        }
1141        (
1142            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1143            BackendRepr::SimdVector { .. },
1144        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1145        (
1146            X86(
1147                X86InlineAsmRegClass::xmm_reg
1148                | X86InlineAsmRegClass::ymm_reg
1149                | X86InlineAsmRegClass::zmm_reg,
1150            ),
1151            BackendRepr::Scalar(s),
1152        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1153            && s.primitive() == Primitive::Float(Float::F128) =>
1154        {
1155            bx.bitcast(value, bx.type_f128())
1156        }
1157        (
1158            X86(
1159                X86InlineAsmRegClass::xmm_reg
1160                | X86InlineAsmRegClass::ymm_reg
1161                | X86InlineAsmRegClass::zmm_reg,
1162            ),
1163            BackendRepr::Scalar(s),
1164        ) if s.primitive() == Primitive::Float(Float::F16) => {
1165            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1166            bx.extract_element(value, bx.const_usize(0))
1167        }
1168        (
1169            X86(
1170                X86InlineAsmRegClass::xmm_reg
1171                | X86InlineAsmRegClass::ymm_reg
1172                | X86InlineAsmRegClass::zmm_reg,
1173            ),
1174            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1175        ) if element.primitive() == Primitive::Float(Float::F16) => {
1176            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1177        }
1178        (
1179            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1180            BackendRepr::Scalar(s),
1181        ) => {
1182            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1183                bx.bitcast(value, bx.cx.type_i32())
1184            } else {
1185                value
1186            }
1187        }
1188        (
1189            Arm(
1190                ArmInlineAsmRegClass::dreg
1191                | ArmInlineAsmRegClass::dreg_low8
1192                | ArmInlineAsmRegClass::dreg_low16,
1193            ),
1194            BackendRepr::Scalar(s),
1195        ) => {
1196            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1197                bx.bitcast(value, bx.cx.type_i64())
1198            } else {
1199                value
1200            }
1201        }
1202        (
1203            Arm(
1204                ArmInlineAsmRegClass::dreg
1205                | ArmInlineAsmRegClass::dreg_low8
1206                | ArmInlineAsmRegClass::dreg_low16
1207                | ArmInlineAsmRegClass::qreg
1208                | ArmInlineAsmRegClass::qreg_low4
1209                | ArmInlineAsmRegClass::qreg_low8,
1210            ),
1211            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1212        ) if element.primitive() == Primitive::Float(Float::F16) => {
1213            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1214        }
1215        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1216            if s.primitive() == Primitive::Float(Float::F16) =>
1217        {
1218            let value = bx.bitcast(value, bx.type_i32());
1219            let value = bx.trunc(value, bx.type_i16());
1220            bx.bitcast(value, bx.type_f16())
1221        }
1222        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1223            match s.primitive() {
1224                // MIPS only supports register-length arithmetics.
1225                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1226                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1227                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1228                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1229                _ => value,
1230            }
1231        }
1232        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1233            if s.primitive() == Primitive::Float(Float::F16)
1234                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1235        {
1236            let value = bx.bitcast(value, bx.type_i32());
1237            let value = bx.trunc(value, bx.type_i16());
1238            bx.bitcast(value, bx.type_f16())
1239        }
1240        (
1241            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1242            BackendRepr::Scalar(s),
1243        ) if s.primitive() == Primitive::Float(Float::F32) => {
1244            let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
1245            bx.extract_element(value, bx.const_usize(0))
1246        }
1247        (
1248            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1249            BackendRepr::Scalar(s),
1250        ) if s.primitive() == Primitive::Float(Float::F64) => {
1251            let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
1252            bx.extract_element(value, bx.const_usize(0))
1253        }
1254        _ => value,
1255    }
1256}
1257
1258/// Output type to use for llvm_fixup_output.
1259fn llvm_fixup_output_type<'ll, 'tcx>(
1260    cx: &CodegenCx<'ll, 'tcx>,
1261    reg: InlineAsmRegClass,
1262    layout: &TyAndLayout<'tcx>,
1263    instance: Instance<'_>,
1264) -> &'ll Type {
1265    use InlineAsmRegClass::*;
1266    match (reg, layout.backend_repr) {
1267        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1268            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1269                cx.type_vector(cx.type_i8(), 8)
1270            } else {
1271                layout.llvm_type(cx)
1272            }
1273        }
1274        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1275            if s.primitive() != Primitive::Float(Float::F128) =>
1276        {
1277            let elem_ty = llvm_asm_scalar_type(cx, s);
1278            let count = 16 / layout.size.bytes();
1279            cx.type_vector(elem_ty, count)
1280        }
1281        (
1282            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1283            BackendRepr::SimdVector { element, count },
1284        ) if layout.size.bytes() == 8 => {
1285            let elem_ty = llvm_asm_scalar_type(cx, element);
1286            cx.type_vector(elem_ty, count * 2)
1287        }
1288        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1289            if s.primitive() == Primitive::Float(Float::F64) =>
1290        {
1291            cx.type_i64()
1292        }
1293        (
1294            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1295            BackendRepr::SimdVector { .. },
1296        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1297        (
1298            X86(
1299                X86InlineAsmRegClass::xmm_reg
1300                | X86InlineAsmRegClass::ymm_reg
1301                | X86InlineAsmRegClass::zmm_reg,
1302            ),
1303            BackendRepr::Scalar(s),
1304        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1305            && s.primitive() == Primitive::Float(Float::F128) =>
1306        {
1307            cx.type_vector(cx.type_i32(), 4)
1308        }
1309        (
1310            X86(
1311                X86InlineAsmRegClass::xmm_reg
1312                | X86InlineAsmRegClass::ymm_reg
1313                | X86InlineAsmRegClass::zmm_reg,
1314            ),
1315            BackendRepr::Scalar(s),
1316        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1317        (
1318            X86(
1319                X86InlineAsmRegClass::xmm_reg
1320                | X86InlineAsmRegClass::ymm_reg
1321                | X86InlineAsmRegClass::zmm_reg,
1322            ),
1323            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1324        ) if element.primitive() == Primitive::Float(Float::F16) => {
1325            cx.type_vector(cx.type_i16(), count)
1326        }
1327        (
1328            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1329            BackendRepr::Scalar(s),
1330        ) => {
1331            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1332                cx.type_f32()
1333            } else {
1334                layout.llvm_type(cx)
1335            }
1336        }
1337        (
1338            Arm(
1339                ArmInlineAsmRegClass::dreg
1340                | ArmInlineAsmRegClass::dreg_low8
1341                | ArmInlineAsmRegClass::dreg_low16,
1342            ),
1343            BackendRepr::Scalar(s),
1344        ) => {
1345            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1346                cx.type_f64()
1347            } else {
1348                layout.llvm_type(cx)
1349            }
1350        }
1351        (
1352            Arm(
1353                ArmInlineAsmRegClass::dreg
1354                | ArmInlineAsmRegClass::dreg_low8
1355                | ArmInlineAsmRegClass::dreg_low16
1356                | ArmInlineAsmRegClass::qreg
1357                | ArmInlineAsmRegClass::qreg_low4
1358                | ArmInlineAsmRegClass::qreg_low8,
1359            ),
1360            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1361        ) if element.primitive() == Primitive::Float(Float::F16) => {
1362            cx.type_vector(cx.type_i16(), count)
1363        }
1364        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1365            if s.primitive() == Primitive::Float(Float::F16) =>
1366        {
1367            cx.type_f32()
1368        }
1369        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1370            match s.primitive() {
1371                // MIPS only supports register-length arithmetics.
1372                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1373                Primitive::Float(Float::F32) => cx.type_i32(),
1374                Primitive::Float(Float::F64) => cx.type_i64(),
1375                _ => layout.llvm_type(cx),
1376            }
1377        }
1378        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1379            if s.primitive() == Primitive::Float(Float::F16)
1380                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1381        {
1382            cx.type_f32()
1383        }
1384        (
1385            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1386            BackendRepr::Scalar(s),
1387        ) if s.primitive() == Primitive::Float(Float::F32) => cx.type_vector(cx.type_f32(), 4),
1388        (
1389            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1390            BackendRepr::Scalar(s),
1391        ) if s.primitive() == Primitive::Float(Float::F64) => cx.type_vector(cx.type_f64(), 2),
1392        _ => layout.llvm_type(cx),
1393    }
1394}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
