rustc_codegen_llvm/
asm.rs

1use std::assert_matches::assert_matches;
2
3use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
4use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
5use rustc_codegen_ssa::mir::operand::OperandValue;
6use rustc_codegen_ssa::traits::*;
7use rustc_data_structures::fx::FxHashMap;
8use rustc_middle::ty::Instance;
9use rustc_middle::ty::layout::TyAndLayout;
10use rustc_middle::{bug, span_bug};
11use rustc_span::{Pos, Span, Symbol, sym};
12use rustc_target::asm::*;
13use smallvec::SmallVec;
14use tracing::debug;
15
16use crate::builder::Builder;
17use crate::common::Funclet;
18use crate::context::CodegenCx;
19use crate::type_::Type;
20use crate::type_of::LayoutLlvmExt;
21use crate::value::Value;
22use crate::{attributes, llvm};
23
24impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
25    fn codegen_inline_asm(
26        &mut self,
27        template: &[InlineAsmTemplatePiece],
28        operands: &[InlineAsmOperandRef<'tcx, Self>],
29        options: InlineAsmOptions,
30        line_spans: &[Span],
31        instance: Instance<'_>,
32        dest: Option<Self::BasicBlock>,
33        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
34    ) {
35        let asm_arch = self.tcx.sess.asm_arch.unwrap();
36
37        // Collect the types of output operands
38        let mut constraints = vec![];
39        let mut clobbers = vec![];
40        let mut output_types = vec![];
41        let mut op_idx = FxHashMap::default();
42        let mut clobbered_x87 = false;
43        for (idx, op) in operands.iter().enumerate() {
44            match *op {
45                InlineAsmOperandRef::Out { reg, late, place } => {
46                    let is_target_supported = |reg_class: InlineAsmRegClass| {
47                        for &(_, feature) in reg_class.supported_types(asm_arch, true) {
48                            if let Some(feature) = feature {
49                                if self
50                                    .tcx
51                                    .asm_target_features(instance.def_id())
52                                    .contains(&feature)
53                                {
54                                    return true;
55                                }
56                            } else {
57                                // Register class is unconditionally supported
58                                return true;
59                            }
60                        }
61                        false
62                    };
63
64                    let mut layout = None;
65                    let ty = if let Some(ref place) = place {
66                        layout = Some(&place.layout);
67                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
68                    } else if matches!(
69                        reg.reg_class(),
70                        InlineAsmRegClass::X86(
71                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
72                        )
73                    ) {
74                        // Special handling for x87/mmx registers: we always
75                        // clobber the whole set if one register is marked as
76                        // clobbered. This is due to the way LLVM handles the
77                        // FP stack in inline assembly.
78                        if !clobbered_x87 {
79                            clobbered_x87 = true;
80                            clobbers.push("~{st}".to_string());
81                            for i in 1..=7 {
82                                clobbers.push(format!("~{{st({})}}", i));
83                            }
84                        }
85                        continue;
86                    } else if !is_target_supported(reg.reg_class())
87                        || reg.reg_class().is_clobber_only(asm_arch, true)
88                    {
89                        // We turn discarded outputs into clobber constraints
90                        // if the target feature needed by the register class is
91                        // disabled. This is necessary otherwise LLVM will try
92                        // to actually allocate a register for the dummy output.
93                        assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
94                        clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
95                        continue;
96                    } else {
97                        // If the output is discarded, we don't really care what
98                        // type is used. We're just using this to tell LLVM to
99                        // reserve the register.
100                        dummy_output_type(self.cx, reg.reg_class())
101                    };
102                    output_types.push(ty);
103                    op_idx.insert(idx, constraints.len());
104                    let prefix = if late { "=" } else { "=&" };
105                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
106                }
107                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
108                    let layout = if let Some(ref out_place) = out_place {
109                        &out_place.layout
110                    } else {
111                        // LLVM required tied operands to have the same type,
112                        // so we just use the type of the input.
113                        &in_value.layout
114                    };
115                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
116                    output_types.push(ty);
117                    op_idx.insert(idx, constraints.len());
118                    let prefix = if late { "=" } else { "=&" };
119                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
120                }
121                _ => {}
122            }
123        }
124
125        // Collect input operands
126        let mut inputs = vec![];
127        for (idx, op) in operands.iter().enumerate() {
128            match *op {
129                InlineAsmOperandRef::In { reg, value } => {
130                    let llval = llvm_fixup_input(
131                        self,
132                        value.immediate(),
133                        reg.reg_class(),
134                        &value.layout,
135                        instance,
136                    );
137                    inputs.push(llval);
138                    op_idx.insert(idx, constraints.len());
139                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
140                }
141                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
142                    let value = llvm_fixup_input(
143                        self,
144                        in_value.immediate(),
145                        reg.reg_class(),
146                        &in_value.layout,
147                        instance,
148                    );
149                    inputs.push(value);
150
151                    // In the case of fixed registers, we have the choice of
152                    // either using a tied operand or duplicating the constraint.
153                    // We prefer the latter because it matches the behavior of
154                    // Clang.
155                    if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
156                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
157                    } else {
158                        constraints.push(format!("{}", op_idx[&idx]));
159                    }
160                }
161                InlineAsmOperandRef::SymFn { instance } => {
162                    inputs.push(self.cx.get_fn(instance));
163                    op_idx.insert(idx, constraints.len());
164                    constraints.push("s".to_string());
165                }
166                InlineAsmOperandRef::SymStatic { def_id } => {
167                    inputs.push(self.cx.get_static(def_id));
168                    op_idx.insert(idx, constraints.len());
169                    constraints.push("s".to_string());
170                }
171                _ => {}
172            }
173        }
174
175        // Build the template string
176        let mut labels = vec![];
177        let mut template_str = String::new();
178        for piece in template {
179            match *piece {
180                InlineAsmTemplatePiece::String(ref s) => {
181                    if s.contains('$') {
182                        for c in s.chars() {
183                            if c == '$' {
184                                template_str.push_str("$$");
185                            } else {
186                                template_str.push(c);
187                            }
188                        }
189                    } else {
190                        template_str.push_str(s)
191                    }
192                }
193                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
194                    match operands[operand_idx] {
195                        InlineAsmOperandRef::In { reg, .. }
196                        | InlineAsmOperandRef::Out { reg, .. }
197                        | InlineAsmOperandRef::InOut { reg, .. } => {
198                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
199                            if let Some(modifier) = modifier {
200                                template_str.push_str(&format!(
201                                    "${{{}:{}}}",
202                                    op_idx[&operand_idx], modifier
203                                ));
204                            } else {
205                                template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
206                            }
207                        }
208                        InlineAsmOperandRef::Const { ref string } => {
209                            // Const operands get injected directly into the template
210                            template_str.push_str(string);
211                        }
212                        InlineAsmOperandRef::SymFn { .. }
213                        | InlineAsmOperandRef::SymStatic { .. } => {
214                            // Only emit the raw symbol name
215                            template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
216                        }
217                        InlineAsmOperandRef::Label { label } => {
218                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
219                            constraints.push("!i".to_owned());
220                            labels.push(label);
221                        }
222                    }
223                }
224            }
225        }
226
227        constraints.append(&mut clobbers);
228        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
229            match asm_arch {
230                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
231                    constraints.push("~{cc}".to_string());
232                }
233                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
234                    constraints.extend_from_slice(&[
235                        "~{dirflag}".to_string(),
236                        "~{fpsr}".to_string(),
237                        "~{flags}".to_string(),
238                    ]);
239                }
240                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
241                    constraints.extend_from_slice(&[
242                        "~{vtype}".to_string(),
243                        "~{vl}".to_string(),
244                        "~{vxsat}".to_string(),
245                        "~{vxrm}".to_string(),
246                    ]);
247                }
248                InlineAsmArch::Avr => {
249                    constraints.push("~{sreg}".to_string());
250                }
251                InlineAsmArch::Nvptx64 => {}
252                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
253                InlineAsmArch::Hexagon => {}
254                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
255                    constraints.extend_from_slice(&[
256                        "~{$fcc0}".to_string(),
257                        "~{$fcc1}".to_string(),
258                        "~{$fcc2}".to_string(),
259                        "~{$fcc3}".to_string(),
260                        "~{$fcc4}".to_string(),
261                        "~{$fcc5}".to_string(),
262                        "~{$fcc6}".to_string(),
263                        "~{$fcc7}".to_string(),
264                    ]);
265                }
266                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
267                InlineAsmArch::S390x => {
268                    constraints.push("~{cc}".to_string());
269                }
270                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
271                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
272                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
273                    constraints.push("~{icc}".to_string());
274                    constraints.push("~{fcc0}".to_string());
275                    constraints.push("~{fcc1}".to_string());
276                    constraints.push("~{fcc2}".to_string());
277                    constraints.push("~{fcc3}".to_string());
278                }
279                InlineAsmArch::SpirV => {}
280                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
281                InlineAsmArch::Bpf => {}
282                InlineAsmArch::Msp430 => {
283                    constraints.push("~{sr}".to_string());
284                }
285                InlineAsmArch::M68k => {
286                    constraints.push("~{ccr}".to_string());
287                }
288                InlineAsmArch::CSKY => {
289                    constraints.push("~{psr}".to_string());
290                }
291            }
292        }
293        if !options.contains(InlineAsmOptions::NOMEM) {
294            // This is actually ignored by LLVM, but it's probably best to keep
295            // it just in case. LLVM instead uses the ReadOnly/ReadNone
296            // attributes on the call instruction to optimize.
297            constraints.push("~{memory}".to_string());
298        }
299        let volatile = !options.contains(InlineAsmOptions::PURE);
300        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
301        let output_type = match &output_types[..] {
302            [] => self.type_void(),
303            [ty] => ty,
304            tys => self.type_struct(tys, false),
305        };
306        let dialect = match asm_arch {
307            InlineAsmArch::X86 | InlineAsmArch::X86_64
308                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
309            {
310                llvm::AsmDialect::Intel
311            }
312            _ => llvm::AsmDialect::Att,
313        };
314        let result = inline_asm_call(
315            self,
316            &template_str,
317            &constraints.join(","),
318            &inputs,
319            output_type,
320            &labels,
321            volatile,
322            alignstack,
323            dialect,
324            line_spans,
325            options.contains(InlineAsmOptions::MAY_UNWIND),
326            dest,
327            catch_funclet,
328        )
329        .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
330
331        let mut attrs = SmallVec::<[_; 2]>::new();
332        if options.contains(InlineAsmOptions::PURE) {
333            if options.contains(InlineAsmOptions::NOMEM) {
334                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
335            } else if options.contains(InlineAsmOptions::READONLY) {
336                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
337            }
338            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
339        } else if options.contains(InlineAsmOptions::NOMEM) {
340            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
341        } else {
342            // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
343        }
344        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
345
346        // Write results to outputs. We need to do this for all possible control flow.
347        //
348        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
349        // is used (because we need to codegen callbr which always needs a destination), so
350        // here we use the NORETURN option to determine if `dest` should be used.
351        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
352            .into_iter()
353            .chain(labels.iter().copied().map(Some))
354        {
355            if let Some(block) = block {
356                self.switch_to_block(block);
357            }
358
359            for (idx, op) in operands.iter().enumerate() {
360                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
361                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
362                {
363                    let value = if output_types.len() == 1 {
364                        result
365                    } else {
366                        self.extract_value(result, op_idx[&idx] as u64)
367                    };
368                    let value =
369                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
370                    OperandValue::Immediate(value).store(self, place);
371                }
372            }
373        }
374    }
375}
376
377impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
378    fn codegen_global_asm(
379        &mut self,
380        template: &[InlineAsmTemplatePiece],
381        operands: &[GlobalAsmOperandRef<'tcx>],
382        options: InlineAsmOptions,
383        _line_spans: &[Span],
384    ) {
385        let asm_arch = self.tcx.sess.asm_arch.unwrap();
386
387        // Build the template string
388        let mut template_str = String::new();
389
390        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
391        // but AT&T can be specified explicitly.
392        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
393            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
394                template_str.push_str(".att_syntax\n")
395            } else {
396                template_str.push_str(".intel_syntax\n")
397            }
398        }
399
400        for piece in template {
401            match *piece {
402                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
403                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
404                    match operands[operand_idx] {
405                        GlobalAsmOperandRef::Const { ref string } => {
406                            // Const operands get injected directly into the
407                            // template. Note that we don't need to escape $
408                            // here unlike normal inline assembly.
409                            template_str.push_str(string);
410                        }
411                        GlobalAsmOperandRef::SymFn { instance } => {
412                            let llval = self.get_fn(instance);
413                            self.add_compiler_used_global(llval);
414                            let symbol = llvm::build_string(|s| unsafe {
415                                llvm::LLVMRustGetMangledName(llval, s);
416                            })
417                            .expect("symbol is not valid UTF-8");
418                            template_str.push_str(&symbol);
419                        }
420                        GlobalAsmOperandRef::SymStatic { def_id } => {
421                            let llval = self
422                                .renamed_statics
423                                .borrow()
424                                .get(&def_id)
425                                .copied()
426                                .unwrap_or_else(|| self.get_static(def_id));
427                            self.add_compiler_used_global(llval);
428                            let symbol = llvm::build_string(|s| unsafe {
429                                llvm::LLVMRustGetMangledName(llval, s);
430                            })
431                            .expect("symbol is not valid UTF-8");
432                            template_str.push_str(&symbol);
433                        }
434                    }
435                }
436            }
437        }
438
439        // Just to play it safe, if intel was used, reset the assembly syntax to att.
440        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
441            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
442        {
443            template_str.push_str("\n.att_syntax\n");
444        }
445
446        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
447    }
448
449    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
450        let llval = self.get_fn(instance);
451        llvm::build_string(|s| unsafe {
452            llvm::LLVMRustGetMangledName(llval, s);
453        })
454        .expect("symbol is not valid UTF-8")
455    }
456}
457
458pub(crate) fn inline_asm_call<'ll>(
459    bx: &mut Builder<'_, 'll, '_>,
460    asm: &str,
461    cons: &str,
462    inputs: &[&'ll Value],
463    output: &'ll llvm::Type,
464    labels: &[&'ll llvm::BasicBlock],
465    volatile: bool,
466    alignstack: bool,
467    dia: llvm::AsmDialect,
468    line_spans: &[Span],
469    unwind: bool,
470    dest: Option<&'ll llvm::BasicBlock>,
471    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
472) -> Option<&'ll Value> {
473    let volatile = if volatile { llvm::True } else { llvm::False };
474    let alignstack = if alignstack { llvm::True } else { llvm::False };
475    let can_throw = if unwind { llvm::True } else { llvm::False };
476
477    let argtys = inputs
478        .iter()
479        .map(|v| {
480            debug!("Asm Input Type: {:?}", *v);
481            bx.cx.val_ty(*v)
482        })
483        .collect::<Vec<_>>();
484
485    debug!("Asm Output Type: {:?}", output);
486    let fty = bx.cx.type_func(&argtys, output);
487
488    // Ask LLVM to verify that the constraints are well-formed.
489    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
490    debug!("constraint verification result: {:?}", constraints_ok);
491    if !constraints_ok {
492        // LLVM has detected an issue with our constraints, so bail out.
493        return None;
494    }
495
496    let v = unsafe {
497        llvm::LLVMGetInlineAsm(
498            fty,
499            asm.as_ptr(),
500            asm.len(),
501            cons.as_ptr(),
502            cons.len(),
503            volatile,
504            alignstack,
505            dia,
506            can_throw,
507        )
508    };
509
510    let call = if !labels.is_empty() {
511        assert!(catch_funclet.is_none());
512        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
513    } else if let Some((catch, funclet)) = catch_funclet {
514        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
515    } else {
516        bx.call(fty, None, None, v, inputs, None, None)
517    };
518
519    // Store mark in a metadata node so we can map LLVM errors
520    // back to source locations. See #17552.
521    let key = "srcloc";
522    let kind = bx.get_md_kind_id(key);
523
524    // `srcloc` contains one 64-bit integer for each line of assembly code,
525    // where the lower 32 bits hold the lo byte position and the upper 32 bits
526    // hold the hi byte position.
527    let mut srcloc = vec![];
528    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
529        // LLVM inserts an extra line to add the ".intel_syntax", so add
530        // a dummy srcloc entry for it.
531        //
532        // Don't do this if we only have 1 line span since that may be
533        // due to the asm template string coming from a macro. LLVM will
534        // default to the first srcloc for lines that don't have an
535        // associated srcloc.
536        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
537    }
538    srcloc.extend(line_spans.iter().map(|span| {
539        llvm::LLVMValueAsMetadata(
540            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
541        )
542    }));
543    let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) };
544    let md = bx.get_metadata_value(md);
545    llvm::LLVMSetMetadata(call, kind, md);
546
547    Some(call)
548}
549
550/// If the register is an xmm/ymm/zmm register then return its index.
551fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
552    use X86InlineAsmReg::*;
553    match reg {
554        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
555            Some(reg as u32 - xmm0 as u32)
556        }
557        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
558            Some(reg as u32 - ymm0 as u32)
559        }
560        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
561            Some(reg as u32 - zmm0 as u32)
562        }
563        _ => None,
564    }
565}
566
567/// If the register is an AArch64 integer register then return its index.
568fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
569    match reg {
570        InlineAsmReg::AArch64(r) => r.reg_index(),
571        _ => None,
572    }
573}
574
575/// If the register is an AArch64 vector register then return its index.
576fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
577    match reg {
578        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
579        _ => None,
580    }
581}
582
583/// Converts a register class to an LLVM constraint code.
584fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
585    use InlineAsmRegClass::*;
586    match reg {
587        // For vector registers LLVM wants the register name to match the type size.
588        InlineAsmRegOrRegClass::Reg(reg) => {
589            if let Some(idx) = xmm_reg_index(reg) {
590                let class = if let Some(layout) = layout {
591                    match layout.size.bytes() {
592                        64 => 'z',
593                        32 => 'y',
594                        _ => 'x',
595                    }
596                } else {
597                    // We use f32 as the type for discarded outputs
598                    'x'
599                };
600                format!("{{{}mm{}}}", class, idx)
601            } else if let Some(idx) = a64_reg_index(reg) {
602                let class = if let Some(layout) = layout {
603                    match layout.size.bytes() {
604                        8 => 'x',
605                        _ => 'w',
606                    }
607                } else {
608                    // We use i32 as the type for discarded outputs
609                    'w'
610                };
611                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
612                    // LLVM doesn't recognize x30. use lr instead.
613                    "{lr}".to_string()
614                } else {
615                    format!("{{{}{}}}", class, idx)
616                }
617            } else if let Some(idx) = a64_vreg_index(reg) {
618                let class = if let Some(layout) = layout {
619                    match layout.size.bytes() {
620                        16 => 'q',
621                        8 => 'd',
622                        4 => 's',
623                        2 => 'h',
624                        1 => 'd', // We fixup i8 to i8x8
625                        _ => unreachable!(),
626                    }
627                } else {
628                    // We use i64x2 as the type for discarded outputs
629                    'q'
630                };
631                format!("{{{}{}}}", class, idx)
632            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
633                // LLVM doesn't recognize r14
634                "{lr}".to_string()
635            } else {
636                format!("{{{}}}", reg.name())
637            }
638        }
639        // The constraints can be retrieved from
640        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
641        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
642            AArch64(AArch64InlineAsmRegClass::reg) => "r",
643            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
644            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
645            AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
646            Arm(ArmInlineAsmRegClass::reg) => "r",
647            Arm(ArmInlineAsmRegClass::sreg)
648            | Arm(ArmInlineAsmRegClass::dreg_low16)
649            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
650            Arm(ArmInlineAsmRegClass::sreg_low16)
651            | Arm(ArmInlineAsmRegClass::dreg_low8)
652            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
653            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
654            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
655            Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
656            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
657            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
658            Mips(MipsInlineAsmRegClass::reg) => "r",
659            Mips(MipsInlineAsmRegClass::freg) => "f",
660            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
661            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
662            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
663            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
664            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
665            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
666            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
667            PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
668                unreachable!("clobber-only")
669            }
670            RiscV(RiscVInlineAsmRegClass::reg) => "r",
671            RiscV(RiscVInlineAsmRegClass::freg) => "f",
672            RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
673            X86(X86InlineAsmRegClass::reg) => "r",
674            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
675            X86(X86InlineAsmRegClass::reg_byte) => "q",
676            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
677            X86(X86InlineAsmRegClass::zmm_reg) => "v",
678            X86(X86InlineAsmRegClass::kreg) => "^Yk",
679            X86(
680                X86InlineAsmRegClass::x87_reg
681                | X86InlineAsmRegClass::mmx_reg
682                | X86InlineAsmRegClass::kreg0
683                | X86InlineAsmRegClass::tmm_reg,
684            ) => unreachable!("clobber-only"),
685            Wasm(WasmInlineAsmRegClass::local) => "r",
686            Bpf(BpfInlineAsmRegClass::reg) => "r",
687            Bpf(BpfInlineAsmRegClass::wreg) => "w",
688            Avr(AvrInlineAsmRegClass::reg) => "r",
689            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
690            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
691            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
692            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
693            S390x(S390xInlineAsmRegClass::reg) => "r",
694            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
695            S390x(S390xInlineAsmRegClass::freg) => "f",
696            S390x(S390xInlineAsmRegClass::vreg) => "v",
697            S390x(S390xInlineAsmRegClass::areg) => {
698                unreachable!("clobber-only")
699            }
700            Sparc(SparcInlineAsmRegClass::reg) => "r",
701            Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
702            Msp430(Msp430InlineAsmRegClass::reg) => "r",
703            M68k(M68kInlineAsmRegClass::reg) => "r",
704            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
705            M68k(M68kInlineAsmRegClass::reg_data) => "d",
706            CSKY(CSKYInlineAsmRegClass::reg) => "r",
707            CSKY(CSKYInlineAsmRegClass::freg) => "f",
708            SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
709            Err => unreachable!(),
710        }
711        .to_string(),
712    }
713}
714
715/// Converts a modifier into LLVM's equivalent modifier.
716fn modifier_to_llvm(
717    arch: InlineAsmArch,
718    reg: InlineAsmRegClass,
719    modifier: Option<char>,
720) -> Option<char> {
721    use InlineAsmRegClass::*;
722    // The modifiers can be retrieved from
723    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
724    match reg {
725        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
726        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
727            if modifier == Some('v') {
728                None
729            } else {
730                modifier
731            }
732        }
733        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
734        Arm(ArmInlineAsmRegClass::reg) => None,
735        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
736        Arm(ArmInlineAsmRegClass::dreg)
737        | Arm(ArmInlineAsmRegClass::dreg_low16)
738        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
739        Arm(ArmInlineAsmRegClass::qreg)
740        | Arm(ArmInlineAsmRegClass::qreg_low8)
741        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
742            if modifier.is_none() {
743                Some('q')
744            } else {
745                modifier
746            }
747        }
748        Hexagon(_) => None,
749        LoongArch(_) => None,
750        Mips(_) => None,
751        Nvptx(_) => None,
752        PowerPC(_) => None,
753        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
754        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
755        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
756            None if arch == InlineAsmArch::X86_64 => Some('q'),
757            None => Some('k'),
758            Some('l') => Some('b'),
759            Some('h') => Some('h'),
760            Some('x') => Some('w'),
761            Some('e') => Some('k'),
762            Some('r') => Some('q'),
763            _ => unreachable!(),
764        },
765        X86(X86InlineAsmRegClass::reg_byte) => None,
766        X86(reg @ X86InlineAsmRegClass::xmm_reg)
767        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
768        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
769            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
770            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
771            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
772            (_, Some('x')) => Some('x'),
773            (_, Some('y')) => Some('t'),
774            (_, Some('z')) => Some('g'),
775            _ => unreachable!(),
776        },
777        X86(X86InlineAsmRegClass::kreg) => None,
778        X86(
779            X86InlineAsmRegClass::x87_reg
780            | X86InlineAsmRegClass::mmx_reg
781            | X86InlineAsmRegClass::kreg0
782            | X86InlineAsmRegClass::tmm_reg,
783        ) => unreachable!("clobber-only"),
784        Wasm(WasmInlineAsmRegClass::local) => None,
785        Bpf(_) => None,
786        Avr(AvrInlineAsmRegClass::reg_pair)
787        | Avr(AvrInlineAsmRegClass::reg_iw)
788        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
789            Some('h') => Some('B'),
790            Some('l') => Some('A'),
791            _ => None,
792        },
793        Avr(_) => None,
794        S390x(_) => None,
795        Sparc(_) => None,
796        Msp430(_) => None,
797        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
798        M68k(_) => None,
799        CSKY(_) => None,
800        Err => unreachable!(),
801    }
802}
803
804/// Type to use for outputs that are discarded. It doesn't really matter what
805/// the type is, as long as it is valid for the constraint code.
806fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
807    use InlineAsmRegClass::*;
808    match reg {
809        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
810        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
811            cx.type_vector(cx.type_i64(), 2)
812        }
813        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
814        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
815        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
816        Arm(ArmInlineAsmRegClass::dreg)
817        | Arm(ArmInlineAsmRegClass::dreg_low16)
818        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
819        Arm(ArmInlineAsmRegClass::qreg)
820        | Arm(ArmInlineAsmRegClass::qreg_low8)
821        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
822        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
823        Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
824        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
825        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
826        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
827        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
828        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
829        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
830        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
831        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
832        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
833        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
834        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
835        PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
836            unreachable!("clobber-only")
837        }
838        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
839        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
840        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
841        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
842        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
843        X86(X86InlineAsmRegClass::xmm_reg)
844        | X86(X86InlineAsmRegClass::ymm_reg)
845        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
846        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
847        X86(
848            X86InlineAsmRegClass::x87_reg
849            | X86InlineAsmRegClass::mmx_reg
850            | X86InlineAsmRegClass::kreg0
851            | X86InlineAsmRegClass::tmm_reg,
852        ) => unreachable!("clobber-only"),
853        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
854        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
855        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
856        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
857        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
858        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
859        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
860        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
861        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
862        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
863        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
864        S390x(S390xInlineAsmRegClass::areg) => {
865            unreachable!("clobber-only")
866        }
867        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
868        Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
869        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
870        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
871        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
872        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
873        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
874        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
875        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
876        Err => unreachable!(),
877    }
878}
879
880/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
881/// the equivalent integer type.
882fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
883    let dl = &cx.tcx.data_layout;
884    match scalar.primitive() {
885        Primitive::Int(Integer::I8, _) => cx.type_i8(),
886        Primitive::Int(Integer::I16, _) => cx.type_i16(),
887        Primitive::Int(Integer::I32, _) => cx.type_i32(),
888        Primitive::Int(Integer::I64, _) => cx.type_i64(),
889        Primitive::Float(Float::F16) => cx.type_f16(),
890        Primitive::Float(Float::F32) => cx.type_f32(),
891        Primitive::Float(Float::F64) => cx.type_f64(),
892        Primitive::Float(Float::F128) => cx.type_f128(),
893        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
894        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
895        _ => unreachable!(),
896    }
897}
898
899fn any_target_feature_enabled(
900    cx: &CodegenCx<'_, '_>,
901    instance: Instance<'_>,
902    features: &[Symbol],
903) -> bool {
904    let enabled = cx.tcx.asm_target_features(instance.def_id());
905    features.iter().any(|feat| enabled.contains(feat))
906}
907
908/// Fix up an input value to work around LLVM bugs.
909fn llvm_fixup_input<'ll, 'tcx>(
910    bx: &mut Builder<'_, 'll, 'tcx>,
911    mut value: &'ll Value,
912    reg: InlineAsmRegClass,
913    layout: &TyAndLayout<'tcx>,
914    instance: Instance<'_>,
915) -> &'ll Value {
916    use InlineAsmRegClass::*;
917    let dl = &bx.tcx.data_layout;
918    match (reg, layout.backend_repr) {
919        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
920            if let Primitive::Int(Integer::I8, _) = s.primitive() {
921                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
922                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
923            } else {
924                value
925            }
926        }
927        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
928            if s.primitive() != Primitive::Float(Float::F128) =>
929        {
930            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
931            let count = 16 / layout.size.bytes();
932            let vec_ty = bx.cx.type_vector(elem_ty, count);
933            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
934            if let Primitive::Pointer(_) = s.primitive() {
935                let t = bx.type_from_integer(dl.ptr_sized_integer());
936                value = bx.ptrtoint(value, t);
937            }
938            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
939        }
940        (
941            AArch64(AArch64InlineAsmRegClass::vreg_low16),
942            BackendRepr::SimdVector { element, count },
943        ) if layout.size.bytes() == 8 => {
944            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
945            let vec_ty = bx.cx.type_vector(elem_ty, count);
946            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
947            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
948        }
949        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
950            if s.primitive() == Primitive::Float(Float::F64) =>
951        {
952            bx.bitcast(value, bx.cx.type_i64())
953        }
954        (
955            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
956            BackendRepr::SimdVector { .. },
957        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
958        (
959            X86(
960                X86InlineAsmRegClass::xmm_reg
961                | X86InlineAsmRegClass::ymm_reg
962                | X86InlineAsmRegClass::zmm_reg,
963            ),
964            BackendRepr::Scalar(s),
965        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
966            && s.primitive() == Primitive::Float(Float::F128) =>
967        {
968            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
969        }
970        (
971            X86(
972                X86InlineAsmRegClass::xmm_reg
973                | X86InlineAsmRegClass::ymm_reg
974                | X86InlineAsmRegClass::zmm_reg,
975            ),
976            BackendRepr::Scalar(s),
977        ) if s.primitive() == Primitive::Float(Float::F16) => {
978            let value = bx.insert_element(
979                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
980                value,
981                bx.const_usize(0),
982            );
983            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
984        }
985        (
986            X86(
987                X86InlineAsmRegClass::xmm_reg
988                | X86InlineAsmRegClass::ymm_reg
989                | X86InlineAsmRegClass::zmm_reg,
990            ),
991            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
992        ) if element.primitive() == Primitive::Float(Float::F16) => {
993            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
994        }
995        (
996            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
997            BackendRepr::Scalar(s),
998        ) => {
999            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1000                bx.bitcast(value, bx.cx.type_f32())
1001            } else {
1002                value
1003            }
1004        }
1005        (
1006            Arm(
1007                ArmInlineAsmRegClass::dreg
1008                | ArmInlineAsmRegClass::dreg_low8
1009                | ArmInlineAsmRegClass::dreg_low16,
1010            ),
1011            BackendRepr::Scalar(s),
1012        ) => {
1013            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1014                bx.bitcast(value, bx.cx.type_f64())
1015            } else {
1016                value
1017            }
1018        }
1019        (
1020            Arm(
1021                ArmInlineAsmRegClass::dreg
1022                | ArmInlineAsmRegClass::dreg_low8
1023                | ArmInlineAsmRegClass::dreg_low16
1024                | ArmInlineAsmRegClass::qreg
1025                | ArmInlineAsmRegClass::qreg_low4
1026                | ArmInlineAsmRegClass::qreg_low8,
1027            ),
1028            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1029        ) if element.primitive() == Primitive::Float(Float::F16) => {
1030            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1031        }
1032        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1033            if s.primitive() == Primitive::Float(Float::F16) =>
1034        {
1035            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1036            let value = bx.bitcast(value, bx.type_i16());
1037            let value = bx.zext(value, bx.type_i32());
1038            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1039            bx.bitcast(value, bx.type_f32())
1040        }
1041        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1042            match s.primitive() {
1043                // MIPS only supports register-length arithmetics.
1044                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1045                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1046                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1047                _ => value,
1048            }
1049        }
1050        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1051            if s.primitive() == Primitive::Float(Float::F16)
1052                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1053        {
1054            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1055            let value = bx.bitcast(value, bx.type_i16());
1056            let value = bx.zext(value, bx.type_i32());
1057            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1058            bx.bitcast(value, bx.type_f32())
1059        }
1060        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1061            if s.primitive() == Primitive::Float(Float::F32) =>
1062        {
1063            let value = bx.insert_element(
1064                bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
1065                value,
1066                bx.const_usize(0),
1067            );
1068            bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
1069        }
1070        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1071            if s.primitive() == Primitive::Float(Float::F64) =>
1072        {
1073            let value = bx.insert_element(
1074                bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
1075                value,
1076                bx.const_usize(0),
1077            );
1078            bx.bitcast(value, bx.type_vector(bx.type_f64(), 2))
1079        }
1080        _ => value,
1081    }
1082}
1083
1084/// Fix up an output value to work around LLVM bugs.
1085fn llvm_fixup_output<'ll, 'tcx>(
1086    bx: &mut Builder<'_, 'll, 'tcx>,
1087    mut value: &'ll Value,
1088    reg: InlineAsmRegClass,
1089    layout: &TyAndLayout<'tcx>,
1090    instance: Instance<'_>,
1091) -> &'ll Value {
1092    use InlineAsmRegClass::*;
1093    match (reg, layout.backend_repr) {
1094        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1095            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1096                bx.extract_element(value, bx.const_i32(0))
1097            } else {
1098                value
1099            }
1100        }
1101        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1102            if s.primitive() != Primitive::Float(Float::F128) =>
1103        {
1104            value = bx.extract_element(value, bx.const_i32(0));
1105            if let Primitive::Pointer(_) = s.primitive() {
1106                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1107            }
1108            value
1109        }
1110        (
1111            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1112            BackendRepr::SimdVector { element, count },
1113        ) if layout.size.bytes() == 8 => {
1114            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1115            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1116            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1117            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1118        }
1119        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1120            if s.primitive() == Primitive::Float(Float::F64) =>
1121        {
1122            bx.bitcast(value, bx.cx.type_f64())
1123        }
1124        (
1125            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1126            BackendRepr::SimdVector { .. },
1127        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1128        (
1129            X86(
1130                X86InlineAsmRegClass::xmm_reg
1131                | X86InlineAsmRegClass::ymm_reg
1132                | X86InlineAsmRegClass::zmm_reg,
1133            ),
1134            BackendRepr::Scalar(s),
1135        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1136            && s.primitive() == Primitive::Float(Float::F128) =>
1137        {
1138            bx.bitcast(value, bx.type_f128())
1139        }
1140        (
1141            X86(
1142                X86InlineAsmRegClass::xmm_reg
1143                | X86InlineAsmRegClass::ymm_reg
1144                | X86InlineAsmRegClass::zmm_reg,
1145            ),
1146            BackendRepr::Scalar(s),
1147        ) if s.primitive() == Primitive::Float(Float::F16) => {
1148            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1149            bx.extract_element(value, bx.const_usize(0))
1150        }
1151        (
1152            X86(
1153                X86InlineAsmRegClass::xmm_reg
1154                | X86InlineAsmRegClass::ymm_reg
1155                | X86InlineAsmRegClass::zmm_reg,
1156            ),
1157            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1158        ) if element.primitive() == Primitive::Float(Float::F16) => {
1159            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1160        }
1161        (
1162            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1163            BackendRepr::Scalar(s),
1164        ) => {
1165            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1166                bx.bitcast(value, bx.cx.type_i32())
1167            } else {
1168                value
1169            }
1170        }
1171        (
1172            Arm(
1173                ArmInlineAsmRegClass::dreg
1174                | ArmInlineAsmRegClass::dreg_low8
1175                | ArmInlineAsmRegClass::dreg_low16,
1176            ),
1177            BackendRepr::Scalar(s),
1178        ) => {
1179            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1180                bx.bitcast(value, bx.cx.type_i64())
1181            } else {
1182                value
1183            }
1184        }
1185        (
1186            Arm(
1187                ArmInlineAsmRegClass::dreg
1188                | ArmInlineAsmRegClass::dreg_low8
1189                | ArmInlineAsmRegClass::dreg_low16
1190                | ArmInlineAsmRegClass::qreg
1191                | ArmInlineAsmRegClass::qreg_low4
1192                | ArmInlineAsmRegClass::qreg_low8,
1193            ),
1194            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1195        ) if element.primitive() == Primitive::Float(Float::F16) => {
1196            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1197        }
1198        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1199            if s.primitive() == Primitive::Float(Float::F16) =>
1200        {
1201            let value = bx.bitcast(value, bx.type_i32());
1202            let value = bx.trunc(value, bx.type_i16());
1203            bx.bitcast(value, bx.type_f16())
1204        }
1205        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1206            match s.primitive() {
1207                // MIPS only supports register-length arithmetics.
1208                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1209                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1210                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1211                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1212                _ => value,
1213            }
1214        }
1215        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1216            if s.primitive() == Primitive::Float(Float::F16)
1217                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1218        {
1219            let value = bx.bitcast(value, bx.type_i32());
1220            let value = bx.trunc(value, bx.type_i16());
1221            bx.bitcast(value, bx.type_f16())
1222        }
1223        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1224            if s.primitive() == Primitive::Float(Float::F32) =>
1225        {
1226            let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
1227            bx.extract_element(value, bx.const_usize(0))
1228        }
1229        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1230            if s.primitive() == Primitive::Float(Float::F64) =>
1231        {
1232            let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
1233            bx.extract_element(value, bx.const_usize(0))
1234        }
1235        _ => value,
1236    }
1237}
1238
1239/// Output type to use for llvm_fixup_output.
1240fn llvm_fixup_output_type<'ll, 'tcx>(
1241    cx: &CodegenCx<'ll, 'tcx>,
1242    reg: InlineAsmRegClass,
1243    layout: &TyAndLayout<'tcx>,
1244    instance: Instance<'_>,
1245) -> &'ll Type {
1246    use InlineAsmRegClass::*;
1247    match (reg, layout.backend_repr) {
1248        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1249            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1250                cx.type_vector(cx.type_i8(), 8)
1251            } else {
1252                layout.llvm_type(cx)
1253            }
1254        }
1255        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1256            if s.primitive() != Primitive::Float(Float::F128) =>
1257        {
1258            let elem_ty = llvm_asm_scalar_type(cx, s);
1259            let count = 16 / layout.size.bytes();
1260            cx.type_vector(elem_ty, count)
1261        }
1262        (
1263            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1264            BackendRepr::SimdVector { element, count },
1265        ) if layout.size.bytes() == 8 => {
1266            let elem_ty = llvm_asm_scalar_type(cx, element);
1267            cx.type_vector(elem_ty, count * 2)
1268        }
1269        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1270            if s.primitive() == Primitive::Float(Float::F64) =>
1271        {
1272            cx.type_i64()
1273        }
1274        (
1275            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1276            BackendRepr::SimdVector { .. },
1277        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1278        (
1279            X86(
1280                X86InlineAsmRegClass::xmm_reg
1281                | X86InlineAsmRegClass::ymm_reg
1282                | X86InlineAsmRegClass::zmm_reg,
1283            ),
1284            BackendRepr::Scalar(s),
1285        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1286            && s.primitive() == Primitive::Float(Float::F128) =>
1287        {
1288            cx.type_vector(cx.type_i32(), 4)
1289        }
1290        (
1291            X86(
1292                X86InlineAsmRegClass::xmm_reg
1293                | X86InlineAsmRegClass::ymm_reg
1294                | X86InlineAsmRegClass::zmm_reg,
1295            ),
1296            BackendRepr::Scalar(s),
1297        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1298        (
1299            X86(
1300                X86InlineAsmRegClass::xmm_reg
1301                | X86InlineAsmRegClass::ymm_reg
1302                | X86InlineAsmRegClass::zmm_reg,
1303            ),
1304            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1305        ) if element.primitive() == Primitive::Float(Float::F16) => {
1306            cx.type_vector(cx.type_i16(), count)
1307        }
1308        (
1309            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1310            BackendRepr::Scalar(s),
1311        ) => {
1312            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1313                cx.type_f32()
1314            } else {
1315                layout.llvm_type(cx)
1316            }
1317        }
1318        (
1319            Arm(
1320                ArmInlineAsmRegClass::dreg
1321                | ArmInlineAsmRegClass::dreg_low8
1322                | ArmInlineAsmRegClass::dreg_low16,
1323            ),
1324            BackendRepr::Scalar(s),
1325        ) => {
1326            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1327                cx.type_f64()
1328            } else {
1329                layout.llvm_type(cx)
1330            }
1331        }
1332        (
1333            Arm(
1334                ArmInlineAsmRegClass::dreg
1335                | ArmInlineAsmRegClass::dreg_low8
1336                | ArmInlineAsmRegClass::dreg_low16
1337                | ArmInlineAsmRegClass::qreg
1338                | ArmInlineAsmRegClass::qreg_low4
1339                | ArmInlineAsmRegClass::qreg_low8,
1340            ),
1341            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1342        ) if element.primitive() == Primitive::Float(Float::F16) => {
1343            cx.type_vector(cx.type_i16(), count)
1344        }
1345        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1346            if s.primitive() == Primitive::Float(Float::F16) =>
1347        {
1348            cx.type_f32()
1349        }
1350        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1351            match s.primitive() {
1352                // MIPS only supports register-length arithmetics.
1353                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1354                Primitive::Float(Float::F32) => cx.type_i32(),
1355                Primitive::Float(Float::F64) => cx.type_i64(),
1356                _ => layout.llvm_type(cx),
1357            }
1358        }
1359        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1360            if s.primitive() == Primitive::Float(Float::F16)
1361                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1362        {
1363            cx.type_f32()
1364        }
1365        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1366            if s.primitive() == Primitive::Float(Float::F32) =>
1367        {
1368            cx.type_vector(cx.type_f32(), 4)
1369        }
1370        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1371            if s.primitive() == Primitive::Float(Float::F64) =>
1372        {
1373            cx.type_vector(cx.type_f64(), 2)
1374        }
1375        _ => layout.llvm_type(cx),
1376    }
1377}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
