rustc_codegen_llvm/
asm.rs

1use std::assert_matches::assert_matches;
2
3use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
4use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
5use rustc_codegen_ssa::mir::operand::OperandValue;
6use rustc_codegen_ssa::traits::*;
7use rustc_data_structures::fx::FxHashMap;
8use rustc_middle::ty::Instance;
9use rustc_middle::ty::layout::TyAndLayout;
10use rustc_middle::{bug, span_bug};
11use rustc_span::{Pos, Span, Symbol, sym};
12use rustc_target::asm::*;
13use smallvec::SmallVec;
14use tracing::debug;
15
16use crate::builder::Builder;
17use crate::common::Funclet;
18use crate::context::CodegenCx;
19use crate::llvm::ToLlvmBool;
20use crate::type_::Type;
21use crate::type_of::LayoutLlvmExt;
22use crate::value::Value;
23use crate::{attributes, llvm};
24
25impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
26    fn codegen_inline_asm(
27        &mut self,
28        template: &[InlineAsmTemplatePiece],
29        operands: &[InlineAsmOperandRef<'tcx, Self>],
30        options: InlineAsmOptions,
31        line_spans: &[Span],
32        instance: Instance<'_>,
33        dest: Option<Self::BasicBlock>,
34        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
35    ) {
36        let asm_arch = self.tcx.sess.asm_arch.unwrap();
37
38        // Collect the types of output operands
39        let mut constraints = vec![];
40        let mut clobbers = vec![];
41        let mut output_types = vec![];
42        let mut op_idx = FxHashMap::default();
43        let mut clobbered_x87 = false;
44        for (idx, op) in operands.iter().enumerate() {
45            match *op {
46                InlineAsmOperandRef::Out { reg, late, place } => {
47                    let is_target_supported = |reg_class: InlineAsmRegClass| {
48                        for &(_, feature) in reg_class.supported_types(asm_arch, true) {
49                            if let Some(feature) = feature {
50                                if self
51                                    .tcx
52                                    .asm_target_features(instance.def_id())
53                                    .contains(&feature)
54                                {
55                                    return true;
56                                }
57                            } else {
58                                // Register class is unconditionally supported
59                                return true;
60                            }
61                        }
62                        false
63                    };
64
65                    let mut layout = None;
66                    let ty = if let Some(ref place) = place {
67                        layout = Some(&place.layout);
68                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
69                    } else if matches!(
70                        reg.reg_class(),
71                        InlineAsmRegClass::X86(
72                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
73                        )
74                    ) {
75                        // Special handling for x87/mmx registers: we always
76                        // clobber the whole set if one register is marked as
77                        // clobbered. This is due to the way LLVM handles the
78                        // FP stack in inline assembly.
79                        if !clobbered_x87 {
80                            clobbered_x87 = true;
81                            clobbers.push("~{st}".to_string());
82                            for i in 1..=7 {
83                                clobbers.push(format!("~{{st({})}}", i));
84                            }
85                        }
86                        continue;
87                    } else if !is_target_supported(reg.reg_class())
88                        || reg.reg_class().is_clobber_only(asm_arch, true)
89                    {
90                        // We turn discarded outputs into clobber constraints
91                        // if the target feature needed by the register class is
92                        // disabled. This is necessary otherwise LLVM will try
93                        // to actually allocate a register for the dummy output.
94                        assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
95                        clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
96                        continue;
97                    } else {
98                        // If the output is discarded, we don't really care what
99                        // type is used. We're just using this to tell LLVM to
100                        // reserve the register.
101                        dummy_output_type(self.cx, reg.reg_class())
102                    };
103                    output_types.push(ty);
104                    op_idx.insert(idx, constraints.len());
105                    let prefix = if late { "=" } else { "=&" };
106                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
107                }
108                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
109                    let layout = if let Some(ref out_place) = out_place {
110                        &out_place.layout
111                    } else {
112                        // LLVM required tied operands to have the same type,
113                        // so we just use the type of the input.
114                        &in_value.layout
115                    };
116                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
117                    output_types.push(ty);
118                    op_idx.insert(idx, constraints.len());
119                    let prefix = if late { "=" } else { "=&" };
120                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
121                }
122                _ => {}
123            }
124        }
125
126        // Collect input operands
127        let mut inputs = vec![];
128        for (idx, op) in operands.iter().enumerate() {
129            match *op {
130                InlineAsmOperandRef::In { reg, value } => {
131                    let llval = llvm_fixup_input(
132                        self,
133                        value.immediate(),
134                        reg.reg_class(),
135                        &value.layout,
136                        instance,
137                    );
138                    inputs.push(llval);
139                    op_idx.insert(idx, constraints.len());
140                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
141                }
142                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
143                    let value = llvm_fixup_input(
144                        self,
145                        in_value.immediate(),
146                        reg.reg_class(),
147                        &in_value.layout,
148                        instance,
149                    );
150                    inputs.push(value);
151
152                    // In the case of fixed registers, we have the choice of
153                    // either using a tied operand or duplicating the constraint.
154                    // We prefer the latter because it matches the behavior of
155                    // Clang.
156                    if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
157                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
158                    } else {
159                        constraints.push(format!("{}", op_idx[&idx]));
160                    }
161                }
162                InlineAsmOperandRef::SymFn { instance } => {
163                    inputs.push(self.cx.get_fn(instance));
164                    op_idx.insert(idx, constraints.len());
165                    constraints.push("s".to_string());
166                }
167                InlineAsmOperandRef::SymStatic { def_id } => {
168                    inputs.push(self.cx.get_static(def_id));
169                    op_idx.insert(idx, constraints.len());
170                    constraints.push("s".to_string());
171                }
172                _ => {}
173            }
174        }
175
176        // Build the template string
177        let mut labels = vec![];
178        let mut template_str = String::new();
179        for piece in template {
180            match *piece {
181                InlineAsmTemplatePiece::String(ref s) => {
182                    if s.contains('$') {
183                        for c in s.chars() {
184                            if c == '$' {
185                                template_str.push_str("$$");
186                            } else {
187                                template_str.push(c);
188                            }
189                        }
190                    } else {
191                        template_str.push_str(s)
192                    }
193                }
194                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
195                    match operands[operand_idx] {
196                        InlineAsmOperandRef::In { reg, .. }
197                        | InlineAsmOperandRef::Out { reg, .. }
198                        | InlineAsmOperandRef::InOut { reg, .. } => {
199                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
200                            if let Some(modifier) = modifier {
201                                template_str.push_str(&format!(
202                                    "${{{}:{}}}",
203                                    op_idx[&operand_idx], modifier
204                                ));
205                            } else {
206                                template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
207                            }
208                        }
209                        InlineAsmOperandRef::Const { ref string } => {
210                            // Const operands get injected directly into the template
211                            template_str.push_str(string);
212                        }
213                        InlineAsmOperandRef::SymFn { .. }
214                        | InlineAsmOperandRef::SymStatic { .. } => {
215                            // Only emit the raw symbol name
216                            template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
217                        }
218                        InlineAsmOperandRef::Label { label } => {
219                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
220                            constraints.push("!i".to_owned());
221                            labels.push(label);
222                        }
223                    }
224                }
225            }
226        }
227
228        constraints.append(&mut clobbers);
229        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
230            match asm_arch {
231                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
232                    constraints.push("~{cc}".to_string());
233                }
234                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
235                    constraints.extend_from_slice(&[
236                        "~{dirflag}".to_string(),
237                        "~{fpsr}".to_string(),
238                        "~{flags}".to_string(),
239                    ]);
240                }
241                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
242                    constraints.extend_from_slice(&[
243                        "~{fflags}".to_string(),
244                        "~{vtype}".to_string(),
245                        "~{vl}".to_string(),
246                        "~{vxsat}".to_string(),
247                        "~{vxrm}".to_string(),
248                    ]);
249                }
250                InlineAsmArch::Avr => {
251                    constraints.push("~{sreg}".to_string());
252                }
253                InlineAsmArch::Nvptx64 => {}
254                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
255                InlineAsmArch::Hexagon => {}
256                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
257                    constraints.extend_from_slice(&[
258                        "~{$fcc0}".to_string(),
259                        "~{$fcc1}".to_string(),
260                        "~{$fcc2}".to_string(),
261                        "~{$fcc3}".to_string(),
262                        "~{$fcc4}".to_string(),
263                        "~{$fcc5}".to_string(),
264                        "~{$fcc6}".to_string(),
265                        "~{$fcc7}".to_string(),
266                    ]);
267                }
268                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
269                InlineAsmArch::S390x => {
270                    constraints.push("~{cc}".to_string());
271                }
272                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
273                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
274                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
275                    constraints.push("~{icc}".to_string());
276                    constraints.push("~{fcc0}".to_string());
277                    constraints.push("~{fcc1}".to_string());
278                    constraints.push("~{fcc2}".to_string());
279                    constraints.push("~{fcc3}".to_string());
280                }
281                InlineAsmArch::SpirV => {}
282                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
283                InlineAsmArch::Bpf => {}
284                InlineAsmArch::Msp430 => {
285                    constraints.push("~{sr}".to_string());
286                }
287                InlineAsmArch::M68k => {
288                    constraints.push("~{ccr}".to_string());
289                }
290                InlineAsmArch::CSKY => {
291                    constraints.push("~{psr}".to_string());
292                }
293            }
294        }
295        if !options.contains(InlineAsmOptions::NOMEM) {
296            // This is actually ignored by LLVM, but it's probably best to keep
297            // it just in case. LLVM instead uses the ReadOnly/ReadNone
298            // attributes on the call instruction to optimize.
299            constraints.push("~{memory}".to_string());
300        }
301        let volatile = !options.contains(InlineAsmOptions::PURE);
302        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
303        let output_type = match &output_types[..] {
304            [] => self.type_void(),
305            [ty] => ty,
306            tys => self.type_struct(tys, false),
307        };
308        let dialect = match asm_arch {
309            InlineAsmArch::X86 | InlineAsmArch::X86_64
310                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
311            {
312                llvm::AsmDialect::Intel
313            }
314            _ => llvm::AsmDialect::Att,
315        };
316        let result = inline_asm_call(
317            self,
318            &template_str,
319            &constraints.join(","),
320            &inputs,
321            output_type,
322            &labels,
323            volatile,
324            alignstack,
325            dialect,
326            line_spans,
327            options.contains(InlineAsmOptions::MAY_UNWIND),
328            dest,
329            catch_funclet,
330        )
331        .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
332
333        let mut attrs = SmallVec::<[_; 2]>::new();
334        if options.contains(InlineAsmOptions::PURE) {
335            if options.contains(InlineAsmOptions::NOMEM) {
336                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
337            } else if options.contains(InlineAsmOptions::READONLY) {
338                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
339            }
340            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
341        } else if options.contains(InlineAsmOptions::NOMEM) {
342            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
343        } else if options.contains(InlineAsmOptions::READONLY) {
344            attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
345        }
346        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
347
348        // Write results to outputs. We need to do this for all possible control flow.
349        //
350        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
351        // is used (because we need to codegen callbr which always needs a destination), so
352        // here we use the NORETURN option to determine if `dest` should be used.
353        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
354            .into_iter()
355            .chain(labels.iter().copied().map(Some))
356        {
357            if let Some(block) = block {
358                self.switch_to_block(block);
359            }
360
361            for (idx, op) in operands.iter().enumerate() {
362                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
363                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
364                {
365                    let value = if output_types.len() == 1 {
366                        result
367                    } else {
368                        self.extract_value(result, op_idx[&idx] as u64)
369                    };
370                    let value =
371                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
372                    OperandValue::Immediate(value).store(self, place);
373                }
374            }
375        }
376    }
377}
378
379impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
380    fn codegen_global_asm(
381        &mut self,
382        template: &[InlineAsmTemplatePiece],
383        operands: &[GlobalAsmOperandRef<'tcx>],
384        options: InlineAsmOptions,
385        _line_spans: &[Span],
386    ) {
387        let asm_arch = self.tcx.sess.asm_arch.unwrap();
388
389        // Build the template string
390        let mut template_str = String::new();
391
392        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
393        // but AT&T can be specified explicitly.
394        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
395            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
396                template_str.push_str(".att_syntax\n")
397            } else {
398                template_str.push_str(".intel_syntax\n")
399            }
400        }
401
402        for piece in template {
403            match *piece {
404                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
405                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
406                    match operands[operand_idx] {
407                        GlobalAsmOperandRef::Const { ref string } => {
408                            // Const operands get injected directly into the
409                            // template. Note that we don't need to escape $
410                            // here unlike normal inline assembly.
411                            template_str.push_str(string);
412                        }
413                        GlobalAsmOperandRef::SymFn { instance } => {
414                            let llval = self.get_fn(instance);
415                            self.add_compiler_used_global(llval);
416                            let symbol = llvm::build_string(|s| unsafe {
417                                llvm::LLVMRustGetMangledName(llval, s);
418                            })
419                            .expect("symbol is not valid UTF-8");
420                            template_str.push_str(&symbol);
421                        }
422                        GlobalAsmOperandRef::SymStatic { def_id } => {
423                            let llval = self
424                                .renamed_statics
425                                .borrow()
426                                .get(&def_id)
427                                .copied()
428                                .unwrap_or_else(|| self.get_static(def_id));
429                            self.add_compiler_used_global(llval);
430                            let symbol = llvm::build_string(|s| unsafe {
431                                llvm::LLVMRustGetMangledName(llval, s);
432                            })
433                            .expect("symbol is not valid UTF-8");
434                            template_str.push_str(&symbol);
435                        }
436                    }
437                }
438            }
439        }
440
441        // Just to play it safe, if intel was used, reset the assembly syntax to att.
442        if matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
443            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
444        {
445            template_str.push_str("\n.att_syntax\n");
446        }
447
448        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
449    }
450
451    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
452        let llval = self.get_fn(instance);
453        llvm::build_string(|s| unsafe {
454            llvm::LLVMRustGetMangledName(llval, s);
455        })
456        .expect("symbol is not valid UTF-8")
457    }
458}
459
460pub(crate) fn inline_asm_call<'ll>(
461    bx: &mut Builder<'_, 'll, '_>,
462    asm: &str,
463    cons: &str,
464    inputs: &[&'ll Value],
465    output: &'ll llvm::Type,
466    labels: &[&'ll llvm::BasicBlock],
467    volatile: bool,
468    alignstack: bool,
469    dia: llvm::AsmDialect,
470    line_spans: &[Span],
471    unwind: bool,
472    dest: Option<&'ll llvm::BasicBlock>,
473    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
474) -> Option<&'ll Value> {
475    let argtys = inputs
476        .iter()
477        .map(|v| {
478            debug!("Asm Input Type: {:?}", *v);
479            bx.cx.val_ty(*v)
480        })
481        .collect::<Vec<_>>();
482
483    debug!("Asm Output Type: {:?}", output);
484    let fty = bx.cx.type_func(&argtys, output);
485
486    // Ask LLVM to verify that the constraints are well-formed.
487    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
488    debug!("constraint verification result: {:?}", constraints_ok);
489    if !constraints_ok {
490        // LLVM has detected an issue with our constraints, so bail out.
491        return None;
492    }
493
494    let v = unsafe {
495        llvm::LLVMGetInlineAsm(
496            fty,
497            asm.as_ptr(),
498            asm.len(),
499            cons.as_ptr(),
500            cons.len(),
501            volatile.to_llvm_bool(),
502            alignstack.to_llvm_bool(),
503            dia,
504            unwind.to_llvm_bool(),
505        )
506    };
507
508    let call = if !labels.is_empty() {
509        assert!(catch_funclet.is_none());
510        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
511    } else if let Some((catch, funclet)) = catch_funclet {
512        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
513    } else {
514        bx.call(fty, None, None, v, inputs, None, None)
515    };
516
517    // Store mark in a metadata node so we can map LLVM errors
518    // back to source locations. See #17552.
519    let key = "srcloc";
520    let kind = bx.get_md_kind_id(key);
521
522    // `srcloc` contains one 64-bit integer for each line of assembly code,
523    // where the lower 32 bits hold the lo byte position and the upper 32 bits
524    // hold the hi byte position.
525    let mut srcloc = vec![];
526    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
527        // LLVM inserts an extra line to add the ".intel_syntax", so add
528        // a dummy srcloc entry for it.
529        //
530        // Don't do this if we only have 1 line span since that may be
531        // due to the asm template string coming from a macro. LLVM will
532        // default to the first srcloc for lines that don't have an
533        // associated srcloc.
534        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
535    }
536    srcloc.extend(line_spans.iter().map(|span| {
537        llvm::LLVMValueAsMetadata(
538            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
539        )
540    }));
541    let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) };
542    let md = bx.get_metadata_value(md);
543    llvm::LLVMSetMetadata(call, kind, md);
544
545    Some(call)
546}
547
548/// If the register is an xmm/ymm/zmm register then return its index.
549fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
550    use X86InlineAsmReg::*;
551    match reg {
552        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
553            Some(reg as u32 - xmm0 as u32)
554        }
555        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
556            Some(reg as u32 - ymm0 as u32)
557        }
558        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
559            Some(reg as u32 - zmm0 as u32)
560        }
561        _ => None,
562    }
563}
564
565/// If the register is an AArch64 integer register then return its index.
566fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
567    match reg {
568        InlineAsmReg::AArch64(r) => r.reg_index(),
569        _ => None,
570    }
571}
572
573/// If the register is an AArch64 vector register then return its index.
574fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
575    match reg {
576        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
577        _ => None,
578    }
579}
580
581/// Converts a register class to an LLVM constraint code.
582fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
583    use InlineAsmRegClass::*;
584    match reg {
585        // For vector registers LLVM wants the register name to match the type size.
586        InlineAsmRegOrRegClass::Reg(reg) => {
587            if let Some(idx) = xmm_reg_index(reg) {
588                let class = if let Some(layout) = layout {
589                    match layout.size.bytes() {
590                        64 => 'z',
591                        32 => 'y',
592                        _ => 'x',
593                    }
594                } else {
595                    // We use f32 as the type for discarded outputs
596                    'x'
597                };
598                format!("{{{}mm{}}}", class, idx)
599            } else if let Some(idx) = a64_reg_index(reg) {
600                let class = if let Some(layout) = layout {
601                    match layout.size.bytes() {
602                        8 => 'x',
603                        _ => 'w',
604                    }
605                } else {
606                    // We use i32 as the type for discarded outputs
607                    'w'
608                };
609                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
610                    // LLVM doesn't recognize x30. use lr instead.
611                    "{lr}".to_string()
612                } else {
613                    format!("{{{}{}}}", class, idx)
614                }
615            } else if let Some(idx) = a64_vreg_index(reg) {
616                let class = if let Some(layout) = layout {
617                    match layout.size.bytes() {
618                        16 => 'q',
619                        8 => 'd',
620                        4 => 's',
621                        2 => 'h',
622                        1 => 'd', // We fixup i8 to i8x8
623                        _ => unreachable!(),
624                    }
625                } else {
626                    // We use i64x2 as the type for discarded outputs
627                    'q'
628                };
629                format!("{{{}{}}}", class, idx)
630            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
631                // LLVM doesn't recognize r14
632                "{lr}".to_string()
633            } else {
634                format!("{{{}}}", reg.name())
635            }
636        }
637        // The constraints can be retrieved from
638        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
639        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
640            AArch64(AArch64InlineAsmRegClass::reg) => "r",
641            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
642            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
643            AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
644            Arm(ArmInlineAsmRegClass::reg) => "r",
645            Arm(ArmInlineAsmRegClass::sreg)
646            | Arm(ArmInlineAsmRegClass::dreg_low16)
647            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
648            Arm(ArmInlineAsmRegClass::sreg_low16)
649            | Arm(ArmInlineAsmRegClass::dreg_low8)
650            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
651            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
652            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
653            Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
654            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
655            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
656            Mips(MipsInlineAsmRegClass::reg) => "r",
657            Mips(MipsInlineAsmRegClass::freg) => "f",
658            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
659            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
660            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
661            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
662            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
663            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
664            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
665            PowerPC(
666                PowerPCInlineAsmRegClass::cr
667                | PowerPCInlineAsmRegClass::ctr
668                | PowerPCInlineAsmRegClass::lr
669                | PowerPCInlineAsmRegClass::xer,
670            ) => {
671                unreachable!("clobber-only")
672            }
673            RiscV(RiscVInlineAsmRegClass::reg) => "r",
674            RiscV(RiscVInlineAsmRegClass::freg) => "f",
675            RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
676            X86(X86InlineAsmRegClass::reg) => "r",
677            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
678            X86(X86InlineAsmRegClass::reg_byte) => "q",
679            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
680            X86(X86InlineAsmRegClass::zmm_reg) => "v",
681            X86(X86InlineAsmRegClass::kreg) => "^Yk",
682            X86(
683                X86InlineAsmRegClass::x87_reg
684                | X86InlineAsmRegClass::mmx_reg
685                | X86InlineAsmRegClass::kreg0
686                | X86InlineAsmRegClass::tmm_reg,
687            ) => unreachable!("clobber-only"),
688            Wasm(WasmInlineAsmRegClass::local) => "r",
689            Bpf(BpfInlineAsmRegClass::reg) => "r",
690            Bpf(BpfInlineAsmRegClass::wreg) => "w",
691            Avr(AvrInlineAsmRegClass::reg) => "r",
692            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
693            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
694            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
695            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
696            S390x(S390xInlineAsmRegClass::reg) => "r",
697            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
698            S390x(S390xInlineAsmRegClass::freg) => "f",
699            S390x(S390xInlineAsmRegClass::vreg) => "v",
700            S390x(S390xInlineAsmRegClass::areg) => {
701                unreachable!("clobber-only")
702            }
703            Sparc(SparcInlineAsmRegClass::reg) => "r",
704            Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
705            Msp430(Msp430InlineAsmRegClass::reg) => "r",
706            M68k(M68kInlineAsmRegClass::reg) => "r",
707            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
708            M68k(M68kInlineAsmRegClass::reg_data) => "d",
709            CSKY(CSKYInlineAsmRegClass::reg) => "r",
710            CSKY(CSKYInlineAsmRegClass::freg) => "f",
711            SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
712            Err => unreachable!(),
713        }
714        .to_string(),
715    }
716}
717
718/// Converts a modifier into LLVM's equivalent modifier.
719fn modifier_to_llvm(
720    arch: InlineAsmArch,
721    reg: InlineAsmRegClass,
722    modifier: Option<char>,
723) -> Option<char> {
724    use InlineAsmRegClass::*;
725    // The modifiers can be retrieved from
726    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
727    match reg {
728        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
729        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
730            if modifier == Some('v') {
731                None
732            } else {
733                modifier
734            }
735        }
736        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
737        Arm(ArmInlineAsmRegClass::reg) => None,
738        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
739        Arm(ArmInlineAsmRegClass::dreg)
740        | Arm(ArmInlineAsmRegClass::dreg_low16)
741        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
742        Arm(ArmInlineAsmRegClass::qreg)
743        | Arm(ArmInlineAsmRegClass::qreg_low8)
744        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
745            if modifier.is_none() {
746                Some('q')
747            } else {
748                modifier
749            }
750        }
751        Hexagon(_) => None,
752        LoongArch(_) => None,
753        Mips(_) => None,
754        Nvptx(_) => None,
755        PowerPC(_) => None,
756        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
757        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
758        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
759            None if arch == InlineAsmArch::X86_64 => Some('q'),
760            None => Some('k'),
761            Some('l') => Some('b'),
762            Some('h') => Some('h'),
763            Some('x') => Some('w'),
764            Some('e') => Some('k'),
765            Some('r') => Some('q'),
766            _ => unreachable!(),
767        },
768        X86(X86InlineAsmRegClass::reg_byte) => None,
769        X86(reg @ X86InlineAsmRegClass::xmm_reg)
770        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
771        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
772            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
773            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
774            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
775            (_, Some('x')) => Some('x'),
776            (_, Some('y')) => Some('t'),
777            (_, Some('z')) => Some('g'),
778            _ => unreachable!(),
779        },
780        X86(X86InlineAsmRegClass::kreg) => None,
781        X86(
782            X86InlineAsmRegClass::x87_reg
783            | X86InlineAsmRegClass::mmx_reg
784            | X86InlineAsmRegClass::kreg0
785            | X86InlineAsmRegClass::tmm_reg,
786        ) => unreachable!("clobber-only"),
787        Wasm(WasmInlineAsmRegClass::local) => None,
788        Bpf(_) => None,
789        Avr(AvrInlineAsmRegClass::reg_pair)
790        | Avr(AvrInlineAsmRegClass::reg_iw)
791        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
792            Some('h') => Some('B'),
793            Some('l') => Some('A'),
794            _ => None,
795        },
796        Avr(_) => None,
797        S390x(_) => None,
798        Sparc(_) => None,
799        Msp430(_) => None,
800        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
801        M68k(_) => None,
802        CSKY(_) => None,
803        Err => unreachable!(),
804    }
805}
806
807/// Type to use for outputs that are discarded. It doesn't really matter what
808/// the type is, as long as it is valid for the constraint code.
809fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
810    use InlineAsmRegClass::*;
811    match reg {
812        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
813        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
814            cx.type_vector(cx.type_i64(), 2)
815        }
816        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
817        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
818        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
819        Arm(ArmInlineAsmRegClass::dreg)
820        | Arm(ArmInlineAsmRegClass::dreg_low16)
821        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
822        Arm(ArmInlineAsmRegClass::qreg)
823        | Arm(ArmInlineAsmRegClass::qreg_low8)
824        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
825        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
826        Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"),
827        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
828        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
829        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
830        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
831        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
832        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
833        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
834        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
835        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
836        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
837        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
838        PowerPC(
839            PowerPCInlineAsmRegClass::cr
840            | PowerPCInlineAsmRegClass::ctr
841            | PowerPCInlineAsmRegClass::lr
842            | PowerPCInlineAsmRegClass::xer,
843        ) => {
844            unreachable!("clobber-only")
845        }
846        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
847        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
848        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
849        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
850        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
851        X86(X86InlineAsmRegClass::xmm_reg)
852        | X86(X86InlineAsmRegClass::ymm_reg)
853        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
854        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
855        X86(
856            X86InlineAsmRegClass::x87_reg
857            | X86InlineAsmRegClass::mmx_reg
858            | X86InlineAsmRegClass::kreg0
859            | X86InlineAsmRegClass::tmm_reg,
860        ) => unreachable!("clobber-only"),
861        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
862        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
863        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
864        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
865        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
866        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
867        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
868        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
869        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
870        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
871        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
872        S390x(S390xInlineAsmRegClass::areg) => {
873            unreachable!("clobber-only")
874        }
875        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
876        Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
877        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
878        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
879        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
880        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
881        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
882        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
883        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
884        Err => unreachable!(),
885    }
886}
887
888/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
889/// the equivalent integer type.
890fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
891    let dl = &cx.tcx.data_layout;
892    match scalar.primitive() {
893        Primitive::Int(Integer::I8, _) => cx.type_i8(),
894        Primitive::Int(Integer::I16, _) => cx.type_i16(),
895        Primitive::Int(Integer::I32, _) => cx.type_i32(),
896        Primitive::Int(Integer::I64, _) => cx.type_i64(),
897        Primitive::Float(Float::F16) => cx.type_f16(),
898        Primitive::Float(Float::F32) => cx.type_f32(),
899        Primitive::Float(Float::F64) => cx.type_f64(),
900        Primitive::Float(Float::F128) => cx.type_f128(),
901        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
902        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
903        _ => unreachable!(),
904    }
905}
906
907fn any_target_feature_enabled(
908    cx: &CodegenCx<'_, '_>,
909    instance: Instance<'_>,
910    features: &[Symbol],
911) -> bool {
912    let enabled = cx.tcx.asm_target_features(instance.def_id());
913    features.iter().any(|feat| enabled.contains(feat))
914}
915
916/// Fix up an input value to work around LLVM bugs.
917fn llvm_fixup_input<'ll, 'tcx>(
918    bx: &mut Builder<'_, 'll, 'tcx>,
919    mut value: &'ll Value,
920    reg: InlineAsmRegClass,
921    layout: &TyAndLayout<'tcx>,
922    instance: Instance<'_>,
923) -> &'ll Value {
924    use InlineAsmRegClass::*;
925    let dl = &bx.tcx.data_layout;
926    match (reg, layout.backend_repr) {
927        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
928            if let Primitive::Int(Integer::I8, _) = s.primitive() {
929                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
930                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
931            } else {
932                value
933            }
934        }
935        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
936            if s.primitive() != Primitive::Float(Float::F128) =>
937        {
938            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
939            let count = 16 / layout.size.bytes();
940            let vec_ty = bx.cx.type_vector(elem_ty, count);
941            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
942            if let Primitive::Pointer(_) = s.primitive() {
943                let t = bx.type_from_integer(dl.ptr_sized_integer());
944                value = bx.ptrtoint(value, t);
945            }
946            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
947        }
948        (
949            AArch64(AArch64InlineAsmRegClass::vreg_low16),
950            BackendRepr::SimdVector { element, count },
951        ) if layout.size.bytes() == 8 => {
952            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
953            let vec_ty = bx.cx.type_vector(elem_ty, count);
954            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
955            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
956        }
957        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
958            if s.primitive() == Primitive::Float(Float::F64) =>
959        {
960            bx.bitcast(value, bx.cx.type_i64())
961        }
962        (
963            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
964            BackendRepr::SimdVector { .. },
965        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
966        (
967            X86(
968                X86InlineAsmRegClass::xmm_reg
969                | X86InlineAsmRegClass::ymm_reg
970                | X86InlineAsmRegClass::zmm_reg,
971            ),
972            BackendRepr::Scalar(s),
973        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
974            && s.primitive() == Primitive::Float(Float::F128) =>
975        {
976            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
977        }
978        (
979            X86(
980                X86InlineAsmRegClass::xmm_reg
981                | X86InlineAsmRegClass::ymm_reg
982                | X86InlineAsmRegClass::zmm_reg,
983            ),
984            BackendRepr::Scalar(s),
985        ) if s.primitive() == Primitive::Float(Float::F16) => {
986            let value = bx.insert_element(
987                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
988                value,
989                bx.const_usize(0),
990            );
991            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
992        }
993        (
994            X86(
995                X86InlineAsmRegClass::xmm_reg
996                | X86InlineAsmRegClass::ymm_reg
997                | X86InlineAsmRegClass::zmm_reg,
998            ),
999            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1000        ) if element.primitive() == Primitive::Float(Float::F16) => {
1001            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1002        }
1003        (
1004            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1005            BackendRepr::Scalar(s),
1006        ) => {
1007            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1008                bx.bitcast(value, bx.cx.type_f32())
1009            } else {
1010                value
1011            }
1012        }
1013        (
1014            Arm(
1015                ArmInlineAsmRegClass::dreg
1016                | ArmInlineAsmRegClass::dreg_low8
1017                | ArmInlineAsmRegClass::dreg_low16,
1018            ),
1019            BackendRepr::Scalar(s),
1020        ) => {
1021            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1022                bx.bitcast(value, bx.cx.type_f64())
1023            } else {
1024                value
1025            }
1026        }
1027        (
1028            Arm(
1029                ArmInlineAsmRegClass::dreg
1030                | ArmInlineAsmRegClass::dreg_low8
1031                | ArmInlineAsmRegClass::dreg_low16
1032                | ArmInlineAsmRegClass::qreg
1033                | ArmInlineAsmRegClass::qreg_low4
1034                | ArmInlineAsmRegClass::qreg_low8,
1035            ),
1036            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1037        ) if element.primitive() == Primitive::Float(Float::F16) => {
1038            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1039        }
1040        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1041            if s.primitive() == Primitive::Float(Float::F16) =>
1042        {
1043            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1044            let value = bx.bitcast(value, bx.type_i16());
1045            let value = bx.zext(value, bx.type_i32());
1046            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1047            bx.bitcast(value, bx.type_f32())
1048        }
1049        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1050            match s.primitive() {
1051                // MIPS only supports register-length arithmetics.
1052                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1053                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1054                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1055                _ => value,
1056            }
1057        }
1058        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1059            if s.primitive() == Primitive::Float(Float::F16)
1060                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1061        {
1062            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1063            let value = bx.bitcast(value, bx.type_i16());
1064            let value = bx.zext(value, bx.type_i32());
1065            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1066            bx.bitcast(value, bx.type_f32())
1067        }
1068        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1069            if s.primitive() == Primitive::Float(Float::F32) =>
1070        {
1071            let value = bx.insert_element(
1072                bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
1073                value,
1074                bx.const_usize(0),
1075            );
1076            bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
1077        }
1078        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1079            if s.primitive() == Primitive::Float(Float::F64) =>
1080        {
1081            let value = bx.insert_element(
1082                bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
1083                value,
1084                bx.const_usize(0),
1085            );
1086            bx.bitcast(value, bx.type_vector(bx.type_f64(), 2))
1087        }
1088        _ => value,
1089    }
1090}
1091
1092/// Fix up an output value to work around LLVM bugs.
1093fn llvm_fixup_output<'ll, 'tcx>(
1094    bx: &mut Builder<'_, 'll, 'tcx>,
1095    mut value: &'ll Value,
1096    reg: InlineAsmRegClass,
1097    layout: &TyAndLayout<'tcx>,
1098    instance: Instance<'_>,
1099) -> &'ll Value {
1100    use InlineAsmRegClass::*;
1101    match (reg, layout.backend_repr) {
1102        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1103            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1104                bx.extract_element(value, bx.const_i32(0))
1105            } else {
1106                value
1107            }
1108        }
1109        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1110            if s.primitive() != Primitive::Float(Float::F128) =>
1111        {
1112            value = bx.extract_element(value, bx.const_i32(0));
1113            if let Primitive::Pointer(_) = s.primitive() {
1114                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1115            }
1116            value
1117        }
1118        (
1119            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1120            BackendRepr::SimdVector { element, count },
1121        ) if layout.size.bytes() == 8 => {
1122            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1123            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1124            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1125            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1126        }
1127        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1128            if s.primitive() == Primitive::Float(Float::F64) =>
1129        {
1130            bx.bitcast(value, bx.cx.type_f64())
1131        }
1132        (
1133            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1134            BackendRepr::SimdVector { .. },
1135        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1136        (
1137            X86(
1138                X86InlineAsmRegClass::xmm_reg
1139                | X86InlineAsmRegClass::ymm_reg
1140                | X86InlineAsmRegClass::zmm_reg,
1141            ),
1142            BackendRepr::Scalar(s),
1143        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1144            && s.primitive() == Primitive::Float(Float::F128) =>
1145        {
1146            bx.bitcast(value, bx.type_f128())
1147        }
1148        (
1149            X86(
1150                X86InlineAsmRegClass::xmm_reg
1151                | X86InlineAsmRegClass::ymm_reg
1152                | X86InlineAsmRegClass::zmm_reg,
1153            ),
1154            BackendRepr::Scalar(s),
1155        ) if s.primitive() == Primitive::Float(Float::F16) => {
1156            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1157            bx.extract_element(value, bx.const_usize(0))
1158        }
1159        (
1160            X86(
1161                X86InlineAsmRegClass::xmm_reg
1162                | X86InlineAsmRegClass::ymm_reg
1163                | X86InlineAsmRegClass::zmm_reg,
1164            ),
1165            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1166        ) if element.primitive() == Primitive::Float(Float::F16) => {
1167            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1168        }
1169        (
1170            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1171            BackendRepr::Scalar(s),
1172        ) => {
1173            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1174                bx.bitcast(value, bx.cx.type_i32())
1175            } else {
1176                value
1177            }
1178        }
1179        (
1180            Arm(
1181                ArmInlineAsmRegClass::dreg
1182                | ArmInlineAsmRegClass::dreg_low8
1183                | ArmInlineAsmRegClass::dreg_low16,
1184            ),
1185            BackendRepr::Scalar(s),
1186        ) => {
1187            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1188                bx.bitcast(value, bx.cx.type_i64())
1189            } else {
1190                value
1191            }
1192        }
1193        (
1194            Arm(
1195                ArmInlineAsmRegClass::dreg
1196                | ArmInlineAsmRegClass::dreg_low8
1197                | ArmInlineAsmRegClass::dreg_low16
1198                | ArmInlineAsmRegClass::qreg
1199                | ArmInlineAsmRegClass::qreg_low4
1200                | ArmInlineAsmRegClass::qreg_low8,
1201            ),
1202            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1203        ) if element.primitive() == Primitive::Float(Float::F16) => {
1204            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1205        }
1206        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1207            if s.primitive() == Primitive::Float(Float::F16) =>
1208        {
1209            let value = bx.bitcast(value, bx.type_i32());
1210            let value = bx.trunc(value, bx.type_i16());
1211            bx.bitcast(value, bx.type_f16())
1212        }
1213        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1214            match s.primitive() {
1215                // MIPS only supports register-length arithmetics.
1216                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1217                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1218                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1219                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1220                _ => value,
1221            }
1222        }
1223        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1224            if s.primitive() == Primitive::Float(Float::F16)
1225                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1226        {
1227            let value = bx.bitcast(value, bx.type_i32());
1228            let value = bx.trunc(value, bx.type_i16());
1229            bx.bitcast(value, bx.type_f16())
1230        }
1231        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1232            if s.primitive() == Primitive::Float(Float::F32) =>
1233        {
1234            let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
1235            bx.extract_element(value, bx.const_usize(0))
1236        }
1237        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1238            if s.primitive() == Primitive::Float(Float::F64) =>
1239        {
1240            let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
1241            bx.extract_element(value, bx.const_usize(0))
1242        }
1243        _ => value,
1244    }
1245}
1246
1247/// Output type to use for llvm_fixup_output.
1248fn llvm_fixup_output_type<'ll, 'tcx>(
1249    cx: &CodegenCx<'ll, 'tcx>,
1250    reg: InlineAsmRegClass,
1251    layout: &TyAndLayout<'tcx>,
1252    instance: Instance<'_>,
1253) -> &'ll Type {
1254    use InlineAsmRegClass::*;
1255    match (reg, layout.backend_repr) {
1256        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1257            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1258                cx.type_vector(cx.type_i8(), 8)
1259            } else {
1260                layout.llvm_type(cx)
1261            }
1262        }
1263        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1264            if s.primitive() != Primitive::Float(Float::F128) =>
1265        {
1266            let elem_ty = llvm_asm_scalar_type(cx, s);
1267            let count = 16 / layout.size.bytes();
1268            cx.type_vector(elem_ty, count)
1269        }
1270        (
1271            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1272            BackendRepr::SimdVector { element, count },
1273        ) if layout.size.bytes() == 8 => {
1274            let elem_ty = llvm_asm_scalar_type(cx, element);
1275            cx.type_vector(elem_ty, count * 2)
1276        }
1277        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1278            if s.primitive() == Primitive::Float(Float::F64) =>
1279        {
1280            cx.type_i64()
1281        }
1282        (
1283            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1284            BackendRepr::SimdVector { .. },
1285        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1286        (
1287            X86(
1288                X86InlineAsmRegClass::xmm_reg
1289                | X86InlineAsmRegClass::ymm_reg
1290                | X86InlineAsmRegClass::zmm_reg,
1291            ),
1292            BackendRepr::Scalar(s),
1293        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1294            && s.primitive() == Primitive::Float(Float::F128) =>
1295        {
1296            cx.type_vector(cx.type_i32(), 4)
1297        }
1298        (
1299            X86(
1300                X86InlineAsmRegClass::xmm_reg
1301                | X86InlineAsmRegClass::ymm_reg
1302                | X86InlineAsmRegClass::zmm_reg,
1303            ),
1304            BackendRepr::Scalar(s),
1305        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1306        (
1307            X86(
1308                X86InlineAsmRegClass::xmm_reg
1309                | X86InlineAsmRegClass::ymm_reg
1310                | X86InlineAsmRegClass::zmm_reg,
1311            ),
1312            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1313        ) if element.primitive() == Primitive::Float(Float::F16) => {
1314            cx.type_vector(cx.type_i16(), count)
1315        }
1316        (
1317            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1318            BackendRepr::Scalar(s),
1319        ) => {
1320            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1321                cx.type_f32()
1322            } else {
1323                layout.llvm_type(cx)
1324            }
1325        }
1326        (
1327            Arm(
1328                ArmInlineAsmRegClass::dreg
1329                | ArmInlineAsmRegClass::dreg_low8
1330                | ArmInlineAsmRegClass::dreg_low16,
1331            ),
1332            BackendRepr::Scalar(s),
1333        ) => {
1334            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1335                cx.type_f64()
1336            } else {
1337                layout.llvm_type(cx)
1338            }
1339        }
1340        (
1341            Arm(
1342                ArmInlineAsmRegClass::dreg
1343                | ArmInlineAsmRegClass::dreg_low8
1344                | ArmInlineAsmRegClass::dreg_low16
1345                | ArmInlineAsmRegClass::qreg
1346                | ArmInlineAsmRegClass::qreg_low4
1347                | ArmInlineAsmRegClass::qreg_low8,
1348            ),
1349            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1350        ) if element.primitive() == Primitive::Float(Float::F16) => {
1351            cx.type_vector(cx.type_i16(), count)
1352        }
1353        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1354            if s.primitive() == Primitive::Float(Float::F16) =>
1355        {
1356            cx.type_f32()
1357        }
1358        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1359            match s.primitive() {
1360                // MIPS only supports register-length arithmetics.
1361                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1362                Primitive::Float(Float::F32) => cx.type_i32(),
1363                Primitive::Float(Float::F64) => cx.type_i64(),
1364                _ => layout.llvm_type(cx),
1365            }
1366        }
1367        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1368            if s.primitive() == Primitive::Float(Float::F16)
1369                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1370        {
1371            cx.type_f32()
1372        }
1373        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1374            if s.primitive() == Primitive::Float(Float::F32) =>
1375        {
1376            cx.type_vector(cx.type_f32(), 4)
1377        }
1378        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
1379            if s.primitive() == Primitive::Float(Float::F64) =>
1380        {
1381            cx.type_vector(cx.type_f64(), 2)
1382        }
1383        _ => layout.llvm_type(cx),
1384    }
1385}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
