rustc_codegen_llvm/
intrinsic.rs

1use std::assert_matches::assert_matches;
2use std::cmp::Ordering;
3
4use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
5use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
6use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
7use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
8use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
9use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
10use rustc_codegen_ssa::traits::*;
11use rustc_hir as hir;
12use rustc_middle::mir::BinOp;
13use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
14use rustc_middle::ty::{self, GenericArgsRef, Ty};
15use rustc_middle::{bug, span_bug};
16use rustc_span::{Span, Symbol, sym};
17use rustc_symbol_mangling::mangle_internal_symbol;
18use rustc_target::spec::PanicStrategy;
19use tracing::debug;
20
21use crate::abi::FnAbiLlvmExt;
22use crate::builder::Builder;
23use crate::context::CodegenCx;
24use crate::llvm::{self, Metadata};
25use crate::type_::Type;
26use crate::type_of::LayoutLlvmExt;
27use crate::va_arg::emit_va_arg;
28use crate::value::Value;
29
30fn call_simple_intrinsic<'ll, 'tcx>(
31    bx: &mut Builder<'_, 'll, 'tcx>,
32    name: Symbol,
33    args: &[OperandRef<'tcx, &'ll Value>],
34) -> Option<&'ll Value> {
35    let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
36        sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]),
37        sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]),
38        sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]),
39        sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]),
40
41        sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]),
42        sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]),
43        sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]),
44        sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]),
45
46        sym::sinf16 => ("llvm.sin", &[bx.type_f16()]),
47        sym::sinf32 => ("llvm.sin", &[bx.type_f32()]),
48        sym::sinf64 => ("llvm.sin", &[bx.type_f64()]),
49        sym::sinf128 => ("llvm.sin", &[bx.type_f128()]),
50
51        sym::cosf16 => ("llvm.cos", &[bx.type_f16()]),
52        sym::cosf32 => ("llvm.cos", &[bx.type_f32()]),
53        sym::cosf64 => ("llvm.cos", &[bx.type_f64()]),
54        sym::cosf128 => ("llvm.cos", &[bx.type_f128()]),
55
56        sym::powf16 => ("llvm.pow", &[bx.type_f16()]),
57        sym::powf32 => ("llvm.pow", &[bx.type_f32()]),
58        sym::powf64 => ("llvm.pow", &[bx.type_f64()]),
59        sym::powf128 => ("llvm.pow", &[bx.type_f128()]),
60
61        sym::expf16 => ("llvm.exp", &[bx.type_f16()]),
62        sym::expf32 => ("llvm.exp", &[bx.type_f32()]),
63        sym::expf64 => ("llvm.exp", &[bx.type_f64()]),
64        sym::expf128 => ("llvm.exp", &[bx.type_f128()]),
65
66        sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]),
67        sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]),
68        sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]),
69        sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]),
70
71        sym::logf16 => ("llvm.log", &[bx.type_f16()]),
72        sym::logf32 => ("llvm.log", &[bx.type_f32()]),
73        sym::logf64 => ("llvm.log", &[bx.type_f64()]),
74        sym::logf128 => ("llvm.log", &[bx.type_f128()]),
75
76        sym::log10f16 => ("llvm.log10", &[bx.type_f16()]),
77        sym::log10f32 => ("llvm.log10", &[bx.type_f32()]),
78        sym::log10f64 => ("llvm.log10", &[bx.type_f64()]),
79        sym::log10f128 => ("llvm.log10", &[bx.type_f128()]),
80
81        sym::log2f16 => ("llvm.log2", &[bx.type_f16()]),
82        sym::log2f32 => ("llvm.log2", &[bx.type_f32()]),
83        sym::log2f64 => ("llvm.log2", &[bx.type_f64()]),
84        sym::log2f128 => ("llvm.log2", &[bx.type_f128()]),
85
86        sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]),
87        sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]),
88        sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]),
89        sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]),
90
91        sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]),
92        sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]),
93        sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]),
94        sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]),
95
96        sym::fabsf16 => ("llvm.fabs", &[bx.type_f16()]),
97        sym::fabsf32 => ("llvm.fabs", &[bx.type_f32()]),
98        sym::fabsf64 => ("llvm.fabs", &[bx.type_f64()]),
99        sym::fabsf128 => ("llvm.fabs", &[bx.type_f128()]),
100
101        sym::minnumf16 => ("llvm.minnum", &[bx.type_f16()]),
102        sym::minnumf32 => ("llvm.minnum", &[bx.type_f32()]),
103        sym::minnumf64 => ("llvm.minnum", &[bx.type_f64()]),
104        sym::minnumf128 => ("llvm.minnum", &[bx.type_f128()]),
105
106        // FIXME: LLVM currently mis-compile those intrinsics, re-enable them
107        // when llvm/llvm-project#{139380,139381,140445} are fixed.
108        //sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]),
109        //sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]),
110        //sym::minimumf64 => ("llvm.minimum", &[bx.type_f64()]),
111        //sym::minimumf128 => ("llvm.minimum", &[cx.type_f128()]),
112        //
113        sym::maxnumf16 => ("llvm.maxnum", &[bx.type_f16()]),
114        sym::maxnumf32 => ("llvm.maxnum", &[bx.type_f32()]),
115        sym::maxnumf64 => ("llvm.maxnum", &[bx.type_f64()]),
116        sym::maxnumf128 => ("llvm.maxnum", &[bx.type_f128()]),
117
118        // FIXME: LLVM currently mis-compile those intrinsics, re-enable them
119        // when llvm/llvm-project#{139380,139381,140445} are fixed.
120        //sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]),
121        //sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]),
122        //sym::maximumf64 => ("llvm.maximum", &[bx.type_f64()]),
123        //sym::maximumf128 => ("llvm.maximum", &[cx.type_f128()]),
124        //
125        sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
126        sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
127        sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
128        sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
129
130        sym::floorf16 => ("llvm.floor", &[bx.type_f16()]),
131        sym::floorf32 => ("llvm.floor", &[bx.type_f32()]),
132        sym::floorf64 => ("llvm.floor", &[bx.type_f64()]),
133        sym::floorf128 => ("llvm.floor", &[bx.type_f128()]),
134
135        sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]),
136        sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]),
137        sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]),
138        sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]),
139
140        sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]),
141        sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]),
142        sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]),
143        sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]),
144
145        // We could use any of `rint`, `nearbyint`, or `roundeven`
146        // for this -- they are all identical in semantics when
147        // assuming the default FP environment.
148        // `rint` is what we used for $forever.
149        sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]),
150        sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]),
151        sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]),
152        sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]),
153
154        sym::roundf16 => ("llvm.round", &[bx.type_f16()]),
155        sym::roundf32 => ("llvm.round", &[bx.type_f32()]),
156        sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
157        sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
158
159        _ => return None,
160    };
161    Some(bx.call_intrinsic(
162        base_name,
163        type_params,
164        &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
165    ))
166}
167
168impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
169    fn codegen_intrinsic_call(
170        &mut self,
171        instance: ty::Instance<'tcx>,
172        args: &[OperandRef<'tcx, &'ll Value>],
173        result: PlaceRef<'tcx, &'ll Value>,
174        span: Span,
175    ) -> Result<(), ty::Instance<'tcx>> {
176        let tcx = self.tcx;
177
178        let name = tcx.item_name(instance.def_id());
179        let fn_args = instance.args;
180
181        let simple = call_simple_intrinsic(self, name, args);
182        let llval = match name {
183            _ if simple.is_some() => simple.unwrap(),
184            sym::ptr_mask => {
185                let ptr = args[0].immediate();
186                self.call_intrinsic(
187                    "llvm.ptrmask",
188                    &[self.val_ty(ptr), self.type_isize()],
189                    &[ptr, args[1].immediate()],
190                )
191            }
192            sym::is_val_statically_known => {
193                if let OperandValue::Immediate(imm) = args[0].val {
194                    self.call_intrinsic(
195                        "llvm.is.constant",
196                        &[args[0].layout.immediate_llvm_type(self.cx)],
197                        &[imm],
198                    )
199                } else {
200                    self.const_bool(false)
201                }
202            }
203            sym::select_unpredictable => {
204                let cond = args[0].immediate();
205                assert_eq!(args[1].layout, args[2].layout);
206                let select = |bx: &mut Self, true_val, false_val| {
207                    let result = bx.select(cond, true_val, false_val);
208                    bx.set_unpredictable(&result);
209                    result
210                };
211                match (args[1].val, args[2].val) {
212                    (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
213                        assert!(true_val.llextra.is_none());
214                        assert!(false_val.llextra.is_none());
215                        assert_eq!(true_val.align, false_val.align);
216                        let ptr = select(self, true_val.llval, false_val.llval);
217                        let selected =
218                            OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
219                        selected.store(self, result);
220                        return Ok(());
221                    }
222                    (OperandValue::Immediate(_), OperandValue::Immediate(_))
223                    | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
224                        let true_val = args[1].immediate_or_packed_pair(self);
225                        let false_val = args[2].immediate_or_packed_pair(self);
226                        select(self, true_val, false_val)
227                    }
228                    (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
229                    _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
230                }
231            }
232            sym::catch_unwind => {
233                catch_unwind_intrinsic(
234                    self,
235                    args[0].immediate(),
236                    args[1].immediate(),
237                    args[2].immediate(),
238                    result,
239                );
240                return Ok(());
241            }
242            sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
243            sym::va_copy => {
244                let dest = args[0].immediate();
245                self.call_intrinsic(
246                    "llvm.va_copy",
247                    &[self.val_ty(dest)],
248                    &[dest, args[1].immediate()],
249                )
250            }
251            sym::va_arg => {
252                match result.layout.backend_repr {
253                    BackendRepr::Scalar(scalar) => {
254                        match scalar.primitive() {
255                            Primitive::Int(..) => {
256                                if self.cx().size_of(result.layout.ty).bytes() < 4 {
257                                    // `va_arg` should not be called on an integer type
258                                    // less than 4 bytes in length. If it is, promote
259                                    // the integer to an `i32` and truncate the result
260                                    // back to the smaller type.
261                                    let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
262                                    self.trunc(promoted_result, result.layout.llvm_type(self))
263                                } else {
264                                    emit_va_arg(self, args[0], result.layout.ty)
265                                }
266                            }
267                            Primitive::Float(Float::F16) => {
268                                bug!("the va_arg intrinsic does not work with `f16`")
269                            }
270                            Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
271                                emit_va_arg(self, args[0], result.layout.ty)
272                            }
273                            // `va_arg` should never be used with the return type f32.
274                            Primitive::Float(Float::F32) => {
275                                bug!("the va_arg intrinsic does not work with `f32`")
276                            }
277                            Primitive::Float(Float::F128) => {
278                                bug!("the va_arg intrinsic does not work with `f128`")
279                            }
280                        }
281                    }
282                    _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
283                }
284            }
285
286            sym::volatile_load | sym::unaligned_volatile_load => {
287                let ptr = args[0].immediate();
288                let load = self.volatile_load(result.layout.llvm_type(self), ptr);
289                let align = if name == sym::unaligned_volatile_load {
290                    1
291                } else {
292                    result.layout.align.abi.bytes() as u32
293                };
294                unsafe {
295                    llvm::LLVMSetAlignment(load, align);
296                }
297                if !result.layout.is_zst() {
298                    self.store_to_place(load, result.val);
299                }
300                return Ok(());
301            }
302            sym::volatile_store => {
303                let dst = args[0].deref(self.cx());
304                args[1].val.volatile_store(self, dst);
305                return Ok(());
306            }
307            sym::unaligned_volatile_store => {
308                let dst = args[0].deref(self.cx());
309                args[1].val.unaligned_volatile_store(self, dst);
310                return Ok(());
311            }
312            sym::prefetch_read_data
313            | sym::prefetch_write_data
314            | sym::prefetch_read_instruction
315            | sym::prefetch_write_instruction => {
316                let (rw, cache_type) = match name {
317                    sym::prefetch_read_data => (0, 1),
318                    sym::prefetch_write_data => (1, 1),
319                    sym::prefetch_read_instruction => (0, 0),
320                    sym::prefetch_write_instruction => (1, 0),
321                    _ => bug!(),
322                };
323                let ptr = args[0].immediate();
324                self.call_intrinsic(
325                    "llvm.prefetch",
326                    &[self.val_ty(ptr)],
327                    &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
328                )
329            }
330            sym::carrying_mul_add => {
331                let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
332
333                let wide_llty = self.type_ix(size.bits() * 2);
334                let args = args.as_array().unwrap();
335                let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
336
337                let wide = if signed {
338                    let prod = self.unchecked_smul(a, b);
339                    let acc = self.unchecked_sadd(prod, c);
340                    self.unchecked_sadd(acc, d)
341                } else {
342                    let prod = self.unchecked_umul(a, b);
343                    let acc = self.unchecked_uadd(prod, c);
344                    self.unchecked_uadd(acc, d)
345                };
346
347                let narrow_llty = self.type_ix(size.bits());
348                let low = self.trunc(wide, narrow_llty);
349                let bits_const = self.const_uint(wide_llty, size.bits());
350                // No need for ashr when signed; LLVM changes it to lshr anyway.
351                let high = self.lshr(wide, bits_const);
352                // FIXME: could be `trunc nuw`, even for signed.
353                let high = self.trunc(high, narrow_llty);
354
355                let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
356                let pair = self.const_poison(pair_llty);
357                let pair = self.insert_value(pair, low, 0);
358                let pair = self.insert_value(pair, high, 1);
359                pair
360            }
361            sym::ctlz
362            | sym::ctlz_nonzero
363            | sym::cttz
364            | sym::cttz_nonzero
365            | sym::ctpop
366            | sym::bswap
367            | sym::bitreverse
368            | sym::rotate_left
369            | sym::rotate_right
370            | sym::saturating_add
371            | sym::saturating_sub => {
372                let ty = args[0].layout.ty;
373                if !ty.is_integral() {
374                    tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
375                        span,
376                        name,
377                        ty,
378                    });
379                    return Ok(());
380                }
381                let (size, signed) = ty.int_size_and_signed(self.tcx);
382                let width = size.bits();
383                let llty = self.type_ix(width);
384                match name {
385                    sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
386                        let y =
387                            self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
388                        let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
389                            "llvm.ctlz"
390                        } else {
391                            "llvm.cttz"
392                        };
393                        let ret =
394                            self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
395                        self.intcast(ret, result.layout.llvm_type(self), false)
396                    }
397                    sym::ctpop => {
398                        let ret =
399                            self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]);
400                        self.intcast(ret, result.layout.llvm_type(self), false)
401                    }
402                    sym::bswap => {
403                        if width == 8 {
404                            args[0].immediate() // byte swap a u8/i8 is just a no-op
405                        } else {
406                            self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
407                        }
408                    }
409                    sym::bitreverse => {
410                        self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
411                    }
412                    sym::rotate_left | sym::rotate_right => {
413                        let is_left = name == sym::rotate_left;
414                        let val = args[0].immediate();
415                        let raw_shift = args[1].immediate();
416                        // rotate = funnel shift with first two args the same
417                        let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
418
419                        // llvm expects shift to be the same type as the values, but rust
420                        // always uses `u32`.
421                        let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
422
423                        self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift])
424                    }
425                    sym::saturating_add | sym::saturating_sub => {
426                        let is_add = name == sym::saturating_add;
427                        let lhs = args[0].immediate();
428                        let rhs = args[1].immediate();
429                        let llvm_name = format!(
430                            "llvm.{}{}.sat",
431                            if signed { 's' } else { 'u' },
432                            if is_add { "add" } else { "sub" },
433                        );
434                        self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
435                    }
436                    _ => bug!(),
437                }
438            }
439
440            sym::raw_eq => {
441                use BackendRepr::*;
442                let tp_ty = fn_args.type_at(0);
443                let layout = self.layout_of(tp_ty).layout;
444                let use_integer_compare = match layout.backend_repr() {
445                    Scalar(_) | ScalarPair(_, _) => true,
446                    SimdVector { .. } => false,
447                    Memory { .. } => {
448                        // For rusty ABIs, small aggregates are actually passed
449                        // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
450                        // so we re-use that same threshold here.
451                        layout.size() <= self.data_layout().pointer_size() * 2
452                    }
453                };
454
455                let a = args[0].immediate();
456                let b = args[1].immediate();
457                if layout.size().bytes() == 0 {
458                    self.const_bool(true)
459                } else if use_integer_compare {
460                    let integer_ty = self.type_ix(layout.size().bits());
461                    let a_val = self.load(integer_ty, a, layout.align().abi);
462                    let b_val = self.load(integer_ty, b, layout.align().abi);
463                    self.icmp(IntPredicate::IntEQ, a_val, b_val)
464                } else {
465                    let n = self.const_usize(layout.size().bytes());
466                    let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
467                    self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
468                }
469            }
470
471            sym::compare_bytes => {
472                // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
473                let cmp = self.call_intrinsic(
474                    "memcmp",
475                    &[],
476                    &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
477                );
478                // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
479                self.sext(cmp, self.type_ix(32))
480            }
481
482            sym::black_box => {
483                args[0].val.store(self, result);
484                let result_val_span = [result.val.llval];
485                // We need to "use" the argument in some way LLVM can't introspect, and on
486                // targets that support it we can typically leverage inline assembly to do
487                // this. LLVM's interpretation of inline assembly is that it's, well, a black
488                // box. This isn't the greatest implementation since it probably deoptimizes
489                // more than we want, but it's so far good enough.
490                //
491                // For zero-sized types, the location pointed to by the result may be
492                // uninitialized. Do not "use" the result in this case; instead just clobber
493                // the memory.
494                let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
495                    ("~{memory}", &[])
496                } else {
497                    ("r,~{memory}", &result_val_span)
498                };
499                crate::asm::inline_asm_call(
500                    self,
501                    "",
502                    constraint,
503                    inputs,
504                    self.type_void(),
505                    &[],
506                    true,
507                    false,
508                    llvm::AsmDialect::Att,
509                    &[span],
510                    false,
511                    None,
512                    None,
513                )
514                .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
515
516                // We have copied the value to `result` already.
517                return Ok(());
518            }
519
520            _ if name.as_str().starts_with("simd_") => {
521                // Unpack non-power-of-2 #[repr(packed, simd)] arguments.
522                // This gives them the expected layout of a regular #[repr(simd)] vector.
523                let mut loaded_args = Vec::new();
524                for arg in args {
525                    loaded_args.push(
526                        // #[repr(packed, simd)] vectors are passed like arrays (as references,
527                        // with reduced alignment and no padding) rather than as immediates.
528                        // We can use a vector load to fix the layout and turn the argument
529                        // into an immediate.
530                        if arg.layout.ty.is_simd()
531                            && let OperandValue::Ref(place) = arg.val
532                        {
533                            let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
534                            let elem_ll_ty = match elem_ty.kind() {
535                                ty::Float(f) => self.type_float_from_ty(*f),
536                                ty::Int(i) => self.type_int_from_ty(*i),
537                                ty::Uint(u) => self.type_uint_from_ty(*u),
538                                ty::RawPtr(_, _) => self.type_ptr(),
539                                _ => unreachable!(),
540                            };
541                            let loaded =
542                                self.load_from_place(self.type_vector(elem_ll_ty, size), place);
543                            OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
544                        } else {
545                            *arg
546                        },
547                    );
548                }
549
550                let llret_ty = if result.layout.ty.is_simd()
551                    && let BackendRepr::Memory { .. } = result.layout.backend_repr
552                {
553                    let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
554                    let elem_ll_ty = match elem_ty.kind() {
555                        ty::Float(f) => self.type_float_from_ty(*f),
556                        ty::Int(i) => self.type_int_from_ty(*i),
557                        ty::Uint(u) => self.type_uint_from_ty(*u),
558                        ty::RawPtr(_, _) => self.type_ptr(),
559                        _ => unreachable!(),
560                    };
561                    self.type_vector(elem_ll_ty, size)
562                } else {
563                    result.layout.llvm_type(self)
564                };
565
566                match generic_simd_intrinsic(
567                    self,
568                    name,
569                    fn_args,
570                    &loaded_args,
571                    result.layout.ty,
572                    llret_ty,
573                    span,
574                ) {
575                    Ok(llval) => llval,
576                    // If there was an error, just skip this invocation... we'll abort compilation
577                    // anyway, but we can keep codegen'ing to find more errors.
578                    Err(()) => return Ok(()),
579                }
580            }
581
582            _ => {
583                debug!("unknown intrinsic '{}' -- falling back to default body", name);
584                // Call the fallback body instead of generating the intrinsic code
585                return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
586            }
587        };
588
589        if result.layout.ty.is_bool() {
590            let val = self.from_immediate(llval);
591            self.store_to_place(val, result.val);
592        } else if !result.layout.ty.is_unit() {
593            self.store_to_place(llval, result.val);
594        }
595        Ok(())
596    }
597
598    fn abort(&mut self) {
599        self.call_intrinsic("llvm.trap", &[], &[]);
600    }
601
602    fn assume(&mut self, val: Self::Value) {
603        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
604            self.call_intrinsic("llvm.assume", &[], &[val]);
605        }
606    }
607
608    fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
609        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
610            self.call_intrinsic(
611                "llvm.expect",
612                &[self.type_i1()],
613                &[cond, self.const_bool(expected)],
614            )
615        } else {
616            cond
617        }
618    }
619
620    fn type_checked_load(
621        &mut self,
622        llvtable: &'ll Value,
623        vtable_byte_offset: u64,
624        typeid: &'ll Metadata,
625    ) -> Self::Value {
626        let typeid = self.get_metadata_value(typeid);
627        let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
628        let type_checked_load = self.call_intrinsic(
629            "llvm.type.checked.load",
630            &[],
631            &[llvtable, vtable_byte_offset, typeid],
632        );
633        self.extract_value(type_checked_load, 0)
634    }
635
636    fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
637        self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list])
638    }
639
640    fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
641        self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list])
642    }
643}
644
645fn catch_unwind_intrinsic<'ll, 'tcx>(
646    bx: &mut Builder<'_, 'll, 'tcx>,
647    try_func: &'ll Value,
648    data: &'ll Value,
649    catch_func: &'ll Value,
650    dest: PlaceRef<'tcx, &'ll Value>,
651) {
652    if bx.sess().panic_strategy() == PanicStrategy::Abort {
653        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
654        bx.call(try_func_ty, None, None, try_func, &[data], None, None);
655        // Return 0 unconditionally from the intrinsic call;
656        // we can never unwind.
657        OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
658    } else if wants_msvc_seh(bx.sess()) {
659        codegen_msvc_try(bx, try_func, data, catch_func, dest);
660    } else if wants_wasm_eh(bx.sess()) {
661        codegen_wasm_try(bx, try_func, data, catch_func, dest);
662    } else if bx.sess().target.os == "emscripten" {
663        codegen_emcc_try(bx, try_func, data, catch_func, dest);
664    } else {
665        codegen_gnu_try(bx, try_func, data, catch_func, dest);
666    }
667}
668
669// MSVC's definition of the `rust_try` function.
670//
671// This implementation uses the new exception handling instructions in LLVM
672// which have support in LLVM for SEH on MSVC targets. Although these
673// instructions are meant to work for all targets, as of the time of this
674// writing, however, LLVM does not recommend the usage of these new instructions
675// as the old ones are still more optimized.
676fn codegen_msvc_try<'ll, 'tcx>(
677    bx: &mut Builder<'_, 'll, 'tcx>,
678    try_func: &'ll Value,
679    data: &'ll Value,
680    catch_func: &'ll Value,
681    dest: PlaceRef<'tcx, &'ll Value>,
682) {
683    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
684        bx.set_personality_fn(bx.eh_personality());
685
686        let normal = bx.append_sibling_block("normal");
687        let catchswitch = bx.append_sibling_block("catchswitch");
688        let catchpad_rust = bx.append_sibling_block("catchpad_rust");
689        let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
690        let caught = bx.append_sibling_block("caught");
691
692        let try_func = llvm::get_param(bx.llfn(), 0);
693        let data = llvm::get_param(bx.llfn(), 1);
694        let catch_func = llvm::get_param(bx.llfn(), 2);
695
696        // We're generating an IR snippet that looks like:
697        //
698        //   declare i32 @rust_try(%try_func, %data, %catch_func) {
699        //      %slot = alloca i8*
700        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
701        //
702        //   normal:
703        //      ret i32 0
704        //
705        //   catchswitch:
706        //      %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller
707        //
708        //   catchpad_rust:
709        //      %tok = catchpad within %cs [%type_descriptor, 8, %slot]
710        //      %ptr = load %slot
711        //      call %catch_func(%data, %ptr)
712        //      catchret from %tok to label %caught
713        //
714        //   catchpad_foreign:
715        //      %tok = catchpad within %cs [null, 64, null]
716        //      call %catch_func(%data, null)
717        //      catchret from %tok to label %caught
718        //
719        //   caught:
720        //      ret i32 1
721        //   }
722        //
723        // This structure follows the basic usage of throw/try/catch in LLVM.
724        // For example, compile this C++ snippet to see what LLVM generates:
725        //
726        //      struct rust_panic {
727        //          rust_panic(const rust_panic&);
728        //          ~rust_panic();
729        //
730        //          void* x[2];
731        //      };
732        //
733        //      int __rust_try(
734        //          void (*try_func)(void*),
735        //          void *data,
736        //          void (*catch_func)(void*, void*) noexcept
737        //      ) {
738        //          try {
739        //              try_func(data);
740        //              return 0;
741        //          } catch(rust_panic& a) {
742        //              catch_func(data, &a);
743        //              return 1;
744        //          } catch(...) {
745        //              catch_func(data, NULL);
746        //              return 1;
747        //          }
748        //      }
749        //
750        // More information can be found in libstd's seh.rs implementation.
751        let ptr_size = bx.tcx().data_layout.pointer_size();
752        let ptr_align = bx.tcx().data_layout.pointer_align().abi;
753        let slot = bx.alloca(ptr_size, ptr_align);
754        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
755        bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
756
757        bx.switch_to_block(normal);
758        bx.ret(bx.const_i32(0));
759
760        bx.switch_to_block(catchswitch);
761        let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
762
763        // We can't use the TypeDescriptor defined in libpanic_unwind because it
764        // might be in another DLL and the SEH encoding only supports specifying
765        // a TypeDescriptor from the current module.
766        //
767        // However this isn't an issue since the MSVC runtime uses string
768        // comparison on the type name to match TypeDescriptors rather than
769        // pointer equality.
770        //
771        // So instead we generate a new TypeDescriptor in each module that uses
772        // `try` and let the linker merge duplicate definitions in the same
773        // module.
774        //
775        // When modifying, make sure that the type_name string exactly matches
776        // the one used in library/panic_unwind/src/seh.rs.
777        let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
778        let type_name = bx.const_bytes(b"rust_panic\0");
779        let type_info =
780            bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
781        let tydesc = bx.declare_global(
782            &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
783            bx.val_ty(type_info),
784        );
785
786        llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
787        if bx.cx.tcx.sess.target.supports_comdat() {
788            llvm::SetUniqueComdat(bx.llmod, tydesc);
789        }
790        llvm::set_initializer(tydesc, type_info);
791
792        // The flag value of 8 indicates that we are catching the exception by
793        // reference instead of by value. We can't use catch by value because
794        // that requires copying the exception object, which we don't support
795        // since our exception object effectively contains a Box.
796        //
797        // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
798        bx.switch_to_block(catchpad_rust);
799        let flags = bx.const_i32(8);
800        let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
801        let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
802        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
803        bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
804        bx.catch_ret(&funclet, caught);
805
806        // The flag value of 64 indicates a "catch-all".
807        bx.switch_to_block(catchpad_foreign);
808        let flags = bx.const_i32(64);
809        let null = bx.const_null(bx.type_ptr());
810        let funclet = bx.catch_pad(cs, &[null, flags, null]);
811        bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
812        bx.catch_ret(&funclet, caught);
813
814        bx.switch_to_block(caught);
815        bx.ret(bx.const_i32(1));
816    });
817
818    // Note that no invoke is used here because by definition this function
819    // can't panic (that's what it's catching).
820    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
821    OperandValue::Immediate(ret).store(bx, dest);
822}
823
824// WASM's definition of the `rust_try` function.
825fn codegen_wasm_try<'ll, 'tcx>(
826    bx: &mut Builder<'_, 'll, 'tcx>,
827    try_func: &'ll Value,
828    data: &'ll Value,
829    catch_func: &'ll Value,
830    dest: PlaceRef<'tcx, &'ll Value>,
831) {
832    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
833        bx.set_personality_fn(bx.eh_personality());
834
835        let normal = bx.append_sibling_block("normal");
836        let catchswitch = bx.append_sibling_block("catchswitch");
837        let catchpad = bx.append_sibling_block("catchpad");
838        let caught = bx.append_sibling_block("caught");
839
840        let try_func = llvm::get_param(bx.llfn(), 0);
841        let data = llvm::get_param(bx.llfn(), 1);
842        let catch_func = llvm::get_param(bx.llfn(), 2);
843
844        // We're generating an IR snippet that looks like:
845        //
846        //   declare i32 @rust_try(%try_func, %data, %catch_func) {
847        //      %slot = alloca i8*
848        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
849        //
850        //   normal:
851        //      ret i32 0
852        //
853        //   catchswitch:
854        //      %cs = catchswitch within none [%catchpad] unwind to caller
855        //
856        //   catchpad:
857        //      %tok = catchpad within %cs [null]
858        //      %ptr = call @llvm.wasm.get.exception(token %tok)
859        //      %sel = call @llvm.wasm.get.ehselector(token %tok)
860        //      call %catch_func(%data, %ptr)
861        //      catchret from %tok to label %caught
862        //
863        //   caught:
864        //      ret i32 1
865        //   }
866        //
867        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
868        bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
869
870        bx.switch_to_block(normal);
871        bx.ret(bx.const_i32(0));
872
873        bx.switch_to_block(catchswitch);
874        let cs = bx.catch_switch(None, None, &[catchpad]);
875
876        bx.switch_to_block(catchpad);
877        let null = bx.const_null(bx.type_ptr());
878        let funclet = bx.catch_pad(cs, &[null]);
879
880        let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
881        let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
882
883        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
884        bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
885        bx.catch_ret(&funclet, caught);
886
887        bx.switch_to_block(caught);
888        bx.ret(bx.const_i32(1));
889    });
890
891    // Note that no invoke is used here because by definition this function
892    // can't panic (that's what it's catching).
893    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
894    OperandValue::Immediate(ret).store(bx, dest);
895}
896
897// Definition of the standard `try` function for Rust using the GNU-like model
898// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
899// instructions).
900//
901// This codegen is a little surprising because we always call a shim
902// function instead of inlining the call to `invoke` manually here. This is done
903// because in LLVM we're only allowed to have one personality per function
904// definition. The call to the `try` intrinsic is being inlined into the
905// function calling it, and that function may already have other personality
906// functions in play. By calling a shim we're guaranteed that our shim will have
907// the right personality function.
908fn codegen_gnu_try<'ll, 'tcx>(
909    bx: &mut Builder<'_, 'll, 'tcx>,
910    try_func: &'ll Value,
911    data: &'ll Value,
912    catch_func: &'ll Value,
913    dest: PlaceRef<'tcx, &'ll Value>,
914) {
915    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
916        // Codegens the shims described above:
917        //
918        //   bx:
919        //      invoke %try_func(%data) normal %normal unwind %catch
920        //
921        //   normal:
922        //      ret 0
923        //
924        //   catch:
925        //      (%ptr, _) = landingpad
926        //      call %catch_func(%data, %ptr)
927        //      ret 1
928        let then = bx.append_sibling_block("then");
929        let catch = bx.append_sibling_block("catch");
930
931        let try_func = llvm::get_param(bx.llfn(), 0);
932        let data = llvm::get_param(bx.llfn(), 1);
933        let catch_func = llvm::get_param(bx.llfn(), 2);
934        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
935        bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
936
937        bx.switch_to_block(then);
938        bx.ret(bx.const_i32(0));
939
940        // Type indicator for the exception being thrown.
941        //
942        // The first value in this tuple is a pointer to the exception object
943        // being thrown. The second value is a "selector" indicating which of
944        // the landing pad clauses the exception's type had been matched to.
945        // rust_try ignores the selector.
946        bx.switch_to_block(catch);
947        let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
948        let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
949        let tydesc = bx.const_null(bx.type_ptr());
950        bx.add_clause(vals, tydesc);
951        let ptr = bx.extract_value(vals, 0);
952        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
953        bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
954        bx.ret(bx.const_i32(1));
955    });
956
957    // Note that no invoke is used here because by definition this function
958    // can't panic (that's what it's catching).
959    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
960    OperandValue::Immediate(ret).store(bx, dest);
961}
962
963// Variant of codegen_gnu_try used for emscripten where Rust panics are
964// implemented using C++ exceptions. Here we use exceptions of a specific type
965// (`struct rust_panic`) to represent Rust panics.
966fn codegen_emcc_try<'ll, 'tcx>(
967    bx: &mut Builder<'_, 'll, 'tcx>,
968    try_func: &'ll Value,
969    data: &'ll Value,
970    catch_func: &'ll Value,
971    dest: PlaceRef<'tcx, &'ll Value>,
972) {
973    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
974        // Codegens the shims described above:
975        //
976        //   bx:
977        //      invoke %try_func(%data) normal %normal unwind %catch
978        //
979        //   normal:
980        //      ret 0
981        //
982        //   catch:
983        //      (%ptr, %selector) = landingpad
984        //      %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic)
985        //      %is_rust_panic = %selector == %rust_typeid
986        //      %catch_data = alloca { i8*, i8 }
987        //      %catch_data[0] = %ptr
988        //      %catch_data[1] = %is_rust_panic
989        //      call %catch_func(%data, %catch_data)
990        //      ret 1
991        let then = bx.append_sibling_block("then");
992        let catch = bx.append_sibling_block("catch");
993
994        let try_func = llvm::get_param(bx.llfn(), 0);
995        let data = llvm::get_param(bx.llfn(), 1);
996        let catch_func = llvm::get_param(bx.llfn(), 2);
997        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
998        bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
999
1000        bx.switch_to_block(then);
1001        bx.ret(bx.const_i32(0));
1002
1003        // Type indicator for the exception being thrown.
1004        //
1005        // The first value in this tuple is a pointer to the exception object
1006        // being thrown. The second value is a "selector" indicating which of
1007        // the landing pad clauses the exception's type had been matched to.
1008        bx.switch_to_block(catch);
1009        let tydesc = bx.eh_catch_typeinfo();
1010        let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1011        let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1012        bx.add_clause(vals, tydesc);
1013        bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1014        let ptr = bx.extract_value(vals, 0);
1015        let selector = bx.extract_value(vals, 1);
1016
1017        // Check if the typeid we got is the one for a Rust panic.
1018        let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]);
1019        let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1020        let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1021
1022        // We need to pass two values to catch_func (ptr and is_rust_panic), so
1023        // create an alloca and pass a pointer to that.
1024        let ptr_size = bx.tcx().data_layout.pointer_size();
1025        let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1026        let i8_align = bx.tcx().data_layout.i8_align.abi;
1027        // Required in order for there to be no padding between the fields.
1028        assert!(i8_align <= ptr_align);
1029        let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1030        bx.store(ptr, catch_data, ptr_align);
1031        let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1032        bx.store(is_rust_panic, catch_data_1, i8_align);
1033
1034        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1035        bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1036        bx.ret(bx.const_i32(1));
1037    });
1038
1039    // Note that no invoke is used here because by definition this function
1040    // can't panic (that's what it's catching).
1041    let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1042    OperandValue::Immediate(ret).store(bx, dest);
1043}
1044
1045// Helper function to give a Block to a closure to codegen a shim function.
1046// This is currently primarily used for the `try` intrinsic functions above.
1047fn gen_fn<'a, 'll, 'tcx>(
1048    cx: &'a CodegenCx<'ll, 'tcx>,
1049    name: &str,
1050    rust_fn_sig: ty::PolyFnSig<'tcx>,
1051    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1052) -> (&'ll Type, &'ll Value) {
1053    let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1054    let llty = fn_abi.llvm_type(cx);
1055    let llfn = cx.declare_fn(name, fn_abi, None);
1056    cx.set_frame_pointer_type(llfn);
1057    cx.apply_target_cpu_attr(llfn);
1058    // FIXME(eddyb) find a nicer way to do this.
1059    llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1060    let llbb = Builder::append_block(cx, llfn, "entry-block");
1061    let bx = Builder::build(cx, llbb);
1062    codegen(bx);
1063    (llty, llfn)
1064}
1065
1066// Helper function used to get a handle to the `__rust_try` function used to
1067// catch exceptions.
1068//
1069// This function is only generated once and is then cached.
1070fn get_rust_try_fn<'a, 'll, 'tcx>(
1071    cx: &'a CodegenCx<'ll, 'tcx>,
1072    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1073) -> (&'ll Type, &'ll Value) {
1074    if let Some(llfn) = cx.rust_try_fn.get() {
1075        return llfn;
1076    }
1077
1078    // Define the type up front for the signature of the rust_try function.
1079    let tcx = cx.tcx;
1080    let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1081    // `unsafe fn(*mut i8) -> ()`
1082    let try_fn_ty = Ty::new_fn_ptr(
1083        tcx,
1084        ty::Binder::dummy(tcx.mk_fn_sig(
1085            [i8p],
1086            tcx.types.unit,
1087            false,
1088            hir::Safety::Unsafe,
1089            ExternAbi::Rust,
1090        )),
1091    );
1092    // `unsafe fn(*mut i8, *mut i8) -> ()`
1093    let catch_fn_ty = Ty::new_fn_ptr(
1094        tcx,
1095        ty::Binder::dummy(tcx.mk_fn_sig(
1096            [i8p, i8p],
1097            tcx.types.unit,
1098            false,
1099            hir::Safety::Unsafe,
1100            ExternAbi::Rust,
1101        )),
1102    );
1103    // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32`
1104    let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
1105        [try_fn_ty, i8p, catch_fn_ty],
1106        tcx.types.i32,
1107        false,
1108        hir::Safety::Unsafe,
1109        ExternAbi::Rust,
1110    ));
1111    let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1112    cx.rust_try_fn.set(Some(rust_try));
1113    rust_try
1114}
1115
1116fn generic_simd_intrinsic<'ll, 'tcx>(
1117    bx: &mut Builder<'_, 'll, 'tcx>,
1118    name: Symbol,
1119    fn_args: GenericArgsRef<'tcx>,
1120    args: &[OperandRef<'tcx, &'ll Value>],
1121    ret_ty: Ty<'tcx>,
1122    llret_ty: &'ll Type,
1123    span: Span,
1124) -> Result<&'ll Value, ()> {
1125    macro_rules! return_error {
1126        ($diag: expr) => {{
1127            bx.sess().dcx().emit_err($diag);
1128            return Err(());
1129        }};
1130    }
1131
1132    macro_rules! require {
1133        ($cond: expr, $diag: expr) => {
1134            if !$cond {
1135                return_error!($diag);
1136            }
1137        };
1138    }
1139
1140    macro_rules! require_simd {
1141        ($ty: expr, $variant:ident) => {{
1142            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1143            $ty.simd_size_and_type(bx.tcx())
1144        }};
1145    }
1146
1147    /// Returns the bitwidth of the `$ty` argument if it is an `Int` or `Uint` type.
1148    macro_rules! require_int_or_uint_ty {
1149        ($ty: expr, $diag: expr) => {
1150            match $ty {
1151                ty::Int(i) => {
1152                    i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1153                }
1154                ty::Uint(i) => {
1155                    i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1156                }
1157                _ => {
1158                    return_error!($diag);
1159                }
1160            }
1161        };
1162    }
1163
1164    /// Converts a vector mask, where each element has a bit width equal to the data elements it is used with,
1165    /// down to an i1 based mask that can be used by llvm intrinsics.
1166    ///
1167    /// The rust simd semantics are that each element should either consist of all ones or all zeroes,
1168    /// but this information is not available to llvm. Truncating the vector effectively uses the lowest bit,
1169    /// but codegen for several targets is better if we consider the highest bit by shifting.
1170    ///
1171    /// For x86 SSE/AVX targets this is beneficial since most instructions with mask parameters only consider the highest bit.
1172    /// So even though on llvm level we have an additional shift, in the final assembly there is no shift or truncate and
1173    /// instead the mask can be used as is.
1174    ///
1175    /// For aarch64 and other targets there is a benefit because a mask from the sign bit can be more
1176    /// efficiently converted to an all ones / all zeroes mask by comparing whether each element is negative.
1177    fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1178        bx: &mut Builder<'a, 'll, 'tcx>,
1179        i_xn: &'ll Value,
1180        in_elem_bitwidth: u64,
1181        in_len: u64,
1182    ) -> &'ll Value {
1183        // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
1184        let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1185        let shift_indices = vec![shift_idx; in_len as _];
1186        let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1187        // Truncate vector to an <i1 x N>
1188        bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1189    }
1190
1191    // Sanity-check: all vector arguments must be immediates.
1192    if cfg!(debug_assertions) {
1193        for arg in args {
1194            if arg.layout.ty.is_simd() {
1195                assert_matches!(arg.val, OperandValue::Immediate(_));
1196            }
1197        }
1198    }
1199
1200    if name == sym::simd_select_bitmask {
1201        let (len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1202
1203        let expected_int_bits = len.max(8).next_power_of_two();
1204        let expected_bytes = len.div_ceil(8);
1205
1206        let mask_ty = args[0].layout.ty;
1207        let mask = match mask_ty.kind() {
1208            ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1209            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1210            ty::Array(elem, len)
1211                if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1212                    && len
1213                        .try_to_target_usize(bx.tcx)
1214                        .expect("expected monomorphic const in codegen")
1215                        == expected_bytes =>
1216            {
1217                let place = PlaceRef::alloca(bx, args[0].layout);
1218                args[0].val.store(bx, place);
1219                let int_ty = bx.type_ix(expected_bytes * 8);
1220                bx.load(int_ty, place.val.llval, Align::ONE)
1221            }
1222            _ => return_error!(InvalidMonomorphization::InvalidBitmask {
1223                span,
1224                name,
1225                mask_ty,
1226                expected_int_bits,
1227                expected_bytes
1228            }),
1229        };
1230
1231        let i1 = bx.type_i1();
1232        let im = bx.type_ix(len);
1233        let i1xn = bx.type_vector(i1, len);
1234        let m_im = bx.trunc(mask, im);
1235        let m_i1s = bx.bitcast(m_im, i1xn);
1236        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1237    }
1238
1239    // every intrinsic below takes a SIMD vector as its first argument
1240    let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput);
1241    let in_ty = args[0].layout.ty;
1242
1243    let comparison = match name {
1244        sym::simd_eq => Some(BinOp::Eq),
1245        sym::simd_ne => Some(BinOp::Ne),
1246        sym::simd_lt => Some(BinOp::Lt),
1247        sym::simd_le => Some(BinOp::Le),
1248        sym::simd_gt => Some(BinOp::Gt),
1249        sym::simd_ge => Some(BinOp::Ge),
1250        _ => None,
1251    };
1252
1253    if let Some(cmp_op) = comparison {
1254        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1255
1256        require!(
1257            in_len == out_len,
1258            InvalidMonomorphization::ReturnLengthInputType {
1259                span,
1260                name,
1261                in_len,
1262                in_ty,
1263                ret_ty,
1264                out_len
1265            }
1266        );
1267        require!(
1268            bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
1269            InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
1270        );
1271
1272        return Ok(compare_simd_types(
1273            bx,
1274            args[0].immediate(),
1275            args[1].immediate(),
1276            in_elem,
1277            llret_ty,
1278            cmp_op,
1279        ));
1280    }
1281
1282    if name == sym::simd_shuffle_const_generic {
1283        let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
1284        let n = idx.len() as u64;
1285
1286        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1287        require!(
1288            out_len == n,
1289            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1290        );
1291        require!(
1292            in_elem == out_ty,
1293            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1294        );
1295
1296        let total_len = in_len * 2;
1297
1298        let indices: Option<Vec<_>> = idx
1299            .iter()
1300            .enumerate()
1301            .map(|(arg_idx, val)| {
1302                let idx = val.unwrap_leaf().to_i32();
1303                if idx >= i32::try_from(total_len).unwrap() {
1304                    bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
1305                        span,
1306                        name,
1307                        arg_idx: arg_idx as u64,
1308                        total_len: total_len.into(),
1309                    });
1310                    None
1311                } else {
1312                    Some(bx.const_i32(idx))
1313                }
1314            })
1315            .collect();
1316        let Some(indices) = indices else {
1317            return Ok(bx.const_null(llret_ty));
1318        };
1319
1320        return Ok(bx.shuffle_vector(
1321            args[0].immediate(),
1322            args[1].immediate(),
1323            bx.const_vector(&indices),
1324        ));
1325    }
1326
1327    if name == sym::simd_shuffle {
1328        // Make sure this is actually a SIMD vector.
1329        let idx_ty = args[2].layout.ty;
1330        let n: u64 = if idx_ty.is_simd()
1331            && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
1332        {
1333            idx_ty.simd_size_and_type(bx.cx.tcx).0
1334        } else {
1335            return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
1336        };
1337
1338        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1339        require!(
1340            out_len == n,
1341            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1342        );
1343        require!(
1344            in_elem == out_ty,
1345            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1346        );
1347
1348        let total_len = u128::from(in_len) * 2;
1349
1350        // Check that the indices are in-bounds.
1351        let indices = args[2].immediate();
1352        for i in 0..n {
1353            let val = bx.const_get_elt(indices, i as u64);
1354            let idx = bx
1355                .const_to_opt_u128(val, true)
1356                .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
1357            if idx >= total_len {
1358                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1359                    span,
1360                    name,
1361                    arg_idx: i,
1362                    total_len,
1363                });
1364            }
1365        }
1366
1367        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
1368    }
1369
1370    if name == sym::simd_insert || name == sym::simd_insert_dyn {
1371        require!(
1372            in_elem == args[2].layout.ty,
1373            InvalidMonomorphization::InsertedType {
1374                span,
1375                name,
1376                in_elem,
1377                in_ty,
1378                out_ty: args[2].layout.ty
1379            }
1380        );
1381
1382        let index_imm = if name == sym::simd_insert {
1383            let idx = bx
1384                .const_to_opt_u128(args[1].immediate(), false)
1385                .expect("typeck should have ensure that this is a const");
1386            if idx >= in_len.into() {
1387                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1388                    span,
1389                    name,
1390                    arg_idx: 1,
1391                    total_len: in_len.into(),
1392                });
1393            }
1394            bx.const_i32(idx as i32)
1395        } else {
1396            args[1].immediate()
1397        };
1398
1399        return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
1400    }
1401    if name == sym::simd_extract || name == sym::simd_extract_dyn {
1402        require!(
1403            ret_ty == in_elem,
1404            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1405        );
1406        let index_imm = if name == sym::simd_extract {
1407            let idx = bx
1408                .const_to_opt_u128(args[1].immediate(), false)
1409                .expect("typeck should have ensure that this is a const");
1410            if idx >= in_len.into() {
1411                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1412                    span,
1413                    name,
1414                    arg_idx: 1,
1415                    total_len: in_len.into(),
1416                });
1417            }
1418            bx.const_i32(idx as i32)
1419        } else {
1420            args[1].immediate()
1421        };
1422
1423        return Ok(bx.extract_element(args[0].immediate(), index_imm));
1424    }
1425
1426    if name == sym::simd_select {
1427        let m_elem_ty = in_elem;
1428        let m_len = in_len;
1429        let (v_len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1430        require!(
1431            m_len == v_len,
1432            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
1433        );
1434        let in_elem_bitwidth = require_int_or_uint_ty!(
1435            m_elem_ty.kind(),
1436            InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1437        );
1438        let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1439        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1440    }
1441
1442    if name == sym::simd_bitmask {
1443        // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a vector mask and
1444        // returns one bit for each lane (which must all be `0` or `!0`) in the form of either:
1445        // * an unsigned integer
1446        // * an array of `u8`
1447        // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
1448        //
1449        // The bit order of the result depends on the byte endianness, LSB-first for little
1450        // endian and MSB-first for big endian.
1451        let expected_int_bits = in_len.max(8).next_power_of_two();
1452        let expected_bytes = in_len.div_ceil(8);
1453
1454        // Integer vector <i{in_bitwidth} x in_len>:
1455        let in_elem_bitwidth = require_int_or_uint_ty!(
1456            in_elem.kind(),
1457            InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
1458        );
1459
1460        let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
1461        // Bitcast <i1 x N> to iN:
1462        let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
1463
1464        match ret_ty.kind() {
1465            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
1466                // Zero-extend iN to the bitmask type:
1467                return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
1468            }
1469            ty::Array(elem, len)
1470                if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1471                    && len
1472                        .try_to_target_usize(bx.tcx)
1473                        .expect("expected monomorphic const in codegen")
1474                        == expected_bytes =>
1475            {
1476                // Zero-extend iN to the array length:
1477                let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
1478
1479                // Convert the integer to a byte array
1480                let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
1481                bx.store(ze, ptr, Align::ONE);
1482                let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
1483                return Ok(bx.load(array_ty, ptr, Align::ONE));
1484            }
1485            _ => return_error!(InvalidMonomorphization::CannotReturn {
1486                span,
1487                name,
1488                ret_ty,
1489                expected_int_bits,
1490                expected_bytes
1491            }),
1492        }
1493    }
1494
1495    fn simd_simple_float_intrinsic<'ll, 'tcx>(
1496        name: Symbol,
1497        in_elem: Ty<'_>,
1498        in_ty: Ty<'_>,
1499        in_len: u64,
1500        bx: &mut Builder<'_, 'll, 'tcx>,
1501        span: Span,
1502        args: &[OperandRef<'tcx, &'ll Value>],
1503    ) -> Result<&'ll Value, ()> {
1504        macro_rules! return_error {
1505            ($diag: expr) => {{
1506                bx.sess().dcx().emit_err($diag);
1507                return Err(());
1508            }};
1509        }
1510
1511        let elem_ty = if let ty::Float(f) = in_elem.kind() {
1512            bx.cx.type_float_from_ty(*f)
1513        } else {
1514            return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
1515        };
1516
1517        let vec_ty = bx.type_vector(elem_ty, in_len);
1518
1519        let intr_name = match name {
1520            sym::simd_ceil => "llvm.ceil",
1521            sym::simd_fabs => "llvm.fabs",
1522            sym::simd_fcos => "llvm.cos",
1523            sym::simd_fexp2 => "llvm.exp2",
1524            sym::simd_fexp => "llvm.exp",
1525            sym::simd_flog10 => "llvm.log10",
1526            sym::simd_flog2 => "llvm.log2",
1527            sym::simd_flog => "llvm.log",
1528            sym::simd_floor => "llvm.floor",
1529            sym::simd_fma => "llvm.fma",
1530            sym::simd_relaxed_fma => "llvm.fmuladd",
1531            sym::simd_fsin => "llvm.sin",
1532            sym::simd_fsqrt => "llvm.sqrt",
1533            sym::simd_round => "llvm.round",
1534            sym::simd_round_ties_even => "llvm.rint",
1535            sym::simd_trunc => "llvm.trunc",
1536            _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
1537        };
1538        Ok(bx.call_intrinsic(
1539            intr_name,
1540            &[vec_ty],
1541            &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
1542        ))
1543    }
1544
1545    if std::matches!(
1546        name,
1547        sym::simd_ceil
1548            | sym::simd_fabs
1549            | sym::simd_fcos
1550            | sym::simd_fexp2
1551            | sym::simd_fexp
1552            | sym::simd_flog10
1553            | sym::simd_flog2
1554            | sym::simd_flog
1555            | sym::simd_floor
1556            | sym::simd_fma
1557            | sym::simd_fsin
1558            | sym::simd_fsqrt
1559            | sym::simd_relaxed_fma
1560            | sym::simd_round
1561            | sym::simd_round_ties_even
1562            | sym::simd_trunc
1563    ) {
1564        return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
1565    }
1566
1567    fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
1568        let elem_ty = match *elem_ty.kind() {
1569            ty::Int(v) => cx.type_int_from_ty(v),
1570            ty::Uint(v) => cx.type_uint_from_ty(v),
1571            ty::Float(v) => cx.type_float_from_ty(v),
1572            ty::RawPtr(_, _) => cx.type_ptr(),
1573            _ => unreachable!(),
1574        };
1575        cx.type_vector(elem_ty, vec_len)
1576    }
1577
1578    if name == sym::simd_gather {
1579        // simd_gather(values: <N x T>, pointers: <N x *_ T>,
1580        //             mask: <N x i{M}>) -> <N x T>
1581        // * N: number of elements in the input vectors
1582        // * T: type of the element to load
1583        // * M: any integer width is supported, will be truncated to i1
1584
1585        // All types must be simd vector types
1586
1587        // The second argument must be a simd vector with an element type that's a pointer
1588        // to the element type of the first argument
1589        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1590        let (out_len, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1591        // The element type of the third argument must be a signed integer type of any width:
1592        let (out_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1593        require_simd!(ret_ty, SimdReturn);
1594
1595        // Of the same length:
1596        require!(
1597            in_len == out_len,
1598            InvalidMonomorphization::SecondArgumentLength {
1599                span,
1600                name,
1601                in_len,
1602                in_ty,
1603                arg_ty: args[1].layout.ty,
1604                out_len
1605            }
1606        );
1607        require!(
1608            in_len == out_len2,
1609            InvalidMonomorphization::ThirdArgumentLength {
1610                span,
1611                name,
1612                in_len,
1613                in_ty,
1614                arg_ty: args[2].layout.ty,
1615                out_len: out_len2
1616            }
1617        );
1618
1619        // The return type must match the first argument type
1620        require!(
1621            ret_ty == in_ty,
1622            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
1623        );
1624
1625        require!(
1626            matches!(
1627                *element_ty1.kind(),
1628                ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
1629            ),
1630            InvalidMonomorphization::ExpectedElementType {
1631                span,
1632                name,
1633                expected_element: element_ty1,
1634                second_arg: args[1].layout.ty,
1635                in_elem,
1636                in_ty,
1637                mutability: ExpectedPointerMutability::Not,
1638            }
1639        );
1640
1641        let mask_elem_bitwidth = require_int_or_uint_ty!(
1642            element_ty2.kind(),
1643            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1644        );
1645
1646        // Alignment of T, must be a constant integer value:
1647        let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1648
1649        // Truncate the mask vector to a vector of i1s:
1650        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1651
1652        // Type of the vector of pointers:
1653        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1654
1655        // Type of the vector of elements:
1656        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1657
1658        return Ok(bx.call_intrinsic(
1659            "llvm.masked.gather",
1660            &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
1661            &[args[1].immediate(), alignment, mask, args[0].immediate()],
1662        ));
1663    }
1664
1665    if name == sym::simd_masked_load {
1666        // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
1667        // * N: number of elements in the input vectors
1668        // * T: type of the element to load
1669        // * M: any integer width is supported, will be truncated to i1
1670        // Loads contiguous elements from memory behind `pointer`, but only for
1671        // those lanes whose `mask` bit is enabled.
1672        // The memory addresses corresponding to the “off” lanes are not accessed.
1673
1674        // The element type of the "mask" argument must be a signed integer type of any width
1675        let mask_ty = in_ty;
1676        let (mask_len, mask_elem) = (in_len, in_elem);
1677
1678        // The second argument must be a pointer matching the element type
1679        let pointer_ty = args[1].layout.ty;
1680
1681        // The last argument is a passthrough vector providing values for disabled lanes
1682        let values_ty = args[2].layout.ty;
1683        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1684
1685        require_simd!(ret_ty, SimdReturn);
1686
1687        // Of the same length:
1688        require!(
1689            values_len == mask_len,
1690            InvalidMonomorphization::ThirdArgumentLength {
1691                span,
1692                name,
1693                in_len: mask_len,
1694                in_ty: mask_ty,
1695                arg_ty: values_ty,
1696                out_len: values_len
1697            }
1698        );
1699
1700        // The return type must match the last argument type
1701        require!(
1702            ret_ty == values_ty,
1703            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
1704        );
1705
1706        require!(
1707            matches!(
1708                *pointer_ty.kind(),
1709                ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
1710            ),
1711            InvalidMonomorphization::ExpectedElementType {
1712                span,
1713                name,
1714                expected_element: values_elem,
1715                second_arg: pointer_ty,
1716                in_elem: values_elem,
1717                in_ty: values_ty,
1718                mutability: ExpectedPointerMutability::Not,
1719            }
1720        );
1721
1722        let m_elem_bitwidth = require_int_or_uint_ty!(
1723            mask_elem.kind(),
1724            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1725        );
1726
1727        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1728
1729        // Alignment of T, must be a constant integer value:
1730        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1731
1732        let llvm_pointer = bx.type_ptr();
1733
1734        // Type of the vector of elements:
1735        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1736
1737        return Ok(bx.call_intrinsic(
1738            "llvm.masked.load",
1739            &[llvm_elem_vec_ty, llvm_pointer],
1740            &[args[1].immediate(), alignment, mask, args[2].immediate()],
1741        ));
1742    }
1743
1744    if name == sym::simd_masked_store {
1745        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
1746        // * N: number of elements in the input vectors
1747        // * T: type of the element to load
1748        // * M: any integer width is supported, will be truncated to i1
1749        // Stores contiguous elements to memory behind `pointer`, but only for
1750        // those lanes whose `mask` bit is enabled.
1751        // The memory addresses corresponding to the “off” lanes are not accessed.
1752
1753        // The element type of the "mask" argument must be a signed integer type of any width
1754        let mask_ty = in_ty;
1755        let (mask_len, mask_elem) = (in_len, in_elem);
1756
1757        // The second argument must be a pointer matching the element type
1758        let pointer_ty = args[1].layout.ty;
1759
1760        // The last argument specifies the values to store to memory
1761        let values_ty = args[2].layout.ty;
1762        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1763
1764        // Of the same length:
1765        require!(
1766            values_len == mask_len,
1767            InvalidMonomorphization::ThirdArgumentLength {
1768                span,
1769                name,
1770                in_len: mask_len,
1771                in_ty: mask_ty,
1772                arg_ty: values_ty,
1773                out_len: values_len
1774            }
1775        );
1776
1777        // The second argument must be a mutable pointer type matching the element type
1778        require!(
1779            matches!(
1780                *pointer_ty.kind(),
1781                ty::RawPtr(p_ty, p_mutbl)
1782                    if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
1783            ),
1784            InvalidMonomorphization::ExpectedElementType {
1785                span,
1786                name,
1787                expected_element: values_elem,
1788                second_arg: pointer_ty,
1789                in_elem: values_elem,
1790                in_ty: values_ty,
1791                mutability: ExpectedPointerMutability::Mut,
1792            }
1793        );
1794
1795        let m_elem_bitwidth = require_int_or_uint_ty!(
1796            mask_elem.kind(),
1797            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1798        );
1799
1800        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1801
1802        // Alignment of T, must be a constant integer value:
1803        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1804
1805        let llvm_pointer = bx.type_ptr();
1806
1807        // Type of the vector of elements:
1808        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1809
1810        return Ok(bx.call_intrinsic(
1811            "llvm.masked.store",
1812            &[llvm_elem_vec_ty, llvm_pointer],
1813            &[args[2].immediate(), args[1].immediate(), alignment, mask],
1814        ));
1815    }
1816
1817    if name == sym::simd_scatter {
1818        // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
1819        //             mask: <N x i{M}>) -> ()
1820        // * N: number of elements in the input vectors
1821        // * T: type of the element to load
1822        // * M: any integer width is supported, will be truncated to i1
1823
1824        // All types must be simd vector types
1825        // The second argument must be a simd vector with an element type that's a pointer
1826        // to the element type of the first argument
1827        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1828        let (element_len1, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1829        let (element_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1830
1831        // Of the same length:
1832        require!(
1833            in_len == element_len1,
1834            InvalidMonomorphization::SecondArgumentLength {
1835                span,
1836                name,
1837                in_len,
1838                in_ty,
1839                arg_ty: args[1].layout.ty,
1840                out_len: element_len1
1841            }
1842        );
1843        require!(
1844            in_len == element_len2,
1845            InvalidMonomorphization::ThirdArgumentLength {
1846                span,
1847                name,
1848                in_len,
1849                in_ty,
1850                arg_ty: args[2].layout.ty,
1851                out_len: element_len2
1852            }
1853        );
1854
1855        require!(
1856            matches!(
1857                *element_ty1.kind(),
1858                ty::RawPtr(p_ty, p_mutbl)
1859                    if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
1860            ),
1861            InvalidMonomorphization::ExpectedElementType {
1862                span,
1863                name,
1864                expected_element: element_ty1,
1865                second_arg: args[1].layout.ty,
1866                in_elem,
1867                in_ty,
1868                mutability: ExpectedPointerMutability::Mut,
1869            }
1870        );
1871
1872        // The element type of the third argument must be an integer type of any width:
1873        let mask_elem_bitwidth = require_int_or_uint_ty!(
1874            element_ty2.kind(),
1875            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1876        );
1877
1878        // Alignment of T, must be a constant integer value:
1879        let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1880
1881        // Truncate the mask vector to a vector of i1s:
1882        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1883
1884        // Type of the vector of pointers:
1885        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1886
1887        // Type of the vector of elements:
1888        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1889
1890        return Ok(bx.call_intrinsic(
1891            "llvm.masked.scatter",
1892            &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
1893            &[args[0].immediate(), args[1].immediate(), alignment, mask],
1894        ));
1895    }
1896
1897    macro_rules! arith_red {
1898        ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
1899         $identity:expr) => {
1900            if name == sym::$name {
1901                require!(
1902                    ret_ty == in_elem,
1903                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1904                );
1905                return match in_elem.kind() {
1906                    ty::Int(_) | ty::Uint(_) => {
1907                        let r = bx.$integer_reduce(args[0].immediate());
1908                        if $ordered {
1909                            // if overflow occurs, the result is the
1910                            // mathematical result modulo 2^n:
1911                            Ok(bx.$op(args[1].immediate(), r))
1912                        } else {
1913                            Ok(bx.$integer_reduce(args[0].immediate()))
1914                        }
1915                    }
1916                    ty::Float(f) => {
1917                        let acc = if $ordered {
1918                            // ordered arithmetic reductions take an accumulator
1919                            args[1].immediate()
1920                        } else {
1921                            // unordered arithmetic reductions use the identity accumulator
1922                            match f.bit_width() {
1923                                32 => bx.const_real(bx.type_f32(), $identity),
1924                                64 => bx.const_real(bx.type_f64(), $identity),
1925                                v => return_error!(
1926                                    InvalidMonomorphization::UnsupportedSymbolOfSize {
1927                                        span,
1928                                        name,
1929                                        symbol: sym::$name,
1930                                        in_ty,
1931                                        in_elem,
1932                                        size: v,
1933                                        ret_ty
1934                                    }
1935                                ),
1936                            }
1937                        };
1938                        Ok(bx.$float_reduce(acc, args[0].immediate()))
1939                    }
1940                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
1941                        span,
1942                        name,
1943                        symbol: sym::$name,
1944                        in_ty,
1945                        in_elem,
1946                        ret_ty
1947                    }),
1948                };
1949            }
1950        };
1951    }
1952
1953    arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
1954    arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
1955    arith_red!(
1956        simd_reduce_add_unordered: vector_reduce_add,
1957        vector_reduce_fadd_reassoc,
1958        false,
1959        add,
1960        -0.0
1961    );
1962    arith_red!(
1963        simd_reduce_mul_unordered: vector_reduce_mul,
1964        vector_reduce_fmul_reassoc,
1965        false,
1966        mul,
1967        1.0
1968    );
1969
1970    macro_rules! minmax_red {
1971        ($name:ident: $int_red:ident, $float_red:ident) => {
1972            if name == sym::$name {
1973                require!(
1974                    ret_ty == in_elem,
1975                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1976                );
1977                return match in_elem.kind() {
1978                    ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
1979                    ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
1980                    ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
1981                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
1982                        span,
1983                        name,
1984                        symbol: sym::$name,
1985                        in_ty,
1986                        in_elem,
1987                        ret_ty
1988                    }),
1989                };
1990            }
1991        };
1992    }
1993
1994    minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
1995    minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
1996
1997    macro_rules! bitwise_red {
1998        ($name:ident : $red:ident, $boolean:expr) => {
1999            if name == sym::$name {
2000                let input = if !$boolean {
2001                    require!(
2002                        ret_ty == in_elem,
2003                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2004                    );
2005                    args[0].immediate()
2006                } else {
2007                    let bitwidth = match in_elem.kind() {
2008                        ty::Int(i) => {
2009                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2010                        }
2011                        ty::Uint(i) => {
2012                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2013                        }
2014                        _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2015                            span,
2016                            name,
2017                            symbol: sym::$name,
2018                            in_ty,
2019                            in_elem,
2020                            ret_ty
2021                        }),
2022                    };
2023
2024                    vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2025                };
2026                return match in_elem.kind() {
2027                    ty::Int(_) | ty::Uint(_) => {
2028                        let r = bx.$red(input);
2029                        Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2030                    }
2031                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2032                        span,
2033                        name,
2034                        symbol: sym::$name,
2035                        in_ty,
2036                        in_elem,
2037                        ret_ty
2038                    }),
2039                };
2040            }
2041        };
2042    }
2043
2044    bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2045    bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2046    bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2047    bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2048    bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2049
2050    if name == sym::simd_cast_ptr {
2051        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2052        require!(
2053            in_len == out_len,
2054            InvalidMonomorphization::ReturnLengthInputType {
2055                span,
2056                name,
2057                in_len,
2058                in_ty,
2059                ret_ty,
2060                out_len
2061            }
2062        );
2063
2064        match in_elem.kind() {
2065            ty::RawPtr(p_ty, _) => {
2066                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2067                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2068                });
2069                require!(
2070                    metadata.is_unit(),
2071                    InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2072                );
2073            }
2074            _ => {
2075                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2076            }
2077        }
2078        match out_elem.kind() {
2079            ty::RawPtr(p_ty, _) => {
2080                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2081                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2082                });
2083                require!(
2084                    metadata.is_unit(),
2085                    InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2086                );
2087            }
2088            _ => {
2089                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2090            }
2091        }
2092
2093        return Ok(args[0].immediate());
2094    }
2095
2096    if name == sym::simd_expose_provenance {
2097        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2098        require!(
2099            in_len == out_len,
2100            InvalidMonomorphization::ReturnLengthInputType {
2101                span,
2102                name,
2103                in_len,
2104                in_ty,
2105                ret_ty,
2106                out_len
2107            }
2108        );
2109
2110        match in_elem.kind() {
2111            ty::RawPtr(_, _) => {}
2112            _ => {
2113                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2114            }
2115        }
2116        match out_elem.kind() {
2117            ty::Uint(ty::UintTy::Usize) => {}
2118            _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2119        }
2120
2121        return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
2122    }
2123
2124    if name == sym::simd_with_exposed_provenance {
2125        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2126        require!(
2127            in_len == out_len,
2128            InvalidMonomorphization::ReturnLengthInputType {
2129                span,
2130                name,
2131                in_len,
2132                in_ty,
2133                ret_ty,
2134                out_len
2135            }
2136        );
2137
2138        match in_elem.kind() {
2139            ty::Uint(ty::UintTy::Usize) => {}
2140            _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
2141        }
2142        match out_elem.kind() {
2143            ty::RawPtr(_, _) => {}
2144            _ => {
2145                return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2146            }
2147        }
2148
2149        return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
2150    }
2151
2152    if name == sym::simd_cast || name == sym::simd_as {
2153        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2154        require!(
2155            in_len == out_len,
2156            InvalidMonomorphization::ReturnLengthInputType {
2157                span,
2158                name,
2159                in_len,
2160                in_ty,
2161                ret_ty,
2162                out_len
2163            }
2164        );
2165        // casting cares about nominal type, not just structural type
2166        if in_elem == out_elem {
2167            return Ok(args[0].immediate());
2168        }
2169
2170        #[derive(Copy, Clone)]
2171        enum Sign {
2172            Unsigned,
2173            Signed,
2174        }
2175        use Sign::*;
2176
2177        enum Style {
2178            Float,
2179            Int(Sign),
2180            Unsupported,
2181        }
2182
2183        let (in_style, in_width) = match in_elem.kind() {
2184            // vectors of pointer-sized integers should've been
2185            // disallowed before here, so this unwrap is safe.
2186            ty::Int(i) => (
2187                Style::Int(Signed),
2188                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2189            ),
2190            ty::Uint(u) => (
2191                Style::Int(Unsigned),
2192                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2193            ),
2194            ty::Float(f) => (Style::Float, f.bit_width()),
2195            _ => (Style::Unsupported, 0),
2196        };
2197        let (out_style, out_width) = match out_elem.kind() {
2198            ty::Int(i) => (
2199                Style::Int(Signed),
2200                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2201            ),
2202            ty::Uint(u) => (
2203                Style::Int(Unsigned),
2204                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2205            ),
2206            ty::Float(f) => (Style::Float, f.bit_width()),
2207            _ => (Style::Unsupported, 0),
2208        };
2209
2210        match (in_style, out_style) {
2211            (Style::Int(sign), Style::Int(_)) => {
2212                return Ok(match in_width.cmp(&out_width) {
2213                    Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
2214                    Ordering::Equal => args[0].immediate(),
2215                    Ordering::Less => match sign {
2216                        Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
2217                        Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
2218                    },
2219                });
2220            }
2221            (Style::Int(Sign::Signed), Style::Float) => {
2222                return Ok(bx.sitofp(args[0].immediate(), llret_ty));
2223            }
2224            (Style::Int(Sign::Unsigned), Style::Float) => {
2225                return Ok(bx.uitofp(args[0].immediate(), llret_ty));
2226            }
2227            (Style::Float, Style::Int(sign)) => {
2228                return Ok(match (sign, name == sym::simd_as) {
2229                    (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
2230                    (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
2231                    (_, true) => bx.cast_float_to_int(
2232                        matches!(sign, Sign::Signed),
2233                        args[0].immediate(),
2234                        llret_ty,
2235                    ),
2236                });
2237            }
2238            (Style::Float, Style::Float) => {
2239                return Ok(match in_width.cmp(&out_width) {
2240                    Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
2241                    Ordering::Equal => args[0].immediate(),
2242                    Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
2243                });
2244            }
2245            _ => { /* Unsupported. Fallthrough. */ }
2246        }
2247        return_error!(InvalidMonomorphization::UnsupportedCast {
2248            span,
2249            name,
2250            in_ty,
2251            in_elem,
2252            ret_ty,
2253            out_elem
2254        });
2255    }
2256    macro_rules! arith_binary {
2257        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2258            $(if name == sym::$name {
2259                match in_elem.kind() {
2260                    $($(ty::$p(_))|* => {
2261                        return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
2262                    })*
2263                    _ => {},
2264                }
2265                return_error!(
2266                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2267                );
2268            })*
2269        }
2270    }
2271    arith_binary! {
2272        simd_add: Uint, Int => add, Float => fadd;
2273        simd_sub: Uint, Int => sub, Float => fsub;
2274        simd_mul: Uint, Int => mul, Float => fmul;
2275        simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
2276        simd_rem: Uint => urem, Int => srem, Float => frem;
2277        simd_shl: Uint, Int => shl;
2278        simd_shr: Uint => lshr, Int => ashr;
2279        simd_and: Uint, Int => and;
2280        simd_or: Uint, Int => or;
2281        simd_xor: Uint, Int => xor;
2282        simd_fmax: Float => maxnum;
2283        simd_fmin: Float => minnum;
2284
2285    }
2286    macro_rules! arith_unary {
2287        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2288            $(if name == sym::$name {
2289                match in_elem.kind() {
2290                    $($(ty::$p(_))|* => {
2291                        return Ok(bx.$call(args[0].immediate()))
2292                    })*
2293                    _ => {},
2294                }
2295                return_error!(
2296                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2297                );
2298            })*
2299        }
2300    }
2301    arith_unary! {
2302        simd_neg: Int => neg, Float => fneg;
2303    }
2304
2305    // Unary integer intrinsics
2306    if matches!(
2307        name,
2308        sym::simd_bswap
2309            | sym::simd_bitreverse
2310            | sym::simd_ctlz
2311            | sym::simd_ctpop
2312            | sym::simd_cttz
2313            | sym::simd_funnel_shl
2314            | sym::simd_funnel_shr
2315    ) {
2316        let vec_ty = bx.cx.type_vector(
2317            match *in_elem.kind() {
2318                ty::Int(i) => bx.cx.type_int_from_ty(i),
2319                ty::Uint(i) => bx.cx.type_uint_from_ty(i),
2320                _ => return_error!(InvalidMonomorphization::UnsupportedOperation {
2321                    span,
2322                    name,
2323                    in_ty,
2324                    in_elem
2325                }),
2326            },
2327            in_len as u64,
2328        );
2329        let llvm_intrinsic = match name {
2330            sym::simd_bswap => "llvm.bswap",
2331            sym::simd_bitreverse => "llvm.bitreverse",
2332            sym::simd_ctlz => "llvm.ctlz",
2333            sym::simd_ctpop => "llvm.ctpop",
2334            sym::simd_cttz => "llvm.cttz",
2335            sym::simd_funnel_shl => "llvm.fshl",
2336            sym::simd_funnel_shr => "llvm.fshr",
2337            _ => unreachable!(),
2338        };
2339        let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
2340
2341        return match name {
2342            // byte swap is no-op for i8/u8
2343            sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2344            sym::simd_ctlz | sym::simd_cttz => {
2345                // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
2346                let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2347                Ok(bx.call_intrinsic(
2348                    llvm_intrinsic,
2349                    &[vec_ty],
2350                    &[args[0].immediate(), dont_poison_on_zero],
2351                ))
2352            }
2353            sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2354                // simple unary argument cases
2355                Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
2356            }
2357            sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
2358                llvm_intrinsic,
2359                &[vec_ty],
2360                &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
2361            )),
2362            _ => unreachable!(),
2363        };
2364    }
2365
2366    if name == sym::simd_arith_offset {
2367        // This also checks that the first operand is a ptr type.
2368        let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
2369            span_bug!(span, "must be called with a vector of pointer types as first argument")
2370        });
2371        let layout = bx.layout_of(pointee);
2372        let ptrs = args[0].immediate();
2373        // The second argument must be a ptr-sized integer.
2374        // (We don't care about the signedness, this is wrapping anyway.)
2375        let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
2376        if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
2377            span_bug!(
2378                span,
2379                "must be called with a vector of pointer-sized integers as second argument"
2380            );
2381        }
2382        let offsets = args[1].immediate();
2383
2384        return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
2385    }
2386
2387    if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
2388        let lhs = args[0].immediate();
2389        let rhs = args[1].immediate();
2390        let is_add = name == sym::simd_saturating_add;
2391        let (signed, elem_ty) = match *in_elem.kind() {
2392            ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
2393            ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
2394            _ => {
2395                return_error!(InvalidMonomorphization::ExpectedVectorElementType {
2396                    span,
2397                    name,
2398                    expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
2399                    vector_type: args[0].layout.ty
2400                });
2401            }
2402        };
2403        let llvm_intrinsic = format!(
2404            "llvm.{}{}.sat",
2405            if signed { 's' } else { 'u' },
2406            if is_add { "add" } else { "sub" },
2407        );
2408        let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
2409
2410        return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
2411    }
2412
2413    span_bug!(span, "unknown SIMD intrinsic");
2414}