Skip to main content

rustc_codegen_llvm/
intrinsic.rs

1use std::cmp::Ordering;
2use std::ffi::c_uint;
3use std::{assert_matches, iter, ptr};
4
5use rustc_abi::{
6    AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, NumScalableVectors,
7    Primitive, Size,
8};
9use rustc_attr_ir::find_attr;
10use rustc_codegen_ssa::RetagInfo;
11use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
12use rustc_codegen_ssa::common::IntPredicate;
13use rustc_codegen_ssa::diagnostics::{ExpectedPointerMutability, InvalidMonomorphization};
14use rustc_codegen_ssa::mir::IntrinsicResult;
15use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
16use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
17use rustc_codegen_ssa::traits::*;
18use rustc_hir as hir;
19use rustc_hir::def_id::LOCAL_CRATE;
20use rustc_lint_defs::builtin::DEPRECATED_LLVM_INTRINSIC;
21use rustc_middle::mir::interpret::{AllocId, Allocation, ErrorHandled, GlobalAlloc, alloc_range};
22use rustc_middle::mir::{self, BinOp};
23use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
24use rustc_middle::ty::offload_meta::OffloadMetadata;
25use rustc_middle::ty::{self, GenericArgs, Instance, SimdAlign, Ty, TyCtxt};
26use rustc_session::diagnostics::feature_err;
27use rustc_span::{ErrorGuaranteed, Span, Symbol, bug, span_bug, sym};
28use rustc_structures::CrateType;
29use rustc_symbol_mangling::{
30    mangle_internal_symbol, mangle_offload_export, symbol_name_for_instance_in_crate,
31};
32use rustc_target::callconv::PassMode;
33use rustc_target::spec::Arch;
34use tracing::debug;
35
36use crate::abi::FnAbiLlvmExt;
37use crate::builder::Builder;
38use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
39use crate::builder::gpu_offload::{self, OffloadKernelDims, declare_omp_get_num_devices};
40use crate::context::CodegenCx;
41use crate::declare::declare_raw_fn;
42use crate::diagnostics::{
43    AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
44    OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
45};
46use crate::intrinsic::ty::typetree::fnc_typetrees;
47use crate::llvm::{self, Attribute, AttributePlace, Metadata, Type, TypeKind, Value};
48use crate::type_of::LayoutLlvmExt;
49use crate::va_arg::emit_va_arg;
50
51fn call_simple_intrinsic<'ll, 'tcx>(
52    bx: &mut Builder<'_, 'll, 'tcx>,
53    name: Symbol,
54    args: &[OperandRef<'tcx, &'ll Value>],
55) -> Option<&'ll Value> {
56    let llvm_version = crate::llvm_util::get_version();
57    // minimum/maximum were broken for f64/f128 before
58    // <https://github.com/llvm/llvm-project/commit/56385af687c3a7a1f67716fb3f819336789a8cab>.
59    // We use the fallback body there.
60    let fixed_minmax = llvm_version >= (23, 0, 0);
61
62    let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
63        sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]),
64        sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]),
65        sym::minimumf64 if fixed_minmax => ("llvm.minimum", &[bx.type_f64()]),
66        sym::minimumf128 if fixed_minmax => ("llvm.minimum", &[bx.type_f128()]),
67
68        sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]),
69        sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]),
70        sym::maximumf64 if fixed_minmax => ("llvm.maximum", &[bx.type_f64()]),
71        sym::maximumf128 if fixed_minmax => ("llvm.maximum", &[bx.type_f128()]),
72
73        sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
74        sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
75        sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
76        sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
77
78        _ => return None,
79    };
80    Some(bx.call_intrinsic(
81        base_name,
82        type_params,
83        &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
84    ))
85}
86
87impl<'ll, 'tcx> Builder<'_, 'll, 'tcx> {
88    fn black_box(&mut self, result: PlaceRef<'tcx, &'ll Value>, span: Span) {
89        let result_val_span = [result.val.llval];
90        // We need to "use" the argument in some way LLVM can't introspect, and on
91        // targets that support it we can typically leverage inline assembly to do
92        // this. LLVM's interpretation of inline assembly is that it's, well, a black
93        // box. This isn't the greatest implementation since it probably deoptimizes
94        // more than we want, but it's so far good enough.
95        //
96        // For zero-sized types, the location pointed to by the result may be
97        // uninitialized. Do not "use" the result in this case; instead just clobber
98        // the memory.
99        let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
100            ("~{memory}", &[])
101        } else {
102            ("r,~{memory}", &result_val_span)
103        };
104        crate::asm::inline_asm_call(
105            self,
106            "",
107            constraint,
108            inputs,
109            self.type_void(),
110            &[],
111            true,
112            false,
113            llvm::AsmDialect::Att,
114            &[span],
115            false,
116            None,
117            None,
118        )
119        .unwrap_or_else(|| ::rustc_span::macros::bug_impl(None,
    format_args!("failed to generate inline asm call for `black_box`"),
    Location::caller())bug!("failed to generate inline asm call for `black_box`"));
120    }
121}
122
123impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
124    fn codegen_intrinsic_call(
125        &mut self,
126        instance: ty::Instance<'tcx>,
127        args: &[OperandRef<'tcx, &'ll Value>],
128        result_layout: ty::layout::TyAndLayout<'tcx>,
129        result_place: Option<PlaceValue<&'ll Value>>,
130        span: Span,
131    ) -> IntrinsicResult<'tcx, &'ll Value> {
132        let tcx = self.tcx;
133        let llvm_version = crate::llvm_util::get_version();
134
135        let name = tcx.item_name(instance.def_id());
136        let fn_args = instance.args;
137
138        let simple = call_simple_intrinsic(self, name, args);
139        let llval = match name {
140            _ if simple.is_some() => simple.unwrap(),
141            sym::minimum_number_nsz_f16
142            | sym::minimum_number_nsz_f32
143            | sym::minimum_number_nsz_f64
144            | sym::minimum_number_nsz_f128
145            | sym::maximum_number_nsz_f16
146            | sym::maximum_number_nsz_f32
147            | sym::maximum_number_nsz_f64
148            | sym::maximum_number_nsz_f128 => {
149                let intrinsic_name = if name.as_str().starts_with("min") {
150                    "llvm.minimumnum"
151                } else {
152                    "llvm.maximumnum"
153                };
154                let call = self.call_intrinsic(
155                    intrinsic_name,
156                    &[args[0].layout.immediate_llvm_type(self.cx)],
157                    &[args[0].immediate(), args[1].immediate()],
158                );
159                // `nsz` on minimumnum/maximumnum is special: its only effect is to make
160                // signed-zero ordering non-deterministic.
161                unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
162                call
163            }
164            sym::ptr_mask => {
165                let ptr = args[0].immediate();
166                self.call_intrinsic(
167                    "llvm.ptrmask",
168                    &[self.val_ty(ptr), self.type_isize()],
169                    &[ptr, args[1].immediate()],
170                )
171            }
172            sym::autodiff => {
173                return codegen_autodiff(self, instance, args, result_layout, result_place);
174            }
175            sym::offload => {
176                if tcx.sess.opts.unstable_opts.offload.is_empty() {
177                    let _ = tcx.dcx().emit_err(OffloadWithoutEnable);
178                }
179
180                if tcx.sess.lto() != rustc_session::config::Lto::Fat {
181                    let _ = tcx.dcx().emit_err(OffloadWithoutFatLTO);
182                }
183
184                codegen_offload(self, tcx, instance, args);
185                // offload *has* a return type, but somehow works without mentioning the place
186                return IntrinsicResult::WroteIntoPlace;
187            }
188            sym::offload_get_num_devices => {
189                let (fn_decl, fn_ty) = declare_omp_get_num_devices(self.cx);
190
191                let llval =
192                    self.call(fn_ty, None, None, fn_decl, ReturnSlot::Direct, &[], None, None);
193
194                return IntrinsicResult::Operand(OperandValue::Immediate(llval));
195            }
196            sym::is_val_statically_known => {
197                if let OperandValue::Immediate(imm) = args[0].val {
198                    self.call_intrinsic(
199                        "llvm.is.constant",
200                        &[args[0].layout.immediate_llvm_type(self.cx)],
201                        &[imm],
202                    )
203                } else {
204                    self.const_bool(false)
205                }
206            }
207            sym::select_unpredictable => {
208                let cond = args[0].immediate();
209                {
    match (&args[1].layout, &args[2].layout) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(args[1].layout, args[2].layout);
210                let select = |bx: &mut Self, true_val, false_val| {
211                    let result = bx.select(cond, true_val, false_val);
212                    bx.set_unpredictable(&result);
213                    result
214                };
215                match (args[1].val, args[2].val) {
216                    (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
217                        if !true_val.llextra.is_none() {
    ::core::panicking::panic("assertion failed: true_val.llextra.is_none()")
};assert!(true_val.llextra.is_none());
218                        if !false_val.llextra.is_none() {
    ::core::panicking::panic("assertion failed: false_val.llextra.is_none()")
};assert!(false_val.llextra.is_none());
219                        {
    match (&true_val.align, &false_val.align) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(true_val.align, false_val.align);
220                        let ptr = select(self, true_val.llval, false_val.llval);
221                        let selected =
222                            OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
223                        let result = PlaceRef { val: result_place.unwrap(), layout: result_layout };
224                        selected.store(self, result);
225                        return IntrinsicResult::WroteIntoPlace;
226                    }
227                    (OperandValue::Immediate(_), OperandValue::Immediate(_))
228                    | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
229                        let true_val = args[1].immediate_or_packed_pair(self);
230                        let false_val = args[2].immediate_or_packed_pair(self);
231                        select(self, true_val, false_val)
232                    }
233                    (OperandValue::ZeroSized, OperandValue::ZeroSized) => {
234                        return IntrinsicResult::Operand(OperandValue::ZeroSized);
235                    }
236                    _ => ::rustc_span::macros::bug_impl(Some(span),
    format_args!("Incompatible OperandValue for select_unpredictable"),
    Location::caller())span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
237                }
238            }
239            sym::catch_unwind => catch_unwind_intrinsic(
240                self,
241                args[0].immediate(),
242                args[1].immediate(),
243                args[2].immediate(),
244            ),
245            sym::breakpoint | sym::resumable_breakpoint => {
246                self.call_intrinsic("llvm.debugtrap", &[], &[])
247            }
248            sym::va_arg => {
249                let target = &self.cx.tcx.sess.target;
250                let stability = target.supports_c_variadic_definitions();
251                if let CVariadicStatus::Unstable { feature } = stability
252                    && !self.tcx.features().enabled(feature)
253                {
254                    let msg =
255                        ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("C-variadic function definitions on this target are unstable"))
    })format!("C-variadic function definitions on this target are unstable");
256                    feature_err(&*self.sess(), feature, span, msg).emit();
257                }
258
259                let BackendRepr::Scalar(scalar) = result_layout.backend_repr else {
260                    ::rustc_span::macros::bug_impl(None,
    format_args!("the va_arg intrinsic does not support non-scalar types"),
    Location::caller())bug!("the va_arg intrinsic does not support non-scalar types")
261                };
262
263                // We reject types that would never be passed as varargs in C because
264                // they get promoted to a larger type, specifically integers smaller than
265                // c_int and float type smaller than c_double.
266                match scalar.primitive() {
267                    Primitive::Pointer(_) => {
268                        // Pointers are always OK.
269                    }
270                    Primitive::Int(..) => {
271                        let int_width = self.cx().size_of(result_layout.ty).bits();
272                        let target_c_int_width = self.cx().sess().target.options.c_int_width;
273                        if int_width < u64::from(target_c_int_width) {
274                            // Smaller integer types are automatically promototed and `va_arg`
275                            // should not be called on them.
276                            ::rustc_span::macros::bug_impl(None,
    format_args!("va_arg got i{0} but needs at least c_int (an i{1})",
        int_width, target_c_int_width), Location::caller());bug!(
277                                "va_arg got i{} but needs at least c_int (an i{})",
278                                int_width,
279                                target_c_int_width
280                            );
281                        }
282                    }
283                    Primitive::Float(Float::F16) => {
284                        ::rustc_span::macros::bug_impl(None,
    format_args!("the va_arg intrinsic does not support `f16`"),
    Location::caller())bug!("the va_arg intrinsic does not support `f16`")
285                    }
286                    Primitive::Float(Float::F16B) => {
287                        ::rustc_span::macros::bug_impl(None,
    format_args!("the va_arg intrinsic does not support `f16b`"),
    Location::caller())bug!("the va_arg intrinsic does not support `f16b`")
288                    }
289                    Primitive::Float(Float::F32) => {
290                        // c_double is actually f32 on avr.
291                        if self.cx().sess().target.arch != Arch::Avr {
292                            ::rustc_span::macros::bug_impl(None,
    format_args!("the va_arg intrinsic does not support `f32` on this target"),
    Location::caller())bug!("the va_arg intrinsic does not support `f32` on this target")
293                        }
294                    }
295                    Primitive::Float(Float::F64) => {
296                        // 64-bit floats are always OK.
297                    }
298                    Primitive::Float(Float::F128) => {
299                        // Supported on some targets, especially where long double is IEEE f128.
300                    }
301                    Primitive::Float(Float::PpcF128) => {
302                        // Supported on powerpc and powerpc64 targets.
303                    }
304                }
305
306                emit_va_arg(self, args[0], result_layout)
307            }
308
309            sym::volatile_load | sym::unaligned_volatile_load => {
310                // Note that we cannot just load the `llvm_type` because we should never load non-scalars.
311                // Trying to do so blows up horribly in some cases -- for example loading a
312                // `MaybeUninint<&dyn Trait>` would load as `{ [i64x2] }` which gives assertions later
313                // (if we're lucky) from things not being pointers that ought to be.
314                let ptr = args[0].immediate();
315                let abi_align = result_layout.align;
316                let ptr_align = if name == sym::volatile_load { abi_align } else { Align::ONE };
317                let need_black_box = llvm_version < (23, 0, 0);
318                if result_layout.is_zst() {
319                    return IntrinsicResult::Operand(OperandValue::ZeroSized);
320                } else if let BackendRepr::Scalar(scalar) = result_layout.backend_repr
321                    && !need_black_box
322                {
323                    let load = self.volatile_load(self.type_from_scalar(scalar), ptr, ptr_align);
324                    self.to_immediate_scalar(load, scalar)
325                } else {
326                    // One day Rust will probably want to define how we split up a volatile load
327                    // of something that's *not* just an ordinary scalar, but for now we can just
328                    // use an LLVM integer type of the correct width and let it split it however.
329                    let llty = self.type_ix(result_layout.size.bits());
330                    let temp = if let Some(result_place) = result_place {
331                        PlaceRef { val: result_place, layout: result_layout }
332                    } else {
333                        PlaceRef::alloca(self, result_layout)
334                    };
335                    let llval = self.volatile_load(llty, ptr, ptr_align);
336                    self.store(llval, temp.val.llval, abi_align);
337                    if need_black_box {
338                        // LLVM up until v22 considers volatile reads `willreturn` and hence can
339                        // move UB from further down up across this read. To prevent that, insert an
340                        // inline asm block that, as far as LLVM is concerned, might not terminate,
341                        // and hence should prevent such reordering.
342                        self.black_box(temp, span);
343                    }
344                    return if result_place.is_none() {
345                        IntrinsicResult::Operand(self.load_operand(temp).val)
346                    } else {
347                        IntrinsicResult::WroteIntoPlace
348                    };
349                }
350            }
351            sym::prefetch_read_data
352            | sym::prefetch_write_data
353            | sym::prefetch_read_instruction
354            | sym::prefetch_write_instruction => {
355                let (rw, cache_type) = match name {
356                    sym::prefetch_read_data => (0, 1),
357                    sym::prefetch_write_data => (1, 1),
358                    sym::prefetch_read_instruction => (0, 0),
359                    sym::prefetch_write_instruction => (1, 0),
360                    _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
    Location::caller())bug!(),
361                };
362                let ptr = args[0].immediate();
363                let locality = fn_args.const_at(1).to_leaf().to_i32();
364                self.call_intrinsic(
365                    "llvm.prefetch.p0",
366                    &[self.val_ty(ptr)],
367                    &[
368                        ptr,
369                        self.const_i32(rw),
370                        self.const_i32(locality),
371                        self.const_i32(cache_type),
372                    ],
373                );
374                return IntrinsicResult::Operand(OperandValue::ZeroSized);
375            }
376            sym::carrying_mul_add => {
377                let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
378
379                let wide_llty = self.type_ix(size.bits() * 2);
380                let args = args.as_array().unwrap();
381                let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
382
383                let wide = if signed {
384                    let prod = self.unchecked_smul(a, b);
385                    let acc = self.unchecked_sadd(prod, c);
386                    self.unchecked_sadd(acc, d)
387                } else {
388                    let prod = self.unchecked_umul(a, b);
389                    let acc = self.unchecked_uadd(prod, c);
390                    self.unchecked_uadd(acc, d)
391                };
392
393                let narrow_llty = self.type_ix(size.bits());
394                let low = self.trunc(wide, narrow_llty);
395                let bits_const = self.const_uint(wide_llty, size.bits());
396                // No need for ashr when signed; LLVM changes it to lshr anyway.
397                let high = self.lshr(wide, bits_const);
398                // FIXME: could be `trunc nuw`, even for signed.
399                let high = self.trunc(high, narrow_llty);
400
401                let pair_llty = self.type_struct(&[narrow_llty, narrow_llty]);
402                let pair = self.const_poison(pair_llty);
403                let pair = self.insert_value(pair, low, 0);
404                let pair = self.insert_value(pair, high, 1);
405                pair
406            }
407
408            sym::ctlz
409            | sym::ctlz_nonzero
410            | sym::cttz
411            | sym::cttz_nonzero
412            | sym::ctpop
413            | sym::bswap
414            | sym::bitreverse
415            | sym::carryless_mul
416            | sym::integer_max
417            | sym::integer_min
418            | sym::saturating_add
419            | sym::saturating_sub
420            | sym::unchecked_funnel_shl
421            | sym::unchecked_funnel_shr => {
422                let ty = args[0].layout.ty;
423                if !ty.is_integral() {
424                    let err = tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
425                        span,
426                        name,
427                        ty,
428                    });
429                    return IntrinsicResult::Err(err);
430                }
431                let (size, signed) = ty.int_size_and_signed(self.tcx);
432                let width = size.bits();
433                let llty = self.type_ix(width);
434                match name {
435                    sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
436                        let y =
437                            self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
438                        let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
439                            "llvm.ctlz"
440                        } else {
441                            "llvm.cttz"
442                        };
443                        let ret =
444                            self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
445                        self.intcast(ret, result_layout.llvm_type(self), false)
446                    }
447                    sym::ctpop => self.ctpop(args[0].immediate()),
448                    sym::bswap => {
449                        if width == 8 {
450                            args[0].immediate() // byte swap a u8/i8 is just a no-op
451                        } else {
452                            self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
453                        }
454                    }
455                    sym::bitreverse => {
456                        self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
457                    }
458                    sym::carryless_mul => {
459                        let lhs = args[0].immediate();
460                        let rhs = args[1].immediate();
461                        self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
462                    }
463                    sym::integer_min | sym::integer_max => {
464                        let lhs = args[0].immediate();
465                        let rhs = args[1].immediate();
466                        let llvm_name = match (name, signed) {
467                            (sym::integer_max, false) => "llvm.umax",
468                            (sym::integer_max, true) => "llvm.smax",
469                            (sym::integer_min, false) => "llvm.umin",
470                            (sym::integer_min, true) => "llvm.smin",
471                            _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
    Location::caller())bug!(),
472                        };
473                        self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
474                    }
475                    sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
476                        let is_left = name == sym::unchecked_funnel_shl;
477                        let lhs = args[0].immediate();
478                        let rhs = args[1].immediate();
479                        let raw_shift = args[2].immediate();
480                        let llvm_name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("llvm.fsh{0}",
                if is_left { 'l' } else { 'r' }))
    })format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
481
482                        // llvm expects shift to be the same type as the values, but rust
483                        // always uses `u32`.
484                        let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false);
485
486                        self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift])
487                    }
488                    sym::saturating_add | sym::saturating_sub => {
489                        let is_add = name == sym::saturating_add;
490                        let lhs = args[0].immediate();
491                        let rhs = args[1].immediate();
492                        let llvm_name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
                if signed { 's' } else { 'u' },
                if is_add { "add" } else { "sub" }))
    })format!(
493                            "llvm.{}{}.sat",
494                            if signed { 's' } else { 'u' },
495                            if is_add { "add" } else { "sub" },
496                        );
497                        self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
498                    }
499                    _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
    Location::caller())bug!(),
500                }
501            }
502
503            sym::fabs
504            | sym::sqrt
505            | sym::powf
506            | sym::fma
507            | sym::fmuladd
508            | sym::floor
509            | sym::ceil
510            | sym::trunc
511            | sym::round
512            | sym::round_ties_even
513            | sym::exp
514            | sym::exp2
515            | sym::log
516            | sym::log10
517            | sym::log2
518            | sym::sin
519            | sym::cos
520            | sym::powi => {
521                let ty = args[0].layout.ty;
522                let ty::Float(f) = ty.kind() else {
523                    ::rustc_span::macros::bug_impl(Some(span),
    format_args!("the `{0}` intrinsic requires a floating-point argument, got {1:?}",
        name, ty), Location::caller());span_bug!(
524                        span,
525                        "the `{}` intrinsic requires a floating-point argument, got {:?}",
526                        name,
527                        ty
528                    );
529                };
530                let llty = self.type_float_from_ty(*f);
531                let llvm_name = match name {
532                    sym::fabs => "llvm.fabs",
533                    sym::sqrt => "llvm.sqrt",
534                    sym::powf => "llvm.pow",
535                    sym::fma => "llvm.fma",
536                    sym::fmuladd => "llvm.fmuladd",
537                    sym::floor => "llvm.floor",
538                    sym::ceil => "llvm.ceil",
539                    sym::trunc => "llvm.trunc",
540                    sym::round => "llvm.round",
541                    // We could use any of `rint`, `nearbyint`, or `roundeven`
542                    // for this -- they are all identical in semantics when
543                    // assuming the default FP environment.
544                    // `rint` is what we used for $forever.
545                    sym::round_ties_even => "llvm.rint",
546                    sym::exp => "llvm.exp",
547                    sym::exp2 => "llvm.exp2",
548                    sym::log => "llvm.log",
549                    sym::log10 => "llvm.log10",
550                    sym::log2 => "llvm.log2",
551                    sym::sin => "llvm.sin",
552                    sym::cos => "llvm.cos",
553                    sym::powi => "llvm.powi",
554
555                    _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
    Location::caller())bug!(),
556                };
557
558                let params: &[&'ll Type] = match name {
559                    sym::powi => &[llty, self.type_i32()],
560                    _ => &[llty],
561                };
562
563                self.call_intrinsic(
564                    llvm_name,
565                    params,
566                    &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
567                )
568            }
569
570            sym::raw_eq => {
571                use BackendRepr::*;
572                let tp_ty = fn_args.type_at(0);
573                let layout = self.layout_of(tp_ty).layout;
574                let use_integer_compare = match layout.backend_repr() {
575                    Scalar(_) | ScalarPair { a: _, b: _, b_offset: _ } => true,
576                    SimdVector { .. } => false,
577                    SimdScalableVector { .. } => {
578                        let err = tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
579                            span,
580                            name: sym::raw_eq,
581                            ty: tp_ty,
582                        });
583                        return IntrinsicResult::Err(err);
584                    }
585                    Memory { .. } => {
586                        // For rusty ABIs, small aggregates are actually passed
587                        // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
588                        // so we re-use that same threshold here.
589                        layout.size() <= self.data_layout().pointer_size() * 2
590                    }
591                };
592
593                let a = args[0].immediate();
594                let b = args[1].immediate();
595                if layout.size().bytes() == 0 {
596                    self.const_bool(true)
597                } else if use_integer_compare {
598                    let integer_ty = self.type_ix(layout.size().bits());
599                    let a_val = self.load(integer_ty, a, layout.align());
600                    let b_val = self.load(integer_ty, b, layout.align());
601                    self.icmp(IntPredicate::IntEQ, a_val, b_val)
602                } else {
603                    let n = self.const_usize(layout.size().bytes());
604                    let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
605                    self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
606                }
607            }
608
609            sym::compare_bytes => {
610                // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
611                let cmp = self.call_intrinsic(
612                    "memcmp",
613                    &[],
614                    &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
615                );
616                // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
617                self.sext(cmp, self.type_ix(32))
618            }
619
620            sym::black_box => {
621                // This `unwrap` is justified by `intrinsic_call_expects_place_always` declaring
622                // this intrinsic as always needing a return place.
623                let result = PlaceRef { val: result_place.unwrap(), layout: result_layout };
624                args[0].val.store(self, result);
625                self.black_box(result, span);
626
627                // We have copied the value to `result` already.
628                return IntrinsicResult::WroteIntoPlace;
629            }
630
631            sym::gpu_launch_sized_workgroup_mem => {
632                // Generate an anonymous global per call, with these properties:
633                // 1. The global is in the address space for workgroup memory
634                // 2. It is an `external` global
635                // 3. It is correctly aligned for the pointee `T`
636                // All instances of extern addrspace(gpu_workgroup) globals are merged in the LLVM backend.
637                // The name is irrelevant.
638                // See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared
639                let name = if llvm_version < (23, 0, 0) && tcx.sess.target.arch == Arch::Nvptx64 {
640                    // The auto-assigned name for extern shared globals in the nvptx backend does
641                    // not compile in ptxas. Workaround this issue by assigning a name.
642                    // Fixed in LLVM 23.
643                    "gpu_launch_sized_workgroup_mem"
644                } else {
645                    ""
646                };
647                let global = self.declare_global_in_addrspace(
648                    name,
649                    self.type_array(self.type_i8(), 0),
650                    AddressSpace::GPU_WORKGROUP,
651                );
652                let ty::RawPtr(inner_ty, _) = result_layout.ty.kind() else { ::core::panicking::unreachable_default_msg()unreachable!() };
653                // The alignment of the global is used to specify the *minimum* alignment that
654                // must be obeyed by the GPU runtime.
655                // When multiple of these global variables are used by a kernel, the maximum alignment is taken.
656                // See https://github.com/llvm/llvm-project/blob/a271d07488a85ce677674bbe8101b10efff58c95/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp#L821
657                let alignment = self.align_of(*inner_ty).bytes() as u32;
658                unsafe {
659                    // FIXME Workaround the above issue by taking maximum alignment if the global existed
660                    if tcx.sess.target.arch == Arch::Nvptx64 {
661                        if alignment > llvm::LLVMGetAlignment(global) {
662                            llvm::LLVMSetAlignment(global, alignment);
663                        }
664                    } else {
665                        llvm::LLVMSetAlignment(global, alignment);
666                    }
667                }
668                self.cx().const_pointercast(global, self.type_ptr())
669            }
670
671            sym::amdgpu_dispatch_ptr => {
672                let val = self.call_intrinsic("llvm.amdgcn.dispatch.ptr", &[], &[]);
673                // Relying on `LLVMBuildPointerCast` to produce an addrspacecast
674                self.pointercast(val, self.type_ptr())
675            }
676
677            sym::sve_tuple_create2 => {
678                {
    match self.layout_of(fn_args.type_at(0)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(1), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
679                    self.layout_of(fn_args.type_at(0)).backend_repr,
680                    BackendRepr::SimdScalableVector {
681                        number_of_vectors: NumScalableVectors(1),
682                        ..
683                    }
684                );
685                let tuple_ty = self.layout_of(fn_args.type_at(1));
686                {
    match tuple_ty.backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(2), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
687                    tuple_ty.backend_repr,
688                    BackendRepr::SimdScalableVector {
689                        number_of_vectors: NumScalableVectors(2),
690                        ..
691                    }
692                );
693                let ret = self.const_poison(self.backend_type(tuple_ty));
694                let ret = self.insert_value(ret, args[0].immediate(), 0);
695                self.insert_value(ret, args[1].immediate(), 1)
696            }
697
698            sym::sve_tuple_create3 => {
699                {
    match self.layout_of(fn_args.type_at(0)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(1), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
700                    self.layout_of(fn_args.type_at(0)).backend_repr,
701                    BackendRepr::SimdScalableVector {
702                        number_of_vectors: NumScalableVectors(1),
703                        ..
704                    }
705                );
706                let tuple_ty = self.layout_of(fn_args.type_at(1));
707                {
    match tuple_ty.backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(3), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(3), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
708                    tuple_ty.backend_repr,
709                    BackendRepr::SimdScalableVector {
710                        number_of_vectors: NumScalableVectors(3),
711                        ..
712                    }
713                );
714                let ret = self.const_poison(self.backend_type(tuple_ty));
715                let ret = self.insert_value(ret, args[0].immediate(), 0);
716                let ret = self.insert_value(ret, args[1].immediate(), 1);
717                self.insert_value(ret, args[2].immediate(), 2)
718            }
719
720            sym::sve_tuple_create4 => {
721                {
    match self.layout_of(fn_args.type_at(0)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(1), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
722                    self.layout_of(fn_args.type_at(0)).backend_repr,
723                    BackendRepr::SimdScalableVector {
724                        number_of_vectors: NumScalableVectors(1),
725                        ..
726                    }
727                );
728                let tuple_ty = self.layout_of(fn_args.type_at(1));
729                {
    match tuple_ty.backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(4), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(4), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
730                    tuple_ty.backend_repr,
731                    BackendRepr::SimdScalableVector {
732                        number_of_vectors: NumScalableVectors(4),
733                        ..
734                    }
735                );
736                let ret = self.const_poison(self.backend_type(tuple_ty));
737                let ret = self.insert_value(ret, args[0].immediate(), 0);
738                let ret = self.insert_value(ret, args[1].immediate(), 1);
739                let ret = self.insert_value(ret, args[2].immediate(), 2);
740                self.insert_value(ret, args[3].immediate(), 3)
741            }
742
743            sym::sve_tuple_get => {
744                {
    match self.layout_of(fn_args.type_at(0)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
            .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
745                    self.layout_of(fn_args.type_at(0)).backend_repr,
746                    BackendRepr::SimdScalableVector {
747                        number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
748                        ..
749                    }
750                );
751                {
    match self.layout_of(fn_args.type_at(1)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(1), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
752                    self.layout_of(fn_args.type_at(1)).backend_repr,
753                    BackendRepr::SimdScalableVector {
754                        number_of_vectors: NumScalableVectors(1),
755                        ..
756                    }
757                );
758                self.extract_value(
759                    args[0].immediate(),
760                    fn_args.const_at(2).to_leaf().to_i32() as u64,
761                )
762            }
763
764            sym::sve_tuple_set => {
765                {
    match self.layout_of(fn_args.type_at(0)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
            .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
766                    self.layout_of(fn_args.type_at(0)).backend_repr,
767                    BackendRepr::SimdScalableVector {
768                        number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
769                        ..
770                    }
771                );
772                {
    match self.layout_of(fn_args.type_at(1)).backend_repr {
        BackendRepr::SimdScalableVector {
            number_of_vectors: NumScalableVectors(1), .. } => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
773                    self.layout_of(fn_args.type_at(1)).backend_repr,
774                    BackendRepr::SimdScalableVector {
775                        number_of_vectors: NumScalableVectors(1),
776                        ..
777                    }
778                );
779                self.insert_value(
780                    args[0].immediate(),
781                    args[1].immediate(),
782                    fn_args.const_at(2).to_leaf().to_i32() as u64,
783                )
784            }
785
786            _ if name.as_str().starts_with("simd_") => {
787                // Unpack non-power-of-2 #[repr(packed, simd)] arguments.
788                // This gives them the expected layout of a regular #[repr(simd)] vector.
789                let mut loaded_args = Vec::new();
790                for arg in args {
791                    loaded_args.push(
792                        // #[repr(packed, simd)] vectors are passed like arrays (as references,
793                        // with reduced alignment and no padding) rather than as immediates.
794                        // We can use a vector load to fix the layout and turn the argument
795                        // into an immediate.
796                        if arg.layout.ty.is_simd()
797                            && let OperandValue::Ref(place) = arg.val
798                        {
799                            let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
800                            let elem_ll_ty = match elem_ty.kind() {
801                                ty::Float(f) => self.type_float_from_ty(*f),
802                                ty::Int(i) => self.type_int_from_ty(*i),
803                                ty::Uint(u) => self.type_uint_from_ty(*u),
804                                ty::RawPtr(_, _) => self.type_ptr(),
805                                _ => ::core::panicking::unreachable_default_msg()unreachable!(),
806                            };
807                            let loaded =
808                                self.load_from_place(self.type_vector(elem_ll_ty, size), place);
809                            OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
810                        } else {
811                            *arg
812                        },
813                    );
814                }
815
816                let llret_ty = if result_layout.ty.is_simd()
817                    && let BackendRepr::Memory { .. } = result_layout.backend_repr
818                {
819                    let (size, elem_ty) = result_layout.ty.simd_size_and_type(self.tcx());
820                    let elem_ll_ty = match elem_ty.kind() {
821                        ty::Float(f) => self.type_float_from_ty(*f),
822                        ty::Int(i) => self.type_int_from_ty(*i),
823                        ty::Uint(u) => self.type_uint_from_ty(*u),
824                        ty::RawPtr(_, _) => self.type_ptr(),
825                        _ => ::core::panicking::unreachable_default_msg()unreachable!(),
826                    };
827                    self.type_vector(elem_ll_ty, size)
828                } else {
829                    result_layout.llvm_type(self)
830                };
831
832                match generic_simd_intrinsic(
833                    self,
834                    name,
835                    fn_args,
836                    &loaded_args,
837                    result_layout.ty,
838                    llret_ty,
839                    span,
840                ) {
841                    Ok(llval) => llval,
842                    // If there was an error, just skip this invocation... we'll abort compilation
843                    // anyway, but we can keep codegen'ing to find more errors.
844                    Err(err) => return IntrinsicResult::Err(err),
845                }
846            }
847
848            sym::return_address => {
849                match self.sess().target.arch {
850                    // Expand this list as needed
851                    Arch::Wasm32 | Arch::Wasm64 => {
852                        let ty = self.type_ptr();
853                        self.const_null(ty)
854                    }
855                    _ => {
856                        let ty = self.type_ix(32);
857                        let val = self.const_int(ty, 0);
858
859                        let type_params: &[&'ll Type] =
860                            if llvm_version < (23, 0, 0) { &[] } else { &[self.type_ptr()] };
861
862                        self.call_intrinsic("llvm.returnaddress", type_params, &[val])
863                    }
864                }
865            }
866
867            sym::codeview_annotation => {
868                if self.sess().target.uses_pdb_debuginfo() {
869                    codegen_codeview_annotation(self, instance, span);
870                }
871                return IntrinsicResult::Operand(OperandValue::ZeroSized);
872            }
873
874            _ => {
875                {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs:875",
                        "rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs"),
                        ::tracing_core::__macro_support::Option::Some(875u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("unknown intrinsic \'{0}\' -- falling back to default body",
                                                    name) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("unknown intrinsic '{}' -- falling back to default body", name);
876                // Call the fallback body instead of generating the intrinsic code
877                let fallback = ty::Instance::new_raw(instance.def_id(), instance.args);
878                return IntrinsicResult::Fallback(fallback);
879            }
880        };
881
882        if let BackendRepr::Memory { .. } = result_layout.backend_repr {
883            // We have an llvm immediate, but that's not what cg_ssa expects,
884            // so write it into the place (that always exists for memory)
885            if !result_layout.is_zst() {
886                self.store_to_place(llval, result_place.unwrap());
887            }
888            IntrinsicResult::WroteIntoPlace
889        } else {
890            IntrinsicResult::Operand(
891                OperandRef::from_immediate_or_packed_pair(self, llval, result_layout).val,
892            )
893        }
894    }
895
896    fn codegen_llvm_intrinsic_call(
897        &mut self,
898        instance: ty::Instance<'tcx>,
899        args: &[OperandRef<'tcx, Self::Value>],
900        _is_cleanup: bool,
901    ) -> Self::Value {
902        let tcx = self.tcx();
903
904        let fn_ty = instance.ty(tcx, self.typing_env());
905        let fn_sig = match *fn_ty.kind() {
906            ty::FnDef(def_id, args) => tcx.instantiate_bound_regions_with_erased(
907                tcx.fn_sig(def_id).instantiate(tcx, args.no_bound_vars().unwrap()).skip_norm_wip(),
908            ),
909            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
910        };
911        if !!fn_sig.c_variadic() {
    ::core::panicking::panic("assertion failed: !fn_sig.c_variadic()")
};assert!(!fn_sig.c_variadic());
912
913        let ret_layout = self.layout_of(fn_sig.output());
914        let llreturn_ty = if ret_layout.is_zst() {
915            self.type_void()
916        } else {
917            ret_layout.immediate_llvm_type(self)
918        };
919
920        let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
921        for &arg in fn_sig.inputs() {
922            let arg_layout = self.layout_of(arg);
923            if arg_layout.is_zst() {
924                continue;
925            }
926            llargument_tys.push(arg_layout.immediate_llvm_type(self));
927        }
928
929        let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) {
930            llfn
931        } else {
932            let sym = tcx.symbol_name(instance).name;
933
934            let llfn = if let Some(llfn) = self.get_declared_value(sym) {
935                llfn
936            } else {
937                intrinsic_fn(self, sym, llreturn_ty, llargument_tys, instance)
938            };
939
940            self.intrinsic_instances.borrow_mut().insert(instance, llfn);
941
942            llfn
943        };
944        let fn_ty = self.get_type_of_global(fn_ptr);
945
946        let mut llargs = args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>();
947
948        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs:948",
                        "rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs"),
                        ::tracing_core::__macro_support::Option::Some(948u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call intrinsic {0:?} with args ({1:?})",
                                                    instance, llargs) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("call intrinsic {:?} with args ({:?})", instance, llargs);
949
950        for (dest_ty, arg) in iter::zip(self.func_params_types(fn_ty), &mut llargs) {
951            let src_ty = self.val_ty(arg);
952            if !can_autocast(self, src_ty, dest_ty) {
    {
        ::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with {1:?} (found) in `{2:?}",
                dest_ty, src_ty, fn_ptr));
    }
};assert!(
953                can_autocast(self, src_ty, dest_ty),
954                "Cannot match `{dest_ty:?}` (expected) with {src_ty:?} (found) in `{fn_ptr:?}"
955            );
956
957            *arg = autocast(self, arg, src_ty, dest_ty);
958        }
959
960        let llret = unsafe {
961            llvm::LLVMBuildCallWithOperandBundles(
962                self.llbuilder(),
963                fn_ty,
964                fn_ptr,
965                llargs.as_ptr(),
966                llargs.len() as c_uint,
967                ptr::dangling(),
968                0,
969                c"".as_ptr(),
970            )
971        };
972
973        let src_ty = self.val_ty(llret);
974        let dest_ty = llreturn_ty;
975        if !can_autocast(self, dest_ty, src_ty) {
    {
        ::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with `{1:?}` (found) in `{2:?}`",
                src_ty, dest_ty, fn_ptr));
    }
};assert!(
976            can_autocast(self, dest_ty, src_ty),
977            "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
978        );
979
980        autocast(self, llret, src_ty, dest_ty)
981    }
982
983    fn abort_immediate(&mut self) {
984        self.call_intrinsic("llvm.trap", &[], &[]);
985    }
986
987    fn assume(&mut self, val: Self::Value) {
988        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
989            self.call_intrinsic("llvm.assume", &[], &[val]);
990        }
991    }
992
993    fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
994        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
995            self.call_intrinsic(
996                "llvm.expect",
997                &[self.type_i1()],
998                &[cond, self.const_bool(expected)],
999            )
1000        } else {
1001            cond
1002        }
1003    }
1004
1005    fn ctpop(&mut self, val: Self::Value) -> Self::Value {
1006        let ret = self.call_intrinsic("llvm.ctpop", &[self.val_ty(val)], &[val]);
1007        self.intcast(ret, self.type_i32(), false)
1008    }
1009
1010    fn type_checked_load(
1011        &mut self,
1012        llvtable: &'ll Value,
1013        vtable_byte_offset: u64,
1014        typeid: &[u8],
1015    ) -> Self::Value {
1016        let typeid = self.create_metadata(typeid);
1017        let typeid = self.get_metadata_value(typeid);
1018        let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
1019        let type_checked_load = self.call_intrinsic(
1020            "llvm.type.checked.load",
1021            &[],
1022            &[llvtable, vtable_byte_offset, typeid],
1023        );
1024        self.extract_value(type_checked_load, 0)
1025    }
1026
1027    fn va_start(&mut self, va_list: &'ll Value) {
1028        self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list]);
1029    }
1030
1031    fn retag_reg(&mut self, ptr: Self::Value, info: &RetagInfo<Self::Value>) -> Self::Value {
1032        codegen_retag_inner(self, "__rust_retag_reg", ptr, info)
1033    }
1034
1035    fn retag_mem(&mut self, ptr: Self::Value, info: &RetagInfo<Self::Value>) {
1036        codegen_retag_inner(self, "__rust_retag_mem", ptr, info);
1037    }
1038}
1039
1040fn llvm_arch_for(rust_arch: &Arch) -> Option<&'static str> {
1041    Some(match rust_arch {
1042        Arch::AArch64 | Arch::Arm64EC => "aarch64",
1043        Arch::AmdGpu => "amdgcn",
1044        Arch::Arm => "arm",
1045        Arch::Bpf => "bpf",
1046        Arch::Hexagon => "hexagon",
1047        Arch::LoongArch32 | Arch::LoongArch64 => "loongarch",
1048        Arch::Mips | Arch::Mips32r6 | Arch::Mips64 | Arch::Mips64r6 => "mips",
1049        Arch::Nvptx64 => "nvvm",
1050        Arch::PowerPC | Arch::PowerPC64 => "ppc",
1051        Arch::RiscV32 | Arch::RiscV64 => "riscv",
1052        Arch::S390x => "s390",
1053        Arch::SpirV => "spv",
1054        Arch::Wasm32 | Arch::Wasm64 => "wasm",
1055        Arch::X86 | Arch::X86_64 => "x86",
1056        _ => return None, // fallback for unknown archs
1057    })
1058}
1059
1060fn can_autocast<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
1061    if rust_ty == llvm_ty {
1062        return true;
1063    }
1064
1065    match cx.type_kind(llvm_ty) {
1066        // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
1067        // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
1068        // as, well, packed structs, so they won't match with those either)
1069        TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => {
1070            let rust_element_tys = cx.struct_element_types(rust_ty);
1071            let llvm_element_tys = cx.struct_element_types(llvm_ty);
1072
1073            if rust_element_tys.len() != llvm_element_tys.len() {
1074                return false;
1075            }
1076
1077            iter::zip(rust_element_tys, llvm_element_tys).all(
1078                |(rust_element_ty, llvm_element_ty)| {
1079                    can_autocast(cx, rust_element_ty, llvm_element_ty)
1080                },
1081            )
1082        }
1083        _ if let Some((llvm_element_ty, element_count)) =
1084            cx.vector_element_type_and_length(llvm_ty) =>
1085        {
1086            if llvm_element_ty == cx.type_bf16() {
1087                rust_ty == cx.type_vector(cx.type_i16(), element_count)
1088            } else if llvm_element_ty == cx.type_i1() {
1089                let int_width = element_count.next_power_of_two().max(8);
1090                rust_ty == cx.type_ix(int_width)
1091            } else {
1092                false
1093            }
1094        }
1095        TypeKind::BFloat => rust_ty == cx.type_i16(),
1096        TypeKind::X86_AMX
1097            if let Some((element_ty, element_count)) =
1098                cx.vector_element_type_and_length(rust_ty) =>
1099        {
1100            let element_size_bits = if let Some(float) = cx.float_type(element_ty) {
1101                float.size().bits()
1102            } else if let Some(bits) = cx.int_width(element_ty) {
1103                bits
1104            } else if cx.type_kind(element_ty) == TypeKind::Pointer {
1105                cx.data_layout().pointer_size().bits()
1106            } else {
1107                ::rustc_span::macros::bug_impl(None,
    format_args!("Vector element type `{0:?}` not one of integer, float or pointer",
        element_ty), Location::caller())bug!("Vector element type `{element_ty:?}` not one of integer, float or pointer")
1108            };
1109
1110            element_size_bits * element_count == 8192
1111        }
1112        _ => false,
1113    }
1114}
1115
1116fn autocast<'ll>(
1117    bx: &mut Builder<'_, 'll, '_>,
1118    val: &'ll Value,
1119    src_ty: &'ll Type,
1120    dest_ty: &'ll Type,
1121) -> &'ll Value {
1122    if src_ty == dest_ty {
1123        return val;
1124    }
1125    match (bx.type_kind(src_ty), bx.type_kind(dest_ty)) {
1126        // re-pack structs
1127        (TypeKind::Struct, TypeKind::Struct) => {
1128            let mut ret = bx.const_poison(dest_ty);
1129            for (idx, (src_element_ty, dest_element_ty)) in
1130                iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty))
1131                    .enumerate()
1132            {
1133                let elt = bx.extract_value(val, idx as u64);
1134                let casted_elt = autocast(bx, elt, src_element_ty, dest_element_ty);
1135                ret = bx.insert_value(ret, casted_elt, idx as u64);
1136            }
1137            ret
1138        }
1139        // cast from the i1xN vector type to the primitive type
1140        (TypeKind::Vector, TypeKind::Integer)
1141            if let Some((elem, vector_length)) = bx.vector_element_type_and_length(src_ty)
1142                && elem == bx.type_i1() =>
1143        {
1144            let int_width = vector_length.next_power_of_two().max(8);
1145
1146            let val = if vector_length == int_width {
1147                val
1148            } else {
1149                // zero-extends vector
1150                let shuffle_indices = match vector_length {
1151                    0 => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("zero length vectors are not allowed")));
}unreachable!("zero length vectors are not allowed"),
1152                    1 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [0, 1, 1, 1, 1, 1, 1, 1]))vec![0, 1, 1, 1, 1, 1, 1, 1],
1153                    2 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [0, 1, 2, 2, 2, 2, 2, 2]))vec![0, 1, 2, 2, 2, 2, 2, 2],
1154                    3 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [0, 1, 2, 3, 3, 3, 3, 3]))vec![0, 1, 2, 3, 3, 3, 3, 3],
1155                    4.. => (0..int_width as i32).collect(),
1156                };
1157                let shuffle_mask =
1158                    shuffle_indices.into_iter().map(|i| bx.const_i32(i)).collect::<Vec<_>>();
1159                bx.shuffle_vector(val, bx.const_null(src_ty), bx.const_vector(&shuffle_mask))
1160            };
1161            bx.bitcast(val, dest_ty)
1162        }
1163        // cast from the primitive type to the i1xN vector type
1164        (TypeKind::Integer, TypeKind::Vector)
1165            if let Some((elem, vector_length)) = bx.vector_element_type_and_length(dest_ty)
1166                && elem == bx.type_i1() =>
1167        {
1168            let int_width = vector_length.next_power_of_two().max(8);
1169
1170            let intermediate_ty = bx.type_vector(bx.type_i1(), int_width);
1171            let intermediate = bx.bitcast(val, intermediate_ty);
1172
1173            if vector_length == int_width {
1174                intermediate
1175            } else {
1176                let shuffle_mask: Vec<_> =
1177                    (0..vector_length).map(|i| bx.const_i32(i as i32)).collect();
1178                bx.shuffle_vector(
1179                    intermediate,
1180                    bx.const_poison(intermediate_ty),
1181                    bx.const_vector(&shuffle_mask),
1182                )
1183            }
1184        }
1185        (TypeKind::Vector, TypeKind::X86_AMX) => {
1186            bx.call_intrinsic("llvm.x86.cast.vector.to.tile", &[src_ty], &[val])
1187        }
1188        (TypeKind::X86_AMX, TypeKind::Vector) => {
1189            bx.call_intrinsic("llvm.x86.cast.tile.to.vector", &[dest_ty], &[val])
1190        }
1191        _ => bx.bitcast(val, dest_ty), // for `bf16(xN)` <-> `u16(xN)`
1192    }
1193}
1194
1195fn intrinsic_fn<'ll, 'tcx>(
1196    bx: &Builder<'_, 'll, 'tcx>,
1197    name: &str,
1198    rust_return_ty: &'ll Type,
1199    rust_argument_tys: Vec<&'ll Type>,
1200    instance: ty::Instance<'tcx>,
1201) -> &'ll Value {
1202    let tcx = bx.tcx;
1203
1204    let rust_fn_ty = bx.type_func(&rust_argument_tys, rust_return_ty);
1205
1206    let intrinsic = llvm::Intrinsic::lookup(name.as_bytes());
1207
1208    if let Some(intrinsic) = intrinsic
1209        && intrinsic.is_target_specific()
1210    {
1211        let (llvm_arch, _) = name[5..].split_once('.').unwrap();
1212        let rust_arch = &tcx.sess.target.arch;
1213
1214        if let Some(correct_llvm_arch) = llvm_arch_for(rust_arch)
1215            && llvm_arch != correct_llvm_arch
1216        {
1217            tcx.dcx().emit_fatal(IntrinsicWrongArch {
1218                name,
1219                target_arch: rust_arch.desc(),
1220                span: tcx.def_span(instance.def_id()),
1221            });
1222        }
1223    }
1224
1225    if let Some(intrinsic) = intrinsic
1226        && !intrinsic.is_overloaded()
1227    {
1228        // FIXME: also do this for overloaded intrinsics
1229        let llfn = intrinsic.get_declaration(bx.llmod, &[]);
1230        let llvm_fn_ty = bx.get_type_of_global(llfn);
1231
1232        let llvm_return_ty = bx.get_return_type(llvm_fn_ty);
1233        let llvm_argument_tys = bx.func_params_types(llvm_fn_ty);
1234        let llvm_is_variadic = bx.func_is_variadic(llvm_fn_ty);
1235
1236        let is_correct_signature = !llvm_is_variadic
1237            && rust_argument_tys.len() == llvm_argument_tys.len()
1238            && iter::once((rust_return_ty, llvm_return_ty))
1239                .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
1240                .all(|(rust_ty, llvm_ty)| can_autocast(bx, rust_ty, llvm_ty));
1241
1242        if !is_correct_signature {
1243            tcx.dcx().emit_fatal(IntrinsicSignatureMismatch {
1244                name,
1245                llvm_fn_ty: &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0:?}", llvm_fn_ty))
    })format!("{llvm_fn_ty:?}"),
1246                rust_fn_ty: &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0:?}", rust_fn_ty))
    })format!("{rust_fn_ty:?}"),
1247                span: tcx.def_span(instance.def_id()),
1248            });
1249        }
1250
1251        return llfn;
1252    }
1253
1254    // Function addresses in Rust are never significant, allowing functions to be merged.
1255    let llfn = declare_raw_fn(
1256        bx,
1257        name,
1258        llvm::CCallConv,
1259        llvm::UnnamedAddr::Global,
1260        llvm::Visibility::Default,
1261        rust_fn_ty,
1262    );
1263
1264    if intrinsic.is_none() {
1265        let mut new_llfn = None;
1266        let can_upgrade = unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn) };
1267
1268        if !can_upgrade {
1269            // This is either plain wrong, or this can be caused by incompatible LLVM versions
1270            tcx.dcx().emit_fatal(UnknownIntrinsic { name, span: tcx.def_span(instance.def_id()) });
1271        } else if let Some(def_id) = instance.def_id().as_local() {
1272            // we can emit diagnostics only for local crates
1273            let hir_id = tcx.local_def_id_to_hir_id(def_id);
1274
1275            // not all intrinsics are upgraded to some other intrinsics, most are upgraded to instruction sequences
1276            let msg = if let Some(new_llfn) = new_llfn {
1277                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("using deprecated intrinsic `{1}`, `{0}` can be used instead",
                str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap(),
                name))
    })format!(
1278                    "using deprecated intrinsic `{name}`, `{}` can be used instead",
1279                    str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap()
1280                )
1281            } else {
1282                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("using deprecated intrinsic `{0}`",
                name))
    })format!("using deprecated intrinsic `{name}`")
1283            };
1284
1285            tcx.emit_node_lint(
1286                DEPRECATED_LLVM_INTRINSIC,
1287                hir_id,
1288                rustc_errors::DiagDecorator(|d| {
1289                    d.primary_message(msg).span(tcx.hir_span(hir_id));
1290                }),
1291            );
1292        }
1293    }
1294
1295    llfn
1296}
1297
1298fn catch_unwind_intrinsic<'ll, 'tcx>(
1299    bx: &mut Builder<'_, 'll, 'tcx>,
1300    try_func: &'ll Value,
1301    data: &'ll Value,
1302    catch_func: &'ll Value,
1303) -> &'ll Value {
1304    if !bx.sess().panic_strategy().unwinds() {
1305        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1306        bx.call(try_func_ty, None, None, try_func, ReturnSlot::Direct, &[data], None, None);
1307        // Return 0 unconditionally from the intrinsic call;
1308        // we can never unwind.
1309        bx.const_bool(false)
1310    } else if wants_msvc_seh(&bx.sess().target) {
1311        codegen_msvc_try(bx, try_func, data, catch_func)
1312    } else if wants_wasm_eh(&bx.sess().target) {
1313        codegen_wasm_try(bx, try_func, data, catch_func)
1314    } else {
1315        codegen_gnu_try(bx, try_func, data, catch_func)
1316    }
1317}
1318
1319// MSVC's definition of the `rust_try` function.
1320//
1321// This implementation uses the new exception handling instructions in LLVM
1322// which have support in LLVM for SEH on MSVC targets. Although these
1323// instructions are meant to work for all targets, as of the time of this
1324// writing, however, LLVM does not recommend the usage of these new instructions
1325// as the old ones are still more optimized.
1326fn codegen_msvc_try<'ll, 'tcx>(
1327    bx: &mut Builder<'_, 'll, 'tcx>,
1328    try_func: &'ll Value,
1329    data: &'ll Value,
1330    catch_func: &'ll Value,
1331) -> &'ll Value {
1332    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1333        bx.set_personality_fn(bx.eh_personality());
1334
1335        let normal = bx.append_sibling_block("normal");
1336        let catchswitch = bx.append_sibling_block("catchswitch");
1337        let catchpad_rust = bx.append_sibling_block("catchpad_rust");
1338        let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
1339        let caught = bx.append_sibling_block("caught");
1340
1341        let try_func = llvm::get_param(bx.llfn(), 0);
1342        let data = llvm::get_param(bx.llfn(), 1);
1343        let catch_func = llvm::get_param(bx.llfn(), 2);
1344
1345        // We're generating an IR snippet that looks like:
1346        //
1347        //   declare bool @rust_try(%try_func, %data, %catch_func) {
1348        //      %slot = alloca i8*
1349        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
1350        //
1351        //   normal:
1352        //      ret i1 false
1353        //
1354        //   catchswitch:
1355        //      %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller
1356        //
1357        //   catchpad_rust:
1358        //      %tok = catchpad within %cs [%type_descriptor, 8, %slot]
1359        //      %ptr = load %slot
1360        //      call %catch_func(%data, %ptr)
1361        //      catchret from %tok to label %caught
1362        //
1363        //   catchpad_foreign:
1364        //      %tok = catchpad within %cs [null, 64, null]
1365        //      call %catch_func(%data, null)
1366        //      catchret from %tok to label %caught
1367        //
1368        //   caught:
1369        //      ret i1 true
1370        //   }
1371        //
1372        // This structure follows the basic usage of throw/try/catch in LLVM.
1373        // For example, compile this C++ snippet to see what LLVM generates:
1374        //
1375        //      struct rust_panic {
1376        //          rust_panic(const rust_panic&);
1377        //          ~rust_panic();
1378        //
1379        //          void* x[2];
1380        //      };
1381        //
1382        //      int __rust_try(
1383        //          void (*try_func)(void*),
1384        //          void *data,
1385        //          void (*catch_func)(void*, void*) noexcept
1386        //      ) {
1387        //          try {
1388        //              try_func(data);
1389        //              return 0;
1390        //          } catch(rust_panic& a) {
1391        //              catch_func(data, &a);
1392        //              return 1;
1393        //          } catch(...) {
1394        //              catch_func(data, NULL);
1395        //              return 1;
1396        //          }
1397        //      }
1398        //
1399        // More information can be found in libstd's seh.rs implementation.
1400        let ptr_size = bx.tcx().data_layout.pointer_size();
1401        let ptr_align = bx.tcx().data_layout.pointer_align();
1402        let slot = bx.alloca(ptr_size, ptr_align);
1403        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1404        bx.invoke(
1405            try_func_ty,
1406            None,
1407            None,
1408            try_func,
1409            ReturnSlot::Direct,
1410            &[data],
1411            normal,
1412            catchswitch,
1413            None,
1414            None,
1415        );
1416
1417        bx.switch_to_block(normal);
1418        bx.ret(bx.const_bool(false));
1419
1420        bx.switch_to_block(catchswitch);
1421        let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
1422
1423        // We can't use the TypeDescriptor defined in libpanic_unwind because it
1424        // might be in another DLL and the SEH encoding only supports specifying
1425        // a TypeDescriptor from the current module.
1426        //
1427        // However this isn't an issue since the MSVC runtime uses string
1428        // comparison on the type name to match TypeDescriptors rather than
1429        // pointer equality.
1430        //
1431        // So instead we generate a new TypeDescriptor in each module that uses
1432        // `try` and let the linker merge duplicate definitions in the same
1433        // module.
1434        //
1435        // When modifying, make sure that the type_name string exactly matches
1436        // the one used in library/panic_unwind/src/seh.rs.
1437        let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
1438        let type_name = bx.const_bytes(b"rust_panic\0");
1439        let type_info =
1440            bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
1441        let tydesc = bx.declare_global(
1442            &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
1443            bx.val_ty(type_info),
1444        );
1445
1446        llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
1447        if bx.cx.tcx.sess.target.supports_comdat() {
1448            llvm::SetUniqueComdat(bx.llmod, tydesc);
1449        }
1450        llvm::set_initializer(tydesc, type_info);
1451
1452        // The flag value of 8 indicates that we are catching the exception by
1453        // reference instead of by value. We can't use catch by value because
1454        // that requires copying the exception object, which we don't support
1455        // since our exception object effectively contains a Box.
1456        //
1457        // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
1458        bx.switch_to_block(catchpad_rust);
1459        let flags = bx.const_i32(8);
1460        let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
1461        let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
1462        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1463        bx.call(
1464            catch_ty,
1465            None,
1466            None,
1467            catch_func,
1468            ReturnSlot::Direct,
1469            &[data, ptr],
1470            Some(&funclet),
1471            None,
1472        );
1473        bx.catch_ret(&funclet, caught);
1474
1475        // The flag value of 64 indicates a "catch-all".
1476        bx.switch_to_block(catchpad_foreign);
1477        let flags = bx.const_i32(64);
1478        let null = bx.const_null(bx.type_ptr());
1479        let funclet = bx.catch_pad(cs, &[null, flags, null]);
1480        bx.call(
1481            catch_ty,
1482            None,
1483            None,
1484            catch_func,
1485            ReturnSlot::Direct,
1486            &[data, null],
1487            Some(&funclet),
1488            None,
1489        );
1490        bx.catch_ret(&funclet, caught);
1491
1492        bx.switch_to_block(caught);
1493        bx.ret(bx.const_bool(true));
1494    });
1495
1496    // Note that no invoke is used here because by definition this function
1497    // can't panic (that's what it's catching).
1498    let ret = bx.call(
1499        llty,
1500        None,
1501        None,
1502        llfn,
1503        ReturnSlot::Direct,
1504        &[try_func, data, catch_func],
1505        None,
1506        None,
1507    );
1508    ret
1509}
1510
1511// WASM's definition of the `rust_try` function.
1512fn codegen_wasm_try<'ll, 'tcx>(
1513    bx: &mut Builder<'_, 'll, 'tcx>,
1514    try_func: &'ll Value,
1515    data: &'ll Value,
1516    catch_func: &'ll Value,
1517) -> &'ll Value {
1518    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1519        bx.set_personality_fn(bx.eh_personality());
1520
1521        let normal = bx.append_sibling_block("normal");
1522        let catchswitch = bx.append_sibling_block("catchswitch");
1523        let catchpad = bx.append_sibling_block("catchpad");
1524        let caught = bx.append_sibling_block("caught");
1525
1526        let try_func = llvm::get_param(bx.llfn(), 0);
1527        let data = llvm::get_param(bx.llfn(), 1);
1528        let catch_func = llvm::get_param(bx.llfn(), 2);
1529
1530        // We're generating an IR snippet that looks like:
1531        //
1532        //   declare i1 @rust_try(%try_func, %data, %catch_func) {
1533        //      %slot = alloca i8*
1534        //      invoke %try_func(%data) to label %normal unwind label %catchswitch
1535        //
1536        //   normal:
1537        //      ret i1 false
1538        //
1539        //   catchswitch:
1540        //      %cs = catchswitch within none [%catchpad] unwind to caller
1541        //
1542        //   catchpad:
1543        //      %tok = catchpad within %cs [null]
1544        //      %ptr = call @llvm.wasm.get.exception(token %tok)
1545        //      %sel = call @llvm.wasm.get.ehselector(token %tok)
1546        //      call %catch_func(%data, %ptr)
1547        //      catchret from %tok to label %caught
1548        //
1549        //   caught:
1550        //      ret i1 true
1551        //   }
1552        //
1553        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1554        bx.invoke(
1555            try_func_ty,
1556            None,
1557            None,
1558            try_func,
1559            ReturnSlot::Direct,
1560            &[data],
1561            normal,
1562            catchswitch,
1563            None,
1564            None,
1565        );
1566
1567        bx.switch_to_block(normal);
1568        bx.ret(bx.const_bool(false));
1569
1570        bx.switch_to_block(catchswitch);
1571        let cs = bx.catch_switch(None, None, &[catchpad]);
1572
1573        bx.switch_to_block(catchpad);
1574        let null = bx.const_null(bx.type_ptr());
1575        let funclet = bx.catch_pad(cs, &[null]);
1576
1577        let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
1578        let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
1579
1580        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1581        bx.call(
1582            catch_ty,
1583            None,
1584            None,
1585            catch_func,
1586            ReturnSlot::Direct,
1587            &[data, ptr],
1588            Some(&funclet),
1589            None,
1590        );
1591        bx.catch_ret(&funclet, caught);
1592
1593        bx.switch_to_block(caught);
1594        bx.ret(bx.const_bool(true));
1595    });
1596
1597    // Note that no invoke is used here because by definition this function
1598    // can't panic (that's what it's catching).
1599    let ret = bx.call(
1600        llty,
1601        None,
1602        None,
1603        llfn,
1604        ReturnSlot::Direct,
1605        &[try_func, data, catch_func],
1606        None,
1607        None,
1608    );
1609    ret
1610}
1611
1612// Definition of the standard `try` function for Rust using the GNU-like model
1613// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
1614// instructions).
1615//
1616// This codegen is a little surprising because we always call a shim
1617// function instead of inlining the call to `invoke` manually here. This is done
1618// because in LLVM we're only allowed to have one personality per function
1619// definition. The call to the `try` intrinsic is being inlined into the
1620// function calling it, and that function may already have other personality
1621// functions in play. By calling a shim we're guaranteed that our shim will have
1622// the right personality function.
1623fn codegen_gnu_try<'ll, 'tcx>(
1624    bx: &mut Builder<'_, 'll, 'tcx>,
1625    try_func: &'ll Value,
1626    data: &'ll Value,
1627    catch_func: &'ll Value,
1628) -> &'ll Value {
1629    let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1630        // Codegens the shims described above:
1631        //
1632        //   bx:
1633        //      invoke %try_func(%data) normal %normal unwind %catch
1634        //
1635        //   normal:
1636        //      ret 0
1637        //
1638        //   catch:
1639        //      (%ptr, _) = landingpad
1640        //      call %catch_func(%data, %ptr)
1641        //      ret 1
1642        let then = bx.append_sibling_block("then");
1643        let catch = bx.append_sibling_block("catch");
1644
1645        let try_func = llvm::get_param(bx.llfn(), 0);
1646        let data = llvm::get_param(bx.llfn(), 1);
1647        let catch_func = llvm::get_param(bx.llfn(), 2);
1648        let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1649        bx.invoke(
1650            try_func_ty,
1651            None,
1652            None,
1653            try_func,
1654            ReturnSlot::Direct,
1655            &[data],
1656            then,
1657            catch,
1658            None,
1659            None,
1660        );
1661
1662        bx.switch_to_block(then);
1663        bx.ret(bx.const_bool(false));
1664
1665        // Type indicator for the exception being thrown.
1666        //
1667        // The first value in this tuple is a pointer to the exception object
1668        // being thrown. The second value is a "selector" indicating which of
1669        // the landing pad clauses the exception's type had been matched to.
1670        // rust_try ignores the selector.
1671        bx.switch_to_block(catch);
1672        let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()]);
1673        let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
1674        let tydesc = bx.const_null(bx.type_ptr());
1675        bx.add_clause(vals, tydesc);
1676        let ptr = bx.extract_value(vals, 0);
1677        let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1678        bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None);
1679        bx.ret(bx.const_bool(true));
1680    });
1681
1682    // Note that no invoke is used here because by definition this function
1683    // can't panic (that's what it's catching).
1684    let ret = bx.call(
1685        llty,
1686        None,
1687        None,
1688        llfn,
1689        ReturnSlot::Direct,
1690        &[try_func, data, catch_func],
1691        None,
1692        None,
1693    );
1694    ret
1695}
1696
1697// Helper function to give a Block to a closure to codegen a shim function.
1698// This is currently primarily used for the `try` intrinsic functions above.
1699fn gen_fn<'a, 'll, 'tcx>(
1700    cx: &'a CodegenCx<'ll, 'tcx>,
1701    name: &str,
1702    rust_fn_sig: ty::PolyFnSig<'tcx>,
1703    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1704) -> (&'ll Type, &'ll Value) {
1705    let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1706    let llty = fn_abi.llvm_type(cx);
1707    let llfn = cx.declare_fn(name, fn_abi, None);
1708    cx.set_frame_pointer_type(llfn);
1709    cx.apply_target_cpu_attr(llfn);
1710    // FIXME(eddyb) find a nicer way to do this.
1711    llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1712    let llbb = Builder::append_block(cx, llfn, "entry-block");
1713    let bx = Builder::build(cx, llbb);
1714    codegen(bx);
1715    (llty, llfn)
1716}
1717
1718// Helper function used to get a handle to the `__rust_try` function used to
1719// catch exceptions.
1720//
1721// This function is only generated once and is then cached.
1722fn get_rust_try_fn<'a, 'll, 'tcx>(
1723    cx: &'a CodegenCx<'ll, 'tcx>,
1724    codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1725) -> (&'ll Type, &'ll Value) {
1726    if let Some(llfn) = cx.rust_try_fn.get() {
1727        return llfn;
1728    }
1729
1730    // Define the type up front for the signature of the rust_try function.
1731    let tcx = cx.tcx;
1732    let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1733    // `unsafe fn(*mut Data) -> ()`
1734    let try_fn_ty = Ty::new_fn_ptr(
1735        tcx,
1736        ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p], tcx.types.unit, hir::Safety::Unsafe)),
1737    );
1738    // `unsafe fn(*mut Data, *mut i8) -> ()`
1739    let catch_fn_ty = Ty::new_fn_ptr(
1740        tcx,
1741        ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p, i8p], tcx.types.unit, hir::Safety::Unsafe)),
1742    );
1743    // `unsafe fn(unsafe fn(*mut Data) -> (), *mut Data, unsafe fn(*mut Data, *mut i8) -> ()) -> bool`
1744    let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig_rust_abi(
1745        [try_fn_ty, i8p, catch_fn_ty],
1746        tcx.types.bool,
1747        hir::Safety::Unsafe,
1748    ));
1749    let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1750
1751    if cx.sess().pointer_authentication() {
1752        let cfg = cx.sess().pointer_auth_config.as_ref().unwrap();
1753        let attrs: Vec<&Attribute> =
1754            cfg.fn_attrs().into_iter().map(|name| llvm::CreateAttrString(cx.llcx, name)).collect();
1755
1756        let (_ty, rust_try_fn) = rust_try;
1757        crate::attributes::apply_to_llfn(rust_try_fn, AttributePlace::Function, &attrs);
1758    }
1759
1760    cx.rust_try_fn.set(Some(rust_try));
1761    rust_try
1762}
1763
1764fn codegen_retag_inner<'ll, 'tcx>(
1765    bx: &mut Builder<'_, 'll, 'tcx>,
1766    name: &'static str,
1767    ptr: &'ll Value,
1768    info: &RetagInfo<&'ll Value>,
1769) -> &'ll Value {
1770    let size = bx.const_usize(info.size.bytes());
1771    let perms = bx.const_u8(info.flags.bits());
1772
1773    bx.call_intrinsic(
1774        name,
1775        // Retag intrinsics have special handling within `CodegenCx::declare_intrinsic`
1776        // to ensure that each form has the correct return type.
1777        &[bx.type_ptr(), bx.val_ty(size), bx.type_i8(), bx.type_ptr(), bx.type_ptr()],
1778        &[ptr, size, perms, info.im_layout, info.pin_layout],
1779    )
1780}
1781
1782fn codegen_autodiff<'ll, 'tcx>(
1783    bx: &mut Builder<'_, 'll, 'tcx>,
1784    instance: ty::Instance<'tcx>,
1785    args: &[OperandRef<'tcx, &'ll Value>],
1786    result_layout: ty::layout::TyAndLayout<'tcx>,
1787    result_place: Option<PlaceValue<&'ll Value>>,
1788) -> IntrinsicResult<'tcx, &'ll Value> {
1789    let tcx = bx.tcx;
1790    if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
1791        let _ = tcx.dcx().emit_err(AutoDiffWithoutEnable);
1792    }
1793
1794    let ct = tcx.crate_types();
1795    let lto = tcx.sess.lto();
1796    if ct.len() == 1 && ct.contains(&CrateType::Executable) {
1797        if lto != rustc_session::config::Lto::Fat {
1798            let _ = tcx.dcx().emit_err(AutoDiffWithoutLto);
1799        }
1800    } else {
1801        if lto != rustc_session::config::Lto::Fat && !tcx.sess.opts.cg.linker_plugin_lto.enabled() {
1802            let _ = tcx.dcx().emit_err(AutoDiffWithoutLto);
1803        }
1804    }
1805
1806    let fn_args = instance.args;
1807    let callee_ty = instance.ty(tcx, bx.typing_env());
1808
1809    let sig = callee_ty.fn_sig(tcx).skip_binder();
1810
1811    let ret_ty = sig.output();
1812    let llret_ty = bx.layout_of(ret_ty).llvm_type(bx);
1813
1814    let source_fn_ptr_ty = fn_args.into_type_list(tcx)[0];
1815    let fn_to_diff = args[0].immediate();
1816
1817    let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() {
1818        ty::FnDef(def_id, diff_args) => (def_id, diff_args.no_bound_vars().unwrap()),
1819        _ => ::rustc_span::macros::bug_impl(None, format_args!("invalid args"),
    Location::caller())bug!("invalid args"),
1820    };
1821
1822    let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) {
1823        Ok(Some(instance)) => instance,
1824        Ok(None) => ::rustc_span::macros::bug_impl(None,
    format_args!("could not resolve ({0:?}, {1:?}) to a specific autodiff instance",
        diff_id, diff_args), Location::caller())bug!(
1825            "could not resolve ({:?}, {:?}) to a specific autodiff instance",
1826            diff_id,
1827            diff_args
1828        ),
1829        Err(err) => {
1830            // An error has already been emitted
1831            return IntrinsicResult::Err(err);
1832        }
1833    };
1834
1835    let val_arr = get_args_from_tuple(bx, args[2], fn_diff);
1836    let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE);
1837
1838    let Some(Some(mut diff_attrs)) =
1839        {
    {
        'done:
            {
            for i in
                ::rustc_attr_ir::HasAttrs::get_attrs(fn_diff.def_id(), &tcx) {
                #[allow(unused_imports)]
                use ::rustc_attr_ir::AttributeKind::*;
                let i: &::rustc_attr_ir::Attribute = i;
                match i {
                    ::rustc_attr_ir::Attribute::Parsed(RustcAutodiff(attr)) => {
                        break 'done Some(attr.clone());
                    }
                    ::rustc_attr_ir::Attribute::Tool(..) |
                        ::rustc_attr_ir::Attribute::DeriveHelper(..) =>
                        {}
                        #[deny(unreachable_patterns)]
                        _ => {}
                }
            }
            None
        }
    }
}find_attr!(tcx, fn_diff.def_id(), RustcAutodiff(attr) => attr.clone())
1840    else {
1841        ::rustc_span::macros::bug_impl(None,
    format_args!("could not find autodiff attrs"), Location::caller())bug!("could not find autodiff attrs")
1842    };
1843
1844    adjust_activity_to_abi(
1845        tcx,
1846        source_fn_ptr_ty,
1847        bx.cx.typing_env(),
1848        &mut diff_attrs.input_activity,
1849    );
1850
1851    let fnc_tree = fnc_typetrees(tcx, source_fn_ptr_ty);
1852
1853    // Build body
1854    generate_enzyme_call(
1855        bx,
1856        fn_to_diff,
1857        &diff_symbol,
1858        llret_ty,
1859        &val_arr,
1860        &diff_attrs,
1861        result_layout,
1862        result_place,
1863        fnc_tree,
1864    )
1865}
1866
1867// Generates the LLVM code to offload a Rust function to a target device (e.g., GPU).
1868// For each kernel call, it generates the necessary globals (including metadata such as
1869// size and pass mode), manages memory mapping to and from the device, handles all
1870// data transfers, and launches the kernel on the target device.
1871fn codegen_offload<'ll, 'tcx>(
1872    bx: &mut Builder<'_, 'll, 'tcx>,
1873    tcx: TyCtxt<'tcx>,
1874    instance: ty::Instance<'tcx>,
1875    args: &[OperandRef<'tcx, &'ll Value>],
1876) {
1877    let cx = bx.cx;
1878    let fn_args = instance.args;
1879
1880    let (target_id, target_args) = match fn_args.into_type_list(tcx)[0].kind() {
1881        ty::FnDef(def_id, params) => (def_id, params.no_bound_vars().unwrap()),
1882        _ => ::rustc_span::macros::bug_impl(None,
    format_args!("invalid offload intrinsic arg"), Location::caller())bug!("invalid offload intrinsic arg"),
1883    };
1884
1885    let fn_target = match Instance::try_resolve(tcx, cx.typing_env(), *target_id, target_args) {
1886        Ok(Some(instance)) => instance,
1887        Ok(None) => ::rustc_span::macros::bug_impl(None,
    format_args!("could not resolve ({0:?}, {1:?}) to a specific offload instance",
        target_id, target_args), Location::caller())bug!(
1888            "could not resolve ({:?}, {:?}) to a specific offload instance",
1889            target_id,
1890            target_args
1891        ),
1892        Err(_) => {
1893            // An error has already been emitted
1894            return;
1895        }
1896    };
1897
1898    let offload_dims = OffloadKernelDims::from_operands(bx, &args[1], &args[2]);
1899    let dyn_cache = match args[3].val {
1900        OperandValue::Immediate(val) => val,
1901        _ => { ::core::panicking::panic_fmt(format_args!("unparsable")); }panic!("unparsable"),
1902    };
1903    let device_id = match args[4].val {
1904        OperandValue::Immediate(val) => val,
1905        _ => { ::core::panicking::panic_fmt(format_args!("unparsable")); }panic!("unparsable"),
1906    };
1907    let args = get_args_from_tuple(bx, args[5], fn_target);
1908    let target_symbol = mangle_offload_export(tcx, fn_target);
1909
1910    let sig = tcx.fn_sig(fn_target.def_id()).instantiate(tcx, fn_target.args).skip_norm_wip();
1911    let sig = tcx.instantiate_bound_regions_with_erased(sig);
1912    let inputs = sig.inputs();
1913
1914    let fn_abi = cx.fn_abi_of_instance(fn_target, ty::List::empty());
1915
1916    let mut metadata = Vec::new();
1917    let mut types = Vec::new();
1918
1919    for (i, arg_abi) in fn_abi.args.iter().enumerate() {
1920        let ty = inputs[i];
1921        let decomposed = OffloadMetadata::handle_abi(cx, tcx, ty, arg_abi);
1922
1923        for (meta, entry_ty) in decomposed {
1924            metadata.push(meta);
1925            types.push(bx.cx.layout_of(entry_ty).llvm_type(bx.cx));
1926        }
1927    }
1928
1929    let offload_globals_ref = cx.offload_globals.borrow();
1930    let offload_globals = match offload_globals_ref.as_ref() {
1931        Some(globals) => globals,
1932        None => {
1933            // Offload is not initialized, cannot continue
1934            return;
1935        }
1936    };
1937    let offload_data =
1938        gpu_offload::gen_define_handling(&cx, &metadata, target_symbol, offload_globals);
1939    gpu_offload::gen_call_handling(
1940        bx,
1941        &offload_data,
1942        &args,
1943        &types,
1944        &metadata,
1945        offload_globals,
1946        &offload_dims,
1947        &dyn_cache,
1948        &device_id,
1949    );
1950}
1951
1952fn get_args_from_tuple<'ll, 'tcx>(
1953    bx: &mut Builder<'_, 'll, 'tcx>,
1954    tuple_op: OperandRef<'tcx, &'ll Value>,
1955    fn_instance: Instance<'tcx>,
1956) -> Vec<&'ll Value> {
1957    let cx = bx.cx;
1958    let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty());
1959
1960    match tuple_op.val {
1961        OperandValue::Immediate(val) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [val]))vec![val],
1962        OperandValue::Pair(v1, v2) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [v1, v2]))vec![v1, v2],
1963        OperandValue::Ref(ptr) => {
1964            let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout };
1965
1966            let mut result = Vec::with_capacity(fn_abi.args.len());
1967            let mut tuple_index = 0;
1968
1969            for arg in &fn_abi.args {
1970                match arg.mode {
1971                    PassMode::Ignore => {}
1972                    PassMode::Direct(_) | PassMode::Cast { .. } => {
1973                        let field = tuple_place.project_field(bx, tuple_index);
1974                        let llvm_ty = field.layout.llvm_type(bx.cx);
1975                        let val = bx.load(llvm_ty, field.val.llval, field.val.align);
1976                        result.push(val);
1977                        tuple_index += 1;
1978                    }
1979                    PassMode::Pair(_, _) => {
1980                        let field = tuple_place.project_field(bx, tuple_index);
1981                        let llvm_ty = field.layout.llvm_type(bx.cx);
1982                        let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align);
1983                        result.push(bx.extract_value(pair_val, 0));
1984                        result.push(bx.extract_value(pair_val, 1));
1985                        tuple_index += 1;
1986                    }
1987                    PassMode::Indirect { .. } => {
1988                        let field = tuple_place.project_field(bx, tuple_index);
1989                        result.push(field.val.llval);
1990                        tuple_index += 1;
1991                    }
1992                    PassMode::IndirectUnsized { .. } => {
1993                        ::rustc_span::macros::bug_impl(None,
    format_args!("autodiff/offload args must not be unsized"),
    Location::caller());bug!("autodiff/offload args must not be unsized");
1994                    }
1995                }
1996            }
1997
1998            result
1999        }
2000
2001        OperandValue::ZeroSized => ::alloc::vec::Vec::new()vec![],
2002    }
2003}
2004
2005fn generic_simd_intrinsic<'ll, 'tcx>(
2006    bx: &mut Builder<'_, 'll, 'tcx>,
2007    name: Symbol,
2008    fn_args: GenericArgs<'tcx>,
2009    args: &[OperandRef<'tcx, &'ll Value>],
2010    ret_ty: Ty<'tcx>,
2011    llret_ty: &'ll Type,
2012    span: Span,
2013) -> Result<&'ll Value, ErrorGuaranteed> {
2014    macro_rules! return_error {
2015        ($diag: expr) => {{
2016            let err = bx.sess().dcx().emit_err($diag);
2017            return Err(err);
2018        }};
2019    }
2020
2021    macro_rules! require {
2022        ($cond: expr, $diag: expr) => {
2023            if !$cond {
2024                return_error!($diag);
2025            }
2026        };
2027    }
2028
2029    macro_rules! require_simd {
2030        ($ty: expr, $variant:ident) => {{
2031            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
2032            $ty.simd_size_and_type(bx.tcx())
2033        }};
2034    }
2035
2036    macro_rules! require_simd_or_scalable {
2037        ($ty: expr, $variant:ident) => {{
2038            require!(
2039                $ty.is_simd() || $ty.is_scalable_vector(),
2040                InvalidMonomorphization::$variant { span, name, ty: $ty }
2041            );
2042            if $ty.is_simd() {
2043                let (len, ty) = $ty.simd_size_and_type(bx.tcx());
2044                (len, ty, None)
2045            } else {
2046                let (count, ty, num_vecs) =
2047                    $ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
2048                (count as u64, ty, Some(num_vecs))
2049            }
2050        }};
2051    }
2052
2053    /// Returns the bitwidth of the `$ty` argument if it is an `Int` or `Uint` type.
2054    macro_rules! require_int_or_uint_ty {
2055        ($ty: expr, $diag: expr) => {
2056            match $ty {
2057                ty::Int(i) => {
2058                    i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2059                }
2060                ty::Uint(i) => {
2061                    i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2062                }
2063                _ => {
2064                    return_error!($diag);
2065                }
2066            }
2067        };
2068    }
2069
2070    /// Converts a vector mask, where each element has a bit width equal to the data elements it is used with,
2071    /// down to an i1 based mask that can be used by llvm intrinsics.
2072    ///
2073    /// The rust simd semantics are that each element should either consist of all ones or all zeroes,
2074    /// but this information is not available to llvm. Truncating the vector effectively uses the lowest bit,
2075    /// but codegen for several targets is better if we consider the highest bit by shifting.
2076    ///
2077    /// For x86 SSE/AVX targets this is beneficial since most instructions with mask parameters only consider the highest bit.
2078    /// So even though on llvm level we have an additional shift, in the final assembly there is no shift or truncate and
2079    /// instead the mask can be used as is.
2080    ///
2081    /// For aarch64 and other targets there is a benefit because a mask from the sign bit can be more
2082    /// efficiently converted to an all ones / all zeroes mask by comparing whether each element is negative.
2083    fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
2084        bx: &mut Builder<'a, 'll, 'tcx>,
2085        i_xn: &'ll Value,
2086        in_elem_bitwidth: u64,
2087        in_len: u64,
2088    ) -> &'ll Value {
2089        // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
2090        let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
2091        let shift_indices = ::alloc::vec::from_elem_global(shift_idx, in_len as _)vec![shift_idx; in_len as _];
2092        let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
2093        // Truncate vector to an <i1 x N>
2094        bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
2095    }
2096
2097    // Sanity-check: all vector arguments must be immediates.
2098    if truecfg!(debug_assertions) {
2099        for arg in args {
2100            if arg.layout.ty.is_simd() {
2101                {
    match arg.val {
        OperandValue::Immediate(_) => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "OperandValue::Immediate(_)", ::core::option::Option::None);
        }
    }
};assert_matches!(arg.val, OperandValue::Immediate(_));
2102            }
2103        }
2104    }
2105
2106    if name == sym::simd_select_bitmask {
2107        let (len, _) = {
    if !args[1].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
                        span,
                        name,
                        ty: args[1].layout.ty,
                    });
            return Err(err);
        };
    };
    args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdArgument);
2108
2109        let expected_int_bits = len.max(8).next_power_of_two();
2110        let expected_bytes = len.div_ceil(8);
2111
2112        let mask_ty = args[0].layout.ty;
2113        let mask = match mask_ty.kind() {
2114            ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
2115            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
2116            ty::Array(elem, len)
2117                if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
    ty::Uint(ty::UintTy::U8) => true,
    _ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2118                    && len
2119                        .try_to_target_usize(bx.tcx)
2120                        .expect("expected monomorphic const in codegen")
2121                        == expected_bytes =>
2122            {
2123                let place = PlaceRef::alloca(bx, args[0].layout);
2124                args[0].val.store(bx, place);
2125                let int_ty = bx.type_ix(expected_bytes * 8);
2126                bx.load(int_ty, place.val.llval, Align::ONE)
2127            }
2128            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::InvalidBitmask {
                span,
                name,
                mask_ty,
                expected_int_bits,
                expected_bytes,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::InvalidBitmask {
2129                span,
2130                name,
2131                mask_ty,
2132                expected_int_bits,
2133                expected_bytes
2134            }),
2135        };
2136
2137        let i1 = bx.type_i1();
2138        let im = bx.type_ix(len);
2139        let i1xn = bx.type_vector(i1, len);
2140        let m_im = bx.trunc(mask, im);
2141        let m_i1s = bx.bitcast(m_im, i1xn);
2142        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2143    }
2144
2145    if name == sym::simd_splat {
2146        let (out_len, out_ty) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2147
2148        if !(args[0].layout.ty == out_ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
                    span,
                    name,
                    expected_element: out_ty,
                    vector_type: ret_ty,
                });
        return Err(err);
    };
};require!(
2149            args[0].layout.ty == out_ty,
2150            InvalidMonomorphization::ExpectedVectorElementType {
2151                span,
2152                name,
2153                expected_element: out_ty,
2154                vector_type: ret_ty,
2155            }
2156        );
2157
2158        // `insertelement <N x elem> poison, elem %x, i32 0`
2159        let poison_vec = bx.const_poison(llret_ty);
2160        let idx0 = bx.const_i32(0);
2161        let v0 = bx.insert_element(poison_vec, args[0].immediate(), idx0);
2162
2163        // `shufflevector <N x elem> v0, <N x elem> poison, <N x i32> zeroinitializer`
2164        // The masks is all zeros, so this splats lane 0 (which has our element in it).
2165        let mask_ty = bx.type_vector(bx.type_i32(), out_len);
2166        let splat = bx.shuffle_vector(v0, poison_vec, bx.const_null(mask_ty));
2167
2168        return Ok(splat);
2169    }
2170
2171    let supports_scalable = match name {
2172        sym::simd_cast | sym::simd_select => true,
2173        _ => false,
2174    };
2175
2176    // Every intrinsic below takes a SIMD vector as its first argument. Some intrinsics also accept
2177    // scalable vectors. `require_simd_or_scalable` is used regardless as it'll do the right thing
2178    // for non-scalable vectors, and an additional check to prohibit scalable vectors for those
2179    // intrinsics that do not support them is added.
2180    if !supports_scalable {
2181        let _ = {
    if !args[0].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
                        span,
                        name,
                        ty: args[0].layout.ty,
                    });
            return Err(err);
        };
    };
    args[0].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[0].layout.ty, SimdInput);
2182    }
2183    let (in_len, in_elem, in_num_vecs) = {
    if !(args[0].layout.ty.is_simd() ||
                args[0].layout.ty.is_scalable_vector()) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
                        span,
                        name,
                        ty: args[0].layout.ty,
                    });
            return Err(err);
        };
    };
    if args[0].layout.ty.is_simd() {
        let (len, ty) = args[0].layout.ty.simd_size_and_type(bx.tcx());
        (len, ty, None)
    } else {
        let (count, ty, num_vecs) =
            args[0].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
        (count as u64, ty, Some(num_vecs))
    }
}require_simd_or_scalable!(args[0].layout.ty, SimdInput);
2184    let in_ty = args[0].layout.ty;
2185
2186    let comparison = match name {
2187        sym::simd_eq => Some(BinOp::Eq),
2188        sym::simd_ne => Some(BinOp::Ne),
2189        sym::simd_lt => Some(BinOp::Lt),
2190        sym::simd_le => Some(BinOp::Le),
2191        sym::simd_gt => Some(BinOp::Gt),
2192        sym::simd_ge => Some(BinOp::Ge),
2193        _ => None,
2194    };
2195
2196    if let Some(cmp_op) = comparison {
2197        let (out_len, out_ty) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2198
2199        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
                    span,
                    name,
                    in_len,
                    in_ty,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
2200            in_len == out_len,
2201            InvalidMonomorphization::ReturnLengthInputType {
2202                span,
2203                name,
2204                in_len,
2205                in_ty,
2206                ret_ty,
2207                out_len
2208            }
2209        );
2210        if !bx.vector_element_type(llret_ty).is_some_and(|elem|
                bx.int_width(elem).is_some()) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnIntegerType {
                    span,
                    name,
                    ret_ty,
                    out_ty,
                });
        return Err(err);
    };
};require!(
2211            bx.vector_element_type(llret_ty).is_some_and(|elem| bx.int_width(elem).is_some()),
2212            InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
2213        );
2214
2215        return Ok(compare_simd_types(
2216            bx,
2217            args[0].immediate(),
2218            args[1].immediate(),
2219            in_elem,
2220            llret_ty,
2221            cmp_op,
2222        ));
2223    }
2224
2225    if name == sym::simd_shuffle_const_generic {
2226        let idx = fn_args[2].expect_const().to_branch();
2227        let n = idx.len() as u64;
2228
2229        let (out_len, out_ty) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2230        if !(out_len == n) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
                    span,
                    name,
                    in_len: n,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
2231            out_len == n,
2232            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2233        );
2234        if !(in_elem == out_ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
                    span,
                    name,
                    in_elem,
                    in_ty,
                    ret_ty,
                    out_ty,
                });
        return Err(err);
    };
};require!(
2235            in_elem == out_ty,
2236            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2237        );
2238
2239        let total_len = in_len * 2;
2240
2241        let indices: Option<Vec<_>> = idx
2242            .iter()
2243            .enumerate()
2244            .map(|(arg_idx, val)| {
2245                let idx = val.to_leaf().to_i32();
2246                if idx >= i32::try_from(total_len).unwrap() {
2247                    bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
2248                        span,
2249                        name,
2250                        arg_idx: arg_idx as u64,
2251                        total_len: total_len.into(),
2252                    });
2253                    None
2254                } else {
2255                    Some(bx.const_i32(idx))
2256                }
2257            })
2258            .collect();
2259        let Some(indices) = indices else {
2260            return Ok(bx.const_null(llret_ty));
2261        };
2262
2263        return Ok(bx.shuffle_vector(
2264            args[0].immediate(),
2265            args[1].immediate(),
2266            bx.const_vector(&indices),
2267        ));
2268    }
2269
2270    if name == sym::simd_shuffle {
2271        // Make sure this is actually a SIMD vector.
2272        let idx_ty = args[2].layout.ty;
2273        let n: u64 = if idx_ty.is_simd()
2274            && #[allow(non_exhaustive_omitted_patterns)] match idx_ty.simd_size_and_type(bx.cx.tcx).1.kind()
    {
    ty::Uint(ty::UintTy::U32) => true,
    _ => false,
}matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
2275        {
2276            idx_ty.simd_size_and_type(bx.cx.tcx).0
2277        } else {
2278            {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdShuffle {
                span,
                name,
                ty: idx_ty,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
2279        };
2280
2281        let (out_len, out_ty) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2282        if !(out_len == n) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
                    span,
                    name,
                    in_len: n,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
2283            out_len == n,
2284            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2285        );
2286        if !(in_elem == out_ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
                    span,
                    name,
                    in_elem,
                    in_ty,
                    ret_ty,
                    out_ty,
                });
        return Err(err);
    };
};require!(
2287            in_elem == out_ty,
2288            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2289        );
2290
2291        let total_len = u128::from(in_len) * 2;
2292
2293        // Check that the indices are in-bounds.
2294        let indices = args[2].immediate();
2295        for i in 0..n {
2296            let val = bx.const_get_elt(indices, i as u64);
2297            let idx = bx
2298                .const_to_opt_u128(val, true)
2299                .unwrap_or_else(|| ::rustc_span::macros::bug_impl(None,
    format_args!("typeck should have already ensured that these are const"),
    Location::caller())bug!("typeck should have already ensured that these are const"));
2300            if idx >= total_len {
2301                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
                span,
                name,
                arg_idx: i,
                total_len,
            });
    return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2302                    span,
2303                    name,
2304                    arg_idx: i,
2305                    total_len,
2306                });
2307            }
2308        }
2309
2310        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
2311    }
2312
2313    if name == sym::simd_insert || name == sym::simd_insert_dyn {
2314        if !(in_elem == args[2].layout.ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::InsertedType {
                    span,
                    name,
                    in_elem,
                    in_ty,
                    out_ty: args[2].layout.ty,
                });
        return Err(err);
    };
};require!(
2315            in_elem == args[2].layout.ty,
2316            InvalidMonomorphization::InsertedType {
2317                span,
2318                name,
2319                in_elem,
2320                in_ty,
2321                out_ty: args[2].layout.ty
2322            }
2323        );
2324
2325        let index_imm = if name == sym::simd_insert {
2326            let idx = bx
2327                .const_to_opt_u128(args[1].immediate(), false)
2328                .expect("typeck should have ensure that this is a const");
2329            if idx >= in_len.into() {
2330                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
                span,
                name,
                arg_idx: 1,
                total_len: in_len.into(),
            });
    return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2331                    span,
2332                    name,
2333                    arg_idx: 1,
2334                    total_len: in_len.into(),
2335                });
2336            }
2337            bx.const_i32(idx as i32)
2338        } else {
2339            args[1].immediate()
2340        };
2341
2342        return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
2343    }
2344    if name == sym::simd_extract || name == sym::simd_extract_dyn {
2345        if !(ret_ty == in_elem) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                    span,
                    name,
                    in_elem,
                    in_ty,
                    ret_ty,
                });
        return Err(err);
    };
};require!(
2346            ret_ty == in_elem,
2347            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2348        );
2349        let index_imm = if name == sym::simd_extract {
2350            let idx = bx
2351                .const_to_opt_u128(args[1].immediate(), false)
2352                .expect("typeck should have ensure that this is a const");
2353            if idx >= in_len.into() {
2354                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
                span,
                name,
                arg_idx: 1,
                total_len: in_len.into(),
            });
    return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2355                    span,
2356                    name,
2357                    arg_idx: 1,
2358                    total_len: in_len.into(),
2359                });
2360            }
2361            bx.const_i32(idx as i32)
2362        } else {
2363            args[1].immediate()
2364        };
2365
2366        return Ok(bx.extract_element(args[0].immediate(), index_imm));
2367    }
2368
2369    if name == sym::simd_select {
2370        let m_elem_ty = in_elem;
2371        let m_len = in_len;
2372        let (v_len, _, _) = {
    if !(args[1].layout.ty.is_simd() ||
                args[1].layout.ty.is_scalable_vector()) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
                        span,
                        name,
                        ty: args[1].layout.ty,
                    });
            return Err(err);
        };
    };
    if args[1].layout.ty.is_simd() {
        let (len, ty) = args[1].layout.ty.simd_size_and_type(bx.tcx());
        (len, ty, None)
    } else {
        let (count, ty, num_vecs) =
            args[1].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
        (count as u64, ty, Some(num_vecs))
    }
}require_simd_or_scalable!(args[1].layout.ty, SimdArgument);
2373        if !(m_len == v_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::MismatchedLengths {
                    span,
                    name,
                    m_len,
                    v_len,
                });
        return Err(err);
    };
};require!(
2374            m_len == v_len,
2375            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
2376        );
2377
2378        let m_i1s = if args[1].layout.ty.is_scalable_vector() {
2379            match m_elem_ty.kind() {
2380                ty::Bool => {}
2381                _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                span,
                name,
                ty: m_elem_ty,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::MaskWrongElementType {
2382                    span,
2383                    name,
2384                    ty: m_elem_ty
2385                }),
2386            };
2387            let i1 = bx.type_i1();
2388            let i1xn = bx.type_scalable_vector(i1, m_len as u64);
2389            bx.trunc(args[0].immediate(), i1xn)
2390        } else {
2391            let in_elem_bitwidth = match m_elem_ty.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: m_elem_ty,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2392                m_elem_ty.kind(),
2393                InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
2394            );
2395            vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
2396        };
2397
2398        return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2399    }
2400
2401    if name == sym::simd_bitmask {
2402        // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a vector mask and
2403        // returns one bit for each lane (which must all be `0` or `!0`) in the form of either:
2404        // * an unsigned integer
2405        // * an array of `u8`
2406        // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
2407        //
2408        // The bit order of the result depends on the byte endianness, LSB-first for little
2409        // endian and MSB-first for big endian.
2410        let expected_int_bits = in_len.max(8).next_power_of_two();
2411        let expected_bytes = in_len.div_ceil(8);
2412
2413        // Integer vector <i{in_bitwidth} x in_len>:
2414        let in_elem_bitwidth = match in_elem.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: in_elem,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2415            in_elem.kind(),
2416            InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
2417        );
2418
2419        let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
2420        // Bitcast <i1 x N> to iN:
2421        let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
2422
2423        match ret_ty.kind() {
2424            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
2425                // Zero-extend iN to the bitmask type:
2426                return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
2427            }
2428            ty::Array(elem, len)
2429                if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
    ty::Uint(ty::UintTy::U8) => true,
    _ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2430                    && len
2431                        .try_to_target_usize(bx.tcx)
2432                        .expect("expected monomorphic const in codegen")
2433                        == expected_bytes =>
2434            {
2435                // Zero-extend iN to the array length:
2436                let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
2437
2438                // Convert the integer to a byte array
2439                let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
2440                bx.store(ze, ptr, Align::ONE);
2441                let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
2442                return Ok(bx.load(array_ty, ptr, Align::ONE));
2443            }
2444            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::CannotReturn {
                span,
                name,
                ret_ty,
                expected_int_bits,
                expected_bytes,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::CannotReturn {
2445                span,
2446                name,
2447                ret_ty,
2448                expected_int_bits,
2449                expected_bytes
2450            }),
2451        }
2452    }
2453
2454    fn simd_simple_float_intrinsic<'ll, 'tcx>(
2455        name: Symbol,
2456        in_elem: Ty<'_>,
2457        in_ty: Ty<'_>,
2458        in_len: u64,
2459        bx: &mut Builder<'_, 'll, 'tcx>,
2460        span: Span,
2461        args: &[OperandRef<'tcx, &'ll Value>],
2462    ) -> Result<&'ll Value, ErrorGuaranteed> {
2463        macro_rules! return_error {
2464            ($diag: expr) => {{
2465                let err = bx.sess().dcx().emit_err($diag);
2466                return Err(err);
2467            }};
2468        }
2469
2470        let ty::Float(f) = in_elem.kind() else {
2471            {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
                span,
                name,
                ty: in_ty,
            });
    return Err(err);
};return_error!(InvalidMonomorphization::BasicFloatType { span, name, ty: in_ty });
2472        };
2473        let elem_ty = bx.cx.type_float_from_ty(*f);
2474
2475        let vec_ty = bx.type_vector(elem_ty, in_len);
2476
2477        let intr_name = match name {
2478            sym::simd_ceil => "llvm.ceil",
2479            sym::simd_fabs => "llvm.fabs",
2480            sym::simd_fcos => "llvm.cos",
2481            sym::simd_fexp2 => "llvm.exp2",
2482            sym::simd_fexp => "llvm.exp",
2483            sym::simd_flog10 => "llvm.log10",
2484            sym::simd_flog2 => "llvm.log2",
2485            sym::simd_flog => "llvm.log",
2486            sym::simd_floor => "llvm.floor",
2487            sym::simd_fma => "llvm.fma",
2488            sym::simd_relaxed_fma => "llvm.fmuladd",
2489            sym::simd_fsin => "llvm.sin",
2490            sym::simd_fsqrt => "llvm.sqrt",
2491            sym::simd_round => "llvm.round",
2492            sym::simd_round_ties_even => "llvm.rint",
2493            sym::simd_trunc => "llvm.trunc",
2494            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::UnrecognizedIntrinsic {
                span,
                name,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
2495        };
2496        Ok(bx.call_intrinsic(
2497            intr_name,
2498            &[vec_ty],
2499            &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
2500        ))
2501    }
2502
2503    if #[allow(non_exhaustive_omitted_patterns)] match name {
    sym::simd_ceil | sym::simd_fabs | sym::simd_fcos | sym::simd_fexp2 |
        sym::simd_fexp | sym::simd_flog10 | sym::simd_flog2 | sym::simd_flog |
        sym::simd_floor | sym::simd_fma | sym::simd_fsin | sym::simd_fsqrt |
        sym::simd_relaxed_fma | sym::simd_round | sym::simd_round_ties_even |
        sym::simd_trunc => true,
    _ => false,
}std::matches!(
2504        name,
2505        sym::simd_ceil
2506            | sym::simd_fabs
2507            | sym::simd_fcos
2508            | sym::simd_fexp2
2509            | sym::simd_fexp
2510            | sym::simd_flog10
2511            | sym::simd_flog2
2512            | sym::simd_flog
2513            | sym::simd_floor
2514            | sym::simd_fma
2515            | sym::simd_fsin
2516            | sym::simd_fsqrt
2517            | sym::simd_relaxed_fma
2518            | sym::simd_round
2519            | sym::simd_round_ties_even
2520            | sym::simd_trunc
2521    ) {
2522        return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
2523    }
2524
2525    fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
2526        let elem_ty = match *elem_ty.kind() {
2527            ty::Int(v) => cx.type_int_from_ty(v),
2528            ty::Uint(v) => cx.type_uint_from_ty(v),
2529            ty::Float(v) => cx.type_float_from_ty(v),
2530            ty::RawPtr(_, _) => cx.type_ptr(),
2531            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
2532        };
2533        cx.type_vector(elem_ty, vec_len)
2534    }
2535
2536    if name == sym::simd_gather {
2537        // simd_gather(values: <N x T>, pointers: <N x *_ T>,
2538        //             mask: <N x i{M}>) -> <N x T>
2539        // * N: number of elements in the input vectors
2540        // * T: type of the element to load
2541        // * M: any integer width is supported, will be truncated to i1
2542
2543        // All types must be simd vector types
2544
2545        // The second argument must be a simd vector with an element type that's a pointer
2546        // to the element type of the first argument
2547        let (_, element_ty0) = {
    if !in_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
                        span,
                        name,
                        ty: in_ty,
                    });
            return Err(err);
        };
    };
    in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2548        let (out_len, element_ty1) = {
    if !args[1].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
                        span,
                        name,
                        ty: args[1].layout.ty,
                    });
            return Err(err);
        };
    };
    args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2549        // The element type of the third argument must be a signed integer type of any width:
2550        let (out_len2, element_ty2) = {
    if !args[2].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
                        span,
                        name,
                        ty: args[2].layout.ty,
                    });
            return Err(err);
        };
    };
    args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2551        {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2552
2553        // Of the same length:
2554        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
                    span,
                    name,
                    in_len,
                    in_ty,
                    arg_ty: args[1].layout.ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
2555            in_len == out_len,
2556            InvalidMonomorphization::SecondArgumentLength {
2557                span,
2558                name,
2559                in_len,
2560                in_ty,
2561                arg_ty: args[1].layout.ty,
2562                out_len
2563            }
2564        );
2565        if !(in_len == out_len2) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
                    span,
                    name,
                    in_len,
                    in_ty,
                    arg_ty: args[2].layout.ty,
                    out_len: out_len2,
                });
        return Err(err);
    };
};require!(
2566            in_len == out_len2,
2567            InvalidMonomorphization::ThirdArgumentLength {
2568                span,
2569                name,
2570                in_len,
2571                in_ty,
2572                arg_ty: args[2].layout.ty,
2573                out_len: out_len2
2574            }
2575        );
2576
2577        // The return type must match the first argument type
2578        if !(ret_ty == in_ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
                    span,
                    name,
                    in_ty,
                    ret_ty,
                });
        return Err(err);
    };
};require!(
2579            ret_ty == in_ty,
2580            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
2581        );
2582
2583        if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
            ty::RawPtr(p_ty, _) if
                p_ty == in_elem && p_ty.kind() == element_ty0.kind() => true,
            _ => false,
        } {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
                    span,
                    name,
                    expected_element: element_ty1,
                    second_arg: args[1].layout.ty,
                    in_elem,
                    in_ty,
                    mutability: ExpectedPointerMutability::Not,
                });
        return Err(err);
    };
};require!(
2584            matches!(
2585                *element_ty1.kind(),
2586                ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
2587            ),
2588            InvalidMonomorphization::ExpectedElementType {
2589                span,
2590                name,
2591                expected_element: element_ty1,
2592                second_arg: args[1].layout.ty,
2593                in_elem,
2594                in_ty,
2595                mutability: ExpectedPointerMutability::Not,
2596            }
2597        );
2598
2599        let mask_elem_bitwidth = match element_ty2.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: element_ty2,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2600            element_ty2.kind(),
2601            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2602        );
2603
2604        // Alignment of T, must be a constant integer value:
2605        let alignment = bx.align_of(in_elem).bytes();
2606
2607        // Truncate the mask vector to a vector of i1s:
2608        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2609
2610        // Type of the vector of pointers:
2611        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2612
2613        // Type of the vector of elements:
2614        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2615
2616        let call = bx.call_intrinsic(
2617            "llvm.masked.gather",
2618            &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2619            &[args[1].immediate(), mask, args[0].immediate()],
2620        );
2621        crate::attributes::apply_to_callsite(
2622            call,
2623            crate::llvm::AttributePlace::Argument(0),
2624            &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2625        );
2626        return Ok(call);
2627    }
2628
2629    fn llvm_alignment<'ll, 'tcx>(
2630        bx: &mut Builder<'_, 'll, 'tcx>,
2631        alignment: SimdAlign,
2632        vector_ty: Ty<'tcx>,
2633        element_ty: Ty<'tcx>,
2634    ) -> u64 {
2635        match alignment {
2636            SimdAlign::Unaligned => 1,
2637            SimdAlign::Element => bx.align_of(element_ty).bytes(),
2638            SimdAlign::Vector => bx.align_of(vector_ty).bytes(),
2639        }
2640    }
2641
2642    if name == sym::simd_masked_load {
2643        // simd_masked_load<_, _, _, const ALIGN: SimdAlign>(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
2644        // * N: number of elements in the input vectors
2645        // * T: type of the element to load
2646        // * M: any integer width is supported, will be truncated to i1
2647        // Loads contiguous elements from memory behind `pointer`, but only for
2648        // those lanes whose `mask` bit is enabled.
2649        // The memory addresses corresponding to the “off” lanes are not accessed.
2650
2651        let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2652
2653        // The element type of the "mask" argument must be a signed integer type of any width
2654        let mask_ty = in_ty;
2655        let (mask_len, mask_elem) = (in_len, in_elem);
2656
2657        // The second argument must be a pointer matching the element type
2658        let pointer_ty = args[1].layout.ty;
2659
2660        // The last argument is a passthrough vector providing values for disabled lanes
2661        let values_ty = args[2].layout.ty;
2662        let (values_len, values_elem) = {
    if !values_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
                        span,
                        name,
                        ty: values_ty,
                    });
            return Err(err);
        };
    };
    values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2663
2664        {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2665
2666        // Of the same length:
2667        if !(values_len == mask_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
                    span,
                    name,
                    in_len: mask_len,
                    in_ty: mask_ty,
                    arg_ty: values_ty,
                    out_len: values_len,
                });
        return Err(err);
    };
};require!(
2668            values_len == mask_len,
2669            InvalidMonomorphization::ThirdArgumentLength {
2670                span,
2671                name,
2672                in_len: mask_len,
2673                in_ty: mask_ty,
2674                arg_ty: values_ty,
2675                out_len: values_len
2676            }
2677        );
2678
2679        // The return type must match the last argument type
2680        if !(ret_ty == values_ty) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
                    span,
                    name,
                    in_ty: values_ty,
                    ret_ty,
                });
        return Err(err);
    };
};require!(
2681            ret_ty == values_ty,
2682            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
2683        );
2684
2685        if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
            ty::RawPtr(p_ty, _) if
                p_ty == values_elem && p_ty.kind() == values_elem.kind() =>
                true,
            _ => false,
        } {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
                    span,
                    name,
                    expected_element: values_elem,
                    second_arg: pointer_ty,
                    in_elem: values_elem,
                    in_ty: values_ty,
                    mutability: ExpectedPointerMutability::Not,
                });
        return Err(err);
    };
};require!(
2686            matches!(
2687                *pointer_ty.kind(),
2688                ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
2689            ),
2690            InvalidMonomorphization::ExpectedElementType {
2691                span,
2692                name,
2693                expected_element: values_elem,
2694                second_arg: pointer_ty,
2695                in_elem: values_elem,
2696                in_ty: values_ty,
2697                mutability: ExpectedPointerMutability::Not,
2698            }
2699        );
2700
2701        let m_elem_bitwidth = match mask_elem.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: mask_elem,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2702            mask_elem.kind(),
2703            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2704        );
2705
2706        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2707
2708        // Alignment of T, must be a constant integer value:
2709        let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2710
2711        let llvm_pointer = bx.type_ptr();
2712
2713        // Type of the vector of elements:
2714        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2715
2716        let call = bx.call_intrinsic(
2717            "llvm.masked.load",
2718            &[llvm_elem_vec_ty, llvm_pointer],
2719            &[args[1].immediate(), mask, args[2].immediate()],
2720        );
2721        crate::attributes::apply_to_callsite(
2722            call,
2723            crate::llvm::AttributePlace::Argument(0),
2724            &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2725        );
2726        return Ok(call);
2727    }
2728
2729    if name == sym::simd_masked_store {
2730        // simd_masked_store<_, _, _, const ALIGN: SimdAlign>(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
2731        // * N: number of elements in the input vectors
2732        // * T: type of the element to load
2733        // * M: any integer width is supported, will be truncated to i1
2734        // Stores contiguous elements to memory behind `pointer`, but only for
2735        // those lanes whose `mask` bit is enabled.
2736        // The memory addresses corresponding to the “off” lanes are not accessed.
2737
2738        let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2739
2740        // The element type of the "mask" argument must be a signed integer type of any width
2741        let mask_ty = in_ty;
2742        let (mask_len, mask_elem) = (in_len, in_elem);
2743
2744        // The second argument must be a pointer matching the element type
2745        let pointer_ty = args[1].layout.ty;
2746
2747        // The last argument specifies the values to store to memory
2748        let values_ty = args[2].layout.ty;
2749        let (values_len, values_elem) = {
    if !values_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
                        span,
                        name,
                        ty: values_ty,
                    });
            return Err(err);
        };
    };
    values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2750
2751        // Of the same length:
2752        if !(values_len == mask_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
                    span,
                    name,
                    in_len: mask_len,
                    in_ty: mask_ty,
                    arg_ty: values_ty,
                    out_len: values_len,
                });
        return Err(err);
    };
};require!(
2753            values_len == mask_len,
2754            InvalidMonomorphization::ThirdArgumentLength {
2755                span,
2756                name,
2757                in_len: mask_len,
2758                in_ty: mask_ty,
2759                arg_ty: values_ty,
2760                out_len: values_len
2761            }
2762        );
2763
2764        // The second argument must be a mutable pointer type matching the element type
2765        if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
            ty::RawPtr(p_ty, p_mutbl) if
                p_ty == values_elem && p_ty.kind() == values_elem.kind() &&
                    p_mutbl.is_mut() => true,
            _ => false,
        } {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
                    span,
                    name,
                    expected_element: values_elem,
                    second_arg: pointer_ty,
                    in_elem: values_elem,
                    in_ty: values_ty,
                    mutability: ExpectedPointerMutability::Mut,
                });
        return Err(err);
    };
};require!(
2766            matches!(
2767                *pointer_ty.kind(),
2768                ty::RawPtr(p_ty, p_mutbl)
2769                    if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
2770            ),
2771            InvalidMonomorphization::ExpectedElementType {
2772                span,
2773                name,
2774                expected_element: values_elem,
2775                second_arg: pointer_ty,
2776                in_elem: values_elem,
2777                in_ty: values_ty,
2778                mutability: ExpectedPointerMutability::Mut,
2779            }
2780        );
2781
2782        let m_elem_bitwidth = match mask_elem.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: mask_elem,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2783            mask_elem.kind(),
2784            InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2785        );
2786
2787        let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2788
2789        // Alignment of T, must be a constant integer value:
2790        let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2791
2792        let llvm_pointer = bx.type_ptr();
2793
2794        // Type of the vector of elements:
2795        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2796
2797        let call = bx.call_intrinsic(
2798            "llvm.masked.store",
2799            &[llvm_elem_vec_ty, llvm_pointer],
2800            &[args[2].immediate(), args[1].immediate(), mask],
2801        );
2802        crate::attributes::apply_to_callsite(
2803            call,
2804            crate::llvm::AttributePlace::Argument(1),
2805            &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2806        );
2807        return Ok(call);
2808    }
2809
2810    if name == sym::simd_scatter {
2811        // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
2812        //             mask: <N x i{M}>) -> ()
2813        // * N: number of elements in the input vectors
2814        // * T: type of the element to load
2815        // * M: any integer width is supported, will be truncated to i1
2816
2817        // All types must be simd vector types
2818        // The second argument must be a simd vector with an element type that's a pointer
2819        // to the element type of the first argument
2820        let (_, element_ty0) = {
    if !in_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
                        span,
                        name,
                        ty: in_ty,
                    });
            return Err(err);
        };
    };
    in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2821        let (element_len1, element_ty1) = {
    if !args[1].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
                        span,
                        name,
                        ty: args[1].layout.ty,
                    });
            return Err(err);
        };
    };
    args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2822        let (element_len2, element_ty2) = {
    if !args[2].layout.ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
                        span,
                        name,
                        ty: args[2].layout.ty,
                    });
            return Err(err);
        };
    };
    args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2823
2824        // Of the same length:
2825        if !(in_len == element_len1) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
                    span,
                    name,
                    in_len,
                    in_ty,
                    arg_ty: args[1].layout.ty,
                    out_len: element_len1,
                });
        return Err(err);
    };
};require!(
2826            in_len == element_len1,
2827            InvalidMonomorphization::SecondArgumentLength {
2828                span,
2829                name,
2830                in_len,
2831                in_ty,
2832                arg_ty: args[1].layout.ty,
2833                out_len: element_len1
2834            }
2835        );
2836        if !(in_len == element_len2) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
                    span,
                    name,
                    in_len,
                    in_ty,
                    arg_ty: args[2].layout.ty,
                    out_len: element_len2,
                });
        return Err(err);
    };
};require!(
2837            in_len == element_len2,
2838            InvalidMonomorphization::ThirdArgumentLength {
2839                span,
2840                name,
2841                in_len,
2842                in_ty,
2843                arg_ty: args[2].layout.ty,
2844                out_len: element_len2
2845            }
2846        );
2847
2848        if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
            ty::RawPtr(p_ty, p_mutbl) if
                p_ty == in_elem && p_mutbl.is_mut() &&
                    p_ty.kind() == element_ty0.kind() => true,
            _ => false,
        } {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
                    span,
                    name,
                    expected_element: element_ty1,
                    second_arg: args[1].layout.ty,
                    in_elem,
                    in_ty,
                    mutability: ExpectedPointerMutability::Mut,
                });
        return Err(err);
    };
};require!(
2849            matches!(
2850                *element_ty1.kind(),
2851                ty::RawPtr(p_ty, p_mutbl)
2852                    if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
2853            ),
2854            InvalidMonomorphization::ExpectedElementType {
2855                span,
2856                name,
2857                expected_element: element_ty1,
2858                second_arg: args[1].layout.ty,
2859                in_elem,
2860                in_ty,
2861                mutability: ExpectedPointerMutability::Mut,
2862            }
2863        );
2864
2865        // The element type of the third argument must be an integer type of any width:
2866        let mask_elem_bitwidth = match element_ty2.kind() {
    ty::Int(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    ty::Uint(i) => {
        i.bit_width().unwrap_or_else(||
                bx.data_layout().pointer_size().bits())
    }
    _ => {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
                        span,
                        name,
                        ty: element_ty2,
                    });
            return Err(err);
        };
    }
}require_int_or_uint_ty!(
2867            element_ty2.kind(),
2868            InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2869        );
2870
2871        // Alignment of T, must be a constant integer value:
2872        let alignment = bx.align_of(in_elem).bytes();
2873
2874        // Truncate the mask vector to a vector of i1s:
2875        let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2876
2877        // Type of the vector of pointers:
2878        let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2879
2880        // Type of the vector of elements:
2881        let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2882
2883        let call = bx.call_intrinsic(
2884            "llvm.masked.scatter",
2885            &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2886            &[args[0].immediate(), args[1].immediate(), mask],
2887        );
2888        crate::attributes::apply_to_callsite(
2889            call,
2890            crate::llvm::AttributePlace::Argument(1),
2891            &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2892        );
2893        return Ok(call);
2894    }
2895
2896    macro_rules! arith_red {
2897        ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2898         $identity:expr) => {
2899            if name == sym::$name {
2900                require!(
2901                    ret_ty == in_elem,
2902                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2903                );
2904                return match in_elem.kind() {
2905                    ty::Int(_) | ty::Uint(_) => {
2906                        let r = bx.$integer_reduce(args[0].immediate());
2907                        if $ordered {
2908                            // if overflow occurs, the result is the
2909                            // mathematical result modulo 2^n:
2910                            Ok(bx.$op(args[1].immediate(), r))
2911                        } else {
2912                            Ok(bx.$integer_reduce(args[0].immediate()))
2913                        }
2914                    }
2915                    ty::Float(f) => {
2916                        let acc = if $ordered {
2917                            // ordered arithmetic reductions take an accumulator
2918                            args[1].immediate()
2919                        } else {
2920                            // unordered arithmetic reductions use the identity accumulator
2921                            match f.bit_width() {
2922                                32 => bx.const_real(bx.type_f32(), $identity),
2923                                64 => bx.const_real(bx.type_f64(), $identity),
2924                                v => return_error!(
2925                                    InvalidMonomorphization::UnsupportedSymbolOfSize {
2926                                        span,
2927                                        name,
2928                                        symbol: sym::$name,
2929                                        in_ty,
2930                                        in_elem,
2931                                        size: v,
2932                                        ret_ty
2933                                    }
2934                                ),
2935                            }
2936                        };
2937                        Ok(bx.$float_reduce(acc, args[0].immediate()))
2938                    }
2939                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2940                        span,
2941                        name,
2942                        symbol: sym::$name,
2943                        in_ty,
2944                        in_elem,
2945                        ret_ty
2946                    }),
2947                };
2948            }
2949        };
2950    }
2951
2952    if name == sym::simd_reduce_add_ordered {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_add(args[0].immediate());
                if true {
                    Ok(bx.add(args[1].immediate(), r))
                } else { Ok(bx.vector_reduce_add(args[0].immediate())) }
            }
            ty::Float(f) => {
                let acc =
                    if true {
                        args[1].immediate()
                    } else {
                        match f.bit_width() {
                            32 => bx.const_real(bx.type_f32(), -0.0),
                            64 => bx.const_real(bx.type_f64(), -0.0),
                            v => {
                                let err =
                                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
                                            span,
                                            name,
                                            symbol: sym::simd_reduce_add_ordered,
                                            in_ty,
                                            in_elem,
                                            size: v,
                                            ret_ty,
                                        });
                                return Err(err);
                            }
                        }
                    };
                Ok(bx.vector_reduce_fadd(acc, args[0].immediate()))
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_add_ordered,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2953    if name == sym::simd_reduce_mul_ordered {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_mul(args[0].immediate());
                if true {
                    Ok(bx.mul(args[1].immediate(), r))
                } else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
            }
            ty::Float(f) => {
                let acc =
                    if true {
                        args[1].immediate()
                    } else {
                        match f.bit_width() {
                            32 => bx.const_real(bx.type_f32(), 1.0),
                            64 => bx.const_real(bx.type_f64(), 1.0),
                            v => {
                                let err =
                                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
                                            span,
                                            name,
                                            symbol: sym::simd_reduce_mul_ordered,
                                            in_ty,
                                            in_elem,
                                            size: v,
                                            ret_ty,
                                        });
                                return Err(err);
                            }
                        }
                    };
                Ok(bx.vector_reduce_fmul(acc, args[0].immediate()))
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_mul_ordered,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2954    if name == sym::simd_reduce_add_unordered {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_add(args[0].immediate());
                if false {
                    Ok(bx.add(args[1].immediate(), r))
                } else { Ok(bx.vector_reduce_add(args[0].immediate())) }
            }
            ty::Float(f) => {
                let acc =
                    if false {
                        args[1].immediate()
                    } else {
                        match f.bit_width() {
                            32 => bx.const_real(bx.type_f32(), -0.0),
                            64 => bx.const_real(bx.type_f64(), -0.0),
                            v => {
                                let err =
                                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
                                            span,
                                            name,
                                            symbol: sym::simd_reduce_add_unordered,
                                            in_ty,
                                            in_elem,
                                            size: v,
                                            ret_ty,
                                        });
                                return Err(err);
                            }
                        }
                    };
                Ok(bx.vector_reduce_fadd_reassoc(acc, args[0].immediate()))
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_add_unordered,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};arith_red!(
2955        simd_reduce_add_unordered: vector_reduce_add,
2956        vector_reduce_fadd_reassoc,
2957        false,
2958        add,
2959        -0.0
2960    );
2961    if name == sym::simd_reduce_mul_unordered {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_mul(args[0].immediate());
                if false {
                    Ok(bx.mul(args[1].immediate(), r))
                } else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
            }
            ty::Float(f) => {
                let acc =
                    if false {
                        args[1].immediate()
                    } else {
                        match f.bit_width() {
                            32 => bx.const_real(bx.type_f32(), 1.0),
                            64 => bx.const_real(bx.type_f64(), 1.0),
                            v => {
                                let err =
                                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
                                            span,
                                            name,
                                            symbol: sym::simd_reduce_mul_unordered,
                                            in_ty,
                                            in_elem,
                                            size: v,
                                            ret_ty,
                                        });
                                return Err(err);
                            }
                        }
                    };
                Ok(bx.vector_reduce_fmul_reassoc(acc, args[0].immediate()))
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_mul_unordered,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};arith_red!(
2962        simd_reduce_mul_unordered: vector_reduce_mul,
2963        vector_reduce_fmul_reassoc,
2964        false,
2965        mul,
2966        1.0
2967    );
2968
2969    macro_rules! minmax_red {
2970        ($name:ident: $int_red:ident) => {
2971            if name == sym::$name {
2972                require!(
2973                    ret_ty == in_elem,
2974                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2975                );
2976                return match in_elem.kind() {
2977                    ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2978                    ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2979                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2980                        span,
2981                        name,
2982                        symbol: sym::$name,
2983                        in_ty,
2984                        in_elem,
2985                        ret_ty
2986                    }),
2987                };
2988            }
2989        };
2990    }
2991
2992    // Currently no support for float due to <https://github.com/llvm/llvm-project/issues/185827>.
2993    if name == sym::simd_reduce_min {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_i) =>
                Ok(bx.vector_reduce_min(args[0].immediate(), true)),
            ty::Uint(_u) =>
                Ok(bx.vector_reduce_min(args[0].immediate(), false)),
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_min,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};minmax_red!(simd_reduce_min: vector_reduce_min);
2994    if name == sym::simd_reduce_max {
    if !(ret_ty == in_elem) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                        span,
                        name,
                        in_elem,
                        in_ty,
                        ret_ty,
                    });
            return Err(err);
        };
    };
    return match in_elem.kind() {
            ty::Int(_i) =>
                Ok(bx.vector_reduce_max(args[0].immediate(), true)),
            ty::Uint(_u) =>
                Ok(bx.vector_reduce_max(args[0].immediate(), false)),
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_max,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};minmax_red!(simd_reduce_max: vector_reduce_max);
2995
2996    macro_rules! bitwise_red {
2997        ($name:ident : $red:ident, $boolean:expr) => {
2998            if name == sym::$name {
2999                let input = if !$boolean {
3000                    require!(
3001                        ret_ty == in_elem,
3002                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
3003                    );
3004                    args[0].immediate()
3005                } else {
3006                    let bitwidth = match in_elem.kind() {
3007                        ty::Int(i) => {
3008                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
3009                        }
3010                        ty::Uint(i) => {
3011                            i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
3012                        }
3013                        _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
3014                            span,
3015                            name,
3016                            symbol: sym::$name,
3017                            in_ty,
3018                            in_elem,
3019                            ret_ty
3020                        }),
3021                    };
3022
3023                    vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
3024                };
3025                return match in_elem.kind() {
3026                    ty::Int(_) | ty::Uint(_) => {
3027                        let r = bx.$red(input);
3028                        Ok(r)
3029                    }
3030                    _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
3031                        span,
3032                        name,
3033                        symbol: sym::$name,
3034                        in_ty,
3035                        in_elem,
3036                        ret_ty
3037                    }),
3038                };
3039            }
3040        };
3041    }
3042
3043    if name == sym::simd_reduce_and {
    let input =
        if !false {
            if !(ret_ty == in_elem) {
                {
                    let err =
                        bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                                span,
                                name,
                                in_elem,
                                in_ty,
                                ret_ty,
                            });
                    return Err(err);
                };
            };
            args[0].immediate()
        } else {
            let bitwidth =
                match in_elem.kind() {
                    ty::Int(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    ty::Uint(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    _ => {
                        let err =
                            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                                    span,
                                    name,
                                    symbol: sym::simd_reduce_and,
                                    in_ty,
                                    in_elem,
                                    ret_ty,
                                });
                        return Err(err);
                    }
                };
            vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
                in_len as _)
        };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_and(input);
                Ok(r)
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_and,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};bitwise_red!(simd_reduce_and: vector_reduce_and, false);
3044    if name == sym::simd_reduce_or {
    let input =
        if !false {
            if !(ret_ty == in_elem) {
                {
                    let err =
                        bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                                span,
                                name,
                                in_elem,
                                in_ty,
                                ret_ty,
                            });
                    return Err(err);
                };
            };
            args[0].immediate()
        } else {
            let bitwidth =
                match in_elem.kind() {
                    ty::Int(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    ty::Uint(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    _ => {
                        let err =
                            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                                    span,
                                    name,
                                    symbol: sym::simd_reduce_or,
                                    in_ty,
                                    in_elem,
                                    ret_ty,
                                });
                        return Err(err);
                    }
                };
            vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
                in_len as _)
        };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_or(input);
                Ok(r)
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_or,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};bitwise_red!(simd_reduce_or: vector_reduce_or, false);
3045    if name == sym::simd_reduce_xor {
    let input =
        if !false {
            if !(ret_ty == in_elem) {
                {
                    let err =
                        bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                                span,
                                name,
                                in_elem,
                                in_ty,
                                ret_ty,
                            });
                    return Err(err);
                };
            };
            args[0].immediate()
        } else {
            let bitwidth =
                match in_elem.kind() {
                    ty::Int(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    ty::Uint(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    _ => {
                        let err =
                            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                                    span,
                                    name,
                                    symbol: sym::simd_reduce_xor,
                                    in_ty,
                                    in_elem,
                                    ret_ty,
                                });
                        return Err(err);
                    }
                };
            vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
                in_len as _)
        };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_xor(input);
                Ok(r)
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_xor,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
3046    if name == sym::simd_reduce_all {
    let input =
        if !true {
            if !(ret_ty == in_elem) {
                {
                    let err =
                        bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                                span,
                                name,
                                in_elem,
                                in_ty,
                                ret_ty,
                            });
                    return Err(err);
                };
            };
            args[0].immediate()
        } else {
            let bitwidth =
                match in_elem.kind() {
                    ty::Int(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    ty::Uint(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    _ => {
                        let err =
                            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                                    span,
                                    name,
                                    symbol: sym::simd_reduce_all,
                                    in_ty,
                                    in_elem,
                                    ret_ty,
                                });
                        return Err(err);
                    }
                };
            vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
                in_len as _)
        };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_and(input);
                Ok(r)
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_all,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};bitwise_red!(simd_reduce_all: vector_reduce_and, true);
3047    if name == sym::simd_reduce_any {
    let input =
        if !true {
            if !(ret_ty == in_elem) {
                {
                    let err =
                        bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
                                span,
                                name,
                                in_elem,
                                in_ty,
                                ret_ty,
                            });
                    return Err(err);
                };
            };
            args[0].immediate()
        } else {
            let bitwidth =
                match in_elem.kind() {
                    ty::Int(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    ty::Uint(i) => {
                        i.bit_width().unwrap_or_else(||
                                bx.data_layout().pointer_size().bits())
                    }
                    _ => {
                        let err =
                            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                                    span,
                                    name,
                                    symbol: sym::simd_reduce_any,
                                    in_ty,
                                    in_elem,
                                    ret_ty,
                                });
                        return Err(err);
                    }
                };
            vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
                in_len as _)
        };
    return match in_elem.kind() {
            ty::Int(_) | ty::Uint(_) => {
                let r = bx.vector_reduce_or(input);
                Ok(r)
            }
            _ => {
                let err =
                    bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
                            span,
                            name,
                            symbol: sym::simd_reduce_any,
                            in_ty,
                            in_elem,
                            ret_ty,
                        });
                return Err(err);
            }
        };
};bitwise_red!(simd_reduce_any: vector_reduce_or, true);
3048
3049    if name == sym::simd_cast_ptr {
3050        let (out_len, out_elem) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3051        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
                    span,
                    name,
                    in_len,
                    in_ty,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
3052            in_len == out_len,
3053            InvalidMonomorphization::ReturnLengthInputType {
3054                span,
3055                name,
3056                in_len,
3057                in_ty,
3058                ret_ty,
3059                out_len
3060            }
3061        );
3062
3063        match in_elem.kind() {
3064            ty::RawPtr(p_ty, _) => {
3065                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
3066                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
3067                });
3068                if !metadata.is_unit() {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
                    span,
                    name,
                    ty: in_elem,
                });
        return Err(err);
    };
};require!(
3069                    metadata.is_unit(),
3070                    InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
3071                );
3072            }
3073            _ => {
3074                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
                span,
                name,
                ty: in_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
3075            }
3076        }
3077        match out_elem.kind() {
3078            ty::RawPtr(p_ty, _) => {
3079                let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
3080                    bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
3081                });
3082                if !metadata.is_unit() {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
                    span,
                    name,
                    ty: out_elem,
                });
        return Err(err);
    };
};require!(
3083                    metadata.is_unit(),
3084                    InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
3085                );
3086            }
3087            _ => {
3088                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
                span,
                name,
                ty: out_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3089            }
3090        }
3091
3092        return Ok(args[0].immediate());
3093    }
3094
3095    if name == sym::simd_expose_provenance {
3096        let (out_len, out_elem) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3097        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
                    span,
                    name,
                    in_len,
                    in_ty,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
3098            in_len == out_len,
3099            InvalidMonomorphization::ReturnLengthInputType {
3100                span,
3101                name,
3102                in_len,
3103                in_ty,
3104                ret_ty,
3105                out_len
3106            }
3107        );
3108
3109        match in_elem.kind() {
3110            ty::RawPtr(_, _) => {}
3111            _ => {
3112                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
                span,
                name,
                ty: in_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
3113            }
3114        }
3115        match out_elem.kind() {
3116            ty::Uint(ty::UintTy::Usize) => {}
3117            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
                span,
                name,
                ty: out_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
3118        }
3119
3120        return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
3121    }
3122
3123    if name == sym::simd_with_exposed_provenance {
3124        let (out_len, out_elem) = {
    if !ret_ty.is_simd() {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3125        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
                    span,
                    name,
                    in_len,
                    in_ty,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
3126            in_len == out_len,
3127            InvalidMonomorphization::ReturnLengthInputType {
3128                span,
3129                name,
3130                in_len,
3131                in_ty,
3132                ret_ty,
3133                out_len
3134            }
3135        );
3136
3137        match in_elem.kind() {
3138            ty::Uint(ty::UintTy::Usize) => {}
3139            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
                span,
                name,
                ty: in_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
3140        }
3141        match out_elem.kind() {
3142            ty::RawPtr(_, _) => {}
3143            _ => {
3144                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
                span,
                name,
                ty: out_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3145            }
3146        }
3147
3148        return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
3149    }
3150
3151    if name == sym::simd_cast || name == sym::simd_as {
3152        let (out_len, out_elem, out_num_vecs) = {
    if !(ret_ty.is_simd() || ret_ty.is_scalable_vector()) {
        {
            let err =
                bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
                        span,
                        name,
                        ty: ret_ty,
                    });
            return Err(err);
        };
    };
    if ret_ty.is_simd() {
        let (len, ty) = ret_ty.simd_size_and_type(bx.tcx());
        (len, ty, None)
    } else {
        let (count, ty, num_vecs) =
            ret_ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
        (count as u64, ty, Some(num_vecs))
    }
}require_simd_or_scalable!(ret_ty, SimdReturn);
3153        if !(in_len == out_len) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
                    span,
                    name,
                    in_len,
                    in_ty,
                    ret_ty,
                    out_len,
                });
        return Err(err);
    };
};require!(
3154            in_len == out_len,
3155            InvalidMonomorphization::ReturnLengthInputType {
3156                span,
3157                name,
3158                in_len,
3159                in_ty,
3160                ret_ty,
3161                out_len
3162            }
3163        );
3164        if !(in_num_vecs == out_num_vecs) {
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnNumVecsInputType {
                    span,
                    name,
                    in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
                    in_ty,
                    ret_ty,
                    out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1)),
                });
        return Err(err);
    };
};require!(
3165            in_num_vecs == out_num_vecs,
3166            InvalidMonomorphization::ReturnNumVecsInputType {
3167                span,
3168                name,
3169                in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
3170                in_ty,
3171                ret_ty,
3172                out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1))
3173            }
3174        );
3175
3176        // Casting cares about nominal type, not just structural type
3177        if in_elem == out_elem {
3178            return Ok(args[0].immediate());
3179        }
3180
3181        #[derive(#[automatically_derived]impl ::core::marker::Copy for Sign { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for Sign {
}
#[automatically_derived]impl ::core::clone::Clone for Sign {
    #[inline]
    fn clone(&self) -> Self { *self }
}Clone)]
3182        enum Sign {
3183            Unsigned,
3184            Signed,
3185        }
3186        use Sign::*;
3187
3188        enum Style {
3189            Float,
3190            Int(Sign),
3191            Unsupported,
3192        }
3193
3194        let (in_style, in_width) = match in_elem.kind() {
3195            // vectors of pointer-sized integers should've been
3196            // disallowed before here, so this unwrap is safe.
3197            ty::Int(i) => (
3198                Style::Int(Signed),
3199                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3200            ),
3201            ty::Uint(u) => (
3202                Style::Int(Unsigned),
3203                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3204            ),
3205            ty::Float(f) => (Style::Float, f.bit_width()),
3206            _ => (Style::Unsupported, 0),
3207        };
3208        let (out_style, out_width) = match out_elem.kind() {
3209            ty::Int(i) => (
3210                Style::Int(Signed),
3211                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3212            ),
3213            ty::Uint(u) => (
3214                Style::Int(Unsigned),
3215                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3216            ),
3217            ty::Float(f) => (Style::Float, f.bit_width()),
3218            _ => (Style::Unsupported, 0),
3219        };
3220
3221        match (in_style, out_style) {
3222            (Style::Int(sign), Style::Int(_)) => {
3223                return Ok(match in_width.cmp(&out_width) {
3224                    Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
3225                    Ordering::Equal => args[0].immediate(),
3226                    Ordering::Less => match sign {
3227                        Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
3228                        Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
3229                    },
3230                });
3231            }
3232            (Style::Int(Sign::Signed), Style::Float) => {
3233                return Ok(bx.sitofp(args[0].immediate(), llret_ty));
3234            }
3235            (Style::Int(Sign::Unsigned), Style::Float) => {
3236                return Ok(bx.uitofp(args[0].immediate(), llret_ty));
3237            }
3238            (Style::Float, Style::Int(sign)) => {
3239                return Ok(match (sign, name == sym::simd_as) {
3240                    (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
3241                    (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
3242                    (_, true) => bx.cast_float_to_int(
3243                        #[allow(non_exhaustive_omitted_patterns)] match sign {
    Sign::Signed => true,
    _ => false,
}matches!(sign, Sign::Signed),
3244                        args[0].immediate(),
3245                        llret_ty,
3246                    ),
3247                });
3248            }
3249            (Style::Float, Style::Float) => {
3250                return Ok(match in_width.cmp(&out_width) {
3251                    Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
3252                    Ordering::Equal => args[0].immediate(),
3253                    Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
3254                });
3255            }
3256            _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedCast {
                span,
                name,
                in_ty,
                in_elem,
                ret_ty,
                out_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::UnsupportedCast {
3257                span,
3258                name,
3259                in_ty,
3260                in_elem,
3261                ret_ty,
3262                out_elem
3263            }),
3264        }
3265    }
3266    macro_rules! arith_binary {
3267        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3268            $(if name == sym::$name {
3269                match in_elem.kind() {
3270                    $($(ty::$p(_))|* => {
3271                        return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
3272                    })*
3273                    _ => {},
3274                }
3275                return_error!(
3276                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3277                );
3278            })*
3279        }
3280    }
3281    if name == sym::simd_add {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.add(args[0].immediate(), args[1].immediate()))
        }
        ty::Float(_) => {
            return Ok(bx.fadd(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_sub {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.sub(args[0].immediate(), args[1].immediate()))
        }
        ty::Float(_) => {
            return Ok(bx.fsub(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_mul {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.mul(args[0].immediate(), args[1].immediate()))
        }
        ty::Float(_) => {
            return Ok(bx.fmul(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_div {
    match in_elem.kind() {
        ty::Uint(_) => {
            return Ok(bx.udiv(args[0].immediate(), args[1].immediate()))
        }
        ty::Int(_) => {
            return Ok(bx.sdiv(args[0].immediate(), args[1].immediate()))
        }
        ty::Float(_) => {
            return Ok(bx.fdiv(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_rem {
    match in_elem.kind() {
        ty::Uint(_) => {
            return Ok(bx.urem(args[0].immediate(), args[1].immediate()))
        }
        ty::Int(_) => {
            return Ok(bx.srem(args[0].immediate(), args[1].immediate()))
        }
        ty::Float(_) => {
            return Ok(bx.frem(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_shl {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.shl(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_shr {
    match in_elem.kind() {
        ty::Uint(_) => {
            return Ok(bx.lshr(args[0].immediate(), args[1].immediate()))
        }
        ty::Int(_) => {
            return Ok(bx.ashr(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_and {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.and(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_or {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.or(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_xor {
    match in_elem.kind() {
        ty::Uint(_) | ty::Int(_) => {
            return Ok(bx.xor(args[0].immediate(), args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_maximum_number_nsz {
    match in_elem.kind() {
        ty::Float(_) => {
            return Ok(bx.maximum_number_nsz(args[0].immediate(),
                        args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}
if name == sym::simd_minimum_number_nsz {
    match in_elem.kind() {
        ty::Float(_) => {
            return Ok(bx.minimum_number_nsz(args[0].immediate(),
                        args[1].immediate()))
        }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}arith_binary! {
3282        simd_add: Uint, Int => add, Float => fadd;
3283        simd_sub: Uint, Int => sub, Float => fsub;
3284        simd_mul: Uint, Int => mul, Float => fmul;
3285        simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
3286        simd_rem: Uint => urem, Int => srem, Float => frem;
3287        simd_shl: Uint, Int => shl;
3288        simd_shr: Uint => lshr, Int => ashr;
3289        simd_and: Uint, Int => and;
3290        simd_or: Uint, Int => or;
3291        simd_xor: Uint, Int => xor;
3292        simd_maximum_number_nsz: Float => maximum_number_nsz;
3293        simd_minimum_number_nsz: Float => minimum_number_nsz;
3294
3295    }
3296    macro_rules! arith_unary {
3297        ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3298            $(if name == sym::$name {
3299                match in_elem.kind() {
3300                    $($(ty::$p(_))|* => {
3301                        return Ok(bx.$call(args[0].immediate()))
3302                    })*
3303                    _ => {},
3304                }
3305                return_error!(
3306                    InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3307                );
3308            })*
3309        }
3310    }
3311    if name == sym::simd_neg {
    match in_elem.kind() {
        ty::Int(_) => { return Ok(bx.neg(args[0].immediate())) }
        ty::Float(_) => { return Ok(bx.fneg(args[0].immediate())) }
        _ => {}
    }
    {
        let err =
            bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                    span,
                    name,
                    in_ty,
                    in_elem,
                });
        return Err(err);
    };
}arith_unary! {
3312        simd_neg: Int => neg, Float => fneg;
3313    }
3314
3315    // Unary integer intrinsics
3316    if #[allow(non_exhaustive_omitted_patterns)] match name {
    sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop
        | sym::simd_cttz | sym::simd_carryless_mul | sym::simd_funnel_shl |
        sym::simd_funnel_shr => true,
    _ => false,
}matches!(
3317        name,
3318        sym::simd_bswap
3319            | sym::simd_bitreverse
3320            | sym::simd_ctlz
3321            | sym::simd_ctpop
3322            | sym::simd_cttz
3323            | sym::simd_carryless_mul
3324            | sym::simd_funnel_shl
3325            | sym::simd_funnel_shr
3326    ) {
3327        let vec_ty = bx.cx.type_vector(
3328            match *in_elem.kind() {
3329                ty::Int(i) => bx.cx.type_int_from_ty(i),
3330                ty::Uint(i) => bx.cx.type_uint_from_ty(i),
3331                _ => {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
                span,
                name,
                in_ty,
                in_elem,
            });
    return Err(err);
}return_error!(InvalidMonomorphization::UnsupportedOperation {
3332                    span,
3333                    name,
3334                    in_ty,
3335                    in_elem
3336                }),
3337            },
3338            in_len as u64,
3339        );
3340        let llvm_intrinsic = match name {
3341            sym::simd_bswap => "llvm.bswap",
3342            sym::simd_bitreverse => "llvm.bitreverse",
3343            sym::simd_ctlz => "llvm.ctlz",
3344            sym::simd_ctpop => "llvm.ctpop",
3345            sym::simd_cttz => "llvm.cttz",
3346            sym::simd_funnel_shl => "llvm.fshl",
3347            sym::simd_funnel_shr => "llvm.fshr",
3348            sym::simd_carryless_mul => "llvm.clmul",
3349            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
3350        };
3351        let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
3352
3353        return match name {
3354            // byte swap is no-op for i8/u8
3355            sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
3356            sym::simd_ctlz | sym::simd_cttz => {
3357                // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
3358                let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
3359                Ok(bx.call_intrinsic(
3360                    llvm_intrinsic,
3361                    &[vec_ty],
3362                    &[args[0].immediate(), dont_poison_on_zero],
3363                ))
3364            }
3365            sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
3366                // simple unary argument cases
3367                Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
3368            }
3369            sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
3370                llvm_intrinsic,
3371                &[vec_ty],
3372                &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
3373            )),
3374            sym::simd_carryless_mul => Ok(bx.call_intrinsic(
3375                llvm_intrinsic,
3376                &[vec_ty],
3377                &[args[0].immediate(), args[1].immediate()],
3378            )),
3379            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
3380        };
3381    }
3382
3383    if name == sym::simd_arith_offset {
3384        // This also checks that the first operand is a ptr type.
3385        let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
3386            ::rustc_span::macros::bug_impl(Some(span),
    format_args!("must be called with a vector of pointer types as first argument"),
    Location::caller())span_bug!(span, "must be called with a vector of pointer types as first argument")
3387        });
3388        let layout = bx.layout_of(pointee);
3389        let ptrs = args[0].immediate();
3390        // The second argument must be a ptr-sized integer.
3391        // (We don't care about the signedness, this is wrapping anyway.)
3392        let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
3393        if !#[allow(non_exhaustive_omitted_patterns)] match offsets_elem.kind() {
    ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize) => true,
    _ => false,
}matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
3394            ::rustc_span::macros::bug_impl(Some(span),
    format_args!("must be called with a vector of pointer-sized integers as second argument"),
    Location::caller());span_bug!(
3395                span,
3396                "must be called with a vector of pointer-sized integers as second argument"
3397            );
3398        }
3399        let offsets = args[1].immediate();
3400
3401        return Ok(bx.gep(layout.size, ptrs, offsets));
3402    }
3403
3404    if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
3405        let lhs = args[0].immediate();
3406        let rhs = args[1].immediate();
3407        let is_add = name == sym::simd_saturating_add;
3408        let (signed, elem_ty) = match *in_elem.kind() {
3409            ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
3410            ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
3411            _ => {
3412                {
    let err =
        bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
                span,
                name,
                expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
                vector_type: args[0].layout.ty,
            });
    return Err(err);
};return_error!(InvalidMonomorphization::ExpectedVectorElementType {
3413                    span,
3414                    name,
3415                    expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
3416                    vector_type: args[0].layout.ty
3417                });
3418            }
3419        };
3420        let llvm_intrinsic = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
                if signed { 's' } else { 'u' },
                if is_add { "add" } else { "sub" }))
    })format!(
3421            "llvm.{}{}.sat",
3422            if signed { 's' } else { 'u' },
3423            if is_add { "add" } else { "sub" },
3424        );
3425        let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
3426
3427        return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
3428    }
3429
3430    ::rustc_span::macros::bug_impl(Some(span),
    format_args!("unknown SIMD intrinsic"), Location::caller());span_bug!(span, "unknown SIMD intrinsic");
3431}
3432
3433fn codegen_codeview_annotation<'ll, 'tcx>(
3434    bx: &mut Builder<'_, 'll, 'tcx>,
3435    instance: ty::Instance<'tcx>,
3436    span: Span,
3437) {
3438    // This function lowers the `codeview_annotation` intrinsic,
3439    // which is declared like this:
3440    //
3441    // trait CodeViewAnnotationArgs {
3442    //     const ARGS: &[&str];
3443    // }
3444    //
3445    // fn codeview_annotation<T: CodeViewAnnotationArgs>() {}
3446    //
3447    // It finds the `T: CodeViewAnnotationArgs` trait bound, locates
3448    // `ARGS`, const evaluates it, reads the strings off the resulting
3449    // allocation and lowers them.
3450
3451    const ARGS_CONST_NAME: &str = "`CodeViewAnnotationArgs::ARGS`";
3452
3453    let tcx = bx.tcx();
3454
3455    // Locate `ARGS`
3456    let Some((args_const_def_id, generic_args)) = tcx
3457        .explicit_clauses_of(instance.def_id())
3458        .instantiate_own(tcx, instance.args)
3459        .filter_map(|(clause, _)| clause.skip_norm_wip().as_trait_clause())
3460        .find_map(|trait_clause| {
3461            let trait_ref = trait_clause.skip_binder().trait_ref;
3462            tcx.associated_items(trait_ref.def_id)
3463                .in_definition_order()
3464                .find(|item| {
3465                    item.tag() == ty::AssocTag::Const
3466                        && tcx.item_name(item.def_id).as_str() == "ARGS"
3467                })
3468                .map(|item| (item.def_id, trait_ref.args))
3469        })
3470    else {
3471        ::rustc_span::macros::bug_impl(Some(span),
    format_args!("could not find {0}", ARGS_CONST_NAME), Location::caller());span_bug!(span, "could not find {ARGS_CONST_NAME}");
3472    };
3473
3474    // Const evaluate `ARGS`
3475    let args_const_val = match tcx.const_eval_resolve(
3476        bx.typing_env(),
3477        mir::UnevaluatedConst::new(args_const_def_id, generic_args),
3478        span,
3479    ) {
3480        Ok(val) => val,
3481        Err(ErrorHandled::Reported(..)) => return,
3482        Err(ErrorHandled::TooGeneric(_)) => {
3483            ::rustc_span::macros::bug_impl(Some(span),
    format_args!("{0} is too generic", ARGS_CONST_NAME), Location::caller())span_bug!(span, "{ARGS_CONST_NAME} is too generic")
3484        }
3485    };
3486
3487    // Read the strings
3488    let array_alloc = match args_const_val {
3489        mir::ConstValue::Indirect { alloc_id, offset } => read_slice_pointer(tcx, alloc_id, offset)
3490            .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
    format_args!("invalid {0} slice pointer", ARGS_CONST_NAME),
    Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} slice pointer")),
3491        _ => ::rustc_span::macros::bug_impl(Some(span),
    format_args!("unexpected {0} value: {1:?}", ARGS_CONST_NAME,
        args_const_val), Location::caller())span_bug!(span, "unexpected {ARGS_CONST_NAME} value: {args_const_val:?}"),
3492    };
3493
3494    let strings = array_alloc.into_iter().flat_map(|(array_alloc_id, array_offset, array_len)| {
3495        (0..array_len).map(move |index| {
3496            let elem_size = tcx.data_layout.pointer_size().bytes() * 2; // Multiplying by 2 for fat pointers
3497            let elem_offset = array_offset + Size::from_bytes(index * elem_size);
3498
3499            let (string_alloc_id, string_offset, string_len) =
3500                match read_slice_pointer(tcx, array_alloc_id, elem_offset) {
3501                    Ok(Some(slice)) => slice,
3502                    Ok(None) => return &[] as &[u8],
3503                    Err(()) => ::rustc_span::macros::bug_impl(Some(span),
    format_args!("invalid {0} string pointer", ARGS_CONST_NAME),
    Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string pointer"),
3504                };
3505
3506            resolve_alloc(tcx, string_alloc_id)
3507                .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
    format_args!("invalid {0} string allocation", ARGS_CONST_NAME),
    Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string allocation"))
3508                .get_bytes_strip_provenance(
3509                    &tcx,
3510                    alloc_range(string_offset, Size::from_bytes(string_len)),
3511                )
3512                .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
    format_args!("invalid {0} string bytes", ARGS_CONST_NAME),
    Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string bytes"))
3513        })
3514    });
3515
3516    // Lower the strings
3517    let md_strings: Vec<&Metadata> = strings.map(|s| bx.cx.create_metadata(s)).collect();
3518    let md_tuple = bx.cx.md_node_in_context(&md_strings);
3519    let md_value = bx.cx.get_metadata_value(md_tuple);
3520    let (fn_ty, intrinsic_fn) = bx.cx.get_intrinsic("llvm.codeview.annotation".into(), &[]);
3521
3522    bx.call(fn_ty, None, None, intrinsic_fn, ReturnSlot::Direct, &[md_value], None, None);
3523}
3524
3525// Reads the slice pointer stored at `offset` in `alloc_id` and
3526// returns its pointee's allocation ID, relative byte offset
3527// and slice length. Returns `None` for an empty slice.
3528fn read_slice_pointer<'tcx>(
3529    tcx: TyCtxt<'tcx>,
3530    alloc_id: AllocId,
3531    offset: Size,
3532) -> Result<Option<(AllocId, Size, u64)>, ()> {
3533    let alloc = resolve_alloc(tcx, alloc_id)?;
3534    let pointer_size = tcx.data_layout.pointer_size();
3535
3536    let pointer = alloc
3537        .read_scalar(&tcx, alloc_range(offset, pointer_size), true)
3538        .map_err(|_| ())?
3539        .to_pointer(&tcx);
3540
3541    let len = alloc
3542        .read_scalar(&tcx, alloc_range(offset + pointer_size, pointer_size), false)
3543        .map_err(|_| ())?
3544        .to_target_usize(&tcx)
3545        .discard_err()
3546        .ok_or(())?;
3547
3548    if len == 0 {
3549        return Ok(None);
3550    }
3551
3552    let (provenance, offset) =
3553        pointer.into_pointer_or_addr().map_err(|_| ())?.prov_and_relative_offset();
3554
3555    Ok(Some((provenance.alloc_id(), offset, len)))
3556}
3557
3558fn resolve_alloc<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId) -> Result<&'tcx Allocation, ()> {
3559    let alloc = match tcx.global_alloc(alloc_id) {
3560        GlobalAlloc::Memory(alloc) => alloc,
3561        GlobalAlloc::Static(def_id) => tcx.eval_static_initializer(def_id).map_err(|_| ())?,
3562        _ => return Err(()),
3563    }
3564    .inner();
3565
3566    Ok(alloc)
3567}