1use std::cmp::Ordering;
2use std::ffi::c_uint;
3use std::{assert_matches, iter, ptr};
4
5use rustc_abi::{
6 Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive, Size,
7 WrappingRange,
8};
9use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
10use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
11use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
14use rustc_codegen_ssa::traits::*;
15use rustc_hir as hir;
16use rustc_hir::def_id::LOCAL_CRATE;
17use rustc_hir::find_attr;
18use rustc_middle::mir::BinOp;
19use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
20use rustc_middle::ty::offload_meta::OffloadMetadata;
21use rustc_middle::ty::{
22 self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv, Unnormalized,
23};
24use rustc_middle::{bug, span_bug};
25use rustc_session::config::CrateType;
26use rustc_session::lint::builtin::DEPRECATED_LLVM_INTRINSIC;
27use rustc_span::{Span, Symbol, sym};
28use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate};
29use rustc_target::callconv::PassMode;
30use rustc_target::spec::{Arch, Os};
31use tracing::debug;
32
33use crate::abi::FnAbiLlvmExt;
34use crate::builder::Builder;
35use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
36use crate::builder::gpu_offload::{
37 OffloadKernelDims, gen_call_handling, gen_define_handling, register_offload,
38};
39use crate::context::CodegenCx;
40use crate::declare::declare_raw_fn;
41use crate::errors::{
42 AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
43 OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
44};
45use crate::llvm::{self, Type, Value};
46use crate::type_of::LayoutLlvmExt;
47use crate::va_arg::emit_va_arg;
48
49fn call_simple_intrinsic<'ll, 'tcx>(
50 bx: &mut Builder<'_, 'll, 'tcx>,
51 name: Symbol,
52 args: &[OperandRef<'tcx, &'ll Value>],
53) -> Option<&'ll Value> {
54 let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
55 sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]),
56 sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]),
57 sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]),
58 sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]),
59
60 sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]),
61 sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]),
62 sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]),
63 sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]),
64
65 sym::sinf16 => ("llvm.sin", &[bx.type_f16()]),
66 sym::sinf32 => ("llvm.sin", &[bx.type_f32()]),
67 sym::sinf64 => ("llvm.sin", &[bx.type_f64()]),
68 sym::sinf128 => ("llvm.sin", &[bx.type_f128()]),
69
70 sym::cosf16 => ("llvm.cos", &[bx.type_f16()]),
71 sym::cosf32 => ("llvm.cos", &[bx.type_f32()]),
72 sym::cosf64 => ("llvm.cos", &[bx.type_f64()]),
73 sym::cosf128 => ("llvm.cos", &[bx.type_f128()]),
74
75 sym::powf16 => ("llvm.pow", &[bx.type_f16()]),
76 sym::powf32 => ("llvm.pow", &[bx.type_f32()]),
77 sym::powf64 => ("llvm.pow", &[bx.type_f64()]),
78 sym::powf128 => ("llvm.pow", &[bx.type_f128()]),
79
80 sym::expf16 => ("llvm.exp", &[bx.type_f16()]),
81 sym::expf32 => ("llvm.exp", &[bx.type_f32()]),
82 sym::expf64 => ("llvm.exp", &[bx.type_f64()]),
83 sym::expf128 => ("llvm.exp", &[bx.type_f128()]),
84
85 sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]),
86 sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]),
87 sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]),
88 sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]),
89
90 sym::logf16 => ("llvm.log", &[bx.type_f16()]),
91 sym::logf32 => ("llvm.log", &[bx.type_f32()]),
92 sym::logf64 => ("llvm.log", &[bx.type_f64()]),
93 sym::logf128 => ("llvm.log", &[bx.type_f128()]),
94
95 sym::log10f16 => ("llvm.log10", &[bx.type_f16()]),
96 sym::log10f32 => ("llvm.log10", &[bx.type_f32()]),
97 sym::log10f64 => ("llvm.log10", &[bx.type_f64()]),
98 sym::log10f128 => ("llvm.log10", &[bx.type_f128()]),
99
100 sym::log2f16 => ("llvm.log2", &[bx.type_f16()]),
101 sym::log2f32 => ("llvm.log2", &[bx.type_f32()]),
102 sym::log2f64 => ("llvm.log2", &[bx.type_f64()]),
103 sym::log2f128 => ("llvm.log2", &[bx.type_f128()]),
104
105 sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]),
106 sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]),
107 sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]),
108 sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]),
109
110 sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]),
111 sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]),
112 sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]),
113 sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]),
114
115 sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
130 sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
131 sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
132 sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
133
134 sym::floorf16 => ("llvm.floor", &[bx.type_f16()]),
135 sym::floorf32 => ("llvm.floor", &[bx.type_f32()]),
136 sym::floorf64 => ("llvm.floor", &[bx.type_f64()]),
137 sym::floorf128 => ("llvm.floor", &[bx.type_f128()]),
138
139 sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]),
140 sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]),
141 sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]),
142 sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]),
143
144 sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]),
145 sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]),
146 sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]),
147 sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]),
148
149 sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]),
154 sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]),
155 sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]),
156 sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]),
157
158 sym::roundf16 => ("llvm.round", &[bx.type_f16()]),
159 sym::roundf32 => ("llvm.round", &[bx.type_f32()]),
160 sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
161 sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
162
163 _ => return None,
164 };
165 Some(bx.call_intrinsic(
166 base_name,
167 type_params,
168 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
169 ))
170}
171
172impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
173 fn codegen_intrinsic_call(
174 &mut self,
175 instance: ty::Instance<'tcx>,
176 args: &[OperandRef<'tcx, &'ll Value>],
177 result: PlaceRef<'tcx, &'ll Value>,
178 span: Span,
179 ) -> Result<(), ty::Instance<'tcx>> {
180 let tcx = self.tcx;
181
182 let name = tcx.item_name(instance.def_id());
183 let fn_args = instance.args;
184
185 let simple = call_simple_intrinsic(self, name, args);
186 let llval = match name {
187 _ if simple.is_some() => simple.unwrap(),
188 sym::minimum_number_nsz_f16
189 | sym::minimum_number_nsz_f32
190 | sym::minimum_number_nsz_f64
191 | sym::minimum_number_nsz_f128
192 | sym::maximum_number_nsz_f16
193 | sym::maximum_number_nsz_f32
194 | sym::maximum_number_nsz_f64
195 | sym::maximum_number_nsz_f128
196 if crate::llvm_util::get_version() >= (22, 0, 0) =>
198 {
199 let intrinsic_name = if name.as_str().starts_with("min") {
200 "llvm.minimumnum"
201 } else {
202 "llvm.maximumnum"
203 };
204 let call = self.call_intrinsic(
205 intrinsic_name,
206 &[args[0].layout.immediate_llvm_type(self.cx)],
207 &[args[0].immediate(), args[1].immediate()],
208 );
209 unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
212 call
213 }
214 sym::ptr_mask => {
215 let ptr = args[0].immediate();
216 self.call_intrinsic(
217 "llvm.ptrmask",
218 &[self.val_ty(ptr), self.type_isize()],
219 &[ptr, args[1].immediate()],
220 )
221 }
222 sym::autodiff => {
223 codegen_autodiff(self, tcx, instance, args, result);
224 return Ok(());
225 }
226 sym::offload => {
227 if tcx.sess.opts.unstable_opts.offload.is_empty() {
228 let _ = tcx.dcx().emit_almost_fatal(OffloadWithoutEnable);
229 }
230
231 if tcx.sess.lto() != rustc_session::config::Lto::Fat {
232 let _ = tcx.dcx().emit_almost_fatal(OffloadWithoutFatLTO);
233 }
234
235 codegen_offload(self, tcx, instance, args);
236 return Ok(());
237 }
238 sym::is_val_statically_known => {
239 if let OperandValue::Immediate(imm) = args[0].val {
240 self.call_intrinsic(
241 "llvm.is.constant",
242 &[args[0].layout.immediate_llvm_type(self.cx)],
243 &[imm],
244 )
245 } else {
246 self.const_bool(false)
247 }
248 }
249 sym::select_unpredictable => {
250 let cond = args[0].immediate();
251 match (&args[1].layout, &args[2].layout) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val, &*right_val,
::core::option::Option::None);
}
}
};assert_eq!(args[1].layout, args[2].layout);
252 let select = |bx: &mut Self, true_val, false_val| {
253 let result = bx.select(cond, true_val, false_val);
254 bx.set_unpredictable(&result);
255 result
256 };
257 match (args[1].val, args[2].val) {
258 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
259 if !true_val.llextra.is_none() {
::core::panicking::panic("assertion failed: true_val.llextra.is_none()")
};assert!(true_val.llextra.is_none());
260 if !false_val.llextra.is_none() {
::core::panicking::panic("assertion failed: false_val.llextra.is_none()")
};assert!(false_val.llextra.is_none());
261 match (&true_val.align, &false_val.align) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val, &*right_val,
::core::option::Option::None);
}
}
};assert_eq!(true_val.align, false_val.align);
262 let ptr = select(self, true_val.llval, false_val.llval);
263 let selected =
264 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
265 selected.store(self, result);
266 return Ok(());
267 }
268 (OperandValue::Immediate(_), OperandValue::Immediate(_))
269 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
270 let true_val = args[1].immediate_or_packed_pair(self);
271 let false_val = args[2].immediate_or_packed_pair(self);
272 select(self, true_val, false_val)
273 }
274 (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
275 _ => ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("Incompatible OperandValue for select_unpredictable"))span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
276 }
277 }
278 sym::catch_unwind => {
279 catch_unwind_intrinsic(
280 self,
281 args[0].immediate(),
282 args[1].immediate(),
283 args[2].immediate(),
284 result,
285 );
286 return Ok(());
287 }
288 sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
289 sym::va_arg => {
290 let BackendRepr::Scalar(scalar) = result.layout.backend_repr else {
291 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support non-scalar types"))bug!("the va_arg intrinsic does not support non-scalar types")
292 };
293
294 match scalar.primitive() {
298 Primitive::Pointer(_) => {
299 }
301 Primitive::Int(Integer::I128, _) => {
302 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `i128`/`u128`"))bug!("the va_arg intrinsic does not support `i128`/`u128`")
305 }
306 Primitive::Int(..) => {
307 let int_width = self.cx().size_of(result.layout.ty).bits();
308 let target_c_int_width = self.cx().sess().target.options.c_int_width;
309 if int_width < u64::from(target_c_int_width) {
310 ::rustc_middle::util::bug::bug_fmt(format_args!("va_arg got i{0} but needs at least c_int (an i{1})",
int_width, target_c_int_width));bug!(
313 "va_arg got i{} but needs at least c_int (an i{})",
314 int_width,
315 target_c_int_width
316 );
317 }
318 }
319 Primitive::Float(Float::F16) => {
320 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f16`"))bug!("the va_arg intrinsic does not support `f16`")
321 }
322 Primitive::Float(Float::F32) => {
323 if self.cx().sess().target.arch != Arch::Avr {
325 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f32` on this target"))bug!("the va_arg intrinsic does not support `f32` on this target")
326 }
327 }
328 Primitive::Float(Float::F64) => {
329 }
331 Primitive::Float(Float::F128) => {
332 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f128`"))bug!("the va_arg intrinsic does not support `f128`")
334 }
335 }
336
337 emit_va_arg(self, args[0], result.layout.ty)
338 }
339
340 sym::volatile_load | sym::unaligned_volatile_load => {
341 let ptr = args[0].immediate();
342 let load = self.volatile_load(result.layout.llvm_type(self), ptr);
343 let align = if name == sym::unaligned_volatile_load {
344 1
345 } else {
346 result.layout.align.bytes() as u32
347 };
348 unsafe {
349 llvm::LLVMSetAlignment(load, align);
350 }
351 if !result.layout.is_zst() {
352 self.store_to_place(load, result.val);
353 }
354 return Ok(());
355 }
356 sym::volatile_store => {
357 let dst = args[0].deref(self.cx());
358 args[1].val.volatile_store(self, dst);
359 return Ok(());
360 }
361 sym::unaligned_volatile_store => {
362 let dst = args[0].deref(self.cx());
363 args[1].val.unaligned_volatile_store(self, dst);
364 return Ok(());
365 }
366 sym::prefetch_read_data
367 | sym::prefetch_write_data
368 | sym::prefetch_read_instruction
369 | sym::prefetch_write_instruction => {
370 let (rw, cache_type) = match name {
371 sym::prefetch_read_data => (0, 1),
372 sym::prefetch_write_data => (1, 1),
373 sym::prefetch_read_instruction => (0, 0),
374 sym::prefetch_write_instruction => (1, 0),
375 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("impossible case reached"))bug!(),
376 };
377 let ptr = args[0].immediate();
378 let locality = fn_args.const_at(1).to_leaf().to_i32();
379 self.call_intrinsic(
380 "llvm.prefetch",
381 &[self.val_ty(ptr)],
382 &[
383 ptr,
384 self.const_i32(rw),
385 self.const_i32(locality),
386 self.const_i32(cache_type),
387 ],
388 )
389 }
390 sym::carrying_mul_add => {
391 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
392
393 let wide_llty = self.type_ix(size.bits() * 2);
394 let args = args.as_array().unwrap();
395 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
396
397 let wide = if signed {
398 let prod = self.unchecked_smul(a, b);
399 let acc = self.unchecked_sadd(prod, c);
400 self.unchecked_sadd(acc, d)
401 } else {
402 let prod = self.unchecked_umul(a, b);
403 let acc = self.unchecked_uadd(prod, c);
404 self.unchecked_uadd(acc, d)
405 };
406
407 let narrow_llty = self.type_ix(size.bits());
408 let low = self.trunc(wide, narrow_llty);
409 let bits_const = self.const_uint(wide_llty, size.bits());
410 let high = self.lshr(wide, bits_const);
412 let high = self.trunc(high, narrow_llty);
414
415 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
416 let pair = self.const_poison(pair_llty);
417 let pair = self.insert_value(pair, low, 0);
418 let pair = self.insert_value(pair, high, 1);
419 pair
420 }
421
422 sym::carryless_mul if crate::llvm_util::get_version() >= (22, 0, 0) => {
424 let ty = args[0].layout.ty;
425 if !ty.is_integral() {
426 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
427 span,
428 name,
429 ty,
430 });
431 return Ok(());
432 }
433 let (size, _) = ty.int_size_and_signed(self.tcx);
434 let width = size.bits();
435 let llty = self.type_ix(width);
436
437 let lhs = args[0].immediate();
438 let rhs = args[1].immediate();
439 self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
440 }
441
442 sym::ctlz
443 | sym::ctlz_nonzero
444 | sym::cttz
445 | sym::cttz_nonzero
446 | sym::ctpop
447 | sym::bswap
448 | sym::bitreverse
449 | sym::saturating_add
450 | sym::saturating_sub
451 | sym::unchecked_funnel_shl
452 | sym::unchecked_funnel_shr => {
453 let ty = args[0].layout.ty;
454 if !ty.is_integral() {
455 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
456 span,
457 name,
458 ty,
459 });
460 return Ok(());
461 }
462 let (size, signed) = ty.int_size_and_signed(self.tcx);
463 let width = size.bits();
464 let llty = self.type_ix(width);
465 match name {
466 sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
467 let y =
468 self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
469 let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
470 "llvm.ctlz"
471 } else {
472 "llvm.cttz"
473 };
474 let ret =
475 self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
476 self.intcast(ret, result.layout.llvm_type(self), false)
477 }
478 sym::ctpop => {
479 let ret =
480 self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]);
481 self.intcast(ret, result.layout.llvm_type(self), false)
482 }
483 sym::bswap => {
484 if width == 8 {
485 args[0].immediate() } else {
487 self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
488 }
489 }
490 sym::bitreverse => {
491 self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
492 }
493 sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
494 let is_left = name == sym::unchecked_funnel_shl;
495 let lhs = args[0].immediate();
496 let rhs = args[1].immediate();
497 let raw_shift = args[2].immediate();
498 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.fsh{0}",
if is_left { 'l' } else { 'r' }))
})format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
499
500 let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false);
503
504 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift])
505 }
506 sym::saturating_add | sym::saturating_sub => {
507 let is_add = name == sym::saturating_add;
508 let lhs = args[0].immediate();
509 let rhs = args[1].immediate();
510 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
511 "llvm.{}{}.sat",
512 if signed { 's' } else { 'u' },
513 if is_add { "add" } else { "sub" },
514 );
515 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
516 }
517 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("impossible case reached"))bug!(),
518 }
519 }
520
521 sym::fabs => {
522 let ty = args[0].layout.ty;
523 let ty::Float(f) = ty.kind() else {
524 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("the `fabs` intrinsic requires a floating-point argument, got {0:?}",
ty));span_bug!(span, "the `fabs` intrinsic requires a floating-point argument, got {:?}", ty);
525 };
526 let llty = self.type_float_from_ty(*f);
527 let llvm_name = "llvm.fabs";
528 self.call_intrinsic(
529 llvm_name,
530 &[llty],
531 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
532 )
533 }
534
535 sym::raw_eq => {
536 use BackendRepr::*;
537 let tp_ty = fn_args.type_at(0);
538 let layout = self.layout_of(tp_ty).layout;
539 let use_integer_compare = match layout.backend_repr() {
540 Scalar(_) | ScalarPair(_, _) => true,
541 SimdVector { .. } => false,
542 SimdScalableVector { .. } => {
543 tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
544 span,
545 name: sym::raw_eq,
546 ty: tp_ty,
547 });
548 return Ok(());
549 }
550 Memory { .. } => {
551 layout.size() <= self.data_layout().pointer_size() * 2
555 }
556 };
557
558 let a = args[0].immediate();
559 let b = args[1].immediate();
560 if layout.size().bytes() == 0 {
561 self.const_bool(true)
562 } else if use_integer_compare {
563 let integer_ty = self.type_ix(layout.size().bits());
564 let a_val = self.load(integer_ty, a, layout.align().abi);
565 let b_val = self.load(integer_ty, b, layout.align().abi);
566 self.icmp(IntPredicate::IntEQ, a_val, b_val)
567 } else {
568 let n = self.const_usize(layout.size().bytes());
569 let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
570 self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
571 }
572 }
573
574 sym::compare_bytes => {
575 let cmp = self.call_intrinsic(
577 "memcmp",
578 &[],
579 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
580 );
581 self.sext(cmp, self.type_ix(32))
583 }
584
585 sym::black_box => {
586 args[0].val.store(self, result);
587 let result_val_span = [result.val.llval];
588 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
598 ("~{memory}", &[])
599 } else {
600 ("r,~{memory}", &result_val_span)
601 };
602 crate::asm::inline_asm_call(
603 self,
604 "",
605 constraint,
606 inputs,
607 self.type_void(),
608 &[],
609 true,
610 false,
611 llvm::AsmDialect::Att,
612 &[span],
613 false,
614 None,
615 None,
616 )
617 .unwrap_or_else(|| ::rustc_middle::util::bug::bug_fmt(format_args!("failed to generate inline asm call for `black_box`"))bug!("failed to generate inline asm call for `black_box`"));
618
619 return Ok(());
621 }
622
623 sym::amdgpu_dispatch_ptr => {
624 let val = self.call_intrinsic("llvm.amdgcn.dispatch.ptr", &[], &[]);
625 self.pointercast(val, self.type_ptr())
627 }
628
629 sym::sve_tuple_create2 => {
630 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
631 self.layout_of(fn_args.type_at(0)).backend_repr,
632 BackendRepr::SimdScalableVector {
633 number_of_vectors: NumScalableVectors(1),
634 ..
635 }
636 );
637 let tuple_ty = self.layout_of(fn_args.type_at(1));
638 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
639 tuple_ty.backend_repr,
640 BackendRepr::SimdScalableVector {
641 number_of_vectors: NumScalableVectors(2),
642 ..
643 }
644 );
645 let ret = self.const_poison(self.backend_type(tuple_ty));
646 let ret = self.insert_value(ret, args[0].immediate(), 0);
647 self.insert_value(ret, args[1].immediate(), 1)
648 }
649
650 sym::sve_tuple_create3 => {
651 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
652 self.layout_of(fn_args.type_at(0)).backend_repr,
653 BackendRepr::SimdScalableVector {
654 number_of_vectors: NumScalableVectors(1),
655 ..
656 }
657 );
658 let tuple_ty = self.layout_of(fn_args.type_at(1));
659 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(3), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(3), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
660 tuple_ty.backend_repr,
661 BackendRepr::SimdScalableVector {
662 number_of_vectors: NumScalableVectors(3),
663 ..
664 }
665 );
666 let ret = self.const_poison(self.backend_type(tuple_ty));
667 let ret = self.insert_value(ret, args[0].immediate(), 0);
668 let ret = self.insert_value(ret, args[1].immediate(), 1);
669 self.insert_value(ret, args[2].immediate(), 2)
670 }
671
672 sym::sve_tuple_create4 => {
673 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
674 self.layout_of(fn_args.type_at(0)).backend_repr,
675 BackendRepr::SimdScalableVector {
676 number_of_vectors: NumScalableVectors(1),
677 ..
678 }
679 );
680 let tuple_ty = self.layout_of(fn_args.type_at(1));
681 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(4), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(4), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
682 tuple_ty.backend_repr,
683 BackendRepr::SimdScalableVector {
684 number_of_vectors: NumScalableVectors(4),
685 ..
686 }
687 );
688 let ret = self.const_poison(self.backend_type(tuple_ty));
689 let ret = self.insert_value(ret, args[0].immediate(), 0);
690 let ret = self.insert_value(ret, args[1].immediate(), 1);
691 let ret = self.insert_value(ret, args[2].immediate(), 2);
692 self.insert_value(ret, args[3].immediate(), 3)
693 }
694
695 sym::sve_tuple_get => {
696 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
697 self.layout_of(fn_args.type_at(0)).backend_repr,
698 BackendRepr::SimdScalableVector {
699 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
700 ..
701 }
702 );
703 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
704 self.layout_of(fn_args.type_at(1)).backend_repr,
705 BackendRepr::SimdScalableVector {
706 number_of_vectors: NumScalableVectors(1),
707 ..
708 }
709 );
710 self.extract_value(
711 args[0].immediate(),
712 fn_args.const_at(2).to_leaf().to_i32() as u64,
713 )
714 }
715
716 sym::sve_tuple_set => {
717 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
718 self.layout_of(fn_args.type_at(0)).backend_repr,
719 BackendRepr::SimdScalableVector {
720 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
721 ..
722 }
723 );
724 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
725 self.layout_of(fn_args.type_at(1)).backend_repr,
726 BackendRepr::SimdScalableVector {
727 number_of_vectors: NumScalableVectors(1),
728 ..
729 }
730 );
731 self.insert_value(
732 args[0].immediate(),
733 args[1].immediate(),
734 fn_args.const_at(2).to_leaf().to_i32() as u64,
735 )
736 }
737
738 _ if name.as_str().starts_with("simd_") => {
739 let mut loaded_args = Vec::new();
742 for arg in args {
743 loaded_args.push(
744 if arg.layout.ty.is_simd()
749 && let OperandValue::Ref(place) = arg.val
750 {
751 let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
752 let elem_ll_ty = match elem_ty.kind() {
753 ty::Float(f) => self.type_float_from_ty(*f),
754 ty::Int(i) => self.type_int_from_ty(*i),
755 ty::Uint(u) => self.type_uint_from_ty(*u),
756 ty::RawPtr(_, _) => self.type_ptr(),
757 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
758 };
759 let loaded =
760 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
761 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
762 } else {
763 *arg
764 },
765 );
766 }
767
768 let llret_ty = if result.layout.ty.is_simd()
769 && let BackendRepr::Memory { .. } = result.layout.backend_repr
770 {
771 let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
772 let elem_ll_ty = match elem_ty.kind() {
773 ty::Float(f) => self.type_float_from_ty(*f),
774 ty::Int(i) => self.type_int_from_ty(*i),
775 ty::Uint(u) => self.type_uint_from_ty(*u),
776 ty::RawPtr(_, _) => self.type_ptr(),
777 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
778 };
779 self.type_vector(elem_ll_ty, size)
780 } else {
781 result.layout.llvm_type(self)
782 };
783
784 match generic_simd_intrinsic(
785 self,
786 name,
787 fn_args,
788 &loaded_args,
789 result.layout.ty,
790 llret_ty,
791 span,
792 ) {
793 Ok(llval) => llval,
794 Err(()) => return Ok(()),
797 }
798 }
799
800 _ => {
801 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/intrinsic.rs:801",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(801u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
let mut iter = __CALLSITE.metadata().fields().iter();
__CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
::tracing::__macro_support::Option::Some(&format_args!("unknown intrinsic \'{0}\' -- falling back to default body",
name) as &dyn Value))])
});
} else { ; }
};debug!("unknown intrinsic '{}' -- falling back to default body", name);
802 return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
804 }
805 };
806
807 if result.layout.ty.is_bool() {
808 let val = self.from_immediate(llval);
809 self.store_to_place(val, result.val);
810 } else if !result.layout.ty.is_unit() {
811 self.store_to_place(llval, result.val);
812 }
813 Ok(())
814 }
815
816 fn codegen_llvm_intrinsic_call(
817 &mut self,
818 instance: ty::Instance<'tcx>,
819 args: &[OperandRef<'tcx, Self::Value>],
820 _is_cleanup: bool,
821 ) -> Self::Value {
822 let tcx = self.tcx();
823
824 let fn_ty = instance.ty(tcx, self.typing_env());
825 let fn_sig = match *fn_ty.kind() {
826 ty::FnDef(def_id, args) => tcx.instantiate_bound_regions_with_erased(
827 tcx.fn_sig(def_id).instantiate(tcx, args).skip_norm_wip(),
828 ),
829 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
830 };
831 if !!fn_sig.c_variadic() {
::core::panicking::panic("assertion failed: !fn_sig.c_variadic()")
};assert!(!fn_sig.c_variadic());
832
833 let ret_layout = self.layout_of(fn_sig.output());
834 let llreturn_ty = if ret_layout.is_zst() {
835 self.type_void()
836 } else {
837 ret_layout.immediate_llvm_type(self)
838 };
839
840 let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
841 for &arg in fn_sig.inputs() {
842 let arg_layout = self.layout_of(arg);
843 if arg_layout.is_zst() {
844 continue;
845 }
846 llargument_tys.push(arg_layout.immediate_llvm_type(self));
847 }
848
849 let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) {
850 llfn
851 } else {
852 let sym = tcx.symbol_name(instance).name;
853
854 let llfn = if let Some(llfn) = self.get_declared_value(sym) {
855 llfn
856 } else {
857 intrinsic_fn(self, sym, llreturn_ty, llargument_tys, instance)
858 };
859
860 self.intrinsic_instances.borrow_mut().insert(instance, llfn);
861
862 llfn
863 };
864 let fn_ty = self.get_type_of_global(fn_ptr);
865
866 let mut llargs = ::alloc::vec::Vec::new()vec![];
867
868 for arg in args {
869 match arg.val {
870 OperandValue::ZeroSized => {}
871 OperandValue::Immediate(a) => llargs.push(a),
872 OperandValue::Pair(a, b) => {
873 llargs.push(a);
874 llargs.push(b);
875 }
876 OperandValue::Ref(op_place_val) => {
877 let mut llval = op_place_val.llval;
878 llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align);
884 if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
885 if scalar.is_bool() {
886 self.range_metadata(llval, WrappingRange { start: 0, end: 1 });
887 }
888 llval = self.to_immediate_scalar(llval, scalar);
890 }
891 llargs.push(llval);
892 }
893 }
894 }
895
896 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/intrinsic.rs:896",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(896u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
let mut iter = __CALLSITE.metadata().fields().iter();
__CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
::tracing::__macro_support::Option::Some(&format_args!("call intrinsic {0:?} with args ({1:?})",
instance, llargs) as &dyn Value))])
});
} else { ; }
};debug!("call intrinsic {:?} with args ({:?})", instance, llargs);
897
898 for (dest_ty, arg) in iter::zip(self.func_params_types(fn_ty), &mut llargs) {
899 let src_ty = self.val_ty(arg);
900 if !can_autocast(self, src_ty, dest_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with {1:?} (found) in `{2:?}",
dest_ty, src_ty, fn_ptr));
}
};assert!(
901 can_autocast(self, src_ty, dest_ty),
902 "Cannot match `{dest_ty:?}` (expected) with {src_ty:?} (found) in `{fn_ptr:?}"
903 );
904
905 *arg = autocast(self, arg, src_ty, dest_ty);
906 }
907
908 let llret = unsafe {
909 llvm::LLVMBuildCallWithOperandBundles(
910 self.llbuilder,
911 fn_ty,
912 fn_ptr,
913 llargs.as_ptr(),
914 llargs.len() as c_uint,
915 ptr::dangling(),
916 0,
917 c"".as_ptr(),
918 )
919 };
920
921 let src_ty = self.val_ty(llret);
922 let dest_ty = llreturn_ty;
923 if !can_autocast(self, dest_ty, src_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with `{1:?}` (found) in `{2:?}`",
src_ty, dest_ty, fn_ptr));
}
};assert!(
924 can_autocast(self, dest_ty, src_ty),
925 "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
926 );
927
928 autocast(self, llret, src_ty, dest_ty)
929 }
930
931 fn abort(&mut self) {
932 self.call_intrinsic("llvm.trap", &[], &[]);
933 }
934
935 fn assume(&mut self, val: Self::Value) {
936 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
937 self.call_intrinsic("llvm.assume", &[], &[val]);
938 }
939 }
940
941 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
942 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
943 self.call_intrinsic(
944 "llvm.expect",
945 &[self.type_i1()],
946 &[cond, self.const_bool(expected)],
947 )
948 } else {
949 cond
950 }
951 }
952
953 fn type_checked_load(
954 &mut self,
955 llvtable: &'ll Value,
956 vtable_byte_offset: u64,
957 typeid: &[u8],
958 ) -> Self::Value {
959 let typeid = self.create_metadata(typeid);
960 let typeid = self.get_metadata_value(typeid);
961 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
962 let type_checked_load = self.call_intrinsic(
963 "llvm.type.checked.load",
964 &[],
965 &[llvtable, vtable_byte_offset, typeid],
966 );
967 self.extract_value(type_checked_load, 0)
968 }
969
970 fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
971 self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list])
972 }
973
974 fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
975 self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list])
976 }
977}
978
979fn llvm_arch_for(rust_arch: &Arch) -> Option<&'static str> {
980 Some(match rust_arch {
981 Arch::AArch64 | Arch::Arm64EC => "aarch64",
982 Arch::AmdGpu => "amdgcn",
983 Arch::Arm => "arm",
984 Arch::Bpf => "bpf",
985 Arch::Hexagon => "hexagon",
986 Arch::LoongArch32 | Arch::LoongArch64 => "loongarch",
987 Arch::Mips | Arch::Mips32r6 | Arch::Mips64 | Arch::Mips64r6 => "mips",
988 Arch::Nvptx64 => "nvvm",
989 Arch::PowerPC | Arch::PowerPC64 => "ppc",
990 Arch::RiscV32 | Arch::RiscV64 => "riscv",
991 Arch::S390x => "s390",
992 Arch::SpirV => "spv",
993 Arch::Wasm32 | Arch::Wasm64 => "wasm",
994 Arch::X86 | Arch::X86_64 => "x86",
995 _ => return None, })
997}
998
999fn can_autocast<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
1000 if rust_ty == llvm_ty {
1001 return true;
1002 }
1003
1004 match cx.type_kind(llvm_ty) {
1005 TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => {
1009 let rust_element_tys = cx.struct_element_types(rust_ty);
1010 let llvm_element_tys = cx.struct_element_types(llvm_ty);
1011
1012 if rust_element_tys.len() != llvm_element_tys.len() {
1013 return false;
1014 }
1015
1016 iter::zip(rust_element_tys, llvm_element_tys).all(
1017 |(rust_element_ty, llvm_element_ty)| {
1018 can_autocast(cx, rust_element_ty, llvm_element_ty)
1019 },
1020 )
1021 }
1022 TypeKind::Vector => {
1023 let llvm_element_ty = cx.element_type(llvm_ty);
1024 let element_count = cx.vector_length(llvm_ty) as u64;
1025
1026 if llvm_element_ty == cx.type_bf16() {
1027 rust_ty == cx.type_vector(cx.type_i16(), element_count)
1028 } else if llvm_element_ty == cx.type_i1() {
1029 let int_width = element_count.next_power_of_two().max(8);
1030 rust_ty == cx.type_ix(int_width)
1031 } else {
1032 false
1033 }
1034 }
1035 TypeKind::BFloat => rust_ty == cx.type_i16(),
1036 TypeKind::X86_AMX if cx.type_kind(rust_ty) == TypeKind::Vector => {
1037 let element_ty = cx.element_type(rust_ty);
1038 let element_count = cx.vector_length(rust_ty) as u64;
1039
1040 let element_size_bits = match cx.type_kind(element_ty) {
1041 TypeKind::Half => 16,
1042 TypeKind::Float => 32,
1043 TypeKind::Double => 64,
1044 TypeKind::FP128 => 128,
1045 TypeKind::Integer => cx.int_width(element_ty),
1046 TypeKind::Pointer => cx.int_width(cx.isize_ty),
1047 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("Vector element type `{0:?}` not one of integer, float or pointer",
element_ty))bug!(
1048 "Vector element type `{element_ty:?}` not one of integer, float or pointer"
1049 ),
1050 };
1051
1052 element_size_bits * element_count == 8192
1053 }
1054 _ => false,
1055 }
1056}
1057
1058fn autocast<'ll>(
1059 bx: &mut Builder<'_, 'll, '_>,
1060 val: &'ll Value,
1061 src_ty: &'ll Type,
1062 dest_ty: &'ll Type,
1063) -> &'ll Value {
1064 if src_ty == dest_ty {
1065 return val;
1066 }
1067 match (bx.type_kind(src_ty), bx.type_kind(dest_ty)) {
1068 (TypeKind::Struct, TypeKind::Struct) => {
1070 let mut ret = bx.const_poison(dest_ty);
1071 for (idx, (src_element_ty, dest_element_ty)) in
1072 iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty))
1073 .enumerate()
1074 {
1075 let elt = bx.extract_value(val, idx as u64);
1076 let casted_elt = autocast(bx, elt, src_element_ty, dest_element_ty);
1077 ret = bx.insert_value(ret, casted_elt, idx as u64);
1078 }
1079 ret
1080 }
1081 (TypeKind::Vector, TypeKind::Integer) if bx.element_type(src_ty) == bx.type_i1() => {
1083 let vector_length = bx.vector_length(src_ty) as u64;
1084 let int_width = vector_length.next_power_of_two().max(8);
1085
1086 let val = if vector_length == int_width {
1087 val
1088 } else {
1089 let shuffle_indices = match vector_length {
1091 0 => {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("zero length vectors are not allowed")));
}unreachable!("zero length vectors are not allowed"),
1092 1 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 1, 1, 1, 1, 1, 1]))vec![0, 1, 1, 1, 1, 1, 1, 1],
1093 2 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 2, 2, 2, 2, 2]))vec![0, 1, 2, 2, 2, 2, 2, 2],
1094 3 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 3, 3, 3, 3, 3]))vec![0, 1, 2, 3, 3, 3, 3, 3],
1095 4.. => (0..int_width as i32).collect(),
1096 };
1097 let shuffle_mask =
1098 shuffle_indices.into_iter().map(|i| bx.const_i32(i)).collect::<Vec<_>>();
1099 bx.shuffle_vector(val, bx.const_null(src_ty), bx.const_vector(&shuffle_mask))
1100 };
1101 bx.bitcast(val, dest_ty)
1102 }
1103 (TypeKind::Integer, TypeKind::Vector) if bx.element_type(dest_ty) == bx.type_i1() => {
1105 let vector_length = bx.vector_length(dest_ty) as u64;
1106 let int_width = vector_length.next_power_of_two().max(8);
1107
1108 let intermediate_ty = bx.type_vector(bx.type_i1(), int_width);
1109 let intermediate = bx.bitcast(val, intermediate_ty);
1110
1111 if vector_length == int_width {
1112 intermediate
1113 } else {
1114 let shuffle_mask: Vec<_> =
1115 (0..vector_length).map(|i| bx.const_i32(i as i32)).collect();
1116 bx.shuffle_vector(
1117 intermediate,
1118 bx.const_poison(intermediate_ty),
1119 bx.const_vector(&shuffle_mask),
1120 )
1121 }
1122 }
1123 (TypeKind::Vector, TypeKind::X86_AMX) => {
1124 bx.call_intrinsic("llvm.x86.cast.vector.to.tile", &[src_ty], &[val])
1125 }
1126 (TypeKind::X86_AMX, TypeKind::Vector) => {
1127 bx.call_intrinsic("llvm.x86.cast.tile.to.vector", &[dest_ty], &[val])
1128 }
1129 _ => bx.bitcast(val, dest_ty), }
1131}
1132
1133fn intrinsic_fn<'ll, 'tcx>(
1134 bx: &Builder<'_, 'll, 'tcx>,
1135 name: &str,
1136 rust_return_ty: &'ll Type,
1137 rust_argument_tys: Vec<&'ll Type>,
1138 instance: ty::Instance<'tcx>,
1139) -> &'ll Value {
1140 let tcx = bx.tcx;
1141
1142 let rust_fn_ty = bx.type_func(&rust_argument_tys, rust_return_ty);
1143
1144 let intrinsic = llvm::Intrinsic::lookup(name.as_bytes());
1145
1146 if let Some(intrinsic) = intrinsic
1147 && intrinsic.is_target_specific()
1148 {
1149 let (llvm_arch, _) = name[5..].split_once('.').unwrap();
1150 let rust_arch = &tcx.sess.target.arch;
1151
1152 if let Some(correct_llvm_arch) = llvm_arch_for(rust_arch)
1153 && llvm_arch != correct_llvm_arch
1154 {
1155 tcx.dcx().emit_fatal(IntrinsicWrongArch {
1156 name,
1157 target_arch: rust_arch.desc(),
1158 span: tcx.def_span(instance.def_id()),
1159 });
1160 }
1161 }
1162
1163 if let Some(intrinsic) = intrinsic
1164 && !intrinsic.is_overloaded()
1165 {
1166 let llfn = intrinsic.get_declaration(bx.llmod, &[]);
1168 let llvm_fn_ty = bx.get_type_of_global(llfn);
1169
1170 let llvm_return_ty = bx.get_return_type(llvm_fn_ty);
1171 let llvm_argument_tys = bx.func_params_types(llvm_fn_ty);
1172 let llvm_is_variadic = bx.func_is_variadic(llvm_fn_ty);
1173
1174 let is_correct_signature = !llvm_is_variadic
1175 && rust_argument_tys.len() == llvm_argument_tys.len()
1176 && iter::once((rust_return_ty, llvm_return_ty))
1177 .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
1178 .all(|(rust_ty, llvm_ty)| can_autocast(bx, rust_ty, llvm_ty));
1179
1180 if !is_correct_signature {
1181 tcx.dcx().emit_fatal(IntrinsicSignatureMismatch {
1182 name,
1183 llvm_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", llvm_fn_ty))
})format!("{llvm_fn_ty:?}"),
1184 rust_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", rust_fn_ty))
})format!("{rust_fn_ty:?}"),
1185 span: tcx.def_span(instance.def_id()),
1186 });
1187 }
1188
1189 return llfn;
1190 }
1191
1192 let llfn = declare_raw_fn(
1194 bx,
1195 name,
1196 llvm::CCallConv,
1197 llvm::UnnamedAddr::Global,
1198 llvm::Visibility::Default,
1199 rust_fn_ty,
1200 );
1201
1202 if intrinsic.is_none() {
1203 let mut new_llfn = None;
1204 let can_upgrade = unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn) };
1205
1206 if !can_upgrade {
1207 tcx.dcx().emit_fatal(UnknownIntrinsic { name, span: tcx.def_span(instance.def_id()) });
1209 } else if let Some(def_id) = instance.def_id().as_local() {
1210 let hir_id = tcx.local_def_id_to_hir_id(def_id);
1212
1213 let msg = if let Some(new_llfn) = new_llfn {
1215 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{1}`, `{0}` can be used instead",
str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap(),
name))
})format!(
1216 "using deprecated intrinsic `{name}`, `{}` can be used instead",
1217 str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap()
1218 )
1219 } else {
1220 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{0}`",
name))
})format!("using deprecated intrinsic `{name}`")
1221 };
1222
1223 tcx.emit_node_lint(
1224 DEPRECATED_LLVM_INTRINSIC,
1225 hir_id,
1226 rustc_errors::DiagDecorator(|d| {
1227 d.primary_message(msg).span(tcx.hir_span(hir_id));
1228 }),
1229 );
1230 }
1231 }
1232
1233 llfn
1234}
1235
1236fn catch_unwind_intrinsic<'ll, 'tcx>(
1237 bx: &mut Builder<'_, 'll, 'tcx>,
1238 try_func: &'ll Value,
1239 data: &'ll Value,
1240 catch_func: &'ll Value,
1241 dest: PlaceRef<'tcx, &'ll Value>,
1242) {
1243 if !bx.sess().panic_strategy().unwinds() {
1244 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1245 bx.call(try_func_ty, None, None, try_func, &[data], None, None);
1246 OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
1249 } else if wants_msvc_seh(bx.sess()) {
1250 codegen_msvc_try(bx, try_func, data, catch_func, dest);
1251 } else if wants_wasm_eh(bx.sess()) {
1252 codegen_wasm_try(bx, try_func, data, catch_func, dest);
1253 } else if bx.sess().target.os == Os::Emscripten {
1254 codegen_emcc_try(bx, try_func, data, catch_func, dest);
1255 } else {
1256 codegen_gnu_try(bx, try_func, data, catch_func, dest);
1257 }
1258}
1259
1260fn codegen_msvc_try<'ll, 'tcx>(
1268 bx: &mut Builder<'_, 'll, 'tcx>,
1269 try_func: &'ll Value,
1270 data: &'ll Value,
1271 catch_func: &'ll Value,
1272 dest: PlaceRef<'tcx, &'ll Value>,
1273) {
1274 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1275 bx.set_personality_fn(bx.eh_personality());
1276
1277 let normal = bx.append_sibling_block("normal");
1278 let catchswitch = bx.append_sibling_block("catchswitch");
1279 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
1280 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
1281 let caught = bx.append_sibling_block("caught");
1282
1283 let try_func = llvm::get_param(bx.llfn(), 0);
1284 let data = llvm::get_param(bx.llfn(), 1);
1285 let catch_func = llvm::get_param(bx.llfn(), 2);
1286
1287 let ptr_size = bx.tcx().data_layout.pointer_size();
1343 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1344 let slot = bx.alloca(ptr_size, ptr_align);
1345 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1346 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
1347
1348 bx.switch_to_block(normal);
1349 bx.ret(bx.const_i32(0));
1350
1351 bx.switch_to_block(catchswitch);
1352 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
1353
1354 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
1369 let type_name = bx.const_bytes(b"rust_panic\0");
1370 let type_info =
1371 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
1372 let tydesc = bx.declare_global(
1373 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
1374 bx.val_ty(type_info),
1375 );
1376
1377 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
1378 if bx.cx.tcx.sess.target.supports_comdat() {
1379 llvm::SetUniqueComdat(bx.llmod, tydesc);
1380 }
1381 llvm::set_initializer(tydesc, type_info);
1382
1383 bx.switch_to_block(catchpad_rust);
1390 let flags = bx.const_i32(8);
1391 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
1392 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
1393 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1394 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
1395 bx.catch_ret(&funclet, caught);
1396
1397 bx.switch_to_block(catchpad_foreign);
1399 let flags = bx.const_i32(64);
1400 let null = bx.const_null(bx.type_ptr());
1401 let funclet = bx.catch_pad(cs, &[null, flags, null]);
1402 bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
1403 bx.catch_ret(&funclet, caught);
1404
1405 bx.switch_to_block(caught);
1406 bx.ret(bx.const_i32(1));
1407 });
1408
1409 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1412 OperandValue::Immediate(ret).store(bx, dest);
1413}
1414
1415fn codegen_wasm_try<'ll, 'tcx>(
1417 bx: &mut Builder<'_, 'll, 'tcx>,
1418 try_func: &'ll Value,
1419 data: &'ll Value,
1420 catch_func: &'ll Value,
1421 dest: PlaceRef<'tcx, &'ll Value>,
1422) {
1423 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1424 bx.set_personality_fn(bx.eh_personality());
1425
1426 let normal = bx.append_sibling_block("normal");
1427 let catchswitch = bx.append_sibling_block("catchswitch");
1428 let catchpad = bx.append_sibling_block("catchpad");
1429 let caught = bx.append_sibling_block("caught");
1430
1431 let try_func = llvm::get_param(bx.llfn(), 0);
1432 let data = llvm::get_param(bx.llfn(), 1);
1433 let catch_func = llvm::get_param(bx.llfn(), 2);
1434
1435 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1459 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
1460
1461 bx.switch_to_block(normal);
1462 bx.ret(bx.const_i32(0));
1463
1464 bx.switch_to_block(catchswitch);
1465 let cs = bx.catch_switch(None, None, &[catchpad]);
1466
1467 bx.switch_to_block(catchpad);
1468 let null = bx.const_null(bx.type_ptr());
1469 let funclet = bx.catch_pad(cs, &[null]);
1470
1471 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
1472 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
1473
1474 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1475 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
1476 bx.catch_ret(&funclet, caught);
1477
1478 bx.switch_to_block(caught);
1479 bx.ret(bx.const_i32(1));
1480 });
1481
1482 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1485 OperandValue::Immediate(ret).store(bx, dest);
1486}
1487
1488fn codegen_gnu_try<'ll, 'tcx>(
1500 bx: &mut Builder<'_, 'll, 'tcx>,
1501 try_func: &'ll Value,
1502 data: &'ll Value,
1503 catch_func: &'ll Value,
1504 dest: PlaceRef<'tcx, &'ll Value>,
1505) {
1506 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1507 let then = bx.append_sibling_block("then");
1520 let catch = bx.append_sibling_block("catch");
1521
1522 let try_func = llvm::get_param(bx.llfn(), 0);
1523 let data = llvm::get_param(bx.llfn(), 1);
1524 let catch_func = llvm::get_param(bx.llfn(), 2);
1525 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1526 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1527
1528 bx.switch_to_block(then);
1529 bx.ret(bx.const_i32(0));
1530
1531 bx.switch_to_block(catch);
1538 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1539 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
1540 let tydesc = bx.const_null(bx.type_ptr());
1541 bx.add_clause(vals, tydesc);
1542 let ptr = bx.extract_value(vals, 0);
1543 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1544 bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
1545 bx.ret(bx.const_i32(1));
1546 });
1547
1548 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1551 OperandValue::Immediate(ret).store(bx, dest);
1552}
1553
1554fn codegen_emcc_try<'ll, 'tcx>(
1558 bx: &mut Builder<'_, 'll, 'tcx>,
1559 try_func: &'ll Value,
1560 data: &'ll Value,
1561 catch_func: &'ll Value,
1562 dest: PlaceRef<'tcx, &'ll Value>,
1563) {
1564 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1565 let then = bx.append_sibling_block("then");
1583 let catch = bx.append_sibling_block("catch");
1584
1585 let try_func = llvm::get_param(bx.llfn(), 0);
1586 let data = llvm::get_param(bx.llfn(), 1);
1587 let catch_func = llvm::get_param(bx.llfn(), 2);
1588 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1589 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1590
1591 bx.switch_to_block(then);
1592 bx.ret(bx.const_i32(0));
1593
1594 bx.switch_to_block(catch);
1600 let tydesc = bx.eh_catch_typeinfo();
1601 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1602 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1603 bx.add_clause(vals, tydesc);
1604 bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1605 let ptr = bx.extract_value(vals, 0);
1606 let selector = bx.extract_value(vals, 1);
1607
1608 let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]);
1610 let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1611 let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1612
1613 let ptr_size = bx.tcx().data_layout.pointer_size();
1616 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1617 let i8_align = bx.tcx().data_layout.i8_align;
1618 if !(i8_align <= ptr_align) {
::core::panicking::panic("assertion failed: i8_align <= ptr_align")
};assert!(i8_align <= ptr_align);
1620 let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1621 bx.store(ptr, catch_data, ptr_align);
1622 let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1623 bx.store(is_rust_panic, catch_data_1, i8_align);
1624
1625 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1626 bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1627 bx.ret(bx.const_i32(1));
1628 });
1629
1630 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1633 OperandValue::Immediate(ret).store(bx, dest);
1634}
1635
1636fn gen_fn<'a, 'll, 'tcx>(
1639 cx: &'a CodegenCx<'ll, 'tcx>,
1640 name: &str,
1641 rust_fn_sig: ty::PolyFnSig<'tcx>,
1642 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1643) -> (&'ll Type, &'ll Value) {
1644 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1645 let llty = fn_abi.llvm_type(cx);
1646 let llfn = cx.declare_fn(name, fn_abi, None);
1647 cx.set_frame_pointer_type(llfn);
1648 cx.apply_target_cpu_attr(llfn);
1649 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1651 let llbb = Builder::append_block(cx, llfn, "entry-block");
1652 let bx = Builder::build(cx, llbb);
1653 codegen(bx);
1654 (llty, llfn)
1655}
1656
1657fn get_rust_try_fn<'a, 'll, 'tcx>(
1662 cx: &'a CodegenCx<'ll, 'tcx>,
1663 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1664) -> (&'ll Type, &'ll Value) {
1665 if let Some(llfn) = cx.rust_try_fn.get() {
1666 return llfn;
1667 }
1668
1669 let tcx = cx.tcx;
1671 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1672 let try_fn_ty = Ty::new_fn_ptr(
1674 tcx,
1675 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p], tcx.types.unit, hir::Safety::Unsafe)),
1676 );
1677 let catch_fn_ty = Ty::new_fn_ptr(
1679 tcx,
1680 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p, i8p], tcx.types.unit, hir::Safety::Unsafe)),
1681 );
1682 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig_rust_abi(
1684 [try_fn_ty, i8p, catch_fn_ty],
1685 tcx.types.i32,
1686 hir::Safety::Unsafe,
1687 ));
1688 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1689 cx.rust_try_fn.set(Some(rust_try));
1690 rust_try
1691}
1692
1693fn codegen_autodiff<'ll, 'tcx>(
1694 bx: &mut Builder<'_, 'll, 'tcx>,
1695 tcx: TyCtxt<'tcx>,
1696 instance: ty::Instance<'tcx>,
1697 args: &[OperandRef<'tcx, &'ll Value>],
1698 result: PlaceRef<'tcx, &'ll Value>,
1699) {
1700 if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
1701 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable);
1702 }
1703
1704 let ct = tcx.crate_types();
1705 let lto = tcx.sess.lto();
1706 if ct.len() == 1 && ct.contains(&CrateType::Executable) {
1707 if lto != rustc_session::config::Lto::Fat {
1708 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutLto);
1709 }
1710 } else {
1711 if lto != rustc_session::config::Lto::Fat && !tcx.sess.opts.cg.linker_plugin_lto.enabled() {
1712 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutLto);
1713 }
1714 }
1715
1716 let fn_args = instance.args;
1717 let callee_ty = instance.ty(tcx, bx.typing_env());
1718
1719 let sig = callee_ty.fn_sig(tcx).skip_binder();
1720
1721 let ret_ty = sig.output();
1722 let llret_ty = bx.layout_of(ret_ty).llvm_type(bx);
1723
1724 let source_fn_ptr_ty = fn_args.into_type_list(tcx)[0];
1725 let fn_to_diff = args[0].immediate();
1726
1727 let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() {
1728 ty::FnDef(def_id, diff_args) => (def_id, diff_args),
1729 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("invalid args"))bug!("invalid args"),
1730 };
1731
1732 let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) {
1733 Ok(Some(instance)) => instance,
1734 Ok(None) => ::rustc_middle::util::bug::bug_fmt(format_args!("could not resolve ({0:?}, {1:?}) to a specific autodiff instance",
diff_id, diff_args))bug!(
1735 "could not resolve ({:?}, {:?}) to a specific autodiff instance",
1736 diff_id,
1737 diff_args
1738 ),
1739 Err(_) => {
1740 return;
1742 }
1743 };
1744
1745 let val_arr = get_args_from_tuple(bx, args[2], fn_diff);
1746 let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE);
1747
1748 let Some(Some(mut diff_attrs)) =
1749 {
{
'done:
{
for i in
::rustc_hir::attrs::HasAttrs::get_attrs(fn_diff.def_id(),
&tcx) {
#[allow(unused_imports)]
use rustc_hir::attrs::AttributeKind::*;
let i: &rustc_hir::Attribute = i;
match i {
rustc_hir::Attribute::Parsed(RustcAutodiff(attr)) => {
break 'done Some(attr.clone());
}
rustc_hir::Attribute::Unparsed(..) =>
{}
#[deny(unreachable_patterns)]
_ => {}
}
}
None
}
}
}find_attr!(tcx, fn_diff.def_id(), RustcAutodiff(attr) => attr.clone())
1750 else {
1751 ::rustc_middle::util::bug::bug_fmt(format_args!("could not find autodiff attrs"))bug!("could not find autodiff attrs")
1752 };
1753
1754 adjust_activity_to_abi(
1755 tcx,
1756 source_fn_ptr_ty,
1757 TypingEnv::fully_monomorphized(),
1758 &mut diff_attrs.input_activity,
1759 );
1760
1761 let fnc_tree = rustc_middle::ty::fnc_typetrees(tcx, source_fn_ptr_ty);
1762
1763 generate_enzyme_call(
1765 bx,
1766 bx.cx,
1767 fn_to_diff,
1768 &diff_symbol,
1769 llret_ty,
1770 &val_arr,
1771 &diff_attrs,
1772 result,
1773 fnc_tree,
1774 );
1775}
1776
1777fn codegen_offload<'ll, 'tcx>(
1782 bx: &mut Builder<'_, 'll, 'tcx>,
1783 tcx: TyCtxt<'tcx>,
1784 instance: ty::Instance<'tcx>,
1785 args: &[OperandRef<'tcx, &'ll Value>],
1786) {
1787 let cx = bx.cx;
1788 let fn_args = instance.args;
1789
1790 let (target_id, target_args) = match fn_args.into_type_list(tcx)[0].kind() {
1791 ty::FnDef(def_id, params) => (def_id, params),
1792 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("invalid offload intrinsic arg"))bug!("invalid offload intrinsic arg"),
1793 };
1794
1795 let fn_target = match Instance::try_resolve(tcx, cx.typing_env(), *target_id, target_args) {
1796 Ok(Some(instance)) => instance,
1797 Ok(None) => ::rustc_middle::util::bug::bug_fmt(format_args!("could not resolve ({0:?}, {1:?}) to a specific offload instance",
target_id, target_args))bug!(
1798 "could not resolve ({:?}, {:?}) to a specific offload instance",
1799 target_id,
1800 target_args
1801 ),
1802 Err(_) => {
1803 return;
1805 }
1806 };
1807
1808 let offload_dims = OffloadKernelDims::from_operands(bx, &args[1], &args[2]);
1809 let args = get_args_from_tuple(bx, args[3], fn_target);
1810 let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE);
1811
1812 let sig = tcx.fn_sig(fn_target.def_id()).skip_binder();
1813 let sig = tcx.instantiate_bound_regions_with_erased(sig);
1814 let inputs = sig.inputs();
1815
1816 let metadata = inputs.iter().map(|ty| OffloadMetadata::from_ty(tcx, *ty)).collect::<Vec<_>>();
1817
1818 let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::<Vec<_>>();
1819
1820 let offload_globals_ref = cx.offload_globals.borrow();
1821 let offload_globals = match offload_globals_ref.as_ref() {
1822 Some(globals) => globals,
1823 None => {
1824 return;
1826 }
1827 };
1828 register_offload(cx);
1829 let offload_data = gen_define_handling(&cx, &metadata, target_symbol, offload_globals);
1830 gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals, &offload_dims);
1831}
1832
1833fn get_args_from_tuple<'ll, 'tcx>(
1834 bx: &mut Builder<'_, 'll, 'tcx>,
1835 tuple_op: OperandRef<'tcx, &'ll Value>,
1836 fn_instance: Instance<'tcx>,
1837) -> Vec<&'ll Value> {
1838 let cx = bx.cx;
1839 let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty());
1840
1841 match tuple_op.val {
1842 OperandValue::Immediate(val) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[val]))vec![val],
1843 OperandValue::Pair(v1, v2) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[v1, v2]))vec![v1, v2],
1844 OperandValue::Ref(ptr) => {
1845 let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout };
1846
1847 let mut result = Vec::with_capacity(fn_abi.args.len());
1848 let mut tuple_index = 0;
1849
1850 for arg in &fn_abi.args {
1851 match arg.mode {
1852 PassMode::Ignore => {}
1853 PassMode::Direct(_) | PassMode::Cast { .. } => {
1854 let field = tuple_place.project_field(bx, tuple_index);
1855 let llvm_ty = field.layout.llvm_type(bx.cx);
1856 let val = bx.load(llvm_ty, field.val.llval, field.val.align);
1857 result.push(val);
1858 tuple_index += 1;
1859 }
1860 PassMode::Pair(_, _) => {
1861 let field = tuple_place.project_field(bx, tuple_index);
1862 let llvm_ty = field.layout.llvm_type(bx.cx);
1863 let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align);
1864 result.push(bx.extract_value(pair_val, 0));
1865 result.push(bx.extract_value(pair_val, 1));
1866 tuple_index += 1;
1867 }
1868 PassMode::Indirect { .. } => {
1869 let field = tuple_place.project_field(bx, tuple_index);
1870 result.push(field.val.llval);
1871 tuple_index += 1;
1872 }
1873 }
1874 }
1875
1876 result
1877 }
1878
1879 OperandValue::ZeroSized => ::alloc::vec::Vec::new()vec![],
1880 }
1881}
1882
1883fn generic_simd_intrinsic<'ll, 'tcx>(
1884 bx: &mut Builder<'_, 'll, 'tcx>,
1885 name: Symbol,
1886 fn_args: GenericArgsRef<'tcx>,
1887 args: &[OperandRef<'tcx, &'ll Value>],
1888 ret_ty: Ty<'tcx>,
1889 llret_ty: &'ll Type,
1890 span: Span,
1891) -> Result<&'ll Value, ()> {
1892 macro_rules! return_error {
1893 ($diag: expr) => {{
1894 bx.sess().dcx().emit_err($diag);
1895 return Err(());
1896 }};
1897 }
1898
1899 macro_rules! require {
1900 ($cond: expr, $diag: expr) => {
1901 if !$cond {
1902 return_error!($diag);
1903 }
1904 };
1905 }
1906
1907 macro_rules! require_simd {
1908 ($ty: expr, $variant:ident) => {{
1909 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1910 $ty.simd_size_and_type(bx.tcx())
1911 }};
1912 }
1913
1914 macro_rules! require_simd_or_scalable {
1915 ($ty: expr, $variant:ident) => {{
1916 require!(
1917 $ty.is_simd() || $ty.is_scalable_vector(),
1918 InvalidMonomorphization::$variant { span, name, ty: $ty }
1919 );
1920 if $ty.is_simd() {
1921 let (len, ty) = $ty.simd_size_and_type(bx.tcx());
1922 (len, ty, None)
1923 } else {
1924 let (count, ty, num_vecs) =
1925 $ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
1926 (count as u64, ty, Some(num_vecs))
1927 }
1928 }};
1929 }
1930
1931 macro_rules! require_int_or_uint_ty {
1933 ($ty: expr, $diag: expr) => {
1934 match $ty {
1935 ty::Int(i) => {
1936 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1937 }
1938 ty::Uint(i) => {
1939 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1940 }
1941 _ => {
1942 return_error!($diag);
1943 }
1944 }
1945 };
1946 }
1947
1948 let llvm_version = crate::llvm_util::get_version();
1949
1950 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1964 bx: &mut Builder<'a, 'll, 'tcx>,
1965 i_xn: &'ll Value,
1966 in_elem_bitwidth: u64,
1967 in_len: u64,
1968 ) -> &'ll Value {
1969 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1971 let shift_indices = ::alloc::vec::from_elem(shift_idx, in_len as _)vec![shift_idx; in_len as _];
1972 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1973 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1975 }
1976
1977 if truecfg!(debug_assertions) {
1979 for arg in args {
1980 if arg.layout.ty.is_simd() {
1981 {
match arg.val {
OperandValue::Immediate(_) => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"OperandValue::Immediate(_)", ::core::option::Option::None);
}
}
};assert_matches!(arg.val, OperandValue::Immediate(_));
1982 }
1983 }
1984 }
1985
1986 if name == sym::simd_select_bitmask {
1987 let (len, _) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdArgument);
1988
1989 let expected_int_bits = len.max(8).next_power_of_two();
1990 let expected_bytes = len.div_ceil(8);
1991
1992 let mask_ty = args[0].layout.ty;
1993 let mask = match mask_ty.kind() {
1994 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1995 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1996 ty::Array(elem, len)
1997 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1998 && len
1999 .try_to_target_usize(bx.tcx)
2000 .expect("expected monomorphic const in codegen")
2001 == expected_bytes =>
2002 {
2003 let place = PlaceRef::alloca(bx, args[0].layout);
2004 args[0].val.store(bx, place);
2005 let int_ty = bx.type_ix(expected_bytes * 8);
2006 bx.load(int_ty, place.val.llval, Align::ONE)
2007 }
2008 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::InvalidBitmask {
span,
name,
mask_ty,
expected_int_bits,
expected_bytes,
});
return Err(());
}return_error!(InvalidMonomorphization::InvalidBitmask {
2009 span,
2010 name,
2011 mask_ty,
2012 expected_int_bits,
2013 expected_bytes
2014 }),
2015 };
2016
2017 let i1 = bx.type_i1();
2018 let im = bx.type_ix(len);
2019 let i1xn = bx.type_vector(i1, len);
2020 let m_im = bx.trunc(mask, im);
2021 let m_i1s = bx.bitcast(m_im, i1xn);
2022 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2023 }
2024
2025 if name == sym::simd_splat {
2026 let (_out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2027
2028 if !(args[0].layout.ty == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: out_ty,
vector_type: ret_ty,
});
return Err(());
};
};require!(
2029 args[0].layout.ty == out_ty,
2030 InvalidMonomorphization::ExpectedVectorElementType {
2031 span,
2032 name,
2033 expected_element: out_ty,
2034 vector_type: ret_ty,
2035 }
2036 );
2037
2038 let poison_vec = bx.const_poison(llret_ty);
2040 let idx0 = bx.const_i32(0);
2041 let v0 = bx.insert_element(poison_vec, args[0].immediate(), idx0);
2042
2043 let splat = bx.shuffle_vector(v0, poison_vec, bx.const_null(llret_ty));
2046
2047 return Ok(splat);
2048 }
2049
2050 let supports_scalable = match name {
2051 sym::simd_cast | sym::simd_select => true,
2052 _ => false,
2053 };
2054
2055 if !supports_scalable {
2060 let _ = {
if !args[0].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(());
};
};
args[0].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[0].layout.ty, SimdInput);
2061 }
2062 let (in_len, in_elem, in_num_vecs) = {
if !(args[0].layout.ty.is_simd() ||
args[0].layout.ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(());
};
};
if args[0].layout.ty.is_simd() {
let (len, ty) = args[0].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[0].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[0].layout.ty, SimdInput);
2063 let in_ty = args[0].layout.ty;
2064
2065 let comparison = match name {
2066 sym::simd_eq => Some(BinOp::Eq),
2067 sym::simd_ne => Some(BinOp::Ne),
2068 sym::simd_lt => Some(BinOp::Lt),
2069 sym::simd_le => Some(BinOp::Le),
2070 sym::simd_gt => Some(BinOp::Gt),
2071 sym::simd_ge => Some(BinOp::Ge),
2072 _ => None,
2073 };
2074
2075 if let Some(cmp_op) = comparison {
2076 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2077
2078 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2079 in_len == out_len,
2080 InvalidMonomorphization::ReturnLengthInputType {
2081 span,
2082 name,
2083 in_len,
2084 in_ty,
2085 ret_ty,
2086 out_len
2087 }
2088 );
2089 if !(bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnIntegerType {
span,
name,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2090 bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
2091 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
2092 );
2093
2094 return Ok(compare_simd_types(
2095 bx,
2096 args[0].immediate(),
2097 args[1].immediate(),
2098 in_elem,
2099 llret_ty,
2100 cmp_op,
2101 ));
2102 }
2103
2104 if name == sym::simd_shuffle_const_generic {
2105 let idx = fn_args[2].expect_const().to_branch();
2106 let n = idx.len() as u64;
2107
2108 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2109 if !(out_len == n) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2110 out_len == n,
2111 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2112 );
2113 if !(in_elem == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2114 in_elem == out_ty,
2115 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2116 );
2117
2118 let total_len = in_len * 2;
2119
2120 let indices: Option<Vec<_>> = idx
2121 .iter()
2122 .enumerate()
2123 .map(|(arg_idx, val)| {
2124 let idx = val.to_leaf().to_i32();
2125 if idx >= i32::try_from(total_len).unwrap() {
2126 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
2127 span,
2128 name,
2129 arg_idx: arg_idx as u64,
2130 total_len: total_len.into(),
2131 });
2132 None
2133 } else {
2134 Some(bx.const_i32(idx))
2135 }
2136 })
2137 .collect();
2138 let Some(indices) = indices else {
2139 return Ok(bx.const_null(llret_ty));
2140 };
2141
2142 return Ok(bx.shuffle_vector(
2143 args[0].immediate(),
2144 args[1].immediate(),
2145 bx.const_vector(&indices),
2146 ));
2147 }
2148
2149 if name == sym::simd_shuffle {
2150 let idx_ty = args[2].layout.ty;
2152 let n: u64 = if idx_ty.is_simd()
2153 && #[allow(non_exhaustive_omitted_patterns)] match idx_ty.simd_size_and_type(bx.cx.tcx).1.kind()
{
ty::Uint(ty::UintTy::U32) => true,
_ => false,
}matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
2154 {
2155 idx_ty.simd_size_and_type(bx.cx.tcx).0
2156 } else {
2157 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdShuffle {
span,
name,
ty: idx_ty,
});
return Err(());
}return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
2158 };
2159
2160 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2161 if !(out_len == n) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2162 out_len == n,
2163 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2164 );
2165 if !(in_elem == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2166 in_elem == out_ty,
2167 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2168 );
2169
2170 let total_len = u128::from(in_len) * 2;
2171
2172 let indices = args[2].immediate();
2174 for i in 0..n {
2175 let val = bx.const_get_elt(indices, i as u64);
2176 let idx = bx
2177 .const_to_opt_u128(val, true)
2178 .unwrap_or_else(|| ::rustc_middle::util::bug::bug_fmt(format_args!("typeck should have already ensured that these are const"))bug!("typeck should have already ensured that these are const"));
2179 if idx >= total_len {
2180 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: i,
total_len,
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2181 span,
2182 name,
2183 arg_idx: i,
2184 total_len,
2185 });
2186 }
2187 }
2188
2189 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
2190 }
2191
2192 if name == sym::simd_insert || name == sym::simd_insert_dyn {
2193 if !(in_elem == args[2].layout.ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::InsertedType {
span,
name,
in_elem,
in_ty,
out_ty: args[2].layout.ty,
});
return Err(());
};
};require!(
2194 in_elem == args[2].layout.ty,
2195 InvalidMonomorphization::InsertedType {
2196 span,
2197 name,
2198 in_elem,
2199 in_ty,
2200 out_ty: args[2].layout.ty
2201 }
2202 );
2203
2204 let index_imm = if name == sym::simd_insert {
2205 let idx = bx
2206 .const_to_opt_u128(args[1].immediate(), false)
2207 .expect("typeck should have ensure that this is a const");
2208 if idx >= in_len.into() {
2209 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2210 span,
2211 name,
2212 arg_idx: 1,
2213 total_len: in_len.into(),
2214 });
2215 }
2216 bx.const_i32(idx as i32)
2217 } else {
2218 args[1].immediate()
2219 };
2220
2221 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
2222 }
2223 if name == sym::simd_extract || name == sym::simd_extract_dyn {
2224 if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};require!(
2225 ret_ty == in_elem,
2226 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2227 );
2228 let index_imm = if name == sym::simd_extract {
2229 let idx = bx
2230 .const_to_opt_u128(args[1].immediate(), false)
2231 .expect("typeck should have ensure that this is a const");
2232 if idx >= in_len.into() {
2233 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2234 span,
2235 name,
2236 arg_idx: 1,
2237 total_len: in_len.into(),
2238 });
2239 }
2240 bx.const_i32(idx as i32)
2241 } else {
2242 args[1].immediate()
2243 };
2244
2245 return Ok(bx.extract_element(args[0].immediate(), index_imm));
2246 }
2247
2248 if name == sym::simd_select {
2249 let m_elem_ty = in_elem;
2250 let m_len = in_len;
2251 let (v_len, _, _) = {
if !(args[1].layout.ty.is_simd() ||
args[1].layout.ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
if args[1].layout.ty.is_simd() {
let (len, ty) = args[1].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[1].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[1].layout.ty, SimdArgument);
2252 if !(m_len == v_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MismatchedLengths {
span,
name,
m_len,
v_len,
});
return Err(());
};
};require!(
2253 m_len == v_len,
2254 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
2255 );
2256
2257 let m_i1s = if args[1].layout.ty.is_scalable_vector() {
2258 match m_elem_ty.kind() {
2259 ty::Bool => {}
2260 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(());
}return_error!(InvalidMonomorphization::MaskWrongElementType {
2261 span,
2262 name,
2263 ty: m_elem_ty
2264 }),
2265 };
2266 let i1 = bx.type_i1();
2267 let i1xn = bx.type_scalable_vector(i1, m_len as u64);
2268 bx.trunc(args[0].immediate(), i1xn)
2269 } else {
2270 let in_elem_bitwidth = match m_elem_ty.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2271 m_elem_ty.kind(),
2272 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
2273 );
2274 vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
2275 };
2276
2277 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2278 }
2279
2280 if name == sym::simd_bitmask {
2281 let expected_int_bits = in_len.max(8).next_power_of_two();
2290 let expected_bytes = in_len.div_ceil(8);
2291
2292 let in_elem_bitwidth = match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: in_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2294 in_elem.kind(),
2295 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
2296 );
2297
2298 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
2299 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
2301
2302 match ret_ty.kind() {
2303 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
2304 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
2306 }
2307 ty::Array(elem, len)
2308 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2309 && len
2310 .try_to_target_usize(bx.tcx)
2311 .expect("expected monomorphic const in codegen")
2312 == expected_bytes =>
2313 {
2314 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
2316
2317 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
2319 bx.store(ze, ptr, Align::ONE);
2320 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
2321 return Ok(bx.load(array_ty, ptr, Align::ONE));
2322 }
2323 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::CannotReturn {
span,
name,
ret_ty,
expected_int_bits,
expected_bytes,
});
return Err(());
}return_error!(InvalidMonomorphization::CannotReturn {
2324 span,
2325 name,
2326 ret_ty,
2327 expected_int_bits,
2328 expected_bytes
2329 }),
2330 }
2331 }
2332
2333 fn simd_simple_float_intrinsic<'ll, 'tcx>(
2334 name: Symbol,
2335 in_elem: Ty<'_>,
2336 in_ty: Ty<'_>,
2337 in_len: u64,
2338 bx: &mut Builder<'_, 'll, 'tcx>,
2339 span: Span,
2340 args: &[OperandRef<'tcx, &'ll Value>],
2341 ) -> Result<&'ll Value, ()> {
2342 macro_rules! return_error {
2343 ($diag: expr) => {{
2344 bx.sess().dcx().emit_err($diag);
2345 return Err(());
2346 }};
2347 }
2348
2349 let ty::Float(f) = in_elem.kind() else {
2350 {
bx.sess().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
span,
name,
ty: in_ty,
});
return Err(());
};return_error!(InvalidMonomorphization::BasicFloatType { span, name, ty: in_ty });
2351 };
2352 let elem_ty = bx.cx.type_float_from_ty(*f);
2353
2354 let vec_ty = bx.type_vector(elem_ty, in_len);
2355
2356 let intr_name = match name {
2357 sym::simd_ceil => "llvm.ceil",
2358 sym::simd_fabs => "llvm.fabs",
2359 sym::simd_fcos => "llvm.cos",
2360 sym::simd_fexp2 => "llvm.exp2",
2361 sym::simd_fexp => "llvm.exp",
2362 sym::simd_flog10 => "llvm.log10",
2363 sym::simd_flog2 => "llvm.log2",
2364 sym::simd_flog => "llvm.log",
2365 sym::simd_floor => "llvm.floor",
2366 sym::simd_fma => "llvm.fma",
2367 sym::simd_relaxed_fma => "llvm.fmuladd",
2368 sym::simd_fsin => "llvm.sin",
2369 sym::simd_fsqrt => "llvm.sqrt",
2370 sym::simd_round => "llvm.round",
2371 sym::simd_round_ties_even => "llvm.rint",
2372 sym::simd_trunc => "llvm.trunc",
2373 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnrecognizedIntrinsic {
span,
name,
});
return Err(());
}return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
2374 };
2375 Ok(bx.call_intrinsic(
2376 intr_name,
2377 &[vec_ty],
2378 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
2379 ))
2380 }
2381
2382 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_ceil | sym::simd_fabs | sym::simd_fcos | sym::simd_fexp2 |
sym::simd_fexp | sym::simd_flog10 | sym::simd_flog2 | sym::simd_flog |
sym::simd_floor | sym::simd_fma | sym::simd_fsin | sym::simd_fsqrt |
sym::simd_relaxed_fma | sym::simd_round | sym::simd_round_ties_even |
sym::simd_trunc => true,
_ => false,
}std::matches!(
2383 name,
2384 sym::simd_ceil
2385 | sym::simd_fabs
2386 | sym::simd_fcos
2387 | sym::simd_fexp2
2388 | sym::simd_fexp
2389 | sym::simd_flog10
2390 | sym::simd_flog2
2391 | sym::simd_flog
2392 | sym::simd_floor
2393 | sym::simd_fma
2394 | sym::simd_fsin
2395 | sym::simd_fsqrt
2396 | sym::simd_relaxed_fma
2397 | sym::simd_round
2398 | sym::simd_round_ties_even
2399 | sym::simd_trunc
2400 ) {
2401 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
2402 }
2403
2404 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
2405 let elem_ty = match *elem_ty.kind() {
2406 ty::Int(v) => cx.type_int_from_ty(v),
2407 ty::Uint(v) => cx.type_uint_from_ty(v),
2408 ty::Float(v) => cx.type_float_from_ty(v),
2409 ty::RawPtr(_, _) => cx.type_ptr(),
2410 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
2411 };
2412 cx.type_vector(elem_ty, vec_len)
2413 }
2414
2415 if name == sym::simd_gather {
2416 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(());
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2427 let (out_len, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2428 let (out_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(());
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2430 {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2431
2432 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len,
});
return Err(());
};
};require!(
2434 in_len == out_len,
2435 InvalidMonomorphization::SecondArgumentLength {
2436 span,
2437 name,
2438 in_len,
2439 in_ty,
2440 arg_ty: args[1].layout.ty,
2441 out_len
2442 }
2443 );
2444 if !(in_len == out_len2) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: out_len2,
});
return Err(());
};
};require!(
2445 in_len == out_len2,
2446 InvalidMonomorphization::ThirdArgumentLength {
2447 span,
2448 name,
2449 in_len,
2450 in_ty,
2451 arg_ty: args[2].layout.ty,
2452 out_len: out_len2
2453 }
2454 );
2455
2456 if !(ret_ty == in_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty,
ret_ty,
});
return Err(());
};
};require!(
2458 ret_ty == in_ty,
2459 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
2460 );
2461
2462 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, _) if
p_ty == in_elem && p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(());
};
};require!(
2463 matches!(
2464 *element_ty1.kind(),
2465 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
2466 ),
2467 InvalidMonomorphization::ExpectedElementType {
2468 span,
2469 name,
2470 expected_element: element_ty1,
2471 second_arg: args[1].layout.ty,
2472 in_elem,
2473 in_ty,
2474 mutability: ExpectedPointerMutability::Not,
2475 }
2476 );
2477
2478 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2479 element_ty2.kind(),
2480 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2481 );
2482
2483 let alignment = bx.align_of(in_elem).bytes();
2485
2486 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2488
2489 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2491
2492 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2494
2495 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2496 let alignment = bx.const_i32(alignment as i32);
2497 &[args[1].immediate(), alignment, mask, args[0].immediate()]
2498 } else {
2499 &[args[1].immediate(), mask, args[0].immediate()]
2500 };
2501
2502 let call =
2503 bx.call_intrinsic("llvm.masked.gather", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], args);
2504 if llvm_version >= (22, 0, 0) {
2505 crate::attributes::apply_to_callsite(
2506 call,
2507 crate::llvm::AttributePlace::Argument(0),
2508 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2509 )
2510 }
2511 return Ok(call);
2512 }
2513
2514 fn llvm_alignment<'ll, 'tcx>(
2515 bx: &mut Builder<'_, 'll, 'tcx>,
2516 alignment: SimdAlign,
2517 vector_ty: Ty<'tcx>,
2518 element_ty: Ty<'tcx>,
2519 ) -> u64 {
2520 match alignment {
2521 SimdAlign::Unaligned => 1,
2522 SimdAlign::Element => bx.align_of(element_ty).bytes(),
2523 SimdAlign::Vector => bx.align_of(vector_ty).bytes(),
2524 }
2525 }
2526
2527 if name == sym::simd_masked_load {
2528 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2537
2538 let mask_ty = in_ty;
2540 let (mask_len, mask_elem) = (in_len, in_elem);
2541
2542 let pointer_ty = args[1].layout.ty;
2544
2545 let values_ty = args[2].layout.ty;
2547 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(());
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2548
2549 {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2550
2551 if !(values_len == mask_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(());
};
};require!(
2553 values_len == mask_len,
2554 InvalidMonomorphization::ThirdArgumentLength {
2555 span,
2556 name,
2557 in_len: mask_len,
2558 in_ty: mask_ty,
2559 arg_ty: values_ty,
2560 out_len: values_len
2561 }
2562 );
2563
2564 if !(ret_ty == values_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty: values_ty,
ret_ty,
});
return Err(());
};
};require!(
2566 ret_ty == values_ty,
2567 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
2568 );
2569
2570 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, _) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() =>
true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(());
};
};require!(
2571 matches!(
2572 *pointer_ty.kind(),
2573 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
2574 ),
2575 InvalidMonomorphization::ExpectedElementType {
2576 span,
2577 name,
2578 expected_element: values_elem,
2579 second_arg: pointer_ty,
2580 in_elem: values_elem,
2581 in_ty: values_ty,
2582 mutability: ExpectedPointerMutability::Not,
2583 }
2584 );
2585
2586 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2587 mask_elem.kind(),
2588 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2589 );
2590
2591 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2592
2593 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2595
2596 let llvm_pointer = bx.type_ptr();
2597
2598 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2600
2601 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2602 let alignment = bx.const_i32(alignment as i32);
2603
2604 &[args[1].immediate(), alignment, mask, args[2].immediate()]
2605 } else {
2606 &[args[1].immediate(), mask, args[2].immediate()]
2607 };
2608
2609 let call = bx.call_intrinsic("llvm.masked.load", &[llvm_elem_vec_ty, llvm_pointer], args);
2610 if llvm_version >= (22, 0, 0) {
2611 crate::attributes::apply_to_callsite(
2612 call,
2613 crate::llvm::AttributePlace::Argument(0),
2614 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2615 )
2616 }
2617 return Ok(call);
2618 }
2619
2620 if name == sym::simd_masked_store {
2621 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2630
2631 let mask_ty = in_ty;
2633 let (mask_len, mask_elem) = (in_len, in_elem);
2634
2635 let pointer_ty = args[1].layout.ty;
2637
2638 let values_ty = args[2].layout.ty;
2640 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(());
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2641
2642 if !(values_len == mask_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(());
};
};require!(
2644 values_len == mask_len,
2645 InvalidMonomorphization::ThirdArgumentLength {
2646 span,
2647 name,
2648 in_len: mask_len,
2649 in_ty: mask_ty,
2650 arg_ty: values_ty,
2651 out_len: values_len
2652 }
2653 );
2654
2655 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() &&
p_mutbl.is_mut() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(());
};
};require!(
2657 matches!(
2658 *pointer_ty.kind(),
2659 ty::RawPtr(p_ty, p_mutbl)
2660 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
2661 ),
2662 InvalidMonomorphization::ExpectedElementType {
2663 span,
2664 name,
2665 expected_element: values_elem,
2666 second_arg: pointer_ty,
2667 in_elem: values_elem,
2668 in_ty: values_ty,
2669 mutability: ExpectedPointerMutability::Mut,
2670 }
2671 );
2672
2673 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2674 mask_elem.kind(),
2675 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2676 );
2677
2678 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2679
2680 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2682
2683 let llvm_pointer = bx.type_ptr();
2684
2685 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2687
2688 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2689 let alignment = bx.const_i32(alignment as i32);
2690 &[args[2].immediate(), args[1].immediate(), alignment, mask]
2691 } else {
2692 &[args[2].immediate(), args[1].immediate(), mask]
2693 };
2694
2695 let call = bx.call_intrinsic("llvm.masked.store", &[llvm_elem_vec_ty, llvm_pointer], args);
2696 if llvm_version >= (22, 0, 0) {
2697 crate::attributes::apply_to_callsite(
2698 call,
2699 crate::llvm::AttributePlace::Argument(1),
2700 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2701 )
2702 }
2703 return Ok(call);
2704 }
2705
2706 if name == sym::simd_scatter {
2707 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(());
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2717 let (element_len1, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2718 let (element_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(());
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2719
2720 if !(in_len == element_len1) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len: element_len1,
});
return Err(());
};
};require!(
2722 in_len == element_len1,
2723 InvalidMonomorphization::SecondArgumentLength {
2724 span,
2725 name,
2726 in_len,
2727 in_ty,
2728 arg_ty: args[1].layout.ty,
2729 out_len: element_len1
2730 }
2731 );
2732 if !(in_len == element_len2) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: element_len2,
});
return Err(());
};
};require!(
2733 in_len == element_len2,
2734 InvalidMonomorphization::ThirdArgumentLength {
2735 span,
2736 name,
2737 in_len,
2738 in_ty,
2739 arg_ty: args[2].layout.ty,
2740 out_len: element_len2
2741 }
2742 );
2743
2744 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == in_elem && p_mutbl.is_mut() &&
p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(());
};
};require!(
2745 matches!(
2746 *element_ty1.kind(),
2747 ty::RawPtr(p_ty, p_mutbl)
2748 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
2749 ),
2750 InvalidMonomorphization::ExpectedElementType {
2751 span,
2752 name,
2753 expected_element: element_ty1,
2754 second_arg: args[1].layout.ty,
2755 in_elem,
2756 in_ty,
2757 mutability: ExpectedPointerMutability::Mut,
2758 }
2759 );
2760
2761 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2763 element_ty2.kind(),
2764 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2765 );
2766
2767 let alignment = bx.align_of(in_elem).bytes();
2769
2770 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2772
2773 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2775
2776 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2778 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2779 let alignment = bx.const_i32(alignment as i32);
2780 &[args[0].immediate(), args[1].immediate(), alignment, mask]
2781 } else {
2782 &[args[0].immediate(), args[1].immediate(), mask]
2783 };
2784 let call = bx.call_intrinsic(
2785 "llvm.masked.scatter",
2786 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2787 args,
2788 );
2789 if llvm_version >= (22, 0, 0) {
2790 crate::attributes::apply_to_callsite(
2791 call,
2792 crate::llvm::AttributePlace::Argument(1),
2793 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2794 )
2795 }
2796 return Ok(call);
2797 }
2798
2799 macro_rules! arith_red {
2800 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2801 $identity:expr) => {
2802 if name == sym::$name {
2803 require!(
2804 ret_ty == in_elem,
2805 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2806 );
2807 return match in_elem.kind() {
2808 ty::Int(_) | ty::Uint(_) => {
2809 let r = bx.$integer_reduce(args[0].immediate());
2810 if $ordered {
2811 Ok(bx.$op(args[1].immediate(), r))
2814 } else {
2815 Ok(bx.$integer_reduce(args[0].immediate()))
2816 }
2817 }
2818 ty::Float(f) => {
2819 let acc = if $ordered {
2820 args[1].immediate()
2822 } else {
2823 match f.bit_width() {
2825 32 => bx.const_real(bx.type_f32(), $identity),
2826 64 => bx.const_real(bx.type_f64(), $identity),
2827 v => return_error!(
2828 InvalidMonomorphization::UnsupportedSymbolOfSize {
2829 span,
2830 name,
2831 symbol: sym::$name,
2832 in_ty,
2833 in_elem,
2834 size: v,
2835 ret_ty
2836 }
2837 ),
2838 }
2839 };
2840 Ok(bx.$float_reduce(acc, args[0].immediate()))
2841 }
2842 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2843 span,
2844 name,
2845 symbol: sym::$name,
2846 in_ty,
2847 in_elem,
2848 ret_ty
2849 }),
2850 };
2851 }
2852 };
2853 }
2854
2855 if name == sym::simd_reduce_add_ordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if true {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fadd(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2856 if name == sym::simd_reduce_mul_ordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if true {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fmul(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2857 if name == sym::simd_reduce_add_unordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if false {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fadd_reassoc(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(
2858 simd_reduce_add_unordered: vector_reduce_add,
2859 vector_reduce_fadd_reassoc,
2860 false,
2861 add,
2862 -0.0
2863 );
2864 if name == sym::simd_reduce_mul_unordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if false {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fmul_reassoc(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(
2865 simd_reduce_mul_unordered: vector_reduce_mul,
2866 vector_reduce_fmul_reassoc,
2867 false,
2868 mul,
2869 1.0
2870 );
2871
2872 macro_rules! minmax_red {
2873 ($name:ident: $int_red:ident, $float_red:ident) => {
2874 if name == sym::$name {
2875 require!(
2876 ret_ty == in_elem,
2877 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2878 );
2879 return match in_elem.kind() {
2880 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2881 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2882 ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
2883 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2884 span,
2885 name,
2886 symbol: sym::$name,
2887 in_ty,
2888 in_elem,
2889 ret_ty
2890 }),
2891 };
2892 }
2893 };
2894 }
2895
2896 if name == sym::simd_reduce_min {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_min(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_min(args[0].immediate(), false)),
ty::Float(_f) => Ok(bx.vector_reduce_fmin(args[0].immediate())),
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_min,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
2897 if name == sym::simd_reduce_max {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_max(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_max(args[0].immediate(), false)),
ty::Float(_f) => Ok(bx.vector_reduce_fmax(args[0].immediate())),
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_max,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
2898
2899 macro_rules! bitwise_red {
2900 ($name:ident : $red:ident, $boolean:expr) => {
2901 if name == sym::$name {
2902 let input = if !$boolean {
2903 require!(
2904 ret_ty == in_elem,
2905 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2906 );
2907 args[0].immediate()
2908 } else {
2909 let bitwidth = match in_elem.kind() {
2910 ty::Int(i) => {
2911 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2912 }
2913 ty::Uint(i) => {
2914 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2915 }
2916 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2917 span,
2918 name,
2919 symbol: sym::$name,
2920 in_ty,
2921 in_elem,
2922 ret_ty
2923 }),
2924 };
2925
2926 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2927 };
2928 return match in_elem.kind() {
2929 ty::Int(_) | ty::Uint(_) => {
2930 let r = bx.$red(input);
2931 Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2932 }
2933 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2934 span,
2935 name,
2936 symbol: sym::$name,
2937 in_ty,
2938 in_elem,
2939 ret_ty
2940 }),
2941 };
2942 }
2943 };
2944 }
2945
2946 if name == sym::simd_reduce_and {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2947 if name == sym::simd_reduce_or {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2948 if name == sym::simd_reduce_xor {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_xor(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2949 if name == sym::simd_reduce_all {
let input =
if !true {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(if !true { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2950 if name == sym::simd_reduce_any {
let input =
if !true {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(if !true { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2951
2952 if name == sym::simd_cast_ptr {
2953 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2954 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2955 in_len == out_len,
2956 InvalidMonomorphization::ReturnLengthInputType {
2957 span,
2958 name,
2959 in_len,
2960 in_ty,
2961 ret_ty,
2962 out_len
2963 }
2964 );
2965
2966 match in_elem.kind() {
2967 ty::RawPtr(p_ty, _) => {
2968 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2969 bx.tcx.normalize_erasing_regions(bx.typing_env(), Unnormalized::new_wip(ty))
2970 });
2971 if !metadata.is_unit() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: in_elem,
});
return Err(());
};
};require!(
2972 metadata.is_unit(),
2973 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2974 );
2975 }
2976 _ => {
2977 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2978 }
2979 }
2980 match out_elem.kind() {
2981 ty::RawPtr(p_ty, _) => {
2982 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2983 bx.tcx.normalize_erasing_regions(bx.typing_env(), Unnormalized::new_wip(ty))
2984 });
2985 if !metadata.is_unit() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: out_elem,
});
return Err(());
};
};require!(
2986 metadata.is_unit(),
2987 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2988 );
2989 }
2990 _ => {
2991 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2992 }
2993 }
2994
2995 return Ok(args[0].immediate());
2996 }
2997
2998 if name == sym::simd_expose_provenance {
2999 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3000 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
3001 in_len == out_len,
3002 InvalidMonomorphization::ReturnLengthInputType {
3003 span,
3004 name,
3005 in_len,
3006 in_ty,
3007 ret_ty,
3008 out_len
3009 }
3010 );
3011
3012 match in_elem.kind() {
3013 ty::RawPtr(_, _) => {}
3014 _ => {
3015 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
3016 }
3017 }
3018 match out_elem.kind() {
3019 ty::Uint(ty::UintTy::Usize) => {}
3020 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
3021 }
3022
3023 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
3024 }
3025
3026 if name == sym::simd_with_exposed_provenance {
3027 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3028 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
3029 in_len == out_len,
3030 InvalidMonomorphization::ReturnLengthInputType {
3031 span,
3032 name,
3033 in_len,
3034 in_ty,
3035 ret_ty,
3036 out_len
3037 }
3038 );
3039
3040 match in_elem.kind() {
3041 ty::Uint(ty::UintTy::Usize) => {}
3042 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
3043 }
3044 match out_elem.kind() {
3045 ty::RawPtr(_, _) => {}
3046 _ => {
3047 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3048 }
3049 }
3050
3051 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
3052 }
3053
3054 if name == sym::simd_cast || name == sym::simd_as {
3055 let (out_len, out_elem, out_num_vecs) = {
if !(ret_ty.is_simd() || ret_ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
if ret_ty.is_simd() {
let (len, ty) = ret_ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
ret_ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(ret_ty, SimdReturn);
3056 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
3057 in_len == out_len,
3058 InvalidMonomorphization::ReturnLengthInputType {
3059 span,
3060 name,
3061 in_len,
3062 in_ty,
3063 ret_ty,
3064 out_len
3065 }
3066 );
3067 if !(in_num_vecs == out_num_vecs) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnNumVecsInputType {
span,
name,
in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
in_ty,
ret_ty,
out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1)),
});
return Err(());
};
};require!(
3068 in_num_vecs == out_num_vecs,
3069 InvalidMonomorphization::ReturnNumVecsInputType {
3070 span,
3071 name,
3072 in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
3073 in_ty,
3074 ret_ty,
3075 out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1))
3076 }
3077 );
3078
3079 if in_elem == out_elem {
3081 return Ok(args[0].immediate());
3082 }
3083
3084 #[derive(#[automatically_derived]
impl ::core::marker::Copy for Sign { }Copy, #[automatically_derived]
impl ::core::clone::Clone for Sign {
#[inline]
fn clone(&self) -> Sign { *self }
}Clone)]
3085 enum Sign {
3086 Unsigned,
3087 Signed,
3088 }
3089 use Sign::*;
3090
3091 enum Style {
3092 Float,
3093 Int(Sign),
3094 Unsupported,
3095 }
3096
3097 let (in_style, in_width) = match in_elem.kind() {
3098 ty::Int(i) => (
3101 Style::Int(Signed),
3102 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3103 ),
3104 ty::Uint(u) => (
3105 Style::Int(Unsigned),
3106 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3107 ),
3108 ty::Float(f) => (Style::Float, f.bit_width()),
3109 _ => (Style::Unsupported, 0),
3110 };
3111 let (out_style, out_width) = match out_elem.kind() {
3112 ty::Int(i) => (
3113 Style::Int(Signed),
3114 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3115 ),
3116 ty::Uint(u) => (
3117 Style::Int(Unsigned),
3118 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3119 ),
3120 ty::Float(f) => (Style::Float, f.bit_width()),
3121 _ => (Style::Unsupported, 0),
3122 };
3123
3124 match (in_style, out_style) {
3125 (Style::Int(sign), Style::Int(_)) => {
3126 return Ok(match in_width.cmp(&out_width) {
3127 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
3128 Ordering::Equal => args[0].immediate(),
3129 Ordering::Less => match sign {
3130 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
3131 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
3132 },
3133 });
3134 }
3135 (Style::Int(Sign::Signed), Style::Float) => {
3136 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
3137 }
3138 (Style::Int(Sign::Unsigned), Style::Float) => {
3139 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
3140 }
3141 (Style::Float, Style::Int(sign)) => {
3142 return Ok(match (sign, name == sym::simd_as) {
3143 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
3144 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
3145 (_, true) => bx.cast_float_to_int(
3146 #[allow(non_exhaustive_omitted_patterns)] match sign {
Sign::Signed => true,
_ => false,
}matches!(sign, Sign::Signed),
3147 args[0].immediate(),
3148 llret_ty,
3149 ),
3150 });
3151 }
3152 (Style::Float, Style::Float) => {
3153 return Ok(match in_width.cmp(&out_width) {
3154 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
3155 Ordering::Equal => args[0].immediate(),
3156 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
3157 });
3158 }
3159 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedCast {
span,
name,
in_ty,
in_elem,
ret_ty,
out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::UnsupportedCast {
3160 span,
3161 name,
3162 in_ty,
3163 in_elem,
3164 ret_ty,
3165 out_elem
3166 }),
3167 }
3168 }
3169 macro_rules! arith_binary {
3170 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3171 $(if name == sym::$name {
3172 match in_elem.kind() {
3173 $($(ty::$p(_))|* => {
3174 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
3175 })*
3176 _ => {},
3177 }
3178 return_error!(
3179 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3180 );
3181 })*
3182 }
3183 }
3184 if name == sym::simd_minimum_number_nsz {
match in_elem.kind() {
ty::Float(_) => {
return Ok(bx.minimum_number_nsz(args[0].immediate(),
args[1].immediate()))
}
_ => {}
}
{
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
};
}arith_binary! {
3185 simd_add: Uint, Int => add, Float => fadd;
3186 simd_sub: Uint, Int => sub, Float => fsub;
3187 simd_mul: Uint, Int => mul, Float => fmul;
3188 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
3189 simd_rem: Uint => urem, Int => srem, Float => frem;
3190 simd_shl: Uint, Int => shl;
3191 simd_shr: Uint => lshr, Int => ashr;
3192 simd_and: Uint, Int => and;
3193 simd_or: Uint, Int => or;
3194 simd_xor: Uint, Int => xor;
3195 simd_maximum_number_nsz: Float => maximum_number_nsz;
3196 simd_minimum_number_nsz: Float => minimum_number_nsz;
3197
3198 }
3199 macro_rules! arith_unary {
3200 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3201 $(if name == sym::$name {
3202 match in_elem.kind() {
3203 $($(ty::$p(_))|* => {
3204 return Ok(bx.$call(args[0].immediate()))
3205 })*
3206 _ => {},
3207 }
3208 return_error!(
3209 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3210 );
3211 })*
3212 }
3213 }
3214 if name == sym::simd_neg {
match in_elem.kind() {
ty::Int(_) => { return Ok(bx.neg(args[0].immediate())) }
ty::Float(_) => { return Ok(bx.fneg(args[0].immediate())) }
_ => {}
}
{
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
};
}arith_unary! {
3215 simd_neg: Int => neg, Float => fneg;
3216 }
3217
3218 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop
| sym::simd_cttz | sym::simd_carryless_mul | sym::simd_funnel_shl |
sym::simd_funnel_shr => true,
_ => false,
}matches!(
3220 name,
3221 sym::simd_bswap
3222 | sym::simd_bitreverse
3223 | sym::simd_ctlz
3224 | sym::simd_ctpop
3225 | sym::simd_cttz
3226 | sym::simd_carryless_mul
3227 | sym::simd_funnel_shl
3228 | sym::simd_funnel_shr
3229 ) {
3230 let vec_ty = bx.cx.type_vector(
3231 match *in_elem.kind() {
3232 ty::Int(i) => bx.cx.type_int_from_ty(i),
3233 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
3234 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::UnsupportedOperation {
3235 span,
3236 name,
3237 in_ty,
3238 in_elem
3239 }),
3240 },
3241 in_len as u64,
3242 );
3243 let llvm_intrinsic = match name {
3244 sym::simd_bswap => "llvm.bswap",
3245 sym::simd_bitreverse => "llvm.bitreverse",
3246 sym::simd_ctlz => "llvm.ctlz",
3247 sym::simd_ctpop => "llvm.ctpop",
3248 sym::simd_cttz => "llvm.cttz",
3249 sym::simd_funnel_shl => "llvm.fshl",
3250 sym::simd_funnel_shr => "llvm.fshr",
3251 sym::simd_carryless_mul => "llvm.clmul",
3252 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
3253 };
3254 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
3255
3256 return match name {
3257 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
3259 sym::simd_ctlz | sym::simd_cttz => {
3260 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
3262 Ok(bx.call_intrinsic(
3263 llvm_intrinsic,
3264 &[vec_ty],
3265 &[args[0].immediate(), dont_poison_on_zero],
3266 ))
3267 }
3268 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
3269 Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
3271 }
3272 sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
3273 llvm_intrinsic,
3274 &[vec_ty],
3275 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
3276 )),
3277 sym::simd_carryless_mul => {
3278 if crate::llvm_util::get_version() >= (22, 0, 0) {
3279 Ok(bx.call_intrinsic(
3280 llvm_intrinsic,
3281 &[vec_ty],
3282 &[args[0].immediate(), args[1].immediate()],
3283 ))
3284 } else {
3285 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("`simd_carryless_mul` needs LLVM 22 or higher"));span_bug!(span, "`simd_carryless_mul` needs LLVM 22 or higher");
3286 }
3287 }
3288 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
3289 };
3290 }
3291
3292 if name == sym::simd_arith_offset {
3293 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
3295 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("must be called with a vector of pointer types as first argument"))span_bug!(span, "must be called with a vector of pointer types as first argument")
3296 });
3297 let layout = bx.layout_of(pointee);
3298 let ptrs = args[0].immediate();
3299 let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
3302 if !#[allow(non_exhaustive_omitted_patterns)] match offsets_elem.kind() {
ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize) => true,
_ => false,
}matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
3303 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("must be called with a vector of pointer-sized integers as second argument"));span_bug!(
3304 span,
3305 "must be called with a vector of pointer-sized integers as second argument"
3306 );
3307 }
3308 let offsets = args[1].immediate();
3309
3310 return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
3311 }
3312
3313 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
3314 let lhs = args[0].immediate();
3315 let rhs = args[1].immediate();
3316 let is_add = name == sym::simd_saturating_add;
3317 let (signed, elem_ty) = match *in_elem.kind() {
3318 ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
3319 ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
3320 _ => {
3321 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
vector_type: args[0].layout.ty,
});
return Err(());
};return_error!(InvalidMonomorphization::ExpectedVectorElementType {
3322 span,
3323 name,
3324 expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
3325 vector_type: args[0].layout.ty
3326 });
3327 }
3328 };
3329 let llvm_intrinsic = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
3330 "llvm.{}{}.sat",
3331 if signed { 's' } else { 'u' },
3332 if is_add { "add" } else { "sub" },
3333 );
3334 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
3335
3336 return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
3337 }
3338
3339 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("unknown SIMD intrinsic"));span_bug!(span, "unknown SIMD intrinsic");
3340}