1use std::cmp::Ordering;
2use std::ffi::c_uint;
3use std::{assert_matches, iter, ptr};
4
5use rustc_abi::{
6 Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive, Size,
7 WrappingRange,
8};
9use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
10use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
11use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
14use rustc_codegen_ssa::traits::*;
15use rustc_hir as hir;
16use rustc_hir::def_id::LOCAL_CRATE;
17use rustc_hir::find_attr;
18use rustc_middle::mir::BinOp;
19use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
20use rustc_middle::ty::offload_meta::OffloadMetadata;
21use rustc_middle::ty::{
22 self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv, Unnormalized,
23};
24use rustc_middle::{bug, span_bug};
25use rustc_session::config::CrateType;
26use rustc_session::lint::builtin::DEPRECATED_LLVM_INTRINSIC;
27use rustc_span::{Span, Symbol, sym};
28use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate};
29use rustc_target::callconv::PassMode;
30use rustc_target::spec::{Arch, Os};
31use tracing::debug;
32
33use crate::abi::FnAbiLlvmExt;
34use crate::builder::Builder;
35use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
36use crate::builder::gpu_offload::{
37 OffloadKernelDims, gen_call_handling, gen_define_handling, register_offload,
38};
39use crate::context::CodegenCx;
40use crate::declare::declare_raw_fn;
41use crate::errors::{
42 AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
43 OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
44};
45use crate::llvm::{self, Type, Value};
46use crate::type_of::LayoutLlvmExt;
47use crate::va_arg::emit_va_arg;
48
49fn call_simple_intrinsic<'ll, 'tcx>(
50 bx: &mut Builder<'_, 'll, 'tcx>,
51 name: Symbol,
52 args: &[OperandRef<'tcx, &'ll Value>],
53) -> Option<&'ll Value> {
54 let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
55 sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]),
56 sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]),
57 sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]),
58 sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]),
59
60 sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]),
61 sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]),
62 sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]),
63 sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]),
64
65 sym::sinf16 => ("llvm.sin", &[bx.type_f16()]),
66 sym::sinf32 => ("llvm.sin", &[bx.type_f32()]),
67 sym::sinf64 => ("llvm.sin", &[bx.type_f64()]),
68 sym::sinf128 => ("llvm.sin", &[bx.type_f128()]),
69
70 sym::cosf16 => ("llvm.cos", &[bx.type_f16()]),
71 sym::cosf32 => ("llvm.cos", &[bx.type_f32()]),
72 sym::cosf64 => ("llvm.cos", &[bx.type_f64()]),
73 sym::cosf128 => ("llvm.cos", &[bx.type_f128()]),
74
75 sym::powf16 => ("llvm.pow", &[bx.type_f16()]),
76 sym::powf32 => ("llvm.pow", &[bx.type_f32()]),
77 sym::powf64 => ("llvm.pow", &[bx.type_f64()]),
78 sym::powf128 => ("llvm.pow", &[bx.type_f128()]),
79
80 sym::expf16 => ("llvm.exp", &[bx.type_f16()]),
81 sym::expf32 => ("llvm.exp", &[bx.type_f32()]),
82 sym::expf64 => ("llvm.exp", &[bx.type_f64()]),
83 sym::expf128 => ("llvm.exp", &[bx.type_f128()]),
84
85 sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]),
86 sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]),
87 sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]),
88 sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]),
89
90 sym::logf16 => ("llvm.log", &[bx.type_f16()]),
91 sym::logf32 => ("llvm.log", &[bx.type_f32()]),
92 sym::logf64 => ("llvm.log", &[bx.type_f64()]),
93 sym::logf128 => ("llvm.log", &[bx.type_f128()]),
94
95 sym::log10f16 => ("llvm.log10", &[bx.type_f16()]),
96 sym::log10f32 => ("llvm.log10", &[bx.type_f32()]),
97 sym::log10f64 => ("llvm.log10", &[bx.type_f64()]),
98 sym::log10f128 => ("llvm.log10", &[bx.type_f128()]),
99
100 sym::log2f16 => ("llvm.log2", &[bx.type_f16()]),
101 sym::log2f32 => ("llvm.log2", &[bx.type_f32()]),
102 sym::log2f64 => ("llvm.log2", &[bx.type_f64()]),
103 sym::log2f128 => ("llvm.log2", &[bx.type_f128()]),
104
105 sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]),
106 sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]),
107 sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]),
108 sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]),
109
110 sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]),
111 sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]),
112 sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]),
113 sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]),
114
115 sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
130 sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
131 sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
132 sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
133
134 sym::floorf16 => ("llvm.floor", &[bx.type_f16()]),
135 sym::floorf32 => ("llvm.floor", &[bx.type_f32()]),
136 sym::floorf64 => ("llvm.floor", &[bx.type_f64()]),
137 sym::floorf128 => ("llvm.floor", &[bx.type_f128()]),
138
139 sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]),
140 sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]),
141 sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]),
142 sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]),
143
144 sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]),
145 sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]),
146 sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]),
147 sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]),
148
149 sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]),
154 sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]),
155 sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]),
156 sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]),
157
158 sym::roundf16 => ("llvm.round", &[bx.type_f16()]),
159 sym::roundf32 => ("llvm.round", &[bx.type_f32()]),
160 sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
161 sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
162
163 _ => return None,
164 };
165 Some(bx.call_intrinsic(
166 base_name,
167 type_params,
168 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
169 ))
170}
171
172impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
173 fn codegen_intrinsic_call(
174 &mut self,
175 instance: ty::Instance<'tcx>,
176 args: &[OperandRef<'tcx, &'ll Value>],
177 result: PlaceRef<'tcx, &'ll Value>,
178 span: Span,
179 ) -> Result<(), ty::Instance<'tcx>> {
180 let tcx = self.tcx;
181
182 let name = tcx.item_name(instance.def_id());
183 let fn_args = instance.args;
184
185 let simple = call_simple_intrinsic(self, name, args);
186 let llval = match name {
187 _ if simple.is_some() => simple.unwrap(),
188 sym::minimum_number_nsz_f16
189 | sym::minimum_number_nsz_f32
190 | sym::minimum_number_nsz_f64
191 | sym::minimum_number_nsz_f128
192 | sym::maximum_number_nsz_f16
193 | sym::maximum_number_nsz_f32
194 | sym::maximum_number_nsz_f64
195 | sym::maximum_number_nsz_f128
196 if crate::llvm_util::get_version() >= (22, 0, 0) =>
198 {
199 let intrinsic_name = if name.as_str().starts_with("min") {
200 "llvm.minimumnum"
201 } else {
202 "llvm.maximumnum"
203 };
204 let call = self.call_intrinsic(
205 intrinsic_name,
206 &[args[0].layout.immediate_llvm_type(self.cx)],
207 &[args[0].immediate(), args[1].immediate()],
208 );
209 unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
212 call
213 }
214 sym::ptr_mask => {
215 let ptr = args[0].immediate();
216 self.call_intrinsic(
217 "llvm.ptrmask",
218 &[self.val_ty(ptr), self.type_isize()],
219 &[ptr, args[1].immediate()],
220 )
221 }
222 sym::autodiff => {
223 codegen_autodiff(self, tcx, instance, args, result);
224 return Ok(());
225 }
226 sym::offload => {
227 if tcx.sess.opts.unstable_opts.offload.is_empty() {
228 let _ = tcx.dcx().emit_almost_fatal(OffloadWithoutEnable);
229 }
230
231 if tcx.sess.lto() != rustc_session::config::Lto::Fat {
232 let _ = tcx.dcx().emit_almost_fatal(OffloadWithoutFatLTO);
233 }
234
235 codegen_offload(self, tcx, instance, args);
236 return Ok(());
237 }
238 sym::is_val_statically_known => {
239 if let OperandValue::Immediate(imm) = args[0].val {
240 self.call_intrinsic(
241 "llvm.is.constant",
242 &[args[0].layout.immediate_llvm_type(self.cx)],
243 &[imm],
244 )
245 } else {
246 self.const_bool(false)
247 }
248 }
249 sym::select_unpredictable => {
250 let cond = args[0].immediate();
251 match (&args[1].layout, &args[2].layout) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val, &*right_val,
::core::option::Option::None);
}
}
};assert_eq!(args[1].layout, args[2].layout);
252 let select = |bx: &mut Self, true_val, false_val| {
253 let result = bx.select(cond, true_val, false_val);
254 bx.set_unpredictable(&result);
255 result
256 };
257 match (args[1].val, args[2].val) {
258 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
259 if !true_val.llextra.is_none() {
::core::panicking::panic("assertion failed: true_val.llextra.is_none()")
};assert!(true_val.llextra.is_none());
260 if !false_val.llextra.is_none() {
::core::panicking::panic("assertion failed: false_val.llextra.is_none()")
};assert!(false_val.llextra.is_none());
261 match (&true_val.align, &false_val.align) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val, &*right_val,
::core::option::Option::None);
}
}
};assert_eq!(true_val.align, false_val.align);
262 let ptr = select(self, true_val.llval, false_val.llval);
263 let selected =
264 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
265 selected.store(self, result);
266 return Ok(());
267 }
268 (OperandValue::Immediate(_), OperandValue::Immediate(_))
269 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
270 let true_val = args[1].immediate_or_packed_pair(self);
271 let false_val = args[2].immediate_or_packed_pair(self);
272 select(self, true_val, false_val)
273 }
274 (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
275 _ => ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("Incompatible OperandValue for select_unpredictable"))span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
276 }
277 }
278 sym::catch_unwind => {
279 catch_unwind_intrinsic(
280 self,
281 args[0].immediate(),
282 args[1].immediate(),
283 args[2].immediate(),
284 result,
285 );
286 return Ok(());
287 }
288 sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
289 sym::va_arg => {
290 let BackendRepr::Scalar(scalar) = result.layout.backend_repr else {
291 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support non-scalar types"))bug!("the va_arg intrinsic does not support non-scalar types")
292 };
293
294 match scalar.primitive() {
298 Primitive::Pointer(_) => {
299 }
301 Primitive::Int(Integer::I128, _) => {
302 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `i128`/`u128`"))bug!("the va_arg intrinsic does not support `i128`/`u128`")
305 }
306 Primitive::Int(..) => {
307 let int_width = self.cx().size_of(result.layout.ty).bits();
308 let target_c_int_width = self.cx().sess().target.options.c_int_width;
309 if int_width < u64::from(target_c_int_width) {
310 ::rustc_middle::util::bug::bug_fmt(format_args!("va_arg got i{0} but needs at least c_int (an i{1})",
int_width, target_c_int_width));bug!(
313 "va_arg got i{} but needs at least c_int (an i{})",
314 int_width,
315 target_c_int_width
316 );
317 }
318 }
319 Primitive::Float(Float::F16) => {
320 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f16`"))bug!("the va_arg intrinsic does not support `f16`")
321 }
322 Primitive::Float(Float::F32) => {
323 if self.cx().sess().target.arch != Arch::Avr {
325 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f32` on this target"))bug!("the va_arg intrinsic does not support `f32` on this target")
326 }
327 }
328 Primitive::Float(Float::F64) => {
329 }
331 Primitive::Float(Float::F128) => {
332 ::rustc_middle::util::bug::bug_fmt(format_args!("the va_arg intrinsic does not support `f128`"))bug!("the va_arg intrinsic does not support `f128`")
334 }
335 }
336
337 emit_va_arg(self, args[0], result.layout.ty)
338 }
339
340 sym::volatile_load | sym::unaligned_volatile_load => {
341 let ptr = args[0].immediate();
342 let load = self.volatile_load(result.layout.llvm_type(self), ptr);
343 let align = if name == sym::unaligned_volatile_load {
344 1
345 } else {
346 result.layout.align.bytes() as u32
347 };
348 unsafe {
349 llvm::LLVMSetAlignment(load, align);
350 }
351 if !result.layout.is_zst() {
352 self.store_to_place(load, result.val);
353 }
354 return Ok(());
355 }
356 sym::volatile_store => {
357 let dst = args[0].deref(self.cx());
358 args[1].val.volatile_store(self, dst);
359 return Ok(());
360 }
361 sym::unaligned_volatile_store => {
362 let dst = args[0].deref(self.cx());
363 args[1].val.unaligned_volatile_store(self, dst);
364 return Ok(());
365 }
366 sym::prefetch_read_data
367 | sym::prefetch_write_data
368 | sym::prefetch_read_instruction
369 | sym::prefetch_write_instruction => {
370 let (rw, cache_type) = match name {
371 sym::prefetch_read_data => (0, 1),
372 sym::prefetch_write_data => (1, 1),
373 sym::prefetch_read_instruction => (0, 0),
374 sym::prefetch_write_instruction => (1, 0),
375 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("impossible case reached"))bug!(),
376 };
377 let ptr = args[0].immediate();
378 let locality = fn_args.const_at(1).to_leaf().to_i32();
379 self.call_intrinsic(
380 "llvm.prefetch",
381 &[self.val_ty(ptr)],
382 &[
383 ptr,
384 self.const_i32(rw),
385 self.const_i32(locality),
386 self.const_i32(cache_type),
387 ],
388 )
389 }
390 sym::carrying_mul_add => {
391 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
392
393 let wide_llty = self.type_ix(size.bits() * 2);
394 let args = args.as_array().unwrap();
395 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
396
397 let wide = if signed {
398 let prod = self.unchecked_smul(a, b);
399 let acc = self.unchecked_sadd(prod, c);
400 self.unchecked_sadd(acc, d)
401 } else {
402 let prod = self.unchecked_umul(a, b);
403 let acc = self.unchecked_uadd(prod, c);
404 self.unchecked_uadd(acc, d)
405 };
406
407 let narrow_llty = self.type_ix(size.bits());
408 let low = self.trunc(wide, narrow_llty);
409 let bits_const = self.const_uint(wide_llty, size.bits());
410 let high = self.lshr(wide, bits_const);
412 let high = self.trunc(high, narrow_llty);
414
415 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
416 let pair = self.const_poison(pair_llty);
417 let pair = self.insert_value(pair, low, 0);
418 let pair = self.insert_value(pair, high, 1);
419 pair
420 }
421
422 sym::carryless_mul if crate::llvm_util::get_version() >= (22, 0, 0) => {
424 let ty = args[0].layout.ty;
425 if !ty.is_integral() {
426 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
427 span,
428 name,
429 ty,
430 });
431 return Ok(());
432 }
433 let (size, _) = ty.int_size_and_signed(self.tcx);
434 let width = size.bits();
435 let llty = self.type_ix(width);
436
437 let lhs = args[0].immediate();
438 let rhs = args[1].immediate();
439 self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
440 }
441
442 sym::ctlz
443 | sym::ctlz_nonzero
444 | sym::cttz
445 | sym::cttz_nonzero
446 | sym::ctpop
447 | sym::bswap
448 | sym::bitreverse
449 | sym::saturating_add
450 | sym::saturating_sub
451 | sym::unchecked_funnel_shl
452 | sym::unchecked_funnel_shr => {
453 let ty = args[0].layout.ty;
454 if !ty.is_integral() {
455 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
456 span,
457 name,
458 ty,
459 });
460 return Ok(());
461 }
462 let (size, signed) = ty.int_size_and_signed(self.tcx);
463 let width = size.bits();
464 let llty = self.type_ix(width);
465 match name {
466 sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
467 let y =
468 self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
469 let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
470 "llvm.ctlz"
471 } else {
472 "llvm.cttz"
473 };
474 let ret =
475 self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
476 self.intcast(ret, result.layout.llvm_type(self), false)
477 }
478 sym::ctpop => {
479 let ret =
480 self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]);
481 self.intcast(ret, result.layout.llvm_type(self), false)
482 }
483 sym::bswap => {
484 if width == 8 {
485 args[0].immediate() } else {
487 self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
488 }
489 }
490 sym::bitreverse => {
491 self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
492 }
493 sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
494 let is_left = name == sym::unchecked_funnel_shl;
495 let lhs = args[0].immediate();
496 let rhs = args[1].immediate();
497 let raw_shift = args[2].immediate();
498 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.fsh{0}",
if is_left { 'l' } else { 'r' }))
})format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
499
500 let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false);
503
504 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift])
505 }
506 sym::saturating_add | sym::saturating_sub => {
507 let is_add = name == sym::saturating_add;
508 let lhs = args[0].immediate();
509 let rhs = args[1].immediate();
510 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
511 "llvm.{}{}.sat",
512 if signed { 's' } else { 'u' },
513 if is_add { "add" } else { "sub" },
514 );
515 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
516 }
517 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("impossible case reached"))bug!(),
518 }
519 }
520
521 sym::fabs => {
522 let ty = args[0].layout.ty;
523 let ty::Float(f) = ty.kind() else {
524 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("the `fabs` intrinsic requires a floating-point argument, got {0:?}",
ty));span_bug!(span, "the `fabs` intrinsic requires a floating-point argument, got {:?}", ty);
525 };
526 let llty = self.type_float_from_ty(*f);
527 let llvm_name = "llvm.fabs";
528 self.call_intrinsic(
529 llvm_name,
530 &[llty],
531 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
532 )
533 }
534
535 sym::raw_eq => {
536 use BackendRepr::*;
537 let tp_ty = fn_args.type_at(0);
538 let layout = self.layout_of(tp_ty).layout;
539 let use_integer_compare = match layout.backend_repr() {
540 Scalar(_) | ScalarPair(_, _) => true,
541 SimdVector { .. } => false,
542 SimdScalableVector { .. } => {
543 tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
544 span,
545 name: sym::raw_eq,
546 ty: tp_ty,
547 });
548 return Ok(());
549 }
550 Memory { .. } => {
551 layout.size() <= self.data_layout().pointer_size() * 2
555 }
556 };
557
558 let a = args[0].immediate();
559 let b = args[1].immediate();
560 if layout.size().bytes() == 0 {
561 self.const_bool(true)
562 } else if use_integer_compare {
563 let integer_ty = self.type_ix(layout.size().bits());
564 let a_val = self.load(integer_ty, a, layout.align().abi);
565 let b_val = self.load(integer_ty, b, layout.align().abi);
566 self.icmp(IntPredicate::IntEQ, a_val, b_val)
567 } else {
568 let n = self.const_usize(layout.size().bytes());
569 let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
570 self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
571 }
572 }
573
574 sym::compare_bytes => {
575 let cmp = self.call_intrinsic(
577 "memcmp",
578 &[],
579 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
580 );
581 self.sext(cmp, self.type_ix(32))
583 }
584
585 sym::black_box => {
586 args[0].val.store(self, result);
587 let result_val_span = [result.val.llval];
588 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
598 ("~{memory}", &[])
599 } else {
600 ("r,~{memory}", &result_val_span)
601 };
602 crate::asm::inline_asm_call(
603 self,
604 "",
605 constraint,
606 inputs,
607 self.type_void(),
608 &[],
609 true,
610 false,
611 llvm::AsmDialect::Att,
612 &[span],
613 false,
614 None,
615 None,
616 )
617 .unwrap_or_else(|| ::rustc_middle::util::bug::bug_fmt(format_args!("failed to generate inline asm call for `black_box`"))bug!("failed to generate inline asm call for `black_box`"));
618
619 return Ok(());
621 }
622
623 sym::amdgpu_dispatch_ptr => {
624 let val = self.call_intrinsic("llvm.amdgcn.dispatch.ptr", &[], &[]);
625 self.pointercast(val, self.type_ptr())
627 }
628
629 sym::sve_tuple_create2 => {
630 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
631 self.layout_of(fn_args.type_at(0)).backend_repr,
632 BackendRepr::SimdScalableVector {
633 number_of_vectors: NumScalableVectors(1),
634 ..
635 }
636 );
637 let tuple_ty = self.layout_of(fn_args.type_at(1));
638 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
639 tuple_ty.backend_repr,
640 BackendRepr::SimdScalableVector {
641 number_of_vectors: NumScalableVectors(2),
642 ..
643 }
644 );
645 let ret = self.const_poison(self.backend_type(tuple_ty));
646 let ret = self.insert_value(ret, args[0].immediate(), 0);
647 self.insert_value(ret, args[1].immediate(), 1)
648 }
649
650 sym::sve_tuple_create3 => {
651 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
652 self.layout_of(fn_args.type_at(0)).backend_repr,
653 BackendRepr::SimdScalableVector {
654 number_of_vectors: NumScalableVectors(1),
655 ..
656 }
657 );
658 let tuple_ty = self.layout_of(fn_args.type_at(1));
659 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(3), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(3), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
660 tuple_ty.backend_repr,
661 BackendRepr::SimdScalableVector {
662 number_of_vectors: NumScalableVectors(3),
663 ..
664 }
665 );
666 let ret = self.const_poison(self.backend_type(tuple_ty));
667 let ret = self.insert_value(ret, args[0].immediate(), 0);
668 let ret = self.insert_value(ret, args[1].immediate(), 1);
669 self.insert_value(ret, args[2].immediate(), 2)
670 }
671
672 sym::sve_tuple_create4 => {
673 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
674 self.layout_of(fn_args.type_at(0)).backend_repr,
675 BackendRepr::SimdScalableVector {
676 number_of_vectors: NumScalableVectors(1),
677 ..
678 }
679 );
680 let tuple_ty = self.layout_of(fn_args.type_at(1));
681 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(4), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(4), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
682 tuple_ty.backend_repr,
683 BackendRepr::SimdScalableVector {
684 number_of_vectors: NumScalableVectors(4),
685 ..
686 }
687 );
688 let ret = self.const_poison(self.backend_type(tuple_ty));
689 let ret = self.insert_value(ret, args[0].immediate(), 0);
690 let ret = self.insert_value(ret, args[1].immediate(), 1);
691 let ret = self.insert_value(ret, args[2].immediate(), 2);
692 self.insert_value(ret, args[3].immediate(), 3)
693 }
694
695 sym::sve_tuple_get => {
696 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
697 self.layout_of(fn_args.type_at(0)).backend_repr,
698 BackendRepr::SimdScalableVector {
699 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
700 ..
701 }
702 );
703 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
704 self.layout_of(fn_args.type_at(1)).backend_repr,
705 BackendRepr::SimdScalableVector {
706 number_of_vectors: NumScalableVectors(1),
707 ..
708 }
709 );
710 self.extract_value(
711 args[0].immediate(),
712 fn_args.const_at(2).to_leaf().to_i32() as u64,
713 )
714 }
715
716 sym::sve_tuple_set => {
717 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
718 self.layout_of(fn_args.type_at(0)).backend_repr,
719 BackendRepr::SimdScalableVector {
720 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
721 ..
722 }
723 );
724 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
725 self.layout_of(fn_args.type_at(1)).backend_repr,
726 BackendRepr::SimdScalableVector {
727 number_of_vectors: NumScalableVectors(1),
728 ..
729 }
730 );
731 self.insert_value(
732 args[0].immediate(),
733 args[1].immediate(),
734 fn_args.const_at(2).to_leaf().to_i32() as u64,
735 )
736 }
737
738 _ if name.as_str().starts_with("simd_") => {
739 let mut loaded_args = Vec::new();
742 for arg in args {
743 loaded_args.push(
744 if arg.layout.ty.is_simd()
749 && let OperandValue::Ref(place) = arg.val
750 {
751 let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
752 let elem_ll_ty = match elem_ty.kind() {
753 ty::Float(f) => self.type_float_from_ty(*f),
754 ty::Int(i) => self.type_int_from_ty(*i),
755 ty::Uint(u) => self.type_uint_from_ty(*u),
756 ty::RawPtr(_, _) => self.type_ptr(),
757 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
758 };
759 let loaded =
760 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
761 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
762 } else {
763 *arg
764 },
765 );
766 }
767
768 let llret_ty = if result.layout.ty.is_simd()
769 && let BackendRepr::Memory { .. } = result.layout.backend_repr
770 {
771 let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
772 let elem_ll_ty = match elem_ty.kind() {
773 ty::Float(f) => self.type_float_from_ty(*f),
774 ty::Int(i) => self.type_int_from_ty(*i),
775 ty::Uint(u) => self.type_uint_from_ty(*u),
776 ty::RawPtr(_, _) => self.type_ptr(),
777 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
778 };
779 self.type_vector(elem_ll_ty, size)
780 } else {
781 result.layout.llvm_type(self)
782 };
783
784 match generic_simd_intrinsic(
785 self,
786 name,
787 fn_args,
788 &loaded_args,
789 result.layout.ty,
790 llret_ty,
791 span,
792 ) {
793 Ok(llval) => llval,
794 Err(()) => return Ok(()),
797 }
798 }
799
800 _ => {
801 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/intrinsic.rs:801",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(801u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
let mut iter = __CALLSITE.metadata().fields().iter();
__CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
::tracing::__macro_support::Option::Some(&format_args!("unknown intrinsic \'{0}\' -- falling back to default body",
name) as &dyn Value))])
});
} else { ; }
};debug!("unknown intrinsic '{}' -- falling back to default body", name);
802 return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
804 }
805 };
806
807 if result.layout.ty.is_bool() {
808 let val = self.from_immediate(llval);
809 self.store_to_place(val, result.val);
810 } else if !result.layout.ty.is_unit() {
811 self.store_to_place(llval, result.val);
812 }
813 Ok(())
814 }
815
816 fn codegen_llvm_intrinsic_call(
817 &mut self,
818 instance: ty::Instance<'tcx>,
819 args: &[OperandRef<'tcx, Self::Value>],
820 _is_cleanup: bool,
821 ) -> Self::Value {
822 let tcx = self.tcx();
823
824 let fn_ty = instance.ty(tcx, self.typing_env());
825 let fn_sig = match *fn_ty.kind() {
826 ty::FnDef(def_id, args) => tcx.instantiate_bound_regions_with_erased(
827 tcx.fn_sig(def_id).instantiate(tcx, args).skip_norm_wip(),
828 ),
829 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
830 };
831 if !!fn_sig.c_variadic() {
::core::panicking::panic("assertion failed: !fn_sig.c_variadic()")
};assert!(!fn_sig.c_variadic());
832
833 let ret_layout = self.layout_of(fn_sig.output());
834 let llreturn_ty = if ret_layout.is_zst() {
835 self.type_void()
836 } else {
837 ret_layout.immediate_llvm_type(self)
838 };
839
840 let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
841 for &arg in fn_sig.inputs() {
842 let arg_layout = self.layout_of(arg);
843 if arg_layout.is_zst() {
844 continue;
845 }
846 llargument_tys.push(arg_layout.immediate_llvm_type(self));
847 }
848
849 let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) {
850 llfn
851 } else {
852 let sym = tcx.symbol_name(instance).name;
853
854 let llfn = if let Some(llfn) = self.get_declared_value(sym) {
855 llfn
856 } else {
857 intrinsic_fn(self, sym, llreturn_ty, llargument_tys, instance)
858 };
859
860 self.intrinsic_instances.borrow_mut().insert(instance, llfn);
861
862 llfn
863 };
864 let fn_ty = self.get_type_of_global(fn_ptr);
865
866 let mut llargs = ::alloc::vec::Vec::new()vec![];
867
868 for arg in args {
869 match arg.val {
870 OperandValue::ZeroSized => {}
871 OperandValue::Immediate(a) => llargs.push(a),
872 OperandValue::Pair(a, b) => {
873 llargs.push(a);
874 llargs.push(b);
875 }
876 OperandValue::Ref(op_place_val) => {
877 let mut llval = op_place_val.llval;
878 llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align);
884 if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
885 if scalar.is_bool() {
886 self.range_metadata(llval, WrappingRange { start: 0, end: 1 });
887 }
888 llval = self.to_immediate_scalar(llval, scalar);
890 }
891 llargs.push(llval);
892 }
893 }
894 }
895
896 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/intrinsic.rs:896",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(896u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
let mut iter = __CALLSITE.metadata().fields().iter();
__CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
::tracing::__macro_support::Option::Some(&format_args!("call intrinsic {0:?} with args ({1:?})",
instance, llargs) as &dyn Value))])
});
} else { ; }
};debug!("call intrinsic {:?} with args ({:?})", instance, llargs);
897
898 for (dest_ty, arg) in iter::zip(self.func_params_types(fn_ty), &mut llargs) {
899 let src_ty = self.val_ty(arg);
900 if !can_autocast(self, src_ty, dest_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with {1:?} (found) in `{2:?}",
dest_ty, src_ty, fn_ptr));
}
};assert!(
901 can_autocast(self, src_ty, dest_ty),
902 "Cannot match `{dest_ty:?}` (expected) with {src_ty:?} (found) in `{fn_ptr:?}"
903 );
904
905 *arg = autocast(self, arg, src_ty, dest_ty);
906 }
907
908 let llret = unsafe {
909 llvm::LLVMBuildCallWithOperandBundles(
910 self.llbuilder,
911 fn_ty,
912 fn_ptr,
913 llargs.as_ptr(),
914 llargs.len() as c_uint,
915 ptr::dangling(),
916 0,
917 c"".as_ptr(),
918 )
919 };
920
921 let src_ty = self.val_ty(llret);
922 let dest_ty = llreturn_ty;
923 if !can_autocast(self, dest_ty, src_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with `{1:?}` (found) in `{2:?}`",
src_ty, dest_ty, fn_ptr));
}
};assert!(
924 can_autocast(self, dest_ty, src_ty),
925 "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
926 );
927
928 autocast(self, llret, src_ty, dest_ty)
929 }
930
931 fn abort(&mut self) {
932 self.call_intrinsic("llvm.trap", &[], &[]);
933 }
934
935 fn assume(&mut self, val: Self::Value) {
936 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
937 self.call_intrinsic("llvm.assume", &[], &[val]);
938 }
939 }
940
941 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
942 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
943 self.call_intrinsic(
944 "llvm.expect",
945 &[self.type_i1()],
946 &[cond, self.const_bool(expected)],
947 )
948 } else {
949 cond
950 }
951 }
952
953 fn type_checked_load(
954 &mut self,
955 llvtable: &'ll Value,
956 vtable_byte_offset: u64,
957 typeid: &[u8],
958 ) -> Self::Value {
959 let typeid = self.create_metadata(typeid);
960 let typeid = self.get_metadata_value(typeid);
961 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
962 let type_checked_load = self.call_intrinsic(
963 "llvm.type.checked.load",
964 &[],
965 &[llvtable, vtable_byte_offset, typeid],
966 );
967 self.extract_value(type_checked_load, 0)
968 }
969
970 fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
971 self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list])
972 }
973
974 fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
975 self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list])
976 }
977}
978
979fn llvm_arch_for(rust_arch: &Arch) -> Option<&'static str> {
980 Some(match rust_arch {
981 Arch::AArch64 | Arch::Arm64EC => "aarch64",
982 Arch::AmdGpu => "amdgcn",
983 Arch::Arm => "arm",
984 Arch::Bpf => "bpf",
985 Arch::Hexagon => "hexagon",
986 Arch::LoongArch32 | Arch::LoongArch64 => "loongarch",
987 Arch::Mips | Arch::Mips32r6 | Arch::Mips64 | Arch::Mips64r6 => "mips",
988 Arch::Nvptx64 => "nvvm",
989 Arch::PowerPC | Arch::PowerPC64 => "ppc",
990 Arch::RiscV32 | Arch::RiscV64 => "riscv",
991 Arch::S390x => "s390",
992 Arch::SpirV => "spv",
993 Arch::Wasm32 | Arch::Wasm64 => "wasm",
994 Arch::X86 | Arch::X86_64 => "x86",
995 _ => return None, })
997}
998
999fn can_autocast<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
1000 if rust_ty == llvm_ty {
1001 return true;
1002 }
1003
1004 match cx.type_kind(llvm_ty) {
1005 TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => {
1009 let rust_element_tys = cx.struct_element_types(rust_ty);
1010 let llvm_element_tys = cx.struct_element_types(llvm_ty);
1011
1012 if rust_element_tys.len() != llvm_element_tys.len() {
1013 return false;
1014 }
1015
1016 iter::zip(rust_element_tys, llvm_element_tys).all(
1017 |(rust_element_ty, llvm_element_ty)| {
1018 can_autocast(cx, rust_element_ty, llvm_element_ty)
1019 },
1020 )
1021 }
1022 TypeKind::Vector => {
1023 let llvm_element_ty = cx.element_type(llvm_ty);
1024 let element_count = cx.vector_length(llvm_ty) as u64;
1025
1026 if llvm_element_ty == cx.type_bf16() {
1027 rust_ty == cx.type_vector(cx.type_i16(), element_count)
1028 } else if llvm_element_ty == cx.type_i1() {
1029 let int_width = element_count.next_power_of_two().max(8);
1030 rust_ty == cx.type_ix(int_width)
1031 } else {
1032 false
1033 }
1034 }
1035 TypeKind::BFloat => rust_ty == cx.type_i16(),
1036 _ => false,
1037 }
1038}
1039
1040fn autocast<'ll>(
1041 bx: &mut Builder<'_, 'll, '_>,
1042 val: &'ll Value,
1043 src_ty: &'ll Type,
1044 dest_ty: &'ll Type,
1045) -> &'ll Value {
1046 if src_ty == dest_ty {
1047 return val;
1048 }
1049 match (bx.type_kind(src_ty), bx.type_kind(dest_ty)) {
1050 (TypeKind::Struct, TypeKind::Struct) => {
1052 let mut ret = bx.const_poison(dest_ty);
1053 for (idx, (src_element_ty, dest_element_ty)) in
1054 iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty))
1055 .enumerate()
1056 {
1057 let elt = bx.extract_value(val, idx as u64);
1058 let casted_elt = autocast(bx, elt, src_element_ty, dest_element_ty);
1059 ret = bx.insert_value(ret, casted_elt, idx as u64);
1060 }
1061 ret
1062 }
1063 (TypeKind::Vector, TypeKind::Integer) if bx.element_type(src_ty) == bx.type_i1() => {
1065 let vector_length = bx.vector_length(src_ty) as u64;
1066 let int_width = vector_length.next_power_of_two().max(8);
1067
1068 let val = if vector_length == int_width {
1069 val
1070 } else {
1071 let shuffle_indices = match vector_length {
1073 0 => {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("zero length vectors are not allowed")));
}unreachable!("zero length vectors are not allowed"),
1074 1 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 1, 1, 1, 1, 1, 1]))vec![0, 1, 1, 1, 1, 1, 1, 1],
1075 2 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 2, 2, 2, 2, 2]))vec![0, 1, 2, 2, 2, 2, 2, 2],
1076 3 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 3, 3, 3, 3, 3]))vec![0, 1, 2, 3, 3, 3, 3, 3],
1077 4.. => (0..int_width as i32).collect(),
1078 };
1079 let shuffle_mask =
1080 shuffle_indices.into_iter().map(|i| bx.const_i32(i)).collect::<Vec<_>>();
1081 bx.shuffle_vector(val, bx.const_null(src_ty), bx.const_vector(&shuffle_mask))
1082 };
1083 bx.bitcast(val, dest_ty)
1084 }
1085 (TypeKind::Integer, TypeKind::Vector) if bx.element_type(dest_ty) == bx.type_i1() => {
1087 let vector_length = bx.vector_length(dest_ty) as u64;
1088 let int_width = vector_length.next_power_of_two().max(8);
1089
1090 let intermediate_ty = bx.type_vector(bx.type_i1(), int_width);
1091 let intermediate = bx.bitcast(val, intermediate_ty);
1092
1093 if vector_length == int_width {
1094 intermediate
1095 } else {
1096 let shuffle_mask: Vec<_> =
1097 (0..vector_length).map(|i| bx.const_i32(i as i32)).collect();
1098 bx.shuffle_vector(
1099 intermediate,
1100 bx.const_poison(intermediate_ty),
1101 bx.const_vector(&shuffle_mask),
1102 )
1103 }
1104 }
1105 _ => bx.bitcast(val, dest_ty), }
1107}
1108
1109fn intrinsic_fn<'ll, 'tcx>(
1110 bx: &Builder<'_, 'll, 'tcx>,
1111 name: &str,
1112 rust_return_ty: &'ll Type,
1113 rust_argument_tys: Vec<&'ll Type>,
1114 instance: ty::Instance<'tcx>,
1115) -> &'ll Value {
1116 let tcx = bx.tcx;
1117
1118 let rust_fn_ty = bx.type_func(&rust_argument_tys, rust_return_ty);
1119
1120 let intrinsic = llvm::Intrinsic::lookup(name.as_bytes());
1121
1122 if let Some(intrinsic) = intrinsic
1123 && intrinsic.is_target_specific()
1124 {
1125 let (llvm_arch, _) = name[5..].split_once('.').unwrap();
1126 let rust_arch = &tcx.sess.target.arch;
1127
1128 if let Some(correct_llvm_arch) = llvm_arch_for(rust_arch)
1129 && llvm_arch != correct_llvm_arch
1130 {
1131 tcx.dcx().emit_fatal(IntrinsicWrongArch {
1132 name,
1133 target_arch: rust_arch.desc(),
1134 span: tcx.def_span(instance.def_id()),
1135 });
1136 }
1137 }
1138
1139 if let Some(intrinsic) = intrinsic
1140 && !intrinsic.is_overloaded()
1141 {
1142 let llfn = intrinsic.get_declaration(bx.llmod, &[]);
1144 let llvm_fn_ty = bx.get_type_of_global(llfn);
1145
1146 let llvm_return_ty = bx.get_return_type(llvm_fn_ty);
1147 let llvm_argument_tys = bx.func_params_types(llvm_fn_ty);
1148 let llvm_is_variadic = bx.func_is_variadic(llvm_fn_ty);
1149
1150 let is_correct_signature = !llvm_is_variadic
1151 && rust_argument_tys.len() == llvm_argument_tys.len()
1152 && iter::once((rust_return_ty, llvm_return_ty))
1153 .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
1154 .all(|(rust_ty, llvm_ty)| can_autocast(bx, rust_ty, llvm_ty));
1155
1156 if !is_correct_signature {
1157 tcx.dcx().emit_fatal(IntrinsicSignatureMismatch {
1158 name,
1159 llvm_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", llvm_fn_ty))
})format!("{llvm_fn_ty:?}"),
1160 rust_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", rust_fn_ty))
})format!("{rust_fn_ty:?}"),
1161 span: tcx.def_span(instance.def_id()),
1162 });
1163 }
1164
1165 return llfn;
1166 }
1167
1168 let llfn = declare_raw_fn(
1170 bx,
1171 name,
1172 llvm::CCallConv,
1173 llvm::UnnamedAddr::Global,
1174 llvm::Visibility::Default,
1175 rust_fn_ty,
1176 );
1177
1178 if intrinsic.is_none() {
1179 let mut new_llfn = None;
1180 let can_upgrade = unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn) };
1181
1182 if !can_upgrade {
1183 tcx.dcx().emit_fatal(UnknownIntrinsic { name, span: tcx.def_span(instance.def_id()) });
1185 } else if let Some(def_id) = instance.def_id().as_local() {
1186 let hir_id = tcx.local_def_id_to_hir_id(def_id);
1188
1189 let msg = if let Some(new_llfn) = new_llfn {
1191 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{1}`, `{0}` can be used instead",
str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap(),
name))
})format!(
1192 "using deprecated intrinsic `{name}`, `{}` can be used instead",
1193 str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap()
1194 )
1195 } else {
1196 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{0}`",
name))
})format!("using deprecated intrinsic `{name}`")
1197 };
1198
1199 tcx.emit_node_lint(
1200 DEPRECATED_LLVM_INTRINSIC,
1201 hir_id,
1202 rustc_errors::DiagDecorator(|d| {
1203 d.primary_message(msg).span(tcx.hir_span(hir_id));
1204 }),
1205 );
1206 }
1207 }
1208
1209 llfn
1210}
1211
1212fn catch_unwind_intrinsic<'ll, 'tcx>(
1213 bx: &mut Builder<'_, 'll, 'tcx>,
1214 try_func: &'ll Value,
1215 data: &'ll Value,
1216 catch_func: &'ll Value,
1217 dest: PlaceRef<'tcx, &'ll Value>,
1218) {
1219 if !bx.sess().panic_strategy().unwinds() {
1220 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1221 bx.call(try_func_ty, None, None, try_func, &[data], None, None);
1222 OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
1225 } else if wants_msvc_seh(bx.sess()) {
1226 codegen_msvc_try(bx, try_func, data, catch_func, dest);
1227 } else if wants_wasm_eh(bx.sess()) {
1228 codegen_wasm_try(bx, try_func, data, catch_func, dest);
1229 } else if bx.sess().target.os == Os::Emscripten {
1230 codegen_emcc_try(bx, try_func, data, catch_func, dest);
1231 } else {
1232 codegen_gnu_try(bx, try_func, data, catch_func, dest);
1233 }
1234}
1235
1236fn codegen_msvc_try<'ll, 'tcx>(
1244 bx: &mut Builder<'_, 'll, 'tcx>,
1245 try_func: &'ll Value,
1246 data: &'ll Value,
1247 catch_func: &'ll Value,
1248 dest: PlaceRef<'tcx, &'ll Value>,
1249) {
1250 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1251 bx.set_personality_fn(bx.eh_personality());
1252
1253 let normal = bx.append_sibling_block("normal");
1254 let catchswitch = bx.append_sibling_block("catchswitch");
1255 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
1256 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
1257 let caught = bx.append_sibling_block("caught");
1258
1259 let try_func = llvm::get_param(bx.llfn(), 0);
1260 let data = llvm::get_param(bx.llfn(), 1);
1261 let catch_func = llvm::get_param(bx.llfn(), 2);
1262
1263 let ptr_size = bx.tcx().data_layout.pointer_size();
1319 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1320 let slot = bx.alloca(ptr_size, ptr_align);
1321 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1322 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
1323
1324 bx.switch_to_block(normal);
1325 bx.ret(bx.const_i32(0));
1326
1327 bx.switch_to_block(catchswitch);
1328 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
1329
1330 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
1345 let type_name = bx.const_bytes(b"rust_panic\0");
1346 let type_info =
1347 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
1348 let tydesc = bx.declare_global(
1349 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
1350 bx.val_ty(type_info),
1351 );
1352
1353 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
1354 if bx.cx.tcx.sess.target.supports_comdat() {
1355 llvm::SetUniqueComdat(bx.llmod, tydesc);
1356 }
1357 llvm::set_initializer(tydesc, type_info);
1358
1359 bx.switch_to_block(catchpad_rust);
1366 let flags = bx.const_i32(8);
1367 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
1368 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
1369 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1370 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
1371 bx.catch_ret(&funclet, caught);
1372
1373 bx.switch_to_block(catchpad_foreign);
1375 let flags = bx.const_i32(64);
1376 let null = bx.const_null(bx.type_ptr());
1377 let funclet = bx.catch_pad(cs, &[null, flags, null]);
1378 bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
1379 bx.catch_ret(&funclet, caught);
1380
1381 bx.switch_to_block(caught);
1382 bx.ret(bx.const_i32(1));
1383 });
1384
1385 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1388 OperandValue::Immediate(ret).store(bx, dest);
1389}
1390
1391fn codegen_wasm_try<'ll, 'tcx>(
1393 bx: &mut Builder<'_, 'll, 'tcx>,
1394 try_func: &'ll Value,
1395 data: &'ll Value,
1396 catch_func: &'ll Value,
1397 dest: PlaceRef<'tcx, &'ll Value>,
1398) {
1399 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1400 bx.set_personality_fn(bx.eh_personality());
1401
1402 let normal = bx.append_sibling_block("normal");
1403 let catchswitch = bx.append_sibling_block("catchswitch");
1404 let catchpad = bx.append_sibling_block("catchpad");
1405 let caught = bx.append_sibling_block("caught");
1406
1407 let try_func = llvm::get_param(bx.llfn(), 0);
1408 let data = llvm::get_param(bx.llfn(), 1);
1409 let catch_func = llvm::get_param(bx.llfn(), 2);
1410
1411 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1435 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
1436
1437 bx.switch_to_block(normal);
1438 bx.ret(bx.const_i32(0));
1439
1440 bx.switch_to_block(catchswitch);
1441 let cs = bx.catch_switch(None, None, &[catchpad]);
1442
1443 bx.switch_to_block(catchpad);
1444 let null = bx.const_null(bx.type_ptr());
1445 let funclet = bx.catch_pad(cs, &[null]);
1446
1447 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
1448 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
1449
1450 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1451 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
1452 bx.catch_ret(&funclet, caught);
1453
1454 bx.switch_to_block(caught);
1455 bx.ret(bx.const_i32(1));
1456 });
1457
1458 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1461 OperandValue::Immediate(ret).store(bx, dest);
1462}
1463
1464fn codegen_gnu_try<'ll, 'tcx>(
1476 bx: &mut Builder<'_, 'll, 'tcx>,
1477 try_func: &'ll Value,
1478 data: &'ll Value,
1479 catch_func: &'ll Value,
1480 dest: PlaceRef<'tcx, &'ll Value>,
1481) {
1482 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1483 let then = bx.append_sibling_block("then");
1496 let catch = bx.append_sibling_block("catch");
1497
1498 let try_func = llvm::get_param(bx.llfn(), 0);
1499 let data = llvm::get_param(bx.llfn(), 1);
1500 let catch_func = llvm::get_param(bx.llfn(), 2);
1501 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1502 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1503
1504 bx.switch_to_block(then);
1505 bx.ret(bx.const_i32(0));
1506
1507 bx.switch_to_block(catch);
1514 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1515 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
1516 let tydesc = bx.const_null(bx.type_ptr());
1517 bx.add_clause(vals, tydesc);
1518 let ptr = bx.extract_value(vals, 0);
1519 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1520 bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
1521 bx.ret(bx.const_i32(1));
1522 });
1523
1524 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1527 OperandValue::Immediate(ret).store(bx, dest);
1528}
1529
1530fn codegen_emcc_try<'ll, 'tcx>(
1534 bx: &mut Builder<'_, 'll, 'tcx>,
1535 try_func: &'ll Value,
1536 data: &'ll Value,
1537 catch_func: &'ll Value,
1538 dest: PlaceRef<'tcx, &'ll Value>,
1539) {
1540 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1541 let then = bx.append_sibling_block("then");
1559 let catch = bx.append_sibling_block("catch");
1560
1561 let try_func = llvm::get_param(bx.llfn(), 0);
1562 let data = llvm::get_param(bx.llfn(), 1);
1563 let catch_func = llvm::get_param(bx.llfn(), 2);
1564 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1565 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1566
1567 bx.switch_to_block(then);
1568 bx.ret(bx.const_i32(0));
1569
1570 bx.switch_to_block(catch);
1576 let tydesc = bx.eh_catch_typeinfo();
1577 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1578 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1579 bx.add_clause(vals, tydesc);
1580 bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1581 let ptr = bx.extract_value(vals, 0);
1582 let selector = bx.extract_value(vals, 1);
1583
1584 let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]);
1586 let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1587 let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1588
1589 let ptr_size = bx.tcx().data_layout.pointer_size();
1592 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1593 let i8_align = bx.tcx().data_layout.i8_align;
1594 if !(i8_align <= ptr_align) {
::core::panicking::panic("assertion failed: i8_align <= ptr_align")
};assert!(i8_align <= ptr_align);
1596 let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1597 bx.store(ptr, catch_data, ptr_align);
1598 let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1599 bx.store(is_rust_panic, catch_data_1, i8_align);
1600
1601 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1602 bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1603 bx.ret(bx.const_i32(1));
1604 });
1605
1606 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1609 OperandValue::Immediate(ret).store(bx, dest);
1610}
1611
1612fn gen_fn<'a, 'll, 'tcx>(
1615 cx: &'a CodegenCx<'ll, 'tcx>,
1616 name: &str,
1617 rust_fn_sig: ty::PolyFnSig<'tcx>,
1618 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1619) -> (&'ll Type, &'ll Value) {
1620 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1621 let llty = fn_abi.llvm_type(cx);
1622 let llfn = cx.declare_fn(name, fn_abi, None);
1623 cx.set_frame_pointer_type(llfn);
1624 cx.apply_target_cpu_attr(llfn);
1625 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1627 let llbb = Builder::append_block(cx, llfn, "entry-block");
1628 let bx = Builder::build(cx, llbb);
1629 codegen(bx);
1630 (llty, llfn)
1631}
1632
1633fn get_rust_try_fn<'a, 'll, 'tcx>(
1638 cx: &'a CodegenCx<'ll, 'tcx>,
1639 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1640) -> (&'ll Type, &'ll Value) {
1641 if let Some(llfn) = cx.rust_try_fn.get() {
1642 return llfn;
1643 }
1644
1645 let tcx = cx.tcx;
1647 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1648 let try_fn_ty = Ty::new_fn_ptr(
1650 tcx,
1651 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p], tcx.types.unit, hir::Safety::Unsafe)),
1652 );
1653 let catch_fn_ty = Ty::new_fn_ptr(
1655 tcx,
1656 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p, i8p], tcx.types.unit, hir::Safety::Unsafe)),
1657 );
1658 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig_rust_abi(
1660 [try_fn_ty, i8p, catch_fn_ty],
1661 tcx.types.i32,
1662 hir::Safety::Unsafe,
1663 ));
1664 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1665 cx.rust_try_fn.set(Some(rust_try));
1666 rust_try
1667}
1668
1669fn codegen_autodiff<'ll, 'tcx>(
1670 bx: &mut Builder<'_, 'll, 'tcx>,
1671 tcx: TyCtxt<'tcx>,
1672 instance: ty::Instance<'tcx>,
1673 args: &[OperandRef<'tcx, &'ll Value>],
1674 result: PlaceRef<'tcx, &'ll Value>,
1675) {
1676 if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
1677 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable);
1678 }
1679
1680 let ct = tcx.crate_types();
1681 let lto = tcx.sess.lto();
1682 if ct.len() == 1 && ct.contains(&CrateType::Executable) {
1683 if lto != rustc_session::config::Lto::Fat {
1684 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutLto);
1685 }
1686 } else {
1687 if lto != rustc_session::config::Lto::Fat && !tcx.sess.opts.cg.linker_plugin_lto.enabled() {
1688 let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutLto);
1689 }
1690 }
1691
1692 let fn_args = instance.args;
1693 let callee_ty = instance.ty(tcx, bx.typing_env());
1694
1695 let sig = callee_ty.fn_sig(tcx).skip_binder();
1696
1697 let ret_ty = sig.output();
1698 let llret_ty = bx.layout_of(ret_ty).llvm_type(bx);
1699
1700 let source_fn_ptr_ty = fn_args.into_type_list(tcx)[0];
1701 let fn_to_diff = args[0].immediate();
1702
1703 let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() {
1704 ty::FnDef(def_id, diff_args) => (def_id, diff_args),
1705 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("invalid args"))bug!("invalid args"),
1706 };
1707
1708 let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) {
1709 Ok(Some(instance)) => instance,
1710 Ok(None) => ::rustc_middle::util::bug::bug_fmt(format_args!("could not resolve ({0:?}, {1:?}) to a specific autodiff instance",
diff_id, diff_args))bug!(
1711 "could not resolve ({:?}, {:?}) to a specific autodiff instance",
1712 diff_id,
1713 diff_args
1714 ),
1715 Err(_) => {
1716 return;
1718 }
1719 };
1720
1721 let val_arr = get_args_from_tuple(bx, args[2], fn_diff);
1722 let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE);
1723
1724 let Some(Some(mut diff_attrs)) =
1725 {
{
'done:
{
for i in
::rustc_hir::attrs::HasAttrs::get_attrs(fn_diff.def_id(),
&tcx) {
#[allow(unused_imports)]
use rustc_hir::attrs::AttributeKind::*;
let i: &rustc_hir::Attribute = i;
match i {
rustc_hir::Attribute::Parsed(RustcAutodiff(attr)) => {
break 'done Some(attr.clone());
}
rustc_hir::Attribute::Unparsed(..) =>
{}
#[deny(unreachable_patterns)]
_ => {}
}
}
None
}
}
}find_attr!(tcx, fn_diff.def_id(), RustcAutodiff(attr) => attr.clone())
1726 else {
1727 ::rustc_middle::util::bug::bug_fmt(format_args!("could not find autodiff attrs"))bug!("could not find autodiff attrs")
1728 };
1729
1730 adjust_activity_to_abi(
1731 tcx,
1732 source_fn_ptr_ty,
1733 TypingEnv::fully_monomorphized(),
1734 &mut diff_attrs.input_activity,
1735 );
1736
1737 let fnc_tree = rustc_middle::ty::fnc_typetrees(tcx, source_fn_ptr_ty);
1738
1739 generate_enzyme_call(
1741 bx,
1742 bx.cx,
1743 fn_to_diff,
1744 &diff_symbol,
1745 llret_ty,
1746 &val_arr,
1747 &diff_attrs,
1748 result,
1749 fnc_tree,
1750 );
1751}
1752
1753fn codegen_offload<'ll, 'tcx>(
1758 bx: &mut Builder<'_, 'll, 'tcx>,
1759 tcx: TyCtxt<'tcx>,
1760 instance: ty::Instance<'tcx>,
1761 args: &[OperandRef<'tcx, &'ll Value>],
1762) {
1763 let cx = bx.cx;
1764 let fn_args = instance.args;
1765
1766 let (target_id, target_args) = match fn_args.into_type_list(tcx)[0].kind() {
1767 ty::FnDef(def_id, params) => (def_id, params),
1768 _ => ::rustc_middle::util::bug::bug_fmt(format_args!("invalid offload intrinsic arg"))bug!("invalid offload intrinsic arg"),
1769 };
1770
1771 let fn_target = match Instance::try_resolve(tcx, cx.typing_env(), *target_id, target_args) {
1772 Ok(Some(instance)) => instance,
1773 Ok(None) => ::rustc_middle::util::bug::bug_fmt(format_args!("could not resolve ({0:?}, {1:?}) to a specific offload instance",
target_id, target_args))bug!(
1774 "could not resolve ({:?}, {:?}) to a specific offload instance",
1775 target_id,
1776 target_args
1777 ),
1778 Err(_) => {
1779 return;
1781 }
1782 };
1783
1784 let offload_dims = OffloadKernelDims::from_operands(bx, &args[1], &args[2]);
1785 let args = get_args_from_tuple(bx, args[3], fn_target);
1786 let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE);
1787
1788 let sig = tcx.fn_sig(fn_target.def_id()).skip_binder();
1789 let sig = tcx.instantiate_bound_regions_with_erased(sig);
1790 let inputs = sig.inputs();
1791
1792 let metadata = inputs.iter().map(|ty| OffloadMetadata::from_ty(tcx, *ty)).collect::<Vec<_>>();
1793
1794 let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::<Vec<_>>();
1795
1796 let offload_globals_ref = cx.offload_globals.borrow();
1797 let offload_globals = match offload_globals_ref.as_ref() {
1798 Some(globals) => globals,
1799 None => {
1800 return;
1802 }
1803 };
1804 register_offload(cx);
1805 let offload_data = gen_define_handling(&cx, &metadata, target_symbol, offload_globals);
1806 gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals, &offload_dims);
1807}
1808
1809fn get_args_from_tuple<'ll, 'tcx>(
1810 bx: &mut Builder<'_, 'll, 'tcx>,
1811 tuple_op: OperandRef<'tcx, &'ll Value>,
1812 fn_instance: Instance<'tcx>,
1813) -> Vec<&'ll Value> {
1814 let cx = bx.cx;
1815 let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty());
1816
1817 match tuple_op.val {
1818 OperandValue::Immediate(val) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[val]))vec![val],
1819 OperandValue::Pair(v1, v2) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[v1, v2]))vec![v1, v2],
1820 OperandValue::Ref(ptr) => {
1821 let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout };
1822
1823 let mut result = Vec::with_capacity(fn_abi.args.len());
1824 let mut tuple_index = 0;
1825
1826 for arg in &fn_abi.args {
1827 match arg.mode {
1828 PassMode::Ignore => {}
1829 PassMode::Direct(_) | PassMode::Cast { .. } => {
1830 let field = tuple_place.project_field(bx, tuple_index);
1831 let llvm_ty = field.layout.llvm_type(bx.cx);
1832 let val = bx.load(llvm_ty, field.val.llval, field.val.align);
1833 result.push(val);
1834 tuple_index += 1;
1835 }
1836 PassMode::Pair(_, _) => {
1837 let field = tuple_place.project_field(bx, tuple_index);
1838 let llvm_ty = field.layout.llvm_type(bx.cx);
1839 let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align);
1840 result.push(bx.extract_value(pair_val, 0));
1841 result.push(bx.extract_value(pair_val, 1));
1842 tuple_index += 1;
1843 }
1844 PassMode::Indirect { .. } => {
1845 let field = tuple_place.project_field(bx, tuple_index);
1846 result.push(field.val.llval);
1847 tuple_index += 1;
1848 }
1849 }
1850 }
1851
1852 result
1853 }
1854
1855 OperandValue::ZeroSized => ::alloc::vec::Vec::new()vec![],
1856 }
1857}
1858
1859fn generic_simd_intrinsic<'ll, 'tcx>(
1860 bx: &mut Builder<'_, 'll, 'tcx>,
1861 name: Symbol,
1862 fn_args: GenericArgsRef<'tcx>,
1863 args: &[OperandRef<'tcx, &'ll Value>],
1864 ret_ty: Ty<'tcx>,
1865 llret_ty: &'ll Type,
1866 span: Span,
1867) -> Result<&'ll Value, ()> {
1868 macro_rules! return_error {
1869 ($diag: expr) => {{
1870 bx.sess().dcx().emit_err($diag);
1871 return Err(());
1872 }};
1873 }
1874
1875 macro_rules! require {
1876 ($cond: expr, $diag: expr) => {
1877 if !$cond {
1878 return_error!($diag);
1879 }
1880 };
1881 }
1882
1883 macro_rules! require_simd {
1884 ($ty: expr, $variant:ident) => {{
1885 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1886 $ty.simd_size_and_type(bx.tcx())
1887 }};
1888 }
1889
1890 macro_rules! require_simd_or_scalable {
1891 ($ty: expr, $variant:ident) => {{
1892 require!(
1893 $ty.is_simd() || $ty.is_scalable_vector(),
1894 InvalidMonomorphization::$variant { span, name, ty: $ty }
1895 );
1896 if $ty.is_simd() {
1897 let (len, ty) = $ty.simd_size_and_type(bx.tcx());
1898 (len, ty, None)
1899 } else {
1900 let (count, ty, num_vecs) =
1901 $ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
1902 (count as u64, ty, Some(num_vecs))
1903 }
1904 }};
1905 }
1906
1907 macro_rules! require_int_or_uint_ty {
1909 ($ty: expr, $diag: expr) => {
1910 match $ty {
1911 ty::Int(i) => {
1912 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1913 }
1914 ty::Uint(i) => {
1915 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1916 }
1917 _ => {
1918 return_error!($diag);
1919 }
1920 }
1921 };
1922 }
1923
1924 let llvm_version = crate::llvm_util::get_version();
1925
1926 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1940 bx: &mut Builder<'a, 'll, 'tcx>,
1941 i_xn: &'ll Value,
1942 in_elem_bitwidth: u64,
1943 in_len: u64,
1944 ) -> &'ll Value {
1945 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1947 let shift_indices = ::alloc::vec::from_elem(shift_idx, in_len as _)vec![shift_idx; in_len as _];
1948 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1949 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1951 }
1952
1953 if truecfg!(debug_assertions) {
1955 for arg in args {
1956 if arg.layout.ty.is_simd() {
1957 {
match arg.val {
OperandValue::Immediate(_) => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"OperandValue::Immediate(_)", ::core::option::Option::None);
}
}
};assert_matches!(arg.val, OperandValue::Immediate(_));
1958 }
1959 }
1960 }
1961
1962 if name == sym::simd_select_bitmask {
1963 let (len, _) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdArgument);
1964
1965 let expected_int_bits = len.max(8).next_power_of_two();
1966 let expected_bytes = len.div_ceil(8);
1967
1968 let mask_ty = args[0].layout.ty;
1969 let mask = match mask_ty.kind() {
1970 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1971 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1972 ty::Array(elem, len)
1973 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1974 && len
1975 .try_to_target_usize(bx.tcx)
1976 .expect("expected monomorphic const in codegen")
1977 == expected_bytes =>
1978 {
1979 let place = PlaceRef::alloca(bx, args[0].layout);
1980 args[0].val.store(bx, place);
1981 let int_ty = bx.type_ix(expected_bytes * 8);
1982 bx.load(int_ty, place.val.llval, Align::ONE)
1983 }
1984 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::InvalidBitmask {
span,
name,
mask_ty,
expected_int_bits,
expected_bytes,
});
return Err(());
}return_error!(InvalidMonomorphization::InvalidBitmask {
1985 span,
1986 name,
1987 mask_ty,
1988 expected_int_bits,
1989 expected_bytes
1990 }),
1991 };
1992
1993 let i1 = bx.type_i1();
1994 let im = bx.type_ix(len);
1995 let i1xn = bx.type_vector(i1, len);
1996 let m_im = bx.trunc(mask, im);
1997 let m_i1s = bx.bitcast(m_im, i1xn);
1998 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1999 }
2000
2001 if name == sym::simd_splat {
2002 let (_out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2003
2004 if !(args[0].layout.ty == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: out_ty,
vector_type: ret_ty,
});
return Err(());
};
};require!(
2005 args[0].layout.ty == out_ty,
2006 InvalidMonomorphization::ExpectedVectorElementType {
2007 span,
2008 name,
2009 expected_element: out_ty,
2010 vector_type: ret_ty,
2011 }
2012 );
2013
2014 let poison_vec = bx.const_poison(llret_ty);
2016 let idx0 = bx.const_i32(0);
2017 let v0 = bx.insert_element(poison_vec, args[0].immediate(), idx0);
2018
2019 let splat = bx.shuffle_vector(v0, poison_vec, bx.const_null(llret_ty));
2022
2023 return Ok(splat);
2024 }
2025
2026 let supports_scalable = match name {
2027 sym::simd_cast | sym::simd_select => true,
2028 _ => false,
2029 };
2030
2031 if !supports_scalable {
2036 let _ = {
if !args[0].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(());
};
};
args[0].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[0].layout.ty, SimdInput);
2037 }
2038 let (in_len, in_elem, in_num_vecs) = {
if !(args[0].layout.ty.is_simd() ||
args[0].layout.ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(());
};
};
if args[0].layout.ty.is_simd() {
let (len, ty) = args[0].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[0].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[0].layout.ty, SimdInput);
2039 let in_ty = args[0].layout.ty;
2040
2041 let comparison = match name {
2042 sym::simd_eq => Some(BinOp::Eq),
2043 sym::simd_ne => Some(BinOp::Ne),
2044 sym::simd_lt => Some(BinOp::Lt),
2045 sym::simd_le => Some(BinOp::Le),
2046 sym::simd_gt => Some(BinOp::Gt),
2047 sym::simd_ge => Some(BinOp::Ge),
2048 _ => None,
2049 };
2050
2051 if let Some(cmp_op) = comparison {
2052 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2053
2054 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2055 in_len == out_len,
2056 InvalidMonomorphization::ReturnLengthInputType {
2057 span,
2058 name,
2059 in_len,
2060 in_ty,
2061 ret_ty,
2062 out_len
2063 }
2064 );
2065 if !(bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnIntegerType {
span,
name,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2066 bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
2067 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
2068 );
2069
2070 return Ok(compare_simd_types(
2071 bx,
2072 args[0].immediate(),
2073 args[1].immediate(),
2074 in_elem,
2075 llret_ty,
2076 cmp_op,
2077 ));
2078 }
2079
2080 if name == sym::simd_shuffle_const_generic {
2081 let idx = fn_args[2].expect_const().to_branch();
2082 let n = idx.len() as u64;
2083
2084 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2085 if !(out_len == n) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2086 out_len == n,
2087 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2088 );
2089 if !(in_elem == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2090 in_elem == out_ty,
2091 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2092 );
2093
2094 let total_len = in_len * 2;
2095
2096 let indices: Option<Vec<_>> = idx
2097 .iter()
2098 .enumerate()
2099 .map(|(arg_idx, val)| {
2100 let idx = val.to_leaf().to_i32();
2101 if idx >= i32::try_from(total_len).unwrap() {
2102 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
2103 span,
2104 name,
2105 arg_idx: arg_idx as u64,
2106 total_len: total_len.into(),
2107 });
2108 None
2109 } else {
2110 Some(bx.const_i32(idx))
2111 }
2112 })
2113 .collect();
2114 let Some(indices) = indices else {
2115 return Ok(bx.const_null(llret_ty));
2116 };
2117
2118 return Ok(bx.shuffle_vector(
2119 args[0].immediate(),
2120 args[1].immediate(),
2121 bx.const_vector(&indices),
2122 ));
2123 }
2124
2125 if name == sym::simd_shuffle {
2126 let idx_ty = args[2].layout.ty;
2128 let n: u64 = if idx_ty.is_simd()
2129 && #[allow(non_exhaustive_omitted_patterns)] match idx_ty.simd_size_and_type(bx.cx.tcx).1.kind()
{
ty::Uint(ty::UintTy::U32) => true,
_ => false,
}matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
2130 {
2131 idx_ty.simd_size_and_type(bx.cx.tcx).0
2132 } else {
2133 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdShuffle {
span,
name,
ty: idx_ty,
});
return Err(());
}return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
2134 };
2135
2136 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2137 if !(out_len == n) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2138 out_len == n,
2139 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2140 );
2141 if !(in_elem == out_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(());
};
};require!(
2142 in_elem == out_ty,
2143 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2144 );
2145
2146 let total_len = u128::from(in_len) * 2;
2147
2148 let indices = args[2].immediate();
2150 for i in 0..n {
2151 let val = bx.const_get_elt(indices, i as u64);
2152 let idx = bx
2153 .const_to_opt_u128(val, true)
2154 .unwrap_or_else(|| ::rustc_middle::util::bug::bug_fmt(format_args!("typeck should have already ensured that these are const"))bug!("typeck should have already ensured that these are const"));
2155 if idx >= total_len {
2156 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: i,
total_len,
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2157 span,
2158 name,
2159 arg_idx: i,
2160 total_len,
2161 });
2162 }
2163 }
2164
2165 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
2166 }
2167
2168 if name == sym::simd_insert || name == sym::simd_insert_dyn {
2169 if !(in_elem == args[2].layout.ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::InsertedType {
span,
name,
in_elem,
in_ty,
out_ty: args[2].layout.ty,
});
return Err(());
};
};require!(
2170 in_elem == args[2].layout.ty,
2171 InvalidMonomorphization::InsertedType {
2172 span,
2173 name,
2174 in_elem,
2175 in_ty,
2176 out_ty: args[2].layout.ty
2177 }
2178 );
2179
2180 let index_imm = if name == sym::simd_insert {
2181 let idx = bx
2182 .const_to_opt_u128(args[1].immediate(), false)
2183 .expect("typeck should have ensure that this is a const");
2184 if idx >= in_len.into() {
2185 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2186 span,
2187 name,
2188 arg_idx: 1,
2189 total_len: in_len.into(),
2190 });
2191 }
2192 bx.const_i32(idx as i32)
2193 } else {
2194 args[1].immediate()
2195 };
2196
2197 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
2198 }
2199 if name == sym::simd_extract || name == sym::simd_extract_dyn {
2200 if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};require!(
2201 ret_ty == in_elem,
2202 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2203 );
2204 let index_imm = if name == sym::simd_extract {
2205 let idx = bx
2206 .const_to_opt_u128(args[1].immediate(), false)
2207 .expect("typeck should have ensure that this is a const");
2208 if idx >= in_len.into() {
2209 {
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(());
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2210 span,
2211 name,
2212 arg_idx: 1,
2213 total_len: in_len.into(),
2214 });
2215 }
2216 bx.const_i32(idx as i32)
2217 } else {
2218 args[1].immediate()
2219 };
2220
2221 return Ok(bx.extract_element(args[0].immediate(), index_imm));
2222 }
2223
2224 if name == sym::simd_select {
2225 let m_elem_ty = in_elem;
2226 let m_len = in_len;
2227 let (v_len, _, _) = {
if !(args[1].layout.ty.is_simd() ||
args[1].layout.ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
if args[1].layout.ty.is_simd() {
let (len, ty) = args[1].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[1].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[1].layout.ty, SimdArgument);
2228 if !(m_len == v_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MismatchedLengths {
span,
name,
m_len,
v_len,
});
return Err(());
};
};require!(
2229 m_len == v_len,
2230 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
2231 );
2232
2233 let m_i1s = if args[1].layout.ty.is_scalable_vector() {
2234 match m_elem_ty.kind() {
2235 ty::Bool => {}
2236 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(());
}return_error!(InvalidMonomorphization::MaskWrongElementType {
2237 span,
2238 name,
2239 ty: m_elem_ty
2240 }),
2241 };
2242 let i1 = bx.type_i1();
2243 let i1xn = bx.type_scalable_vector(i1, m_len as u64);
2244 bx.trunc(args[0].immediate(), i1xn)
2245 } else {
2246 let in_elem_bitwidth = match m_elem_ty.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2247 m_elem_ty.kind(),
2248 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
2249 );
2250 vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
2251 };
2252
2253 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2254 }
2255
2256 if name == sym::simd_bitmask {
2257 let expected_int_bits = in_len.max(8).next_power_of_two();
2266 let expected_bytes = in_len.div_ceil(8);
2267
2268 let in_elem_bitwidth = match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: in_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2270 in_elem.kind(),
2271 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
2272 );
2273
2274 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
2275 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
2277
2278 match ret_ty.kind() {
2279 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
2280 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
2282 }
2283 ty::Array(elem, len)
2284 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2285 && len
2286 .try_to_target_usize(bx.tcx)
2287 .expect("expected monomorphic const in codegen")
2288 == expected_bytes =>
2289 {
2290 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
2292
2293 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
2295 bx.store(ze, ptr, Align::ONE);
2296 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
2297 return Ok(bx.load(array_ty, ptr, Align::ONE));
2298 }
2299 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::CannotReturn {
span,
name,
ret_ty,
expected_int_bits,
expected_bytes,
});
return Err(());
}return_error!(InvalidMonomorphization::CannotReturn {
2300 span,
2301 name,
2302 ret_ty,
2303 expected_int_bits,
2304 expected_bytes
2305 }),
2306 }
2307 }
2308
2309 fn simd_simple_float_intrinsic<'ll, 'tcx>(
2310 name: Symbol,
2311 in_elem: Ty<'_>,
2312 in_ty: Ty<'_>,
2313 in_len: u64,
2314 bx: &mut Builder<'_, 'll, 'tcx>,
2315 span: Span,
2316 args: &[OperandRef<'tcx, &'ll Value>],
2317 ) -> Result<&'ll Value, ()> {
2318 macro_rules! return_error {
2319 ($diag: expr) => {{
2320 bx.sess().dcx().emit_err($diag);
2321 return Err(());
2322 }};
2323 }
2324
2325 let ty::Float(f) = in_elem.kind() else {
2326 {
bx.sess().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
span,
name,
ty: in_ty,
});
return Err(());
};return_error!(InvalidMonomorphization::BasicFloatType { span, name, ty: in_ty });
2327 };
2328 let elem_ty = bx.cx.type_float_from_ty(*f);
2329
2330 let vec_ty = bx.type_vector(elem_ty, in_len);
2331
2332 let intr_name = match name {
2333 sym::simd_ceil => "llvm.ceil",
2334 sym::simd_fabs => "llvm.fabs",
2335 sym::simd_fcos => "llvm.cos",
2336 sym::simd_fexp2 => "llvm.exp2",
2337 sym::simd_fexp => "llvm.exp",
2338 sym::simd_flog10 => "llvm.log10",
2339 sym::simd_flog2 => "llvm.log2",
2340 sym::simd_flog => "llvm.log",
2341 sym::simd_floor => "llvm.floor",
2342 sym::simd_fma => "llvm.fma",
2343 sym::simd_relaxed_fma => "llvm.fmuladd",
2344 sym::simd_fsin => "llvm.sin",
2345 sym::simd_fsqrt => "llvm.sqrt",
2346 sym::simd_round => "llvm.round",
2347 sym::simd_round_ties_even => "llvm.rint",
2348 sym::simd_trunc => "llvm.trunc",
2349 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnrecognizedIntrinsic {
span,
name,
});
return Err(());
}return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
2350 };
2351 Ok(bx.call_intrinsic(
2352 intr_name,
2353 &[vec_ty],
2354 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
2355 ))
2356 }
2357
2358 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_ceil | sym::simd_fabs | sym::simd_fcos | sym::simd_fexp2 |
sym::simd_fexp | sym::simd_flog10 | sym::simd_flog2 | sym::simd_flog |
sym::simd_floor | sym::simd_fma | sym::simd_fsin | sym::simd_fsqrt |
sym::simd_relaxed_fma | sym::simd_round | sym::simd_round_ties_even |
sym::simd_trunc => true,
_ => false,
}std::matches!(
2359 name,
2360 sym::simd_ceil
2361 | sym::simd_fabs
2362 | sym::simd_fcos
2363 | sym::simd_fexp2
2364 | sym::simd_fexp
2365 | sym::simd_flog10
2366 | sym::simd_flog2
2367 | sym::simd_flog
2368 | sym::simd_floor
2369 | sym::simd_fma
2370 | sym::simd_fsin
2371 | sym::simd_fsqrt
2372 | sym::simd_relaxed_fma
2373 | sym::simd_round
2374 | sym::simd_round_ties_even
2375 | sym::simd_trunc
2376 ) {
2377 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
2378 }
2379
2380 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
2381 let elem_ty = match *elem_ty.kind() {
2382 ty::Int(v) => cx.type_int_from_ty(v),
2383 ty::Uint(v) => cx.type_uint_from_ty(v),
2384 ty::Float(v) => cx.type_float_from_ty(v),
2385 ty::RawPtr(_, _) => cx.type_ptr(),
2386 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
2387 };
2388 cx.type_vector(elem_ty, vec_len)
2389 }
2390
2391 if name == sym::simd_gather {
2392 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(());
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2403 let (out_len, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2404 let (out_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(());
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2406 {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2407
2408 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len,
});
return Err(());
};
};require!(
2410 in_len == out_len,
2411 InvalidMonomorphization::SecondArgumentLength {
2412 span,
2413 name,
2414 in_len,
2415 in_ty,
2416 arg_ty: args[1].layout.ty,
2417 out_len
2418 }
2419 );
2420 if !(in_len == out_len2) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: out_len2,
});
return Err(());
};
};require!(
2421 in_len == out_len2,
2422 InvalidMonomorphization::ThirdArgumentLength {
2423 span,
2424 name,
2425 in_len,
2426 in_ty,
2427 arg_ty: args[2].layout.ty,
2428 out_len: out_len2
2429 }
2430 );
2431
2432 if !(ret_ty == in_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty,
ret_ty,
});
return Err(());
};
};require!(
2434 ret_ty == in_ty,
2435 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
2436 );
2437
2438 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, _) if
p_ty == in_elem && p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(());
};
};require!(
2439 matches!(
2440 *element_ty1.kind(),
2441 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
2442 ),
2443 InvalidMonomorphization::ExpectedElementType {
2444 span,
2445 name,
2446 expected_element: element_ty1,
2447 second_arg: args[1].layout.ty,
2448 in_elem,
2449 in_ty,
2450 mutability: ExpectedPointerMutability::Not,
2451 }
2452 );
2453
2454 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2455 element_ty2.kind(),
2456 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2457 );
2458
2459 let alignment = bx.align_of(in_elem).bytes();
2461
2462 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2464
2465 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2467
2468 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2470
2471 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2472 let alignment = bx.const_i32(alignment as i32);
2473 &[args[1].immediate(), alignment, mask, args[0].immediate()]
2474 } else {
2475 &[args[1].immediate(), mask, args[0].immediate()]
2476 };
2477
2478 let call =
2479 bx.call_intrinsic("llvm.masked.gather", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], args);
2480 if llvm_version >= (22, 0, 0) {
2481 crate::attributes::apply_to_callsite(
2482 call,
2483 crate::llvm::AttributePlace::Argument(0),
2484 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2485 )
2486 }
2487 return Ok(call);
2488 }
2489
2490 fn llvm_alignment<'ll, 'tcx>(
2491 bx: &mut Builder<'_, 'll, 'tcx>,
2492 alignment: SimdAlign,
2493 vector_ty: Ty<'tcx>,
2494 element_ty: Ty<'tcx>,
2495 ) -> u64 {
2496 match alignment {
2497 SimdAlign::Unaligned => 1,
2498 SimdAlign::Element => bx.align_of(element_ty).bytes(),
2499 SimdAlign::Vector => bx.align_of(vector_ty).bytes(),
2500 }
2501 }
2502
2503 if name == sym::simd_masked_load {
2504 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2513
2514 let mask_ty = in_ty;
2516 let (mask_len, mask_elem) = (in_len, in_elem);
2517
2518 let pointer_ty = args[1].layout.ty;
2520
2521 let values_ty = args[2].layout.ty;
2523 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(());
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2524
2525 {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2526
2527 if !(values_len == mask_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(());
};
};require!(
2529 values_len == mask_len,
2530 InvalidMonomorphization::ThirdArgumentLength {
2531 span,
2532 name,
2533 in_len: mask_len,
2534 in_ty: mask_ty,
2535 arg_ty: values_ty,
2536 out_len: values_len
2537 }
2538 );
2539
2540 if !(ret_ty == values_ty) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty: values_ty,
ret_ty,
});
return Err(());
};
};require!(
2542 ret_ty == values_ty,
2543 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
2544 );
2545
2546 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, _) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() =>
true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(());
};
};require!(
2547 matches!(
2548 *pointer_ty.kind(),
2549 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
2550 ),
2551 InvalidMonomorphization::ExpectedElementType {
2552 span,
2553 name,
2554 expected_element: values_elem,
2555 second_arg: pointer_ty,
2556 in_elem: values_elem,
2557 in_ty: values_ty,
2558 mutability: ExpectedPointerMutability::Not,
2559 }
2560 );
2561
2562 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2563 mask_elem.kind(),
2564 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2565 );
2566
2567 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2568
2569 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2571
2572 let llvm_pointer = bx.type_ptr();
2573
2574 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2576
2577 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2578 let alignment = bx.const_i32(alignment as i32);
2579
2580 &[args[1].immediate(), alignment, mask, args[2].immediate()]
2581 } else {
2582 &[args[1].immediate(), mask, args[2].immediate()]
2583 };
2584
2585 let call = bx.call_intrinsic("llvm.masked.load", &[llvm_elem_vec_ty, llvm_pointer], args);
2586 if llvm_version >= (22, 0, 0) {
2587 crate::attributes::apply_to_callsite(
2588 call,
2589 crate::llvm::AttributePlace::Argument(0),
2590 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2591 )
2592 }
2593 return Ok(call);
2594 }
2595
2596 if name == sym::simd_masked_store {
2597 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2606
2607 let mask_ty = in_ty;
2609 let (mask_len, mask_elem) = (in_len, in_elem);
2610
2611 let pointer_ty = args[1].layout.ty;
2613
2614 let values_ty = args[2].layout.ty;
2616 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(());
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2617
2618 if !(values_len == mask_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(());
};
};require!(
2620 values_len == mask_len,
2621 InvalidMonomorphization::ThirdArgumentLength {
2622 span,
2623 name,
2624 in_len: mask_len,
2625 in_ty: mask_ty,
2626 arg_ty: values_ty,
2627 out_len: values_len
2628 }
2629 );
2630
2631 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() &&
p_mutbl.is_mut() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(());
};
};require!(
2633 matches!(
2634 *pointer_ty.kind(),
2635 ty::RawPtr(p_ty, p_mutbl)
2636 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
2637 ),
2638 InvalidMonomorphization::ExpectedElementType {
2639 span,
2640 name,
2641 expected_element: values_elem,
2642 second_arg: pointer_ty,
2643 in_elem: values_elem,
2644 in_ty: values_ty,
2645 mutability: ExpectedPointerMutability::Mut,
2646 }
2647 );
2648
2649 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2650 mask_elem.kind(),
2651 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2652 );
2653
2654 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2655
2656 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2658
2659 let llvm_pointer = bx.type_ptr();
2660
2661 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2663
2664 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2665 let alignment = bx.const_i32(alignment as i32);
2666 &[args[2].immediate(), args[1].immediate(), alignment, mask]
2667 } else {
2668 &[args[2].immediate(), args[1].immediate(), mask]
2669 };
2670
2671 let call = bx.call_intrinsic("llvm.masked.store", &[llvm_elem_vec_ty, llvm_pointer], args);
2672 if llvm_version >= (22, 0, 0) {
2673 crate::attributes::apply_to_callsite(
2674 call,
2675 crate::llvm::AttributePlace::Argument(1),
2676 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2677 )
2678 }
2679 return Ok(call);
2680 }
2681
2682 if name == sym::simd_scatter {
2683 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(());
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2693 let (element_len1, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(());
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2694 let (element_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(());
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2695
2696 if !(in_len == element_len1) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len: element_len1,
});
return Err(());
};
};require!(
2698 in_len == element_len1,
2699 InvalidMonomorphization::SecondArgumentLength {
2700 span,
2701 name,
2702 in_len,
2703 in_ty,
2704 arg_ty: args[1].layout.ty,
2705 out_len: element_len1
2706 }
2707 );
2708 if !(in_len == element_len2) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: element_len2,
});
return Err(());
};
};require!(
2709 in_len == element_len2,
2710 InvalidMonomorphization::ThirdArgumentLength {
2711 span,
2712 name,
2713 in_len,
2714 in_ty,
2715 arg_ty: args[2].layout.ty,
2716 out_len: element_len2
2717 }
2718 );
2719
2720 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == in_elem && p_mutbl.is_mut() &&
p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(());
};
};require!(
2721 matches!(
2722 *element_ty1.kind(),
2723 ty::RawPtr(p_ty, p_mutbl)
2724 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
2725 ),
2726 InvalidMonomorphization::ExpectedElementType {
2727 span,
2728 name,
2729 expected_element: element_ty1,
2730 second_arg: args[1].layout.ty,
2731 in_elem,
2732 in_ty,
2733 mutability: ExpectedPointerMutability::Mut,
2734 }
2735 );
2736
2737 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(());
};
}
}require_int_or_uint_ty!(
2739 element_ty2.kind(),
2740 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2741 );
2742
2743 let alignment = bx.align_of(in_elem).bytes();
2745
2746 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2748
2749 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2751
2752 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2754 let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2755 let alignment = bx.const_i32(alignment as i32);
2756 &[args[0].immediate(), args[1].immediate(), alignment, mask]
2757 } else {
2758 &[args[0].immediate(), args[1].immediate(), mask]
2759 };
2760 let call = bx.call_intrinsic(
2761 "llvm.masked.scatter",
2762 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2763 args,
2764 );
2765 if llvm_version >= (22, 0, 0) {
2766 crate::attributes::apply_to_callsite(
2767 call,
2768 crate::llvm::AttributePlace::Argument(1),
2769 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2770 )
2771 }
2772 return Ok(call);
2773 }
2774
2775 macro_rules! arith_red {
2776 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2777 $identity:expr) => {
2778 if name == sym::$name {
2779 require!(
2780 ret_ty == in_elem,
2781 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2782 );
2783 return match in_elem.kind() {
2784 ty::Int(_) | ty::Uint(_) => {
2785 let r = bx.$integer_reduce(args[0].immediate());
2786 if $ordered {
2787 Ok(bx.$op(args[1].immediate(), r))
2790 } else {
2791 Ok(bx.$integer_reduce(args[0].immediate()))
2792 }
2793 }
2794 ty::Float(f) => {
2795 let acc = if $ordered {
2796 args[1].immediate()
2798 } else {
2799 match f.bit_width() {
2801 32 => bx.const_real(bx.type_f32(), $identity),
2802 64 => bx.const_real(bx.type_f64(), $identity),
2803 v => return_error!(
2804 InvalidMonomorphization::UnsupportedSymbolOfSize {
2805 span,
2806 name,
2807 symbol: sym::$name,
2808 in_ty,
2809 in_elem,
2810 size: v,
2811 ret_ty
2812 }
2813 ),
2814 }
2815 };
2816 Ok(bx.$float_reduce(acc, args[0].immediate()))
2817 }
2818 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2819 span,
2820 name,
2821 symbol: sym::$name,
2822 in_ty,
2823 in_elem,
2824 ret_ty
2825 }),
2826 };
2827 }
2828 };
2829 }
2830
2831 if name == sym::simd_reduce_add_ordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if true {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fadd(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2832 if name == sym::simd_reduce_mul_ordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if true {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fmul(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2833 if name == sym::simd_reduce_add_unordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if false {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fadd_reassoc(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(
2834 simd_reduce_add_unordered: vector_reduce_add,
2835 vector_reduce_fadd_reassoc,
2836 false,
2837 add,
2838 -0.0
2839 );
2840 if name == sym::simd_reduce_mul_unordered {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if false {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(());
}
}
};
Ok(bx.vector_reduce_fmul_reassoc(acc, args[0].immediate()))
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};arith_red!(
2841 simd_reduce_mul_unordered: vector_reduce_mul,
2842 vector_reduce_fmul_reassoc,
2843 false,
2844 mul,
2845 1.0
2846 );
2847
2848 macro_rules! minmax_red {
2849 ($name:ident: $int_red:ident, $float_red:ident) => {
2850 if name == sym::$name {
2851 require!(
2852 ret_ty == in_elem,
2853 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2854 );
2855 return match in_elem.kind() {
2856 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2857 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2858 ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
2859 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2860 span,
2861 name,
2862 symbol: sym::$name,
2863 in_ty,
2864 in_elem,
2865 ret_ty
2866 }),
2867 };
2868 }
2869 };
2870 }
2871
2872 if name == sym::simd_reduce_min {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_min(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_min(args[0].immediate(), false)),
ty::Float(_f) => Ok(bx.vector_reduce_fmin(args[0].immediate())),
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_min,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
2873 if name == sym::simd_reduce_max {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_max(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_max(args[0].immediate(), false)),
ty::Float(_f) => Ok(bx.vector_reduce_fmax(args[0].immediate())),
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_max,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
2874
2875 macro_rules! bitwise_red {
2876 ($name:ident : $red:ident, $boolean:expr) => {
2877 if name == sym::$name {
2878 let input = if !$boolean {
2879 require!(
2880 ret_ty == in_elem,
2881 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2882 );
2883 args[0].immediate()
2884 } else {
2885 let bitwidth = match in_elem.kind() {
2886 ty::Int(i) => {
2887 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2888 }
2889 ty::Uint(i) => {
2890 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2891 }
2892 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2893 span,
2894 name,
2895 symbol: sym::$name,
2896 in_ty,
2897 in_elem,
2898 ret_ty
2899 }),
2900 };
2901
2902 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2903 };
2904 return match in_elem.kind() {
2905 ty::Int(_) | ty::Uint(_) => {
2906 let r = bx.$red(input);
2907 Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2908 }
2909 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2910 span,
2911 name,
2912 symbol: sym::$name,
2913 in_ty,
2914 in_elem,
2915 ret_ty
2916 }),
2917 };
2918 }
2919 };
2920 }
2921
2922 if name == sym::simd_reduce_and {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2923 if name == sym::simd_reduce_or {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2924 if name == sym::simd_reduce_xor {
let input =
if !false {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_xor(input);
Ok(if !false { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2925 if name == sym::simd_reduce_all {
let input =
if !true {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(if !true { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2926 if name == sym::simd_reduce_any {
let input =
if !true {
if !(ret_ty == in_elem) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(());
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(if !true { r } else { bx.zext(r, bx.type_bool()) })
}
_ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(());
}
};
};bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2927
2928 if name == sym::simd_cast_ptr {
2929 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2930 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2931 in_len == out_len,
2932 InvalidMonomorphization::ReturnLengthInputType {
2933 span,
2934 name,
2935 in_len,
2936 in_ty,
2937 ret_ty,
2938 out_len
2939 }
2940 );
2941
2942 match in_elem.kind() {
2943 ty::RawPtr(p_ty, _) => {
2944 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2945 bx.tcx.normalize_erasing_regions(bx.typing_env(), Unnormalized::new_wip(ty))
2946 });
2947 if !metadata.is_unit() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: in_elem,
});
return Err(());
};
};require!(
2948 metadata.is_unit(),
2949 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2950 );
2951 }
2952 _ => {
2953 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2954 }
2955 }
2956 match out_elem.kind() {
2957 ty::RawPtr(p_ty, _) => {
2958 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2959 bx.tcx.normalize_erasing_regions(bx.typing_env(), Unnormalized::new_wip(ty))
2960 });
2961 if !metadata.is_unit() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: out_elem,
});
return Err(());
};
};require!(
2962 metadata.is_unit(),
2963 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2964 );
2965 }
2966 _ => {
2967 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2968 }
2969 }
2970
2971 return Ok(args[0].immediate());
2972 }
2973
2974 if name == sym::simd_expose_provenance {
2975 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2976 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
2977 in_len == out_len,
2978 InvalidMonomorphization::ReturnLengthInputType {
2979 span,
2980 name,
2981 in_len,
2982 in_ty,
2983 ret_ty,
2984 out_len
2985 }
2986 );
2987
2988 match in_elem.kind() {
2989 ty::RawPtr(_, _) => {}
2990 _ => {
2991 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2992 }
2993 }
2994 match out_elem.kind() {
2995 ty::Uint(ty::UintTy::Usize) => {}
2996 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2997 }
2998
2999 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
3000 }
3001
3002 if name == sym::simd_with_exposed_provenance {
3003 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3004 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
3005 in_len == out_len,
3006 InvalidMonomorphization::ReturnLengthInputType {
3007 span,
3008 name,
3009 in_len,
3010 in_ty,
3011 ret_ty,
3012 out_len
3013 }
3014 );
3015
3016 match in_elem.kind() {
3017 ty::Uint(ty::UintTy::Usize) => {}
3018 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
3019 }
3020 match out_elem.kind() {
3021 ty::RawPtr(_, _) => {}
3022 _ => {
3023 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3024 }
3025 }
3026
3027 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
3028 }
3029
3030 if name == sym::simd_cast || name == sym::simd_as {
3031 let (out_len, out_elem, out_num_vecs) = {
if !(ret_ty.is_simd() || ret_ty.is_scalable_vector()) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(());
};
};
if ret_ty.is_simd() {
let (len, ty) = ret_ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
ret_ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(ret_ty, SimdReturn);
3032 if !(in_len == out_len) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(());
};
};require!(
3033 in_len == out_len,
3034 InvalidMonomorphization::ReturnLengthInputType {
3035 span,
3036 name,
3037 in_len,
3038 in_ty,
3039 ret_ty,
3040 out_len
3041 }
3042 );
3043 if !(in_num_vecs == out_num_vecs) {
{
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnNumVecsInputType {
span,
name,
in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
in_ty,
ret_ty,
out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1)),
});
return Err(());
};
};require!(
3044 in_num_vecs == out_num_vecs,
3045 InvalidMonomorphization::ReturnNumVecsInputType {
3046 span,
3047 name,
3048 in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
3049 in_ty,
3050 ret_ty,
3051 out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1))
3052 }
3053 );
3054
3055 if in_elem == out_elem {
3057 return Ok(args[0].immediate());
3058 }
3059
3060 #[derive(#[automatically_derived]
impl ::core::marker::Copy for Sign { }Copy, #[automatically_derived]
impl ::core::clone::Clone for Sign {
#[inline]
fn clone(&self) -> Sign { *self }
}Clone)]
3061 enum Sign {
3062 Unsigned,
3063 Signed,
3064 }
3065 use Sign::*;
3066
3067 enum Style {
3068 Float,
3069 Int(Sign),
3070 Unsupported,
3071 }
3072
3073 let (in_style, in_width) = match in_elem.kind() {
3074 ty::Int(i) => (
3077 Style::Int(Signed),
3078 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3079 ),
3080 ty::Uint(u) => (
3081 Style::Int(Unsigned),
3082 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3083 ),
3084 ty::Float(f) => (Style::Float, f.bit_width()),
3085 _ => (Style::Unsupported, 0),
3086 };
3087 let (out_style, out_width) = match out_elem.kind() {
3088 ty::Int(i) => (
3089 Style::Int(Signed),
3090 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3091 ),
3092 ty::Uint(u) => (
3093 Style::Int(Unsigned),
3094 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3095 ),
3096 ty::Float(f) => (Style::Float, f.bit_width()),
3097 _ => (Style::Unsupported, 0),
3098 };
3099
3100 match (in_style, out_style) {
3101 (Style::Int(sign), Style::Int(_)) => {
3102 return Ok(match in_width.cmp(&out_width) {
3103 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
3104 Ordering::Equal => args[0].immediate(),
3105 Ordering::Less => match sign {
3106 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
3107 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
3108 },
3109 });
3110 }
3111 (Style::Int(Sign::Signed), Style::Float) => {
3112 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
3113 }
3114 (Style::Int(Sign::Unsigned), Style::Float) => {
3115 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
3116 }
3117 (Style::Float, Style::Int(sign)) => {
3118 return Ok(match (sign, name == sym::simd_as) {
3119 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
3120 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
3121 (_, true) => bx.cast_float_to_int(
3122 #[allow(non_exhaustive_omitted_patterns)] match sign {
Sign::Signed => true,
_ => false,
}matches!(sign, Sign::Signed),
3123 args[0].immediate(),
3124 llret_ty,
3125 ),
3126 });
3127 }
3128 (Style::Float, Style::Float) => {
3129 return Ok(match in_width.cmp(&out_width) {
3130 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
3131 Ordering::Equal => args[0].immediate(),
3132 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
3133 });
3134 }
3135 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedCast {
span,
name,
in_ty,
in_elem,
ret_ty,
out_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::UnsupportedCast {
3136 span,
3137 name,
3138 in_ty,
3139 in_elem,
3140 ret_ty,
3141 out_elem
3142 }),
3143 }
3144 }
3145 macro_rules! arith_binary {
3146 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3147 $(if name == sym::$name {
3148 match in_elem.kind() {
3149 $($(ty::$p(_))|* => {
3150 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
3151 })*
3152 _ => {},
3153 }
3154 return_error!(
3155 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3156 );
3157 })*
3158 }
3159 }
3160 if name == sym::simd_minimum_number_nsz {
match in_elem.kind() {
ty::Float(_) => {
return Ok(bx.minimum_number_nsz(args[0].immediate(),
args[1].immediate()))
}
_ => {}
}
{
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
};
}arith_binary! {
3161 simd_add: Uint, Int => add, Float => fadd;
3162 simd_sub: Uint, Int => sub, Float => fsub;
3163 simd_mul: Uint, Int => mul, Float => fmul;
3164 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
3165 simd_rem: Uint => urem, Int => srem, Float => frem;
3166 simd_shl: Uint, Int => shl;
3167 simd_shr: Uint => lshr, Int => ashr;
3168 simd_and: Uint, Int => and;
3169 simd_or: Uint, Int => or;
3170 simd_xor: Uint, Int => xor;
3171 simd_maximum_number_nsz: Float => maximum_number_nsz;
3172 simd_minimum_number_nsz: Float => minimum_number_nsz;
3173
3174 }
3175 macro_rules! arith_unary {
3176 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3177 $(if name == sym::$name {
3178 match in_elem.kind() {
3179 $($(ty::$p(_))|* => {
3180 return Ok(bx.$call(args[0].immediate()))
3181 })*
3182 _ => {},
3183 }
3184 return_error!(
3185 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3186 );
3187 })*
3188 }
3189 }
3190 if name == sym::simd_neg {
match in_elem.kind() {
ty::Int(_) => { return Ok(bx.neg(args[0].immediate())) }
ty::Float(_) => { return Ok(bx.fneg(args[0].immediate())) }
_ => {}
}
{
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
};
}arith_unary! {
3191 simd_neg: Int => neg, Float => fneg;
3192 }
3193
3194 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop
| sym::simd_cttz | sym::simd_carryless_mul | sym::simd_funnel_shl |
sym::simd_funnel_shr => true,
_ => false,
}matches!(
3196 name,
3197 sym::simd_bswap
3198 | sym::simd_bitreverse
3199 | sym::simd_ctlz
3200 | sym::simd_ctpop
3201 | sym::simd_cttz
3202 | sym::simd_carryless_mul
3203 | sym::simd_funnel_shl
3204 | sym::simd_funnel_shr
3205 ) {
3206 let vec_ty = bx.cx.type_vector(
3207 match *in_elem.kind() {
3208 ty::Int(i) => bx.cx.type_int_from_ty(i),
3209 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
3210 _ => {
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(());
}return_error!(InvalidMonomorphization::UnsupportedOperation {
3211 span,
3212 name,
3213 in_ty,
3214 in_elem
3215 }),
3216 },
3217 in_len as u64,
3218 );
3219 let llvm_intrinsic = match name {
3220 sym::simd_bswap => "llvm.bswap",
3221 sym::simd_bitreverse => "llvm.bitreverse",
3222 sym::simd_ctlz => "llvm.ctlz",
3223 sym::simd_ctpop => "llvm.ctpop",
3224 sym::simd_cttz => "llvm.cttz",
3225 sym::simd_funnel_shl => "llvm.fshl",
3226 sym::simd_funnel_shr => "llvm.fshr",
3227 sym::simd_carryless_mul => "llvm.clmul",
3228 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
3229 };
3230 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
3231
3232 return match name {
3233 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
3235 sym::simd_ctlz | sym::simd_cttz => {
3236 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
3238 Ok(bx.call_intrinsic(
3239 llvm_intrinsic,
3240 &[vec_ty],
3241 &[args[0].immediate(), dont_poison_on_zero],
3242 ))
3243 }
3244 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
3245 Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
3247 }
3248 sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
3249 llvm_intrinsic,
3250 &[vec_ty],
3251 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
3252 )),
3253 sym::simd_carryless_mul => {
3254 if crate::llvm_util::get_version() >= (22, 0, 0) {
3255 Ok(bx.call_intrinsic(
3256 llvm_intrinsic,
3257 &[vec_ty],
3258 &[args[0].immediate(), args[1].immediate()],
3259 ))
3260 } else {
3261 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("`simd_carryless_mul` needs LLVM 22 or higher"));span_bug!(span, "`simd_carryless_mul` needs LLVM 22 or higher");
3262 }
3263 }
3264 _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
3265 };
3266 }
3267
3268 if name == sym::simd_arith_offset {
3269 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
3271 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("must be called with a vector of pointer types as first argument"))span_bug!(span, "must be called with a vector of pointer types as first argument")
3272 });
3273 let layout = bx.layout_of(pointee);
3274 let ptrs = args[0].immediate();
3275 let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
3278 if !#[allow(non_exhaustive_omitted_patterns)] match offsets_elem.kind() {
ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize) => true,
_ => false,
}matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
3279 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("must be called with a vector of pointer-sized integers as second argument"));span_bug!(
3280 span,
3281 "must be called with a vector of pointer-sized integers as second argument"
3282 );
3283 }
3284 let offsets = args[1].immediate();
3285
3286 return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
3287 }
3288
3289 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
3290 let lhs = args[0].immediate();
3291 let rhs = args[1].immediate();
3292 let is_add = name == sym::simd_saturating_add;
3293 let (signed, elem_ty) = match *in_elem.kind() {
3294 ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
3295 ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
3296 _ => {
3297 {
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
vector_type: args[0].layout.ty,
});
return Err(());
};return_error!(InvalidMonomorphization::ExpectedVectorElementType {
3298 span,
3299 name,
3300 expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
3301 vector_type: args[0].layout.ty
3302 });
3303 }
3304 };
3305 let llvm_intrinsic = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
3306 "llvm.{}{}.sat",
3307 if signed { 's' } else { 'u' },
3308 if is_add { "add" } else { "sub" },
3309 );
3310 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
3311
3312 return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
3313 }
3314
3315 ::rustc_middle::util::bug::span_bug_fmt(span,
format_args!("unknown SIMD intrinsic"));span_bug!(span, "unknown SIMD intrinsic");
3316}