1use std::assert_matches::assert_matches;
2use std::cmp::Ordering;
3
4use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
5use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
6use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
7use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
8use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
9use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
10use rustc_codegen_ssa::traits::*;
11use rustc_hir as hir;
12use rustc_middle::mir::BinOp;
13use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
14use rustc_middle::ty::{self, GenericArgsRef, Ty};
15use rustc_middle::{bug, span_bug};
16use rustc_span::{Span, Symbol, sym};
17use rustc_symbol_mangling::mangle_internal_symbol;
18use rustc_target::spec::PanicStrategy;
19use tracing::debug;
20
21use crate::abi::FnAbiLlvmExt;
22use crate::builder::Builder;
23use crate::context::CodegenCx;
24use crate::llvm::{self, Metadata};
25use crate::type_::Type;
26use crate::type_of::LayoutLlvmExt;
27use crate::va_arg::emit_va_arg;
28use crate::value::Value;
29
30fn call_simple_intrinsic<'ll, 'tcx>(
31 bx: &mut Builder<'_, 'll, 'tcx>,
32 name: Symbol,
33 args: &[OperandRef<'tcx, &'ll Value>],
34) -> Option<&'ll Value> {
35 let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
36 sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]),
37 sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]),
38 sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]),
39 sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]),
40
41 sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]),
42 sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]),
43 sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]),
44 sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]),
45
46 sym::sinf16 => ("llvm.sin", &[bx.type_f16()]),
47 sym::sinf32 => ("llvm.sin", &[bx.type_f32()]),
48 sym::sinf64 => ("llvm.sin", &[bx.type_f64()]),
49 sym::sinf128 => ("llvm.sin", &[bx.type_f128()]),
50
51 sym::cosf16 => ("llvm.cos", &[bx.type_f16()]),
52 sym::cosf32 => ("llvm.cos", &[bx.type_f32()]),
53 sym::cosf64 => ("llvm.cos", &[bx.type_f64()]),
54 sym::cosf128 => ("llvm.cos", &[bx.type_f128()]),
55
56 sym::powf16 => ("llvm.pow", &[bx.type_f16()]),
57 sym::powf32 => ("llvm.pow", &[bx.type_f32()]),
58 sym::powf64 => ("llvm.pow", &[bx.type_f64()]),
59 sym::powf128 => ("llvm.pow", &[bx.type_f128()]),
60
61 sym::expf16 => ("llvm.exp", &[bx.type_f16()]),
62 sym::expf32 => ("llvm.exp", &[bx.type_f32()]),
63 sym::expf64 => ("llvm.exp", &[bx.type_f64()]),
64 sym::expf128 => ("llvm.exp", &[bx.type_f128()]),
65
66 sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]),
67 sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]),
68 sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]),
69 sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]),
70
71 sym::logf16 => ("llvm.log", &[bx.type_f16()]),
72 sym::logf32 => ("llvm.log", &[bx.type_f32()]),
73 sym::logf64 => ("llvm.log", &[bx.type_f64()]),
74 sym::logf128 => ("llvm.log", &[bx.type_f128()]),
75
76 sym::log10f16 => ("llvm.log10", &[bx.type_f16()]),
77 sym::log10f32 => ("llvm.log10", &[bx.type_f32()]),
78 sym::log10f64 => ("llvm.log10", &[bx.type_f64()]),
79 sym::log10f128 => ("llvm.log10", &[bx.type_f128()]),
80
81 sym::log2f16 => ("llvm.log2", &[bx.type_f16()]),
82 sym::log2f32 => ("llvm.log2", &[bx.type_f32()]),
83 sym::log2f64 => ("llvm.log2", &[bx.type_f64()]),
84 sym::log2f128 => ("llvm.log2", &[bx.type_f128()]),
85
86 sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]),
87 sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]),
88 sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]),
89 sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]),
90
91 sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]),
92 sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]),
93 sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]),
94 sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]),
95
96 sym::fabsf16 => ("llvm.fabs", &[bx.type_f16()]),
97 sym::fabsf32 => ("llvm.fabs", &[bx.type_f32()]),
98 sym::fabsf64 => ("llvm.fabs", &[bx.type_f64()]),
99 sym::fabsf128 => ("llvm.fabs", &[bx.type_f128()]),
100
101 sym::minnumf16 => ("llvm.minnum", &[bx.type_f16()]),
102 sym::minnumf32 => ("llvm.minnum", &[bx.type_f32()]),
103 sym::minnumf64 => ("llvm.minnum", &[bx.type_f64()]),
104 sym::minnumf128 => ("llvm.minnum", &[bx.type_f128()]),
105
106 sym::maxnumf16 => ("llvm.maxnum", &[bx.type_f16()]),
114 sym::maxnumf32 => ("llvm.maxnum", &[bx.type_f32()]),
115 sym::maxnumf64 => ("llvm.maxnum", &[bx.type_f64()]),
116 sym::maxnumf128 => ("llvm.maxnum", &[bx.type_f128()]),
117
118 sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
126 sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
127 sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
128 sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
129
130 sym::floorf16 => ("llvm.floor", &[bx.type_f16()]),
131 sym::floorf32 => ("llvm.floor", &[bx.type_f32()]),
132 sym::floorf64 => ("llvm.floor", &[bx.type_f64()]),
133 sym::floorf128 => ("llvm.floor", &[bx.type_f128()]),
134
135 sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]),
136 sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]),
137 sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]),
138 sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]),
139
140 sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]),
141 sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]),
142 sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]),
143 sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]),
144
145 sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]),
150 sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]),
151 sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]),
152 sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]),
153
154 sym::roundf16 => ("llvm.round", &[bx.type_f16()]),
155 sym::roundf32 => ("llvm.round", &[bx.type_f32()]),
156 sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
157 sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
158
159 _ => return None,
160 };
161 Some(bx.call_intrinsic(
162 base_name,
163 type_params,
164 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
165 ))
166}
167
168impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
169 fn codegen_intrinsic_call(
170 &mut self,
171 instance: ty::Instance<'tcx>,
172 args: &[OperandRef<'tcx, &'ll Value>],
173 result: PlaceRef<'tcx, &'ll Value>,
174 span: Span,
175 ) -> Result<(), ty::Instance<'tcx>> {
176 let tcx = self.tcx;
177
178 let name = tcx.item_name(instance.def_id());
179 let fn_args = instance.args;
180
181 let simple = call_simple_intrinsic(self, name, args);
182 let llval = match name {
183 _ if simple.is_some() => simple.unwrap(),
184 sym::ptr_mask => {
185 let ptr = args[0].immediate();
186 self.call_intrinsic(
187 "llvm.ptrmask",
188 &[self.val_ty(ptr), self.type_isize()],
189 &[ptr, args[1].immediate()],
190 )
191 }
192 sym::is_val_statically_known => {
193 if let OperandValue::Immediate(imm) = args[0].val {
194 self.call_intrinsic(
195 "llvm.is.constant",
196 &[args[0].layout.immediate_llvm_type(self.cx)],
197 &[imm],
198 )
199 } else {
200 self.const_bool(false)
201 }
202 }
203 sym::select_unpredictable => {
204 let cond = args[0].immediate();
205 assert_eq!(args[1].layout, args[2].layout);
206 let select = |bx: &mut Self, true_val, false_val| {
207 let result = bx.select(cond, true_val, false_val);
208 bx.set_unpredictable(&result);
209 result
210 };
211 match (args[1].val, args[2].val) {
212 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
213 assert!(true_val.llextra.is_none());
214 assert!(false_val.llextra.is_none());
215 assert_eq!(true_val.align, false_val.align);
216 let ptr = select(self, true_val.llval, false_val.llval);
217 let selected =
218 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
219 selected.store(self, result);
220 return Ok(());
221 }
222 (OperandValue::Immediate(_), OperandValue::Immediate(_))
223 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
224 let true_val = args[1].immediate_or_packed_pair(self);
225 let false_val = args[2].immediate_or_packed_pair(self);
226 select(self, true_val, false_val)
227 }
228 (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
229 _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
230 }
231 }
232 sym::catch_unwind => {
233 catch_unwind_intrinsic(
234 self,
235 args[0].immediate(),
236 args[1].immediate(),
237 args[2].immediate(),
238 result,
239 );
240 return Ok(());
241 }
242 sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
243 sym::va_copy => {
244 let dest = args[0].immediate();
245 self.call_intrinsic(
246 "llvm.va_copy",
247 &[self.val_ty(dest)],
248 &[dest, args[1].immediate()],
249 )
250 }
251 sym::va_arg => {
252 match result.layout.backend_repr {
253 BackendRepr::Scalar(scalar) => {
254 match scalar.primitive() {
255 Primitive::Int(..) => {
256 if self.cx().size_of(result.layout.ty).bytes() < 4 {
257 let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
262 self.trunc(promoted_result, result.layout.llvm_type(self))
263 } else {
264 emit_va_arg(self, args[0], result.layout.ty)
265 }
266 }
267 Primitive::Float(Float::F16) => {
268 bug!("the va_arg intrinsic does not work with `f16`")
269 }
270 Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
271 emit_va_arg(self, args[0], result.layout.ty)
272 }
273 Primitive::Float(Float::F32) => {
275 bug!("the va_arg intrinsic does not work with `f32`")
276 }
277 Primitive::Float(Float::F128) => {
278 bug!("the va_arg intrinsic does not work with `f128`")
279 }
280 }
281 }
282 _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
283 }
284 }
285
286 sym::volatile_load | sym::unaligned_volatile_load => {
287 let ptr = args[0].immediate();
288 let load = self.volatile_load(result.layout.llvm_type(self), ptr);
289 let align = if name == sym::unaligned_volatile_load {
290 1
291 } else {
292 result.layout.align.abi.bytes() as u32
293 };
294 unsafe {
295 llvm::LLVMSetAlignment(load, align);
296 }
297 if !result.layout.is_zst() {
298 self.store_to_place(load, result.val);
299 }
300 return Ok(());
301 }
302 sym::volatile_store => {
303 let dst = args[0].deref(self.cx());
304 args[1].val.volatile_store(self, dst);
305 return Ok(());
306 }
307 sym::unaligned_volatile_store => {
308 let dst = args[0].deref(self.cx());
309 args[1].val.unaligned_volatile_store(self, dst);
310 return Ok(());
311 }
312 sym::prefetch_read_data
313 | sym::prefetch_write_data
314 | sym::prefetch_read_instruction
315 | sym::prefetch_write_instruction => {
316 let (rw, cache_type) = match name {
317 sym::prefetch_read_data => (0, 1),
318 sym::prefetch_write_data => (1, 1),
319 sym::prefetch_read_instruction => (0, 0),
320 sym::prefetch_write_instruction => (1, 0),
321 _ => bug!(),
322 };
323 let ptr = args[0].immediate();
324 self.call_intrinsic(
325 "llvm.prefetch",
326 &[self.val_ty(ptr)],
327 &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
328 )
329 }
330 sym::carrying_mul_add => {
331 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
332
333 let wide_llty = self.type_ix(size.bits() * 2);
334 let args = args.as_array().unwrap();
335 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
336
337 let wide = if signed {
338 let prod = self.unchecked_smul(a, b);
339 let acc = self.unchecked_sadd(prod, c);
340 self.unchecked_sadd(acc, d)
341 } else {
342 let prod = self.unchecked_umul(a, b);
343 let acc = self.unchecked_uadd(prod, c);
344 self.unchecked_uadd(acc, d)
345 };
346
347 let narrow_llty = self.type_ix(size.bits());
348 let low = self.trunc(wide, narrow_llty);
349 let bits_const = self.const_uint(wide_llty, size.bits());
350 let high = self.lshr(wide, bits_const);
352 let high = self.trunc(high, narrow_llty);
354
355 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
356 let pair = self.const_poison(pair_llty);
357 let pair = self.insert_value(pair, low, 0);
358 let pair = self.insert_value(pair, high, 1);
359 pair
360 }
361 sym::ctlz
362 | sym::ctlz_nonzero
363 | sym::cttz
364 | sym::cttz_nonzero
365 | sym::ctpop
366 | sym::bswap
367 | sym::bitreverse
368 | sym::rotate_left
369 | sym::rotate_right
370 | sym::saturating_add
371 | sym::saturating_sub => {
372 let ty = args[0].layout.ty;
373 if !ty.is_integral() {
374 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
375 span,
376 name,
377 ty,
378 });
379 return Ok(());
380 }
381 let (size, signed) = ty.int_size_and_signed(self.tcx);
382 let width = size.bits();
383 let llty = self.type_ix(width);
384 match name {
385 sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
386 let y =
387 self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
388 let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
389 "llvm.ctlz"
390 } else {
391 "llvm.cttz"
392 };
393 let ret =
394 self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
395 self.intcast(ret, result.layout.llvm_type(self), false)
396 }
397 sym::ctpop => {
398 let ret =
399 self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]);
400 self.intcast(ret, result.layout.llvm_type(self), false)
401 }
402 sym::bswap => {
403 if width == 8 {
404 args[0].immediate() } else {
406 self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
407 }
408 }
409 sym::bitreverse => {
410 self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
411 }
412 sym::rotate_left | sym::rotate_right => {
413 let is_left = name == sym::rotate_left;
414 let val = args[0].immediate();
415 let raw_shift = args[1].immediate();
416 let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
418
419 let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
422
423 self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift])
424 }
425 sym::saturating_add | sym::saturating_sub => {
426 let is_add = name == sym::saturating_add;
427 let lhs = args[0].immediate();
428 let rhs = args[1].immediate();
429 let llvm_name = format!(
430 "llvm.{}{}.sat",
431 if signed { 's' } else { 'u' },
432 if is_add { "add" } else { "sub" },
433 );
434 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
435 }
436 _ => bug!(),
437 }
438 }
439
440 sym::raw_eq => {
441 use BackendRepr::*;
442 let tp_ty = fn_args.type_at(0);
443 let layout = self.layout_of(tp_ty).layout;
444 let use_integer_compare = match layout.backend_repr() {
445 Scalar(_) | ScalarPair(_, _) => true,
446 SimdVector { .. } => false,
447 Memory { .. } => {
448 layout.size() <= self.data_layout().pointer_size() * 2
452 }
453 };
454
455 let a = args[0].immediate();
456 let b = args[1].immediate();
457 if layout.size().bytes() == 0 {
458 self.const_bool(true)
459 } else if use_integer_compare {
460 let integer_ty = self.type_ix(layout.size().bits());
461 let a_val = self.load(integer_ty, a, layout.align().abi);
462 let b_val = self.load(integer_ty, b, layout.align().abi);
463 self.icmp(IntPredicate::IntEQ, a_val, b_val)
464 } else {
465 let n = self.const_usize(layout.size().bytes());
466 let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
467 self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
468 }
469 }
470
471 sym::compare_bytes => {
472 let cmp = self.call_intrinsic(
474 "memcmp",
475 &[],
476 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
477 );
478 self.sext(cmp, self.type_ix(32))
480 }
481
482 sym::black_box => {
483 args[0].val.store(self, result);
484 let result_val_span = [result.val.llval];
485 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
495 ("~{memory}", &[])
496 } else {
497 ("r,~{memory}", &result_val_span)
498 };
499 crate::asm::inline_asm_call(
500 self,
501 "",
502 constraint,
503 inputs,
504 self.type_void(),
505 &[],
506 true,
507 false,
508 llvm::AsmDialect::Att,
509 &[span],
510 false,
511 None,
512 None,
513 )
514 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
515
516 return Ok(());
518 }
519
520 _ if name.as_str().starts_with("simd_") => {
521 let mut loaded_args = Vec::new();
524 for arg in args {
525 loaded_args.push(
526 if arg.layout.ty.is_simd()
531 && let OperandValue::Ref(place) = arg.val
532 {
533 let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
534 let elem_ll_ty = match elem_ty.kind() {
535 ty::Float(f) => self.type_float_from_ty(*f),
536 ty::Int(i) => self.type_int_from_ty(*i),
537 ty::Uint(u) => self.type_uint_from_ty(*u),
538 ty::RawPtr(_, _) => self.type_ptr(),
539 _ => unreachable!(),
540 };
541 let loaded =
542 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
543 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
544 } else {
545 *arg
546 },
547 );
548 }
549
550 let llret_ty = if result.layout.ty.is_simd()
551 && let BackendRepr::Memory { .. } = result.layout.backend_repr
552 {
553 let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
554 let elem_ll_ty = match elem_ty.kind() {
555 ty::Float(f) => self.type_float_from_ty(*f),
556 ty::Int(i) => self.type_int_from_ty(*i),
557 ty::Uint(u) => self.type_uint_from_ty(*u),
558 ty::RawPtr(_, _) => self.type_ptr(),
559 _ => unreachable!(),
560 };
561 self.type_vector(elem_ll_ty, size)
562 } else {
563 result.layout.llvm_type(self)
564 };
565
566 match generic_simd_intrinsic(
567 self,
568 name,
569 fn_args,
570 &loaded_args,
571 result.layout.ty,
572 llret_ty,
573 span,
574 ) {
575 Ok(llval) => llval,
576 Err(()) => return Ok(()),
579 }
580 }
581
582 _ => {
583 debug!("unknown intrinsic '{}' -- falling back to default body", name);
584 return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
586 }
587 };
588
589 if result.layout.ty.is_bool() {
590 let val = self.from_immediate(llval);
591 self.store_to_place(val, result.val);
592 } else if !result.layout.ty.is_unit() {
593 self.store_to_place(llval, result.val);
594 }
595 Ok(())
596 }
597
598 fn abort(&mut self) {
599 self.call_intrinsic("llvm.trap", &[], &[]);
600 }
601
602 fn assume(&mut self, val: Self::Value) {
603 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
604 self.call_intrinsic("llvm.assume", &[], &[val]);
605 }
606 }
607
608 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
609 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
610 self.call_intrinsic(
611 "llvm.expect",
612 &[self.type_i1()],
613 &[cond, self.const_bool(expected)],
614 )
615 } else {
616 cond
617 }
618 }
619
620 fn type_checked_load(
621 &mut self,
622 llvtable: &'ll Value,
623 vtable_byte_offset: u64,
624 typeid: &'ll Metadata,
625 ) -> Self::Value {
626 let typeid = self.get_metadata_value(typeid);
627 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
628 let type_checked_load = self.call_intrinsic(
629 "llvm.type.checked.load",
630 &[],
631 &[llvtable, vtable_byte_offset, typeid],
632 );
633 self.extract_value(type_checked_load, 0)
634 }
635
636 fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
637 self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list])
638 }
639
640 fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
641 self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list])
642 }
643}
644
645fn catch_unwind_intrinsic<'ll, 'tcx>(
646 bx: &mut Builder<'_, 'll, 'tcx>,
647 try_func: &'ll Value,
648 data: &'ll Value,
649 catch_func: &'ll Value,
650 dest: PlaceRef<'tcx, &'ll Value>,
651) {
652 if bx.sess().panic_strategy() == PanicStrategy::Abort {
653 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
654 bx.call(try_func_ty, None, None, try_func, &[data], None, None);
655 OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
658 } else if wants_msvc_seh(bx.sess()) {
659 codegen_msvc_try(bx, try_func, data, catch_func, dest);
660 } else if wants_wasm_eh(bx.sess()) {
661 codegen_wasm_try(bx, try_func, data, catch_func, dest);
662 } else if bx.sess().target.os == "emscripten" {
663 codegen_emcc_try(bx, try_func, data, catch_func, dest);
664 } else {
665 codegen_gnu_try(bx, try_func, data, catch_func, dest);
666 }
667}
668
669fn codegen_msvc_try<'ll, 'tcx>(
677 bx: &mut Builder<'_, 'll, 'tcx>,
678 try_func: &'ll Value,
679 data: &'ll Value,
680 catch_func: &'ll Value,
681 dest: PlaceRef<'tcx, &'ll Value>,
682) {
683 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
684 bx.set_personality_fn(bx.eh_personality());
685
686 let normal = bx.append_sibling_block("normal");
687 let catchswitch = bx.append_sibling_block("catchswitch");
688 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
689 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
690 let caught = bx.append_sibling_block("caught");
691
692 let try_func = llvm::get_param(bx.llfn(), 0);
693 let data = llvm::get_param(bx.llfn(), 1);
694 let catch_func = llvm::get_param(bx.llfn(), 2);
695
696 let ptr_size = bx.tcx().data_layout.pointer_size();
752 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
753 let slot = bx.alloca(ptr_size, ptr_align);
754 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
755 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
756
757 bx.switch_to_block(normal);
758 bx.ret(bx.const_i32(0));
759
760 bx.switch_to_block(catchswitch);
761 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
762
763 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
778 let type_name = bx.const_bytes(b"rust_panic\0");
779 let type_info =
780 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
781 let tydesc = bx.declare_global(
782 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
783 bx.val_ty(type_info),
784 );
785
786 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
787 if bx.cx.tcx.sess.target.supports_comdat() {
788 llvm::SetUniqueComdat(bx.llmod, tydesc);
789 }
790 llvm::set_initializer(tydesc, type_info);
791
792 bx.switch_to_block(catchpad_rust);
799 let flags = bx.const_i32(8);
800 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
801 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
802 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
803 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
804 bx.catch_ret(&funclet, caught);
805
806 bx.switch_to_block(catchpad_foreign);
808 let flags = bx.const_i32(64);
809 let null = bx.const_null(bx.type_ptr());
810 let funclet = bx.catch_pad(cs, &[null, flags, null]);
811 bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
812 bx.catch_ret(&funclet, caught);
813
814 bx.switch_to_block(caught);
815 bx.ret(bx.const_i32(1));
816 });
817
818 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
821 OperandValue::Immediate(ret).store(bx, dest);
822}
823
824fn codegen_wasm_try<'ll, 'tcx>(
826 bx: &mut Builder<'_, 'll, 'tcx>,
827 try_func: &'ll Value,
828 data: &'ll Value,
829 catch_func: &'ll Value,
830 dest: PlaceRef<'tcx, &'ll Value>,
831) {
832 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
833 bx.set_personality_fn(bx.eh_personality());
834
835 let normal = bx.append_sibling_block("normal");
836 let catchswitch = bx.append_sibling_block("catchswitch");
837 let catchpad = bx.append_sibling_block("catchpad");
838 let caught = bx.append_sibling_block("caught");
839
840 let try_func = llvm::get_param(bx.llfn(), 0);
841 let data = llvm::get_param(bx.llfn(), 1);
842 let catch_func = llvm::get_param(bx.llfn(), 2);
843
844 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
868 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
869
870 bx.switch_to_block(normal);
871 bx.ret(bx.const_i32(0));
872
873 bx.switch_to_block(catchswitch);
874 let cs = bx.catch_switch(None, None, &[catchpad]);
875
876 bx.switch_to_block(catchpad);
877 let null = bx.const_null(bx.type_ptr());
878 let funclet = bx.catch_pad(cs, &[null]);
879
880 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
881 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
882
883 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
884 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
885 bx.catch_ret(&funclet, caught);
886
887 bx.switch_to_block(caught);
888 bx.ret(bx.const_i32(1));
889 });
890
891 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
894 OperandValue::Immediate(ret).store(bx, dest);
895}
896
897fn codegen_gnu_try<'ll, 'tcx>(
909 bx: &mut Builder<'_, 'll, 'tcx>,
910 try_func: &'ll Value,
911 data: &'ll Value,
912 catch_func: &'ll Value,
913 dest: PlaceRef<'tcx, &'ll Value>,
914) {
915 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
916 let then = bx.append_sibling_block("then");
929 let catch = bx.append_sibling_block("catch");
930
931 let try_func = llvm::get_param(bx.llfn(), 0);
932 let data = llvm::get_param(bx.llfn(), 1);
933 let catch_func = llvm::get_param(bx.llfn(), 2);
934 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
935 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
936
937 bx.switch_to_block(then);
938 bx.ret(bx.const_i32(0));
939
940 bx.switch_to_block(catch);
947 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
948 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
949 let tydesc = bx.const_null(bx.type_ptr());
950 bx.add_clause(vals, tydesc);
951 let ptr = bx.extract_value(vals, 0);
952 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
953 bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
954 bx.ret(bx.const_i32(1));
955 });
956
957 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
960 OperandValue::Immediate(ret).store(bx, dest);
961}
962
963fn codegen_emcc_try<'ll, 'tcx>(
967 bx: &mut Builder<'_, 'll, 'tcx>,
968 try_func: &'ll Value,
969 data: &'ll Value,
970 catch_func: &'ll Value,
971 dest: PlaceRef<'tcx, &'ll Value>,
972) {
973 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
974 let then = bx.append_sibling_block("then");
992 let catch = bx.append_sibling_block("catch");
993
994 let try_func = llvm::get_param(bx.llfn(), 0);
995 let data = llvm::get_param(bx.llfn(), 1);
996 let catch_func = llvm::get_param(bx.llfn(), 2);
997 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
998 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
999
1000 bx.switch_to_block(then);
1001 bx.ret(bx.const_i32(0));
1002
1003 bx.switch_to_block(catch);
1009 let tydesc = bx.eh_catch_typeinfo();
1010 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1011 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1012 bx.add_clause(vals, tydesc);
1013 bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1014 let ptr = bx.extract_value(vals, 0);
1015 let selector = bx.extract_value(vals, 1);
1016
1017 let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]);
1019 let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1020 let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1021
1022 let ptr_size = bx.tcx().data_layout.pointer_size();
1025 let ptr_align = bx.tcx().data_layout.pointer_align().abi;
1026 let i8_align = bx.tcx().data_layout.i8_align.abi;
1027 assert!(i8_align <= ptr_align);
1029 let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1030 bx.store(ptr, catch_data, ptr_align);
1031 let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1032 bx.store(is_rust_panic, catch_data_1, i8_align);
1033
1034 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1035 bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1036 bx.ret(bx.const_i32(1));
1037 });
1038
1039 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1042 OperandValue::Immediate(ret).store(bx, dest);
1043}
1044
1045fn gen_fn<'a, 'll, 'tcx>(
1048 cx: &'a CodegenCx<'ll, 'tcx>,
1049 name: &str,
1050 rust_fn_sig: ty::PolyFnSig<'tcx>,
1051 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1052) -> (&'ll Type, &'ll Value) {
1053 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1054 let llty = fn_abi.llvm_type(cx);
1055 let llfn = cx.declare_fn(name, fn_abi, None);
1056 cx.set_frame_pointer_type(llfn);
1057 cx.apply_target_cpu_attr(llfn);
1058 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1060 let llbb = Builder::append_block(cx, llfn, "entry-block");
1061 let bx = Builder::build(cx, llbb);
1062 codegen(bx);
1063 (llty, llfn)
1064}
1065
1066fn get_rust_try_fn<'a, 'll, 'tcx>(
1071 cx: &'a CodegenCx<'ll, 'tcx>,
1072 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1073) -> (&'ll Type, &'ll Value) {
1074 if let Some(llfn) = cx.rust_try_fn.get() {
1075 return llfn;
1076 }
1077
1078 let tcx = cx.tcx;
1080 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1081 let try_fn_ty = Ty::new_fn_ptr(
1083 tcx,
1084 ty::Binder::dummy(tcx.mk_fn_sig(
1085 [i8p],
1086 tcx.types.unit,
1087 false,
1088 hir::Safety::Unsafe,
1089 ExternAbi::Rust,
1090 )),
1091 );
1092 let catch_fn_ty = Ty::new_fn_ptr(
1094 tcx,
1095 ty::Binder::dummy(tcx.mk_fn_sig(
1096 [i8p, i8p],
1097 tcx.types.unit,
1098 false,
1099 hir::Safety::Unsafe,
1100 ExternAbi::Rust,
1101 )),
1102 );
1103 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
1105 [try_fn_ty, i8p, catch_fn_ty],
1106 tcx.types.i32,
1107 false,
1108 hir::Safety::Unsafe,
1109 ExternAbi::Rust,
1110 ));
1111 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1112 cx.rust_try_fn.set(Some(rust_try));
1113 rust_try
1114}
1115
1116fn generic_simd_intrinsic<'ll, 'tcx>(
1117 bx: &mut Builder<'_, 'll, 'tcx>,
1118 name: Symbol,
1119 fn_args: GenericArgsRef<'tcx>,
1120 args: &[OperandRef<'tcx, &'ll Value>],
1121 ret_ty: Ty<'tcx>,
1122 llret_ty: &'ll Type,
1123 span: Span,
1124) -> Result<&'ll Value, ()> {
1125 macro_rules! return_error {
1126 ($diag: expr) => {{
1127 bx.sess().dcx().emit_err($diag);
1128 return Err(());
1129 }};
1130 }
1131
1132 macro_rules! require {
1133 ($cond: expr, $diag: expr) => {
1134 if !$cond {
1135 return_error!($diag);
1136 }
1137 };
1138 }
1139
1140 macro_rules! require_simd {
1141 ($ty: expr, $variant:ident) => {{
1142 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1143 $ty.simd_size_and_type(bx.tcx())
1144 }};
1145 }
1146
1147 macro_rules! require_int_or_uint_ty {
1149 ($ty: expr, $diag: expr) => {
1150 match $ty {
1151 ty::Int(i) => {
1152 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1153 }
1154 ty::Uint(i) => {
1155 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
1156 }
1157 _ => {
1158 return_error!($diag);
1159 }
1160 }
1161 };
1162 }
1163
1164 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1178 bx: &mut Builder<'a, 'll, 'tcx>,
1179 i_xn: &'ll Value,
1180 in_elem_bitwidth: u64,
1181 in_len: u64,
1182 ) -> &'ll Value {
1183 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1185 let shift_indices = vec![shift_idx; in_len as _];
1186 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1187 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1189 }
1190
1191 if cfg!(debug_assertions) {
1193 for arg in args {
1194 if arg.layout.ty.is_simd() {
1195 assert_matches!(arg.val, OperandValue::Immediate(_));
1196 }
1197 }
1198 }
1199
1200 if name == sym::simd_select_bitmask {
1201 let (len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1202
1203 let expected_int_bits = len.max(8).next_power_of_two();
1204 let expected_bytes = len.div_ceil(8);
1205
1206 let mask_ty = args[0].layout.ty;
1207 let mask = match mask_ty.kind() {
1208 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1209 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1210 ty::Array(elem, len)
1211 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1212 && len
1213 .try_to_target_usize(bx.tcx)
1214 .expect("expected monomorphic const in codegen")
1215 == expected_bytes =>
1216 {
1217 let place = PlaceRef::alloca(bx, args[0].layout);
1218 args[0].val.store(bx, place);
1219 let int_ty = bx.type_ix(expected_bytes * 8);
1220 bx.load(int_ty, place.val.llval, Align::ONE)
1221 }
1222 _ => return_error!(InvalidMonomorphization::InvalidBitmask {
1223 span,
1224 name,
1225 mask_ty,
1226 expected_int_bits,
1227 expected_bytes
1228 }),
1229 };
1230
1231 let i1 = bx.type_i1();
1232 let im = bx.type_ix(len);
1233 let i1xn = bx.type_vector(i1, len);
1234 let m_im = bx.trunc(mask, im);
1235 let m_i1s = bx.bitcast(m_im, i1xn);
1236 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1237 }
1238
1239 let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput);
1241 let in_ty = args[0].layout.ty;
1242
1243 let comparison = match name {
1244 sym::simd_eq => Some(BinOp::Eq),
1245 sym::simd_ne => Some(BinOp::Ne),
1246 sym::simd_lt => Some(BinOp::Lt),
1247 sym::simd_le => Some(BinOp::Le),
1248 sym::simd_gt => Some(BinOp::Gt),
1249 sym::simd_ge => Some(BinOp::Ge),
1250 _ => None,
1251 };
1252
1253 if let Some(cmp_op) = comparison {
1254 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1255
1256 require!(
1257 in_len == out_len,
1258 InvalidMonomorphization::ReturnLengthInputType {
1259 span,
1260 name,
1261 in_len,
1262 in_ty,
1263 ret_ty,
1264 out_len
1265 }
1266 );
1267 require!(
1268 bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
1269 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
1270 );
1271
1272 return Ok(compare_simd_types(
1273 bx,
1274 args[0].immediate(),
1275 args[1].immediate(),
1276 in_elem,
1277 llret_ty,
1278 cmp_op,
1279 ));
1280 }
1281
1282 if name == sym::simd_shuffle_const_generic {
1283 let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
1284 let n = idx.len() as u64;
1285
1286 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1287 require!(
1288 out_len == n,
1289 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1290 );
1291 require!(
1292 in_elem == out_ty,
1293 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1294 );
1295
1296 let total_len = in_len * 2;
1297
1298 let indices: Option<Vec<_>> = idx
1299 .iter()
1300 .enumerate()
1301 .map(|(arg_idx, val)| {
1302 let idx = val.unwrap_leaf().to_i32();
1303 if idx >= i32::try_from(total_len).unwrap() {
1304 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
1305 span,
1306 name,
1307 arg_idx: arg_idx as u64,
1308 total_len: total_len.into(),
1309 });
1310 None
1311 } else {
1312 Some(bx.const_i32(idx))
1313 }
1314 })
1315 .collect();
1316 let Some(indices) = indices else {
1317 return Ok(bx.const_null(llret_ty));
1318 };
1319
1320 return Ok(bx.shuffle_vector(
1321 args[0].immediate(),
1322 args[1].immediate(),
1323 bx.const_vector(&indices),
1324 ));
1325 }
1326
1327 if name == sym::simd_shuffle {
1328 let idx_ty = args[2].layout.ty;
1330 let n: u64 = if idx_ty.is_simd()
1331 && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
1332 {
1333 idx_ty.simd_size_and_type(bx.cx.tcx).0
1334 } else {
1335 return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
1336 };
1337
1338 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1339 require!(
1340 out_len == n,
1341 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1342 );
1343 require!(
1344 in_elem == out_ty,
1345 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1346 );
1347
1348 let total_len = u128::from(in_len) * 2;
1349
1350 let indices = args[2].immediate();
1352 for i in 0..n {
1353 let val = bx.const_get_elt(indices, i as u64);
1354 let idx = bx
1355 .const_to_opt_u128(val, true)
1356 .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
1357 if idx >= total_len {
1358 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1359 span,
1360 name,
1361 arg_idx: i,
1362 total_len,
1363 });
1364 }
1365 }
1366
1367 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
1368 }
1369
1370 if name == sym::simd_insert || name == sym::simd_insert_dyn {
1371 require!(
1372 in_elem == args[2].layout.ty,
1373 InvalidMonomorphization::InsertedType {
1374 span,
1375 name,
1376 in_elem,
1377 in_ty,
1378 out_ty: args[2].layout.ty
1379 }
1380 );
1381
1382 let index_imm = if name == sym::simd_insert {
1383 let idx = bx
1384 .const_to_opt_u128(args[1].immediate(), false)
1385 .expect("typeck should have ensure that this is a const");
1386 if idx >= in_len.into() {
1387 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1388 span,
1389 name,
1390 arg_idx: 1,
1391 total_len: in_len.into(),
1392 });
1393 }
1394 bx.const_i32(idx as i32)
1395 } else {
1396 args[1].immediate()
1397 };
1398
1399 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
1400 }
1401 if name == sym::simd_extract || name == sym::simd_extract_dyn {
1402 require!(
1403 ret_ty == in_elem,
1404 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1405 );
1406 let index_imm = if name == sym::simd_extract {
1407 let idx = bx
1408 .const_to_opt_u128(args[1].immediate(), false)
1409 .expect("typeck should have ensure that this is a const");
1410 if idx >= in_len.into() {
1411 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1412 span,
1413 name,
1414 arg_idx: 1,
1415 total_len: in_len.into(),
1416 });
1417 }
1418 bx.const_i32(idx as i32)
1419 } else {
1420 args[1].immediate()
1421 };
1422
1423 return Ok(bx.extract_element(args[0].immediate(), index_imm));
1424 }
1425
1426 if name == sym::simd_select {
1427 let m_elem_ty = in_elem;
1428 let m_len = in_len;
1429 let (v_len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1430 require!(
1431 m_len == v_len,
1432 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
1433 );
1434 let in_elem_bitwidth = require_int_or_uint_ty!(
1435 m_elem_ty.kind(),
1436 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1437 );
1438 let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1439 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1440 }
1441
1442 if name == sym::simd_bitmask {
1443 let expected_int_bits = in_len.max(8).next_power_of_two();
1452 let expected_bytes = in_len.div_ceil(8);
1453
1454 let in_elem_bitwidth = require_int_or_uint_ty!(
1456 in_elem.kind(),
1457 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
1458 );
1459
1460 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
1461 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
1463
1464 match ret_ty.kind() {
1465 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
1466 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
1468 }
1469 ty::Array(elem, len)
1470 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1471 && len
1472 .try_to_target_usize(bx.tcx)
1473 .expect("expected monomorphic const in codegen")
1474 == expected_bytes =>
1475 {
1476 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
1478
1479 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
1481 bx.store(ze, ptr, Align::ONE);
1482 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
1483 return Ok(bx.load(array_ty, ptr, Align::ONE));
1484 }
1485 _ => return_error!(InvalidMonomorphization::CannotReturn {
1486 span,
1487 name,
1488 ret_ty,
1489 expected_int_bits,
1490 expected_bytes
1491 }),
1492 }
1493 }
1494
1495 fn simd_simple_float_intrinsic<'ll, 'tcx>(
1496 name: Symbol,
1497 in_elem: Ty<'_>,
1498 in_ty: Ty<'_>,
1499 in_len: u64,
1500 bx: &mut Builder<'_, 'll, 'tcx>,
1501 span: Span,
1502 args: &[OperandRef<'tcx, &'ll Value>],
1503 ) -> Result<&'ll Value, ()> {
1504 macro_rules! return_error {
1505 ($diag: expr) => {{
1506 bx.sess().dcx().emit_err($diag);
1507 return Err(());
1508 }};
1509 }
1510
1511 let elem_ty = if let ty::Float(f) = in_elem.kind() {
1512 bx.cx.type_float_from_ty(*f)
1513 } else {
1514 return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
1515 };
1516
1517 let vec_ty = bx.type_vector(elem_ty, in_len);
1518
1519 let intr_name = match name {
1520 sym::simd_ceil => "llvm.ceil",
1521 sym::simd_fabs => "llvm.fabs",
1522 sym::simd_fcos => "llvm.cos",
1523 sym::simd_fexp2 => "llvm.exp2",
1524 sym::simd_fexp => "llvm.exp",
1525 sym::simd_flog10 => "llvm.log10",
1526 sym::simd_flog2 => "llvm.log2",
1527 sym::simd_flog => "llvm.log",
1528 sym::simd_floor => "llvm.floor",
1529 sym::simd_fma => "llvm.fma",
1530 sym::simd_relaxed_fma => "llvm.fmuladd",
1531 sym::simd_fsin => "llvm.sin",
1532 sym::simd_fsqrt => "llvm.sqrt",
1533 sym::simd_round => "llvm.round",
1534 sym::simd_round_ties_even => "llvm.rint",
1535 sym::simd_trunc => "llvm.trunc",
1536 _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
1537 };
1538 Ok(bx.call_intrinsic(
1539 intr_name,
1540 &[vec_ty],
1541 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
1542 ))
1543 }
1544
1545 if std::matches!(
1546 name,
1547 sym::simd_ceil
1548 | sym::simd_fabs
1549 | sym::simd_fcos
1550 | sym::simd_fexp2
1551 | sym::simd_fexp
1552 | sym::simd_flog10
1553 | sym::simd_flog2
1554 | sym::simd_flog
1555 | sym::simd_floor
1556 | sym::simd_fma
1557 | sym::simd_fsin
1558 | sym::simd_fsqrt
1559 | sym::simd_relaxed_fma
1560 | sym::simd_round
1561 | sym::simd_round_ties_even
1562 | sym::simd_trunc
1563 ) {
1564 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
1565 }
1566
1567 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
1568 let elem_ty = match *elem_ty.kind() {
1569 ty::Int(v) => cx.type_int_from_ty(v),
1570 ty::Uint(v) => cx.type_uint_from_ty(v),
1571 ty::Float(v) => cx.type_float_from_ty(v),
1572 ty::RawPtr(_, _) => cx.type_ptr(),
1573 _ => unreachable!(),
1574 };
1575 cx.type_vector(elem_ty, vec_len)
1576 }
1577
1578 if name == sym::simd_gather {
1579 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1590 let (out_len, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1591 let (out_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1593 require_simd!(ret_ty, SimdReturn);
1594
1595 require!(
1597 in_len == out_len,
1598 InvalidMonomorphization::SecondArgumentLength {
1599 span,
1600 name,
1601 in_len,
1602 in_ty,
1603 arg_ty: args[1].layout.ty,
1604 out_len
1605 }
1606 );
1607 require!(
1608 in_len == out_len2,
1609 InvalidMonomorphization::ThirdArgumentLength {
1610 span,
1611 name,
1612 in_len,
1613 in_ty,
1614 arg_ty: args[2].layout.ty,
1615 out_len: out_len2
1616 }
1617 );
1618
1619 require!(
1621 ret_ty == in_ty,
1622 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
1623 );
1624
1625 require!(
1626 matches!(
1627 *element_ty1.kind(),
1628 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
1629 ),
1630 InvalidMonomorphization::ExpectedElementType {
1631 span,
1632 name,
1633 expected_element: element_ty1,
1634 second_arg: args[1].layout.ty,
1635 in_elem,
1636 in_ty,
1637 mutability: ExpectedPointerMutability::Not,
1638 }
1639 );
1640
1641 let mask_elem_bitwidth = require_int_or_uint_ty!(
1642 element_ty2.kind(),
1643 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1644 );
1645
1646 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1648
1649 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1651
1652 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1654
1655 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1657
1658 return Ok(bx.call_intrinsic(
1659 "llvm.masked.gather",
1660 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
1661 &[args[1].immediate(), alignment, mask, args[0].immediate()],
1662 ));
1663 }
1664
1665 if name == sym::simd_masked_load {
1666 let mask_ty = in_ty;
1676 let (mask_len, mask_elem) = (in_len, in_elem);
1677
1678 let pointer_ty = args[1].layout.ty;
1680
1681 let values_ty = args[2].layout.ty;
1683 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1684
1685 require_simd!(ret_ty, SimdReturn);
1686
1687 require!(
1689 values_len == mask_len,
1690 InvalidMonomorphization::ThirdArgumentLength {
1691 span,
1692 name,
1693 in_len: mask_len,
1694 in_ty: mask_ty,
1695 arg_ty: values_ty,
1696 out_len: values_len
1697 }
1698 );
1699
1700 require!(
1702 ret_ty == values_ty,
1703 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
1704 );
1705
1706 require!(
1707 matches!(
1708 *pointer_ty.kind(),
1709 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
1710 ),
1711 InvalidMonomorphization::ExpectedElementType {
1712 span,
1713 name,
1714 expected_element: values_elem,
1715 second_arg: pointer_ty,
1716 in_elem: values_elem,
1717 in_ty: values_ty,
1718 mutability: ExpectedPointerMutability::Not,
1719 }
1720 );
1721
1722 let m_elem_bitwidth = require_int_or_uint_ty!(
1723 mask_elem.kind(),
1724 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1725 );
1726
1727 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1728
1729 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1731
1732 let llvm_pointer = bx.type_ptr();
1733
1734 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1736
1737 return Ok(bx.call_intrinsic(
1738 "llvm.masked.load",
1739 &[llvm_elem_vec_ty, llvm_pointer],
1740 &[args[1].immediate(), alignment, mask, args[2].immediate()],
1741 ));
1742 }
1743
1744 if name == sym::simd_masked_store {
1745 let mask_ty = in_ty;
1755 let (mask_len, mask_elem) = (in_len, in_elem);
1756
1757 let pointer_ty = args[1].layout.ty;
1759
1760 let values_ty = args[2].layout.ty;
1762 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1763
1764 require!(
1766 values_len == mask_len,
1767 InvalidMonomorphization::ThirdArgumentLength {
1768 span,
1769 name,
1770 in_len: mask_len,
1771 in_ty: mask_ty,
1772 arg_ty: values_ty,
1773 out_len: values_len
1774 }
1775 );
1776
1777 require!(
1779 matches!(
1780 *pointer_ty.kind(),
1781 ty::RawPtr(p_ty, p_mutbl)
1782 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
1783 ),
1784 InvalidMonomorphization::ExpectedElementType {
1785 span,
1786 name,
1787 expected_element: values_elem,
1788 second_arg: pointer_ty,
1789 in_elem: values_elem,
1790 in_ty: values_ty,
1791 mutability: ExpectedPointerMutability::Mut,
1792 }
1793 );
1794
1795 let m_elem_bitwidth = require_int_or_uint_ty!(
1796 mask_elem.kind(),
1797 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1798 );
1799
1800 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1801
1802 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1804
1805 let llvm_pointer = bx.type_ptr();
1806
1807 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1809
1810 return Ok(bx.call_intrinsic(
1811 "llvm.masked.store",
1812 &[llvm_elem_vec_ty, llvm_pointer],
1813 &[args[2].immediate(), args[1].immediate(), alignment, mask],
1814 ));
1815 }
1816
1817 if name == sym::simd_scatter {
1818 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1828 let (element_len1, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1829 let (element_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1830
1831 require!(
1833 in_len == element_len1,
1834 InvalidMonomorphization::SecondArgumentLength {
1835 span,
1836 name,
1837 in_len,
1838 in_ty,
1839 arg_ty: args[1].layout.ty,
1840 out_len: element_len1
1841 }
1842 );
1843 require!(
1844 in_len == element_len2,
1845 InvalidMonomorphization::ThirdArgumentLength {
1846 span,
1847 name,
1848 in_len,
1849 in_ty,
1850 arg_ty: args[2].layout.ty,
1851 out_len: element_len2
1852 }
1853 );
1854
1855 require!(
1856 matches!(
1857 *element_ty1.kind(),
1858 ty::RawPtr(p_ty, p_mutbl)
1859 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
1860 ),
1861 InvalidMonomorphization::ExpectedElementType {
1862 span,
1863 name,
1864 expected_element: element_ty1,
1865 second_arg: args[1].layout.ty,
1866 in_elem,
1867 in_ty,
1868 mutability: ExpectedPointerMutability::Mut,
1869 }
1870 );
1871
1872 let mask_elem_bitwidth = require_int_or_uint_ty!(
1874 element_ty2.kind(),
1875 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1876 );
1877
1878 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1880
1881 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1883
1884 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1886
1887 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1889
1890 return Ok(bx.call_intrinsic(
1891 "llvm.masked.scatter",
1892 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
1893 &[args[0].immediate(), args[1].immediate(), alignment, mask],
1894 ));
1895 }
1896
1897 macro_rules! arith_red {
1898 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
1899 $identity:expr) => {
1900 if name == sym::$name {
1901 require!(
1902 ret_ty == in_elem,
1903 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1904 );
1905 return match in_elem.kind() {
1906 ty::Int(_) | ty::Uint(_) => {
1907 let r = bx.$integer_reduce(args[0].immediate());
1908 if $ordered {
1909 Ok(bx.$op(args[1].immediate(), r))
1912 } else {
1913 Ok(bx.$integer_reduce(args[0].immediate()))
1914 }
1915 }
1916 ty::Float(f) => {
1917 let acc = if $ordered {
1918 args[1].immediate()
1920 } else {
1921 match f.bit_width() {
1923 32 => bx.const_real(bx.type_f32(), $identity),
1924 64 => bx.const_real(bx.type_f64(), $identity),
1925 v => return_error!(
1926 InvalidMonomorphization::UnsupportedSymbolOfSize {
1927 span,
1928 name,
1929 symbol: sym::$name,
1930 in_ty,
1931 in_elem,
1932 size: v,
1933 ret_ty
1934 }
1935 ),
1936 }
1937 };
1938 Ok(bx.$float_reduce(acc, args[0].immediate()))
1939 }
1940 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
1941 span,
1942 name,
1943 symbol: sym::$name,
1944 in_ty,
1945 in_elem,
1946 ret_ty
1947 }),
1948 };
1949 }
1950 };
1951 }
1952
1953 arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
1954 arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
1955 arith_red!(
1956 simd_reduce_add_unordered: vector_reduce_add,
1957 vector_reduce_fadd_reassoc,
1958 false,
1959 add,
1960 -0.0
1961 );
1962 arith_red!(
1963 simd_reduce_mul_unordered: vector_reduce_mul,
1964 vector_reduce_fmul_reassoc,
1965 false,
1966 mul,
1967 1.0
1968 );
1969
1970 macro_rules! minmax_red {
1971 ($name:ident: $int_red:ident, $float_red:ident) => {
1972 if name == sym::$name {
1973 require!(
1974 ret_ty == in_elem,
1975 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1976 );
1977 return match in_elem.kind() {
1978 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
1979 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
1980 ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
1981 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
1982 span,
1983 name,
1984 symbol: sym::$name,
1985 in_ty,
1986 in_elem,
1987 ret_ty
1988 }),
1989 };
1990 }
1991 };
1992 }
1993
1994 minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
1995 minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
1996
1997 macro_rules! bitwise_red {
1998 ($name:ident : $red:ident, $boolean:expr) => {
1999 if name == sym::$name {
2000 let input = if !$boolean {
2001 require!(
2002 ret_ty == in_elem,
2003 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2004 );
2005 args[0].immediate()
2006 } else {
2007 let bitwidth = match in_elem.kind() {
2008 ty::Int(i) => {
2009 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2010 }
2011 ty::Uint(i) => {
2012 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2013 }
2014 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2015 span,
2016 name,
2017 symbol: sym::$name,
2018 in_ty,
2019 in_elem,
2020 ret_ty
2021 }),
2022 };
2023
2024 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2025 };
2026 return match in_elem.kind() {
2027 ty::Int(_) | ty::Uint(_) => {
2028 let r = bx.$red(input);
2029 Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2030 }
2031 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2032 span,
2033 name,
2034 symbol: sym::$name,
2035 in_ty,
2036 in_elem,
2037 ret_ty
2038 }),
2039 };
2040 }
2041 };
2042 }
2043
2044 bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2045 bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2046 bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2047 bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2048 bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2049
2050 if name == sym::simd_cast_ptr {
2051 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2052 require!(
2053 in_len == out_len,
2054 InvalidMonomorphization::ReturnLengthInputType {
2055 span,
2056 name,
2057 in_len,
2058 in_ty,
2059 ret_ty,
2060 out_len
2061 }
2062 );
2063
2064 match in_elem.kind() {
2065 ty::RawPtr(p_ty, _) => {
2066 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2067 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2068 });
2069 require!(
2070 metadata.is_unit(),
2071 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2072 );
2073 }
2074 _ => {
2075 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2076 }
2077 }
2078 match out_elem.kind() {
2079 ty::RawPtr(p_ty, _) => {
2080 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2081 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2082 });
2083 require!(
2084 metadata.is_unit(),
2085 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2086 );
2087 }
2088 _ => {
2089 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2090 }
2091 }
2092
2093 return Ok(args[0].immediate());
2094 }
2095
2096 if name == sym::simd_expose_provenance {
2097 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2098 require!(
2099 in_len == out_len,
2100 InvalidMonomorphization::ReturnLengthInputType {
2101 span,
2102 name,
2103 in_len,
2104 in_ty,
2105 ret_ty,
2106 out_len
2107 }
2108 );
2109
2110 match in_elem.kind() {
2111 ty::RawPtr(_, _) => {}
2112 _ => {
2113 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2114 }
2115 }
2116 match out_elem.kind() {
2117 ty::Uint(ty::UintTy::Usize) => {}
2118 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2119 }
2120
2121 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
2122 }
2123
2124 if name == sym::simd_with_exposed_provenance {
2125 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2126 require!(
2127 in_len == out_len,
2128 InvalidMonomorphization::ReturnLengthInputType {
2129 span,
2130 name,
2131 in_len,
2132 in_ty,
2133 ret_ty,
2134 out_len
2135 }
2136 );
2137
2138 match in_elem.kind() {
2139 ty::Uint(ty::UintTy::Usize) => {}
2140 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
2141 }
2142 match out_elem.kind() {
2143 ty::RawPtr(_, _) => {}
2144 _ => {
2145 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2146 }
2147 }
2148
2149 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
2150 }
2151
2152 if name == sym::simd_cast || name == sym::simd_as {
2153 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2154 require!(
2155 in_len == out_len,
2156 InvalidMonomorphization::ReturnLengthInputType {
2157 span,
2158 name,
2159 in_len,
2160 in_ty,
2161 ret_ty,
2162 out_len
2163 }
2164 );
2165 if in_elem == out_elem {
2167 return Ok(args[0].immediate());
2168 }
2169
2170 #[derive(Copy, Clone)]
2171 enum Sign {
2172 Unsigned,
2173 Signed,
2174 }
2175 use Sign::*;
2176
2177 enum Style {
2178 Float,
2179 Int(Sign),
2180 Unsupported,
2181 }
2182
2183 let (in_style, in_width) = match in_elem.kind() {
2184 ty::Int(i) => (
2187 Style::Int(Signed),
2188 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2189 ),
2190 ty::Uint(u) => (
2191 Style::Int(Unsigned),
2192 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2193 ),
2194 ty::Float(f) => (Style::Float, f.bit_width()),
2195 _ => (Style::Unsupported, 0),
2196 };
2197 let (out_style, out_width) = match out_elem.kind() {
2198 ty::Int(i) => (
2199 Style::Int(Signed),
2200 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2201 ),
2202 ty::Uint(u) => (
2203 Style::Int(Unsigned),
2204 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2205 ),
2206 ty::Float(f) => (Style::Float, f.bit_width()),
2207 _ => (Style::Unsupported, 0),
2208 };
2209
2210 match (in_style, out_style) {
2211 (Style::Int(sign), Style::Int(_)) => {
2212 return Ok(match in_width.cmp(&out_width) {
2213 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
2214 Ordering::Equal => args[0].immediate(),
2215 Ordering::Less => match sign {
2216 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
2217 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
2218 },
2219 });
2220 }
2221 (Style::Int(Sign::Signed), Style::Float) => {
2222 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
2223 }
2224 (Style::Int(Sign::Unsigned), Style::Float) => {
2225 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
2226 }
2227 (Style::Float, Style::Int(sign)) => {
2228 return Ok(match (sign, name == sym::simd_as) {
2229 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
2230 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
2231 (_, true) => bx.cast_float_to_int(
2232 matches!(sign, Sign::Signed),
2233 args[0].immediate(),
2234 llret_ty,
2235 ),
2236 });
2237 }
2238 (Style::Float, Style::Float) => {
2239 return Ok(match in_width.cmp(&out_width) {
2240 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
2241 Ordering::Equal => args[0].immediate(),
2242 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
2243 });
2244 }
2245 _ => { }
2246 }
2247 return_error!(InvalidMonomorphization::UnsupportedCast {
2248 span,
2249 name,
2250 in_ty,
2251 in_elem,
2252 ret_ty,
2253 out_elem
2254 });
2255 }
2256 macro_rules! arith_binary {
2257 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2258 $(if name == sym::$name {
2259 match in_elem.kind() {
2260 $($(ty::$p(_))|* => {
2261 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
2262 })*
2263 _ => {},
2264 }
2265 return_error!(
2266 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2267 );
2268 })*
2269 }
2270 }
2271 arith_binary! {
2272 simd_add: Uint, Int => add, Float => fadd;
2273 simd_sub: Uint, Int => sub, Float => fsub;
2274 simd_mul: Uint, Int => mul, Float => fmul;
2275 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
2276 simd_rem: Uint => urem, Int => srem, Float => frem;
2277 simd_shl: Uint, Int => shl;
2278 simd_shr: Uint => lshr, Int => ashr;
2279 simd_and: Uint, Int => and;
2280 simd_or: Uint, Int => or;
2281 simd_xor: Uint, Int => xor;
2282 simd_fmax: Float => maxnum;
2283 simd_fmin: Float => minnum;
2284
2285 }
2286 macro_rules! arith_unary {
2287 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2288 $(if name == sym::$name {
2289 match in_elem.kind() {
2290 $($(ty::$p(_))|* => {
2291 return Ok(bx.$call(args[0].immediate()))
2292 })*
2293 _ => {},
2294 }
2295 return_error!(
2296 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2297 );
2298 })*
2299 }
2300 }
2301 arith_unary! {
2302 simd_neg: Int => neg, Float => fneg;
2303 }
2304
2305 if matches!(
2307 name,
2308 sym::simd_bswap
2309 | sym::simd_bitreverse
2310 | sym::simd_ctlz
2311 | sym::simd_ctpop
2312 | sym::simd_cttz
2313 | sym::simd_funnel_shl
2314 | sym::simd_funnel_shr
2315 ) {
2316 let vec_ty = bx.cx.type_vector(
2317 match *in_elem.kind() {
2318 ty::Int(i) => bx.cx.type_int_from_ty(i),
2319 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
2320 _ => return_error!(InvalidMonomorphization::UnsupportedOperation {
2321 span,
2322 name,
2323 in_ty,
2324 in_elem
2325 }),
2326 },
2327 in_len as u64,
2328 );
2329 let llvm_intrinsic = match name {
2330 sym::simd_bswap => "llvm.bswap",
2331 sym::simd_bitreverse => "llvm.bitreverse",
2332 sym::simd_ctlz => "llvm.ctlz",
2333 sym::simd_ctpop => "llvm.ctpop",
2334 sym::simd_cttz => "llvm.cttz",
2335 sym::simd_funnel_shl => "llvm.fshl",
2336 sym::simd_funnel_shr => "llvm.fshr",
2337 _ => unreachable!(),
2338 };
2339 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
2340
2341 return match name {
2342 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2344 sym::simd_ctlz | sym::simd_cttz => {
2345 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2347 Ok(bx.call_intrinsic(
2348 llvm_intrinsic,
2349 &[vec_ty],
2350 &[args[0].immediate(), dont_poison_on_zero],
2351 ))
2352 }
2353 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2354 Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
2356 }
2357 sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
2358 llvm_intrinsic,
2359 &[vec_ty],
2360 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
2361 )),
2362 _ => unreachable!(),
2363 };
2364 }
2365
2366 if name == sym::simd_arith_offset {
2367 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
2369 span_bug!(span, "must be called with a vector of pointer types as first argument")
2370 });
2371 let layout = bx.layout_of(pointee);
2372 let ptrs = args[0].immediate();
2373 let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
2376 if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
2377 span_bug!(
2378 span,
2379 "must be called with a vector of pointer-sized integers as second argument"
2380 );
2381 }
2382 let offsets = args[1].immediate();
2383
2384 return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
2385 }
2386
2387 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
2388 let lhs = args[0].immediate();
2389 let rhs = args[1].immediate();
2390 let is_add = name == sym::simd_saturating_add;
2391 let (signed, elem_ty) = match *in_elem.kind() {
2392 ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
2393 ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
2394 _ => {
2395 return_error!(InvalidMonomorphization::ExpectedVectorElementType {
2396 span,
2397 name,
2398 expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
2399 vector_type: args[0].layout.ty
2400 });
2401 }
2402 };
2403 let llvm_intrinsic = format!(
2404 "llvm.{}{}.sat",
2405 if signed { 's' } else { 'u' },
2406 if is_add { "add" } else { "sub" },
2407 );
2408 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
2409
2410 return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
2411 }
2412
2413 span_bug!(span, "unknown SIMD intrinsic");
2414}