1use std::cmp::Ordering;
2use std::ffi::c_uint;
3use std::{assert_matches, iter, ptr};
4
5use rustc_abi::{
6 AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, NumScalableVectors,
7 Primitive, Size,
8};
9use rustc_attr_ir::find_attr;
10use rustc_codegen_ssa::RetagInfo;
11use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
12use rustc_codegen_ssa::common::IntPredicate;
13use rustc_codegen_ssa::diagnostics::{ExpectedPointerMutability, InvalidMonomorphization};
14use rustc_codegen_ssa::mir::IntrinsicResult;
15use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
16use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
17use rustc_codegen_ssa::traits::*;
18use rustc_hir as hir;
19use rustc_hir::def_id::LOCAL_CRATE;
20use rustc_lint_defs::builtin::DEPRECATED_LLVM_INTRINSIC;
21use rustc_middle::mir::interpret::{AllocId, Allocation, ErrorHandled, GlobalAlloc, alloc_range};
22use rustc_middle::mir::{self, BinOp};
23use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
24use rustc_middle::ty::offload_meta::OffloadMetadata;
25use rustc_middle::ty::{self, GenericArgs, Instance, SimdAlign, Ty, TyCtxt};
26use rustc_session::diagnostics::feature_err;
27use rustc_span::{ErrorGuaranteed, Span, Symbol, bug, span_bug, sym};
28use rustc_structures::CrateType;
29use rustc_symbol_mangling::{
30 mangle_internal_symbol, mangle_offload_export, symbol_name_for_instance_in_crate,
31};
32use rustc_target::callconv::PassMode;
33use rustc_target::spec::Arch;
34use tracing::debug;
35
36use crate::abi::FnAbiLlvmExt;
37use crate::builder::Builder;
38use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
39use crate::builder::gpu_offload::{self, OffloadKernelDims, declare_omp_get_num_devices};
40use crate::context::CodegenCx;
41use crate::declare::declare_raw_fn;
42use crate::diagnostics::{
43 AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
44 OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
45};
46use crate::intrinsic::ty::typetree::fnc_typetrees;
47use crate::llvm::{self, Attribute, AttributePlace, Metadata, Type, TypeKind, Value};
48use crate::type_of::LayoutLlvmExt;
49use crate::va_arg::emit_va_arg;
50
51fn call_simple_intrinsic<'ll, 'tcx>(
52 bx: &mut Builder<'_, 'll, 'tcx>,
53 name: Symbol,
54 args: &[OperandRef<'tcx, &'ll Value>],
55) -> Option<&'ll Value> {
56 let llvm_version = crate::llvm_util::get_version();
57 let fixed_minmax = llvm_version >= (23, 0, 0);
61
62 let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
63 sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]),
64 sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]),
65 sym::minimumf64 if fixed_minmax => ("llvm.minimum", &[bx.type_f64()]),
66 sym::minimumf128 if fixed_minmax => ("llvm.minimum", &[bx.type_f128()]),
67
68 sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]),
69 sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]),
70 sym::maximumf64 if fixed_minmax => ("llvm.maximum", &[bx.type_f64()]),
71 sym::maximumf128 if fixed_minmax => ("llvm.maximum", &[bx.type_f128()]),
72
73 sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
74 sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
75 sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
76 sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
77
78 _ => return None,
79 };
80 Some(bx.call_intrinsic(
81 base_name,
82 type_params,
83 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
84 ))
85}
86
87impl<'ll, 'tcx> Builder<'_, 'll, 'tcx> {
88 fn black_box(&mut self, result: PlaceRef<'tcx, &'ll Value>, span: Span) {
89 let result_val_span = [result.val.llval];
90 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
100 ("~{memory}", &[])
101 } else {
102 ("r,~{memory}", &result_val_span)
103 };
104 crate::asm::inline_asm_call(
105 self,
106 "",
107 constraint,
108 inputs,
109 self.type_void(),
110 &[],
111 true,
112 false,
113 llvm::AsmDialect::Att,
114 &[span],
115 false,
116 None,
117 None,
118 )
119 .unwrap_or_else(|| ::rustc_span::macros::bug_impl(None,
format_args!("failed to generate inline asm call for `black_box`"),
Location::caller())bug!("failed to generate inline asm call for `black_box`"));
120 }
121}
122
123impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
124 fn codegen_intrinsic_call(
125 &mut self,
126 instance: ty::Instance<'tcx>,
127 args: &[OperandRef<'tcx, &'ll Value>],
128 result_layout: ty::layout::TyAndLayout<'tcx>,
129 result_place: Option<PlaceValue<&'ll Value>>,
130 span: Span,
131 ) -> IntrinsicResult<'tcx, &'ll Value> {
132 let tcx = self.tcx;
133 let llvm_version = crate::llvm_util::get_version();
134
135 let name = tcx.item_name(instance.def_id());
136 let fn_args = instance.args;
137
138 let simple = call_simple_intrinsic(self, name, args);
139 let llval = match name {
140 _ if simple.is_some() => simple.unwrap(),
141 sym::minimum_number_nsz_f16
142 | sym::minimum_number_nsz_f32
143 | sym::minimum_number_nsz_f64
144 | sym::minimum_number_nsz_f128
145 | sym::maximum_number_nsz_f16
146 | sym::maximum_number_nsz_f32
147 | sym::maximum_number_nsz_f64
148 | sym::maximum_number_nsz_f128 => {
149 let intrinsic_name = if name.as_str().starts_with("min") {
150 "llvm.minimumnum"
151 } else {
152 "llvm.maximumnum"
153 };
154 let call = self.call_intrinsic(
155 intrinsic_name,
156 &[args[0].layout.immediate_llvm_type(self.cx)],
157 &[args[0].immediate(), args[1].immediate()],
158 );
159 unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
162 call
163 }
164 sym::ptr_mask => {
165 let ptr = args[0].immediate();
166 self.call_intrinsic(
167 "llvm.ptrmask",
168 &[self.val_ty(ptr), self.type_isize()],
169 &[ptr, args[1].immediate()],
170 )
171 }
172 sym::autodiff => {
173 return codegen_autodiff(self, instance, args, result_layout, result_place);
174 }
175 sym::offload => {
176 if tcx.sess.opts.unstable_opts.offload.is_empty() {
177 let _ = tcx.dcx().emit_err(OffloadWithoutEnable);
178 }
179
180 if tcx.sess.lto() != rustc_session::config::Lto::Fat {
181 let _ = tcx.dcx().emit_err(OffloadWithoutFatLTO);
182 }
183
184 codegen_offload(self, tcx, instance, args);
185 return IntrinsicResult::WroteIntoPlace;
187 }
188 sym::offload_get_num_devices => {
189 let (fn_decl, fn_ty) = declare_omp_get_num_devices(self.cx);
190
191 let llval =
192 self.call(fn_ty, None, None, fn_decl, ReturnSlot::Direct, &[], None, None);
193
194 return IntrinsicResult::Operand(OperandValue::Immediate(llval));
195 }
196 sym::is_val_statically_known => {
197 if let OperandValue::Immediate(imm) = args[0].val {
198 self.call_intrinsic(
199 "llvm.is.constant",
200 &[args[0].layout.immediate_llvm_type(self.cx)],
201 &[imm],
202 )
203 } else {
204 self.const_bool(false)
205 }
206 }
207 sym::select_unpredictable => {
208 let cond = args[0].immediate();
209 {
match (&args[1].layout, &args[2].layout) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(args[1].layout, args[2].layout);
210 let select = |bx: &mut Self, true_val, false_val| {
211 let result = bx.select(cond, true_val, false_val);
212 bx.set_unpredictable(&result);
213 result
214 };
215 match (args[1].val, args[2].val) {
216 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
217 if !true_val.llextra.is_none() {
::core::panicking::panic("assertion failed: true_val.llextra.is_none()")
};assert!(true_val.llextra.is_none());
218 if !false_val.llextra.is_none() {
::core::panicking::panic("assertion failed: false_val.llextra.is_none()")
};assert!(false_val.llextra.is_none());
219 {
match (&true_val.align, &false_val.align) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(true_val.align, false_val.align);
220 let ptr = select(self, true_val.llval, false_val.llval);
221 let selected =
222 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
223 let result = PlaceRef { val: result_place.unwrap(), layout: result_layout };
224 selected.store(self, result);
225 return IntrinsicResult::WroteIntoPlace;
226 }
227 (OperandValue::Immediate(_), OperandValue::Immediate(_))
228 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
229 let true_val = args[1].immediate_or_packed_pair(self);
230 let false_val = args[2].immediate_or_packed_pair(self);
231 select(self, true_val, false_val)
232 }
233 (OperandValue::ZeroSized, OperandValue::ZeroSized) => {
234 return IntrinsicResult::Operand(OperandValue::ZeroSized);
235 }
236 _ => ::rustc_span::macros::bug_impl(Some(span),
format_args!("Incompatible OperandValue for select_unpredictable"),
Location::caller())span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
237 }
238 }
239 sym::catch_unwind => catch_unwind_intrinsic(
240 self,
241 args[0].immediate(),
242 args[1].immediate(),
243 args[2].immediate(),
244 ),
245 sym::breakpoint | sym::resumable_breakpoint => {
246 self.call_intrinsic("llvm.debugtrap", &[], &[])
247 }
248 sym::va_arg => {
249 let target = &self.cx.tcx.sess.target;
250 let stability = target.supports_c_variadic_definitions();
251 if let CVariadicStatus::Unstable { feature } = stability
252 && !self.tcx.features().enabled(feature)
253 {
254 let msg =
255 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("C-variadic function definitions on this target are unstable"))
})format!("C-variadic function definitions on this target are unstable");
256 feature_err(&*self.sess(), feature, span, msg).emit();
257 }
258
259 let BackendRepr::Scalar(scalar) = result_layout.backend_repr else {
260 ::rustc_span::macros::bug_impl(None,
format_args!("the va_arg intrinsic does not support non-scalar types"),
Location::caller())bug!("the va_arg intrinsic does not support non-scalar types")
261 };
262
263 match scalar.primitive() {
267 Primitive::Pointer(_) => {
268 }
270 Primitive::Int(..) => {
271 let int_width = self.cx().size_of(result_layout.ty).bits();
272 let target_c_int_width = self.cx().sess().target.options.c_int_width;
273 if int_width < u64::from(target_c_int_width) {
274 ::rustc_span::macros::bug_impl(None,
format_args!("va_arg got i{0} but needs at least c_int (an i{1})",
int_width, target_c_int_width), Location::caller());bug!(
277 "va_arg got i{} but needs at least c_int (an i{})",
278 int_width,
279 target_c_int_width
280 );
281 }
282 }
283 Primitive::Float(Float::F16) => {
284 ::rustc_span::macros::bug_impl(None,
format_args!("the va_arg intrinsic does not support `f16`"),
Location::caller())bug!("the va_arg intrinsic does not support `f16`")
285 }
286 Primitive::Float(Float::F16B) => {
287 ::rustc_span::macros::bug_impl(None,
format_args!("the va_arg intrinsic does not support `f16b`"),
Location::caller())bug!("the va_arg intrinsic does not support `f16b`")
288 }
289 Primitive::Float(Float::F32) => {
290 if self.cx().sess().target.arch != Arch::Avr {
292 ::rustc_span::macros::bug_impl(None,
format_args!("the va_arg intrinsic does not support `f32` on this target"),
Location::caller())bug!("the va_arg intrinsic does not support `f32` on this target")
293 }
294 }
295 Primitive::Float(Float::F64) => {
296 }
298 Primitive::Float(Float::F128) => {
299 }
301 Primitive::Float(Float::PpcF128) => {
302 }
304 }
305
306 emit_va_arg(self, args[0], result_layout)
307 }
308
309 sym::volatile_load | sym::unaligned_volatile_load => {
310 let ptr = args[0].immediate();
315 let abi_align = result_layout.align;
316 let ptr_align = if name == sym::volatile_load { abi_align } else { Align::ONE };
317 let need_black_box = llvm_version < (23, 0, 0);
318 if result_layout.is_zst() {
319 return IntrinsicResult::Operand(OperandValue::ZeroSized);
320 } else if let BackendRepr::Scalar(scalar) = result_layout.backend_repr
321 && !need_black_box
322 {
323 let load = self.volatile_load(self.type_from_scalar(scalar), ptr, ptr_align);
324 self.to_immediate_scalar(load, scalar)
325 } else {
326 let llty = self.type_ix(result_layout.size.bits());
330 let temp = if let Some(result_place) = result_place {
331 PlaceRef { val: result_place, layout: result_layout }
332 } else {
333 PlaceRef::alloca(self, result_layout)
334 };
335 let llval = self.volatile_load(llty, ptr, ptr_align);
336 self.store(llval, temp.val.llval, abi_align);
337 if need_black_box {
338 self.black_box(temp, span);
343 }
344 return if result_place.is_none() {
345 IntrinsicResult::Operand(self.load_operand(temp).val)
346 } else {
347 IntrinsicResult::WroteIntoPlace
348 };
349 }
350 }
351 sym::prefetch_read_data
352 | sym::prefetch_write_data
353 | sym::prefetch_read_instruction
354 | sym::prefetch_write_instruction => {
355 let (rw, cache_type) = match name {
356 sym::prefetch_read_data => (0, 1),
357 sym::prefetch_write_data => (1, 1),
358 sym::prefetch_read_instruction => (0, 0),
359 sym::prefetch_write_instruction => (1, 0),
360 _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
Location::caller())bug!(),
361 };
362 let ptr = args[0].immediate();
363 let locality = fn_args.const_at(1).to_leaf().to_i32();
364 self.call_intrinsic(
365 "llvm.prefetch.p0",
366 &[self.val_ty(ptr)],
367 &[
368 ptr,
369 self.const_i32(rw),
370 self.const_i32(locality),
371 self.const_i32(cache_type),
372 ],
373 );
374 return IntrinsicResult::Operand(OperandValue::ZeroSized);
375 }
376 sym::carrying_mul_add => {
377 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
378
379 let wide_llty = self.type_ix(size.bits() * 2);
380 let args = args.as_array().unwrap();
381 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
382
383 let wide = if signed {
384 let prod = self.unchecked_smul(a, b);
385 let acc = self.unchecked_sadd(prod, c);
386 self.unchecked_sadd(acc, d)
387 } else {
388 let prod = self.unchecked_umul(a, b);
389 let acc = self.unchecked_uadd(prod, c);
390 self.unchecked_uadd(acc, d)
391 };
392
393 let narrow_llty = self.type_ix(size.bits());
394 let low = self.trunc(wide, narrow_llty);
395 let bits_const = self.const_uint(wide_llty, size.bits());
396 let high = self.lshr(wide, bits_const);
398 let high = self.trunc(high, narrow_llty);
400
401 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty]);
402 let pair = self.const_poison(pair_llty);
403 let pair = self.insert_value(pair, low, 0);
404 let pair = self.insert_value(pair, high, 1);
405 pair
406 }
407
408 sym::ctlz
409 | sym::ctlz_nonzero
410 | sym::cttz
411 | sym::cttz_nonzero
412 | sym::ctpop
413 | sym::bswap
414 | sym::bitreverse
415 | sym::carryless_mul
416 | sym::integer_max
417 | sym::integer_min
418 | sym::saturating_add
419 | sym::saturating_sub
420 | sym::unchecked_funnel_shl
421 | sym::unchecked_funnel_shr => {
422 let ty = args[0].layout.ty;
423 if !ty.is_integral() {
424 let err = tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
425 span,
426 name,
427 ty,
428 });
429 return IntrinsicResult::Err(err);
430 }
431 let (size, signed) = ty.int_size_and_signed(self.tcx);
432 let width = size.bits();
433 let llty = self.type_ix(width);
434 match name {
435 sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
436 let y =
437 self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
438 let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
439 "llvm.ctlz"
440 } else {
441 "llvm.cttz"
442 };
443 let ret =
444 self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
445 self.intcast(ret, result_layout.llvm_type(self), false)
446 }
447 sym::ctpop => self.ctpop(args[0].immediate()),
448 sym::bswap => {
449 if width == 8 {
450 args[0].immediate() } else {
452 self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
453 }
454 }
455 sym::bitreverse => {
456 self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
457 }
458 sym::carryless_mul => {
459 let lhs = args[0].immediate();
460 let rhs = args[1].immediate();
461 self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
462 }
463 sym::integer_min | sym::integer_max => {
464 let lhs = args[0].immediate();
465 let rhs = args[1].immediate();
466 let llvm_name = match (name, signed) {
467 (sym::integer_max, false) => "llvm.umax",
468 (sym::integer_max, true) => "llvm.smax",
469 (sym::integer_min, false) => "llvm.umin",
470 (sym::integer_min, true) => "llvm.smin",
471 _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
Location::caller())bug!(),
472 };
473 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
474 }
475 sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
476 let is_left = name == sym::unchecked_funnel_shl;
477 let lhs = args[0].immediate();
478 let rhs = args[1].immediate();
479 let raw_shift = args[2].immediate();
480 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.fsh{0}",
if is_left { 'l' } else { 'r' }))
})format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
481
482 let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false);
485
486 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift])
487 }
488 sym::saturating_add | sym::saturating_sub => {
489 let is_add = name == sym::saturating_add;
490 let lhs = args[0].immediate();
491 let rhs = args[1].immediate();
492 let llvm_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
493 "llvm.{}{}.sat",
494 if signed { 's' } else { 'u' },
495 if is_add { "add" } else { "sub" },
496 );
497 self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
498 }
499 _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
Location::caller())bug!(),
500 }
501 }
502
503 sym::fabs
504 | sym::sqrt
505 | sym::powf
506 | sym::fma
507 | sym::fmuladd
508 | sym::floor
509 | sym::ceil
510 | sym::trunc
511 | sym::round
512 | sym::round_ties_even
513 | sym::exp
514 | sym::exp2
515 | sym::log
516 | sym::log10
517 | sym::log2
518 | sym::sin
519 | sym::cos
520 | sym::powi => {
521 let ty = args[0].layout.ty;
522 let ty::Float(f) = ty.kind() else {
523 ::rustc_span::macros::bug_impl(Some(span),
format_args!("the `{0}` intrinsic requires a floating-point argument, got {1:?}",
name, ty), Location::caller());span_bug!(
524 span,
525 "the `{}` intrinsic requires a floating-point argument, got {:?}",
526 name,
527 ty
528 );
529 };
530 let llty = self.type_float_from_ty(*f);
531 let llvm_name = match name {
532 sym::fabs => "llvm.fabs",
533 sym::sqrt => "llvm.sqrt",
534 sym::powf => "llvm.pow",
535 sym::fma => "llvm.fma",
536 sym::fmuladd => "llvm.fmuladd",
537 sym::floor => "llvm.floor",
538 sym::ceil => "llvm.ceil",
539 sym::trunc => "llvm.trunc",
540 sym::round => "llvm.round",
541 sym::round_ties_even => "llvm.rint",
546 sym::exp => "llvm.exp",
547 sym::exp2 => "llvm.exp2",
548 sym::log => "llvm.log",
549 sym::log10 => "llvm.log10",
550 sym::log2 => "llvm.log2",
551 sym::sin => "llvm.sin",
552 sym::cos => "llvm.cos",
553 sym::powi => "llvm.powi",
554
555 _ => ::rustc_span::macros::bug_impl(None, format_args!("impossible case reached"),
Location::caller())bug!(),
556 };
557
558 let params: &[&'ll Type] = match name {
559 sym::powi => &[llty, self.type_i32()],
560 _ => &[llty],
561 };
562
563 self.call_intrinsic(
564 llvm_name,
565 params,
566 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
567 )
568 }
569
570 sym::raw_eq => {
571 use BackendRepr::*;
572 let tp_ty = fn_args.type_at(0);
573 let layout = self.layout_of(tp_ty).layout;
574 let use_integer_compare = match layout.backend_repr() {
575 Scalar(_) | ScalarPair { a: _, b: _, b_offset: _ } => true,
576 SimdVector { .. } => false,
577 SimdScalableVector { .. } => {
578 let err = tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
579 span,
580 name: sym::raw_eq,
581 ty: tp_ty,
582 });
583 return IntrinsicResult::Err(err);
584 }
585 Memory { .. } => {
586 layout.size() <= self.data_layout().pointer_size() * 2
590 }
591 };
592
593 let a = args[0].immediate();
594 let b = args[1].immediate();
595 if layout.size().bytes() == 0 {
596 self.const_bool(true)
597 } else if use_integer_compare {
598 let integer_ty = self.type_ix(layout.size().bits());
599 let a_val = self.load(integer_ty, a, layout.align());
600 let b_val = self.load(integer_ty, b, layout.align());
601 self.icmp(IntPredicate::IntEQ, a_val, b_val)
602 } else {
603 let n = self.const_usize(layout.size().bytes());
604 let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
605 self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
606 }
607 }
608
609 sym::compare_bytes => {
610 let cmp = self.call_intrinsic(
612 "memcmp",
613 &[],
614 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
615 );
616 self.sext(cmp, self.type_ix(32))
618 }
619
620 sym::black_box => {
621 let result = PlaceRef { val: result_place.unwrap(), layout: result_layout };
624 args[0].val.store(self, result);
625 self.black_box(result, span);
626
627 return IntrinsicResult::WroteIntoPlace;
629 }
630
631 sym::gpu_launch_sized_workgroup_mem => {
632 let name = if llvm_version < (23, 0, 0) && tcx.sess.target.arch == Arch::Nvptx64 {
640 "gpu_launch_sized_workgroup_mem"
644 } else {
645 ""
646 };
647 let global = self.declare_global_in_addrspace(
648 name,
649 self.type_array(self.type_i8(), 0),
650 AddressSpace::GPU_WORKGROUP,
651 );
652 let ty::RawPtr(inner_ty, _) = result_layout.ty.kind() else { ::core::panicking::unreachable_default_msg()unreachable!() };
653 let alignment = self.align_of(*inner_ty).bytes() as u32;
658 unsafe {
659 if tcx.sess.target.arch == Arch::Nvptx64 {
661 if alignment > llvm::LLVMGetAlignment(global) {
662 llvm::LLVMSetAlignment(global, alignment);
663 }
664 } else {
665 llvm::LLVMSetAlignment(global, alignment);
666 }
667 }
668 self.cx().const_pointercast(global, self.type_ptr())
669 }
670
671 sym::amdgpu_dispatch_ptr => {
672 let val = self.call_intrinsic("llvm.amdgcn.dispatch.ptr", &[], &[]);
673 self.pointercast(val, self.type_ptr())
675 }
676
677 sym::sve_tuple_create2 => {
678 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
679 self.layout_of(fn_args.type_at(0)).backend_repr,
680 BackendRepr::SimdScalableVector {
681 number_of_vectors: NumScalableVectors(1),
682 ..
683 }
684 );
685 let tuple_ty = self.layout_of(fn_args.type_at(1));
686 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
687 tuple_ty.backend_repr,
688 BackendRepr::SimdScalableVector {
689 number_of_vectors: NumScalableVectors(2),
690 ..
691 }
692 );
693 let ret = self.const_poison(self.backend_type(tuple_ty));
694 let ret = self.insert_value(ret, args[0].immediate(), 0);
695 self.insert_value(ret, args[1].immediate(), 1)
696 }
697
698 sym::sve_tuple_create3 => {
699 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
700 self.layout_of(fn_args.type_at(0)).backend_repr,
701 BackendRepr::SimdScalableVector {
702 number_of_vectors: NumScalableVectors(1),
703 ..
704 }
705 );
706 let tuple_ty = self.layout_of(fn_args.type_at(1));
707 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(3), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(3), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
708 tuple_ty.backend_repr,
709 BackendRepr::SimdScalableVector {
710 number_of_vectors: NumScalableVectors(3),
711 ..
712 }
713 );
714 let ret = self.const_poison(self.backend_type(tuple_ty));
715 let ret = self.insert_value(ret, args[0].immediate(), 0);
716 let ret = self.insert_value(ret, args[1].immediate(), 1);
717 self.insert_value(ret, args[2].immediate(), 2)
718 }
719
720 sym::sve_tuple_create4 => {
721 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
722 self.layout_of(fn_args.type_at(0)).backend_repr,
723 BackendRepr::SimdScalableVector {
724 number_of_vectors: NumScalableVectors(1),
725 ..
726 }
727 );
728 let tuple_ty = self.layout_of(fn_args.type_at(1));
729 {
match tuple_ty.backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(4), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(4), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
730 tuple_ty.backend_repr,
731 BackendRepr::SimdScalableVector {
732 number_of_vectors: NumScalableVectors(4),
733 ..
734 }
735 );
736 let ret = self.const_poison(self.backend_type(tuple_ty));
737 let ret = self.insert_value(ret, args[0].immediate(), 0);
738 let ret = self.insert_value(ret, args[1].immediate(), 1);
739 let ret = self.insert_value(ret, args[2].immediate(), 2);
740 self.insert_value(ret, args[3].immediate(), 3)
741 }
742
743 sym::sve_tuple_get => {
744 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
745 self.layout_of(fn_args.type_at(0)).backend_repr,
746 BackendRepr::SimdScalableVector {
747 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
748 ..
749 }
750 );
751 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
752 self.layout_of(fn_args.type_at(1)).backend_repr,
753 BackendRepr::SimdScalableVector {
754 number_of_vectors: NumScalableVectors(1),
755 ..
756 }
757 );
758 self.extract_value(
759 args[0].immediate(),
760 fn_args.const_at(2).to_leaf().to_i32() as u64,
761 )
762 }
763
764 sym::sve_tuple_set => {
765 {
match self.layout_of(fn_args.type_at(0)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
.. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
766 self.layout_of(fn_args.type_at(0)).backend_repr,
767 BackendRepr::SimdScalableVector {
768 number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
769 ..
770 }
771 );
772 {
match self.layout_of(fn_args.type_at(1)).backend_repr {
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1), .. } => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"BackendRepr::SimdScalableVector\n{ number_of_vectors: NumScalableVectors(1), .. }",
::core::option::Option::None);
}
}
};assert_matches!(
773 self.layout_of(fn_args.type_at(1)).backend_repr,
774 BackendRepr::SimdScalableVector {
775 number_of_vectors: NumScalableVectors(1),
776 ..
777 }
778 );
779 self.insert_value(
780 args[0].immediate(),
781 args[1].immediate(),
782 fn_args.const_at(2).to_leaf().to_i32() as u64,
783 )
784 }
785
786 _ if name.as_str().starts_with("simd_") => {
787 let mut loaded_args = Vec::new();
790 for arg in args {
791 loaded_args.push(
792 if arg.layout.ty.is_simd()
797 && let OperandValue::Ref(place) = arg.val
798 {
799 let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
800 let elem_ll_ty = match elem_ty.kind() {
801 ty::Float(f) => self.type_float_from_ty(*f),
802 ty::Int(i) => self.type_int_from_ty(*i),
803 ty::Uint(u) => self.type_uint_from_ty(*u),
804 ty::RawPtr(_, _) => self.type_ptr(),
805 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
806 };
807 let loaded =
808 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
809 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
810 } else {
811 *arg
812 },
813 );
814 }
815
816 let llret_ty = if result_layout.ty.is_simd()
817 && let BackendRepr::Memory { .. } = result_layout.backend_repr
818 {
819 let (size, elem_ty) = result_layout.ty.simd_size_and_type(self.tcx());
820 let elem_ll_ty = match elem_ty.kind() {
821 ty::Float(f) => self.type_float_from_ty(*f),
822 ty::Int(i) => self.type_int_from_ty(*i),
823 ty::Uint(u) => self.type_uint_from_ty(*u),
824 ty::RawPtr(_, _) => self.type_ptr(),
825 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
826 };
827 self.type_vector(elem_ll_ty, size)
828 } else {
829 result_layout.llvm_type(self)
830 };
831
832 match generic_simd_intrinsic(
833 self,
834 name,
835 fn_args,
836 &loaded_args,
837 result_layout.ty,
838 llret_ty,
839 span,
840 ) {
841 Ok(llval) => llval,
842 Err(err) => return IntrinsicResult::Err(err),
845 }
846 }
847
848 sym::return_address => {
849 match self.sess().target.arch {
850 Arch::Wasm32 | Arch::Wasm64 => {
852 let ty = self.type_ptr();
853 self.const_null(ty)
854 }
855 _ => {
856 let ty = self.type_ix(32);
857 let val = self.const_int(ty, 0);
858
859 let type_params: &[&'ll Type] =
860 if llvm_version < (23, 0, 0) { &[] } else { &[self.type_ptr()] };
861
862 self.call_intrinsic("llvm.returnaddress", type_params, &[val])
863 }
864 }
865 }
866
867 sym::codeview_annotation => {
868 if self.sess().target.uses_pdb_debuginfo() {
869 codegen_codeview_annotation(self, instance, span);
870 }
871 return IntrinsicResult::Operand(OperandValue::ZeroSized);
872 }
873
874 _ => {
875 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs:875",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(875u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("unknown intrinsic \'{0}\' -- falling back to default body",
name) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("unknown intrinsic '{}' -- falling back to default body", name);
876 let fallback = ty::Instance::new_raw(instance.def_id(), instance.args);
878 return IntrinsicResult::Fallback(fallback);
879 }
880 };
881
882 if let BackendRepr::Memory { .. } = result_layout.backend_repr {
883 if !result_layout.is_zst() {
886 self.store_to_place(llval, result_place.unwrap());
887 }
888 IntrinsicResult::WroteIntoPlace
889 } else {
890 IntrinsicResult::Operand(
891 OperandRef::from_immediate_or_packed_pair(self, llval, result_layout).val,
892 )
893 }
894 }
895
896 fn codegen_llvm_intrinsic_call(
897 &mut self,
898 instance: ty::Instance<'tcx>,
899 args: &[OperandRef<'tcx, Self::Value>],
900 _is_cleanup: bool,
901 ) -> Self::Value {
902 let tcx = self.tcx();
903
904 let fn_ty = instance.ty(tcx, self.typing_env());
905 let fn_sig = match *fn_ty.kind() {
906 ty::FnDef(def_id, args) => tcx.instantiate_bound_regions_with_erased(
907 tcx.fn_sig(def_id).instantiate(tcx, args.no_bound_vars().unwrap()).skip_norm_wip(),
908 ),
909 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
910 };
911 if !!fn_sig.c_variadic() {
::core::panicking::panic("assertion failed: !fn_sig.c_variadic()")
};assert!(!fn_sig.c_variadic());
912
913 let ret_layout = self.layout_of(fn_sig.output());
914 let llreturn_ty = if ret_layout.is_zst() {
915 self.type_void()
916 } else {
917 ret_layout.immediate_llvm_type(self)
918 };
919
920 let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
921 for &arg in fn_sig.inputs() {
922 let arg_layout = self.layout_of(arg);
923 if arg_layout.is_zst() {
924 continue;
925 }
926 llargument_tys.push(arg_layout.immediate_llvm_type(self));
927 }
928
929 let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) {
930 llfn
931 } else {
932 let sym = tcx.symbol_name(instance).name;
933
934 let llfn = if let Some(llfn) = self.get_declared_value(sym) {
935 llfn
936 } else {
937 intrinsic_fn(self, sym, llreturn_ty, llargument_tys, instance)
938 };
939
940 self.intrinsic_instances.borrow_mut().insert(instance, llfn);
941
942 llfn
943 };
944 let fn_ty = self.get_type_of_global(fn_ptr);
945
946 let mut llargs = args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>();
947
948 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs:948",
"rustc_codegen_llvm::intrinsic", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/intrinsic.rs"),
::tracing_core::__macro_support::Option::Some(948u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::intrinsic"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call intrinsic {0:?} with args ({1:?})",
instance, llargs) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("call intrinsic {:?} with args ({:?})", instance, llargs);
949
950 for (dest_ty, arg) in iter::zip(self.func_params_types(fn_ty), &mut llargs) {
951 let src_ty = self.val_ty(arg);
952 if !can_autocast(self, src_ty, dest_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with {1:?} (found) in `{2:?}",
dest_ty, src_ty, fn_ptr));
}
};assert!(
953 can_autocast(self, src_ty, dest_ty),
954 "Cannot match `{dest_ty:?}` (expected) with {src_ty:?} (found) in `{fn_ptr:?}"
955 );
956
957 *arg = autocast(self, arg, src_ty, dest_ty);
958 }
959
960 let llret = unsafe {
961 llvm::LLVMBuildCallWithOperandBundles(
962 self.llbuilder(),
963 fn_ty,
964 fn_ptr,
965 llargs.as_ptr(),
966 llargs.len() as c_uint,
967 ptr::dangling(),
968 0,
969 c"".as_ptr(),
970 )
971 };
972
973 let src_ty = self.val_ty(llret);
974 let dest_ty = llreturn_ty;
975 if !can_autocast(self, dest_ty, src_ty) {
{
::core::panicking::panic_fmt(format_args!("Cannot match `{0:?}` (expected) with `{1:?}` (found) in `{2:?}`",
src_ty, dest_ty, fn_ptr));
}
};assert!(
976 can_autocast(self, dest_ty, src_ty),
977 "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{fn_ptr:?}`"
978 );
979
980 autocast(self, llret, src_ty, dest_ty)
981 }
982
983 fn abort_immediate(&mut self) {
984 self.call_intrinsic("llvm.trap", &[], &[]);
985 }
986
987 fn assume(&mut self, val: Self::Value) {
988 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
989 self.call_intrinsic("llvm.assume", &[], &[val]);
990 }
991 }
992
993 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
994 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
995 self.call_intrinsic(
996 "llvm.expect",
997 &[self.type_i1()],
998 &[cond, self.const_bool(expected)],
999 )
1000 } else {
1001 cond
1002 }
1003 }
1004
1005 fn ctpop(&mut self, val: Self::Value) -> Self::Value {
1006 let ret = self.call_intrinsic("llvm.ctpop", &[self.val_ty(val)], &[val]);
1007 self.intcast(ret, self.type_i32(), false)
1008 }
1009
1010 fn type_checked_load(
1011 &mut self,
1012 llvtable: &'ll Value,
1013 vtable_byte_offset: u64,
1014 typeid: &[u8],
1015 ) -> Self::Value {
1016 let typeid = self.create_metadata(typeid);
1017 let typeid = self.get_metadata_value(typeid);
1018 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
1019 let type_checked_load = self.call_intrinsic(
1020 "llvm.type.checked.load",
1021 &[],
1022 &[llvtable, vtable_byte_offset, typeid],
1023 );
1024 self.extract_value(type_checked_load, 0)
1025 }
1026
1027 fn va_start(&mut self, va_list: &'ll Value) {
1028 self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list]);
1029 }
1030
1031 fn retag_reg(&mut self, ptr: Self::Value, info: &RetagInfo<Self::Value>) -> Self::Value {
1032 codegen_retag_inner(self, "__rust_retag_reg", ptr, info)
1033 }
1034
1035 fn retag_mem(&mut self, ptr: Self::Value, info: &RetagInfo<Self::Value>) {
1036 codegen_retag_inner(self, "__rust_retag_mem", ptr, info);
1037 }
1038}
1039
1040fn llvm_arch_for(rust_arch: &Arch) -> Option<&'static str> {
1041 Some(match rust_arch {
1042 Arch::AArch64 | Arch::Arm64EC => "aarch64",
1043 Arch::AmdGpu => "amdgcn",
1044 Arch::Arm => "arm",
1045 Arch::Bpf => "bpf",
1046 Arch::Hexagon => "hexagon",
1047 Arch::LoongArch32 | Arch::LoongArch64 => "loongarch",
1048 Arch::Mips | Arch::Mips32r6 | Arch::Mips64 | Arch::Mips64r6 => "mips",
1049 Arch::Nvptx64 => "nvvm",
1050 Arch::PowerPC | Arch::PowerPC64 => "ppc",
1051 Arch::RiscV32 | Arch::RiscV64 => "riscv",
1052 Arch::S390x => "s390",
1053 Arch::SpirV => "spv",
1054 Arch::Wasm32 | Arch::Wasm64 => "wasm",
1055 Arch::X86 | Arch::X86_64 => "x86",
1056 _ => return None, })
1058}
1059
1060fn can_autocast<'ll>(cx: &CodegenCx<'ll, '_>, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
1061 if rust_ty == llvm_ty {
1062 return true;
1063 }
1064
1065 match cx.type_kind(llvm_ty) {
1066 TypeKind::Struct if cx.type_kind(rust_ty) == TypeKind::Struct => {
1070 let rust_element_tys = cx.struct_element_types(rust_ty);
1071 let llvm_element_tys = cx.struct_element_types(llvm_ty);
1072
1073 if rust_element_tys.len() != llvm_element_tys.len() {
1074 return false;
1075 }
1076
1077 iter::zip(rust_element_tys, llvm_element_tys).all(
1078 |(rust_element_ty, llvm_element_ty)| {
1079 can_autocast(cx, rust_element_ty, llvm_element_ty)
1080 },
1081 )
1082 }
1083 _ if let Some((llvm_element_ty, element_count)) =
1084 cx.vector_element_type_and_length(llvm_ty) =>
1085 {
1086 if llvm_element_ty == cx.type_bf16() {
1087 rust_ty == cx.type_vector(cx.type_i16(), element_count)
1088 } else if llvm_element_ty == cx.type_i1() {
1089 let int_width = element_count.next_power_of_two().max(8);
1090 rust_ty == cx.type_ix(int_width)
1091 } else {
1092 false
1093 }
1094 }
1095 TypeKind::BFloat => rust_ty == cx.type_i16(),
1096 TypeKind::X86_AMX
1097 if let Some((element_ty, element_count)) =
1098 cx.vector_element_type_and_length(rust_ty) =>
1099 {
1100 let element_size_bits = if let Some(float) = cx.float_type(element_ty) {
1101 float.size().bits()
1102 } else if let Some(bits) = cx.int_width(element_ty) {
1103 bits
1104 } else if cx.type_kind(element_ty) == TypeKind::Pointer {
1105 cx.data_layout().pointer_size().bits()
1106 } else {
1107 ::rustc_span::macros::bug_impl(None,
format_args!("Vector element type `{0:?}` not one of integer, float or pointer",
element_ty), Location::caller())bug!("Vector element type `{element_ty:?}` not one of integer, float or pointer")
1108 };
1109
1110 element_size_bits * element_count == 8192
1111 }
1112 _ => false,
1113 }
1114}
1115
1116fn autocast<'ll>(
1117 bx: &mut Builder<'_, 'll, '_>,
1118 val: &'ll Value,
1119 src_ty: &'ll Type,
1120 dest_ty: &'ll Type,
1121) -> &'ll Value {
1122 if src_ty == dest_ty {
1123 return val;
1124 }
1125 match (bx.type_kind(src_ty), bx.type_kind(dest_ty)) {
1126 (TypeKind::Struct, TypeKind::Struct) => {
1128 let mut ret = bx.const_poison(dest_ty);
1129 for (idx, (src_element_ty, dest_element_ty)) in
1130 iter::zip(bx.struct_element_types(src_ty), bx.struct_element_types(dest_ty))
1131 .enumerate()
1132 {
1133 let elt = bx.extract_value(val, idx as u64);
1134 let casted_elt = autocast(bx, elt, src_element_ty, dest_element_ty);
1135 ret = bx.insert_value(ret, casted_elt, idx as u64);
1136 }
1137 ret
1138 }
1139 (TypeKind::Vector, TypeKind::Integer)
1141 if let Some((elem, vector_length)) = bx.vector_element_type_and_length(src_ty)
1142 && elem == bx.type_i1() =>
1143 {
1144 let int_width = vector_length.next_power_of_two().max(8);
1145
1146 let val = if vector_length == int_width {
1147 val
1148 } else {
1149 let shuffle_indices = match vector_length {
1151 0 => {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("zero length vectors are not allowed")));
}unreachable!("zero length vectors are not allowed"),
1152 1 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 1, 1, 1, 1, 1, 1]))vec![0, 1, 1, 1, 1, 1, 1, 1],
1153 2 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 2, 2, 2, 2, 2]))vec![0, 1, 2, 2, 2, 2, 2, 2],
1154 3 => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[0, 1, 2, 3, 3, 3, 3, 3]))vec![0, 1, 2, 3, 3, 3, 3, 3],
1155 4.. => (0..int_width as i32).collect(),
1156 };
1157 let shuffle_mask =
1158 shuffle_indices.into_iter().map(|i| bx.const_i32(i)).collect::<Vec<_>>();
1159 bx.shuffle_vector(val, bx.const_null(src_ty), bx.const_vector(&shuffle_mask))
1160 };
1161 bx.bitcast(val, dest_ty)
1162 }
1163 (TypeKind::Integer, TypeKind::Vector)
1165 if let Some((elem, vector_length)) = bx.vector_element_type_and_length(dest_ty)
1166 && elem == bx.type_i1() =>
1167 {
1168 let int_width = vector_length.next_power_of_two().max(8);
1169
1170 let intermediate_ty = bx.type_vector(bx.type_i1(), int_width);
1171 let intermediate = bx.bitcast(val, intermediate_ty);
1172
1173 if vector_length == int_width {
1174 intermediate
1175 } else {
1176 let shuffle_mask: Vec<_> =
1177 (0..vector_length).map(|i| bx.const_i32(i as i32)).collect();
1178 bx.shuffle_vector(
1179 intermediate,
1180 bx.const_poison(intermediate_ty),
1181 bx.const_vector(&shuffle_mask),
1182 )
1183 }
1184 }
1185 (TypeKind::Vector, TypeKind::X86_AMX) => {
1186 bx.call_intrinsic("llvm.x86.cast.vector.to.tile", &[src_ty], &[val])
1187 }
1188 (TypeKind::X86_AMX, TypeKind::Vector) => {
1189 bx.call_intrinsic("llvm.x86.cast.tile.to.vector", &[dest_ty], &[val])
1190 }
1191 _ => bx.bitcast(val, dest_ty), }
1193}
1194
1195fn intrinsic_fn<'ll, 'tcx>(
1196 bx: &Builder<'_, 'll, 'tcx>,
1197 name: &str,
1198 rust_return_ty: &'ll Type,
1199 rust_argument_tys: Vec<&'ll Type>,
1200 instance: ty::Instance<'tcx>,
1201) -> &'ll Value {
1202 let tcx = bx.tcx;
1203
1204 let rust_fn_ty = bx.type_func(&rust_argument_tys, rust_return_ty);
1205
1206 let intrinsic = llvm::Intrinsic::lookup(name.as_bytes());
1207
1208 if let Some(intrinsic) = intrinsic
1209 && intrinsic.is_target_specific()
1210 {
1211 let (llvm_arch, _) = name[5..].split_once('.').unwrap();
1212 let rust_arch = &tcx.sess.target.arch;
1213
1214 if let Some(correct_llvm_arch) = llvm_arch_for(rust_arch)
1215 && llvm_arch != correct_llvm_arch
1216 {
1217 tcx.dcx().emit_fatal(IntrinsicWrongArch {
1218 name,
1219 target_arch: rust_arch.desc(),
1220 span: tcx.def_span(instance.def_id()),
1221 });
1222 }
1223 }
1224
1225 if let Some(intrinsic) = intrinsic
1226 && !intrinsic.is_overloaded()
1227 {
1228 let llfn = intrinsic.get_declaration(bx.llmod, &[]);
1230 let llvm_fn_ty = bx.get_type_of_global(llfn);
1231
1232 let llvm_return_ty = bx.get_return_type(llvm_fn_ty);
1233 let llvm_argument_tys = bx.func_params_types(llvm_fn_ty);
1234 let llvm_is_variadic = bx.func_is_variadic(llvm_fn_ty);
1235
1236 let is_correct_signature = !llvm_is_variadic
1237 && rust_argument_tys.len() == llvm_argument_tys.len()
1238 && iter::once((rust_return_ty, llvm_return_ty))
1239 .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
1240 .all(|(rust_ty, llvm_ty)| can_autocast(bx, rust_ty, llvm_ty));
1241
1242 if !is_correct_signature {
1243 tcx.dcx().emit_fatal(IntrinsicSignatureMismatch {
1244 name,
1245 llvm_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", llvm_fn_ty))
})format!("{llvm_fn_ty:?}"),
1246 rust_fn_ty: &::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0:?}", rust_fn_ty))
})format!("{rust_fn_ty:?}"),
1247 span: tcx.def_span(instance.def_id()),
1248 });
1249 }
1250
1251 return llfn;
1252 }
1253
1254 let llfn = declare_raw_fn(
1256 bx,
1257 name,
1258 llvm::CCallConv,
1259 llvm::UnnamedAddr::Global,
1260 llvm::Visibility::Default,
1261 rust_fn_ty,
1262 );
1263
1264 if intrinsic.is_none() {
1265 let mut new_llfn = None;
1266 let can_upgrade = unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn) };
1267
1268 if !can_upgrade {
1269 tcx.dcx().emit_fatal(UnknownIntrinsic { name, span: tcx.def_span(instance.def_id()) });
1271 } else if let Some(def_id) = instance.def_id().as_local() {
1272 let hir_id = tcx.local_def_id_to_hir_id(def_id);
1274
1275 let msg = if let Some(new_llfn) = new_llfn {
1277 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{1}`, `{0}` can be used instead",
str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap(),
name))
})format!(
1278 "using deprecated intrinsic `{name}`, `{}` can be used instead",
1279 str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap()
1280 )
1281 } else {
1282 ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("using deprecated intrinsic `{0}`",
name))
})format!("using deprecated intrinsic `{name}`")
1283 };
1284
1285 tcx.emit_node_lint(
1286 DEPRECATED_LLVM_INTRINSIC,
1287 hir_id,
1288 rustc_errors::DiagDecorator(|d| {
1289 d.primary_message(msg).span(tcx.hir_span(hir_id));
1290 }),
1291 );
1292 }
1293 }
1294
1295 llfn
1296}
1297
1298fn catch_unwind_intrinsic<'ll, 'tcx>(
1299 bx: &mut Builder<'_, 'll, 'tcx>,
1300 try_func: &'ll Value,
1301 data: &'ll Value,
1302 catch_func: &'ll Value,
1303) -> &'ll Value {
1304 if !bx.sess().panic_strategy().unwinds() {
1305 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1306 bx.call(try_func_ty, None, None, try_func, ReturnSlot::Direct, &[data], None, None);
1307 bx.const_bool(false)
1310 } else if wants_msvc_seh(&bx.sess().target) {
1311 codegen_msvc_try(bx, try_func, data, catch_func)
1312 } else if wants_wasm_eh(&bx.sess().target) {
1313 codegen_wasm_try(bx, try_func, data, catch_func)
1314 } else {
1315 codegen_gnu_try(bx, try_func, data, catch_func)
1316 }
1317}
1318
1319fn codegen_msvc_try<'ll, 'tcx>(
1327 bx: &mut Builder<'_, 'll, 'tcx>,
1328 try_func: &'ll Value,
1329 data: &'ll Value,
1330 catch_func: &'ll Value,
1331) -> &'ll Value {
1332 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1333 bx.set_personality_fn(bx.eh_personality());
1334
1335 let normal = bx.append_sibling_block("normal");
1336 let catchswitch = bx.append_sibling_block("catchswitch");
1337 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
1338 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
1339 let caught = bx.append_sibling_block("caught");
1340
1341 let try_func = llvm::get_param(bx.llfn(), 0);
1342 let data = llvm::get_param(bx.llfn(), 1);
1343 let catch_func = llvm::get_param(bx.llfn(), 2);
1344
1345 let ptr_size = bx.tcx().data_layout.pointer_size();
1401 let ptr_align = bx.tcx().data_layout.pointer_align();
1402 let slot = bx.alloca(ptr_size, ptr_align);
1403 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1404 bx.invoke(
1405 try_func_ty,
1406 None,
1407 None,
1408 try_func,
1409 ReturnSlot::Direct,
1410 &[data],
1411 normal,
1412 catchswitch,
1413 None,
1414 None,
1415 );
1416
1417 bx.switch_to_block(normal);
1418 bx.ret(bx.const_bool(false));
1419
1420 bx.switch_to_block(catchswitch);
1421 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
1422
1423 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
1438 let type_name = bx.const_bytes(b"rust_panic\0");
1439 let type_info =
1440 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
1441 let tydesc = bx.declare_global(
1442 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
1443 bx.val_ty(type_info),
1444 );
1445
1446 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
1447 if bx.cx.tcx.sess.target.supports_comdat() {
1448 llvm::SetUniqueComdat(bx.llmod, tydesc);
1449 }
1450 llvm::set_initializer(tydesc, type_info);
1451
1452 bx.switch_to_block(catchpad_rust);
1459 let flags = bx.const_i32(8);
1460 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
1461 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
1462 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1463 bx.call(
1464 catch_ty,
1465 None,
1466 None,
1467 catch_func,
1468 ReturnSlot::Direct,
1469 &[data, ptr],
1470 Some(&funclet),
1471 None,
1472 );
1473 bx.catch_ret(&funclet, caught);
1474
1475 bx.switch_to_block(catchpad_foreign);
1477 let flags = bx.const_i32(64);
1478 let null = bx.const_null(bx.type_ptr());
1479 let funclet = bx.catch_pad(cs, &[null, flags, null]);
1480 bx.call(
1481 catch_ty,
1482 None,
1483 None,
1484 catch_func,
1485 ReturnSlot::Direct,
1486 &[data, null],
1487 Some(&funclet),
1488 None,
1489 );
1490 bx.catch_ret(&funclet, caught);
1491
1492 bx.switch_to_block(caught);
1493 bx.ret(bx.const_bool(true));
1494 });
1495
1496 let ret = bx.call(
1499 llty,
1500 None,
1501 None,
1502 llfn,
1503 ReturnSlot::Direct,
1504 &[try_func, data, catch_func],
1505 None,
1506 None,
1507 );
1508 ret
1509}
1510
1511fn codegen_wasm_try<'ll, 'tcx>(
1513 bx: &mut Builder<'_, 'll, 'tcx>,
1514 try_func: &'ll Value,
1515 data: &'ll Value,
1516 catch_func: &'ll Value,
1517) -> &'ll Value {
1518 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1519 bx.set_personality_fn(bx.eh_personality());
1520
1521 let normal = bx.append_sibling_block("normal");
1522 let catchswitch = bx.append_sibling_block("catchswitch");
1523 let catchpad = bx.append_sibling_block("catchpad");
1524 let caught = bx.append_sibling_block("caught");
1525
1526 let try_func = llvm::get_param(bx.llfn(), 0);
1527 let data = llvm::get_param(bx.llfn(), 1);
1528 let catch_func = llvm::get_param(bx.llfn(), 2);
1529
1530 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1554 bx.invoke(
1555 try_func_ty,
1556 None,
1557 None,
1558 try_func,
1559 ReturnSlot::Direct,
1560 &[data],
1561 normal,
1562 catchswitch,
1563 None,
1564 None,
1565 );
1566
1567 bx.switch_to_block(normal);
1568 bx.ret(bx.const_bool(false));
1569
1570 bx.switch_to_block(catchswitch);
1571 let cs = bx.catch_switch(None, None, &[catchpad]);
1572
1573 bx.switch_to_block(catchpad);
1574 let null = bx.const_null(bx.type_ptr());
1575 let funclet = bx.catch_pad(cs, &[null]);
1576
1577 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
1578 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
1579
1580 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1581 bx.call(
1582 catch_ty,
1583 None,
1584 None,
1585 catch_func,
1586 ReturnSlot::Direct,
1587 &[data, ptr],
1588 Some(&funclet),
1589 None,
1590 );
1591 bx.catch_ret(&funclet, caught);
1592
1593 bx.switch_to_block(caught);
1594 bx.ret(bx.const_bool(true));
1595 });
1596
1597 let ret = bx.call(
1600 llty,
1601 None,
1602 None,
1603 llfn,
1604 ReturnSlot::Direct,
1605 &[try_func, data, catch_func],
1606 None,
1607 None,
1608 );
1609 ret
1610}
1611
1612fn codegen_gnu_try<'ll, 'tcx>(
1624 bx: &mut Builder<'_, 'll, 'tcx>,
1625 try_func: &'ll Value,
1626 data: &'ll Value,
1627 catch_func: &'ll Value,
1628) -> &'ll Value {
1629 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
1630 let then = bx.append_sibling_block("then");
1643 let catch = bx.append_sibling_block("catch");
1644
1645 let try_func = llvm::get_param(bx.llfn(), 0);
1646 let data = llvm::get_param(bx.llfn(), 1);
1647 let catch_func = llvm::get_param(bx.llfn(), 2);
1648 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1649 bx.invoke(
1650 try_func_ty,
1651 None,
1652 None,
1653 try_func,
1654 ReturnSlot::Direct,
1655 &[data],
1656 then,
1657 catch,
1658 None,
1659 None,
1660 );
1661
1662 bx.switch_to_block(then);
1663 bx.ret(bx.const_bool(false));
1664
1665 bx.switch_to_block(catch);
1672 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()]);
1673 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
1674 let tydesc = bx.const_null(bx.type_ptr());
1675 bx.add_clause(vals, tydesc);
1676 let ptr = bx.extract_value(vals, 0);
1677 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1678 bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None);
1679 bx.ret(bx.const_bool(true));
1680 });
1681
1682 let ret = bx.call(
1685 llty,
1686 None,
1687 None,
1688 llfn,
1689 ReturnSlot::Direct,
1690 &[try_func, data, catch_func],
1691 None,
1692 None,
1693 );
1694 ret
1695}
1696
1697fn gen_fn<'a, 'll, 'tcx>(
1700 cx: &'a CodegenCx<'ll, 'tcx>,
1701 name: &str,
1702 rust_fn_sig: ty::PolyFnSig<'tcx>,
1703 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1704) -> (&'ll Type, &'ll Value) {
1705 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1706 let llty = fn_abi.llvm_type(cx);
1707 let llfn = cx.declare_fn(name, fn_abi, None);
1708 cx.set_frame_pointer_type(llfn);
1709 cx.apply_target_cpu_attr(llfn);
1710 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1712 let llbb = Builder::append_block(cx, llfn, "entry-block");
1713 let bx = Builder::build(cx, llbb);
1714 codegen(bx);
1715 (llty, llfn)
1716}
1717
1718fn get_rust_try_fn<'a, 'll, 'tcx>(
1723 cx: &'a CodegenCx<'ll, 'tcx>,
1724 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1725) -> (&'ll Type, &'ll Value) {
1726 if let Some(llfn) = cx.rust_try_fn.get() {
1727 return llfn;
1728 }
1729
1730 let tcx = cx.tcx;
1732 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1733 let try_fn_ty = Ty::new_fn_ptr(
1735 tcx,
1736 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p], tcx.types.unit, hir::Safety::Unsafe)),
1737 );
1738 let catch_fn_ty = Ty::new_fn_ptr(
1740 tcx,
1741 ty::Binder::dummy(tcx.mk_fn_sig_rust_abi([i8p, i8p], tcx.types.unit, hir::Safety::Unsafe)),
1742 );
1743 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig_rust_abi(
1745 [try_fn_ty, i8p, catch_fn_ty],
1746 tcx.types.bool,
1747 hir::Safety::Unsafe,
1748 ));
1749 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1750
1751 if cx.sess().pointer_authentication() {
1752 let cfg = cx.sess().pointer_auth_config.as_ref().unwrap();
1753 let attrs: Vec<&Attribute> =
1754 cfg.fn_attrs().into_iter().map(|name| llvm::CreateAttrString(cx.llcx, name)).collect();
1755
1756 let (_ty, rust_try_fn) = rust_try;
1757 crate::attributes::apply_to_llfn(rust_try_fn, AttributePlace::Function, &attrs);
1758 }
1759
1760 cx.rust_try_fn.set(Some(rust_try));
1761 rust_try
1762}
1763
1764fn codegen_retag_inner<'ll, 'tcx>(
1765 bx: &mut Builder<'_, 'll, 'tcx>,
1766 name: &'static str,
1767 ptr: &'ll Value,
1768 info: &RetagInfo<&'ll Value>,
1769) -> &'ll Value {
1770 let size = bx.const_usize(info.size.bytes());
1771 let perms = bx.const_u8(info.flags.bits());
1772
1773 bx.call_intrinsic(
1774 name,
1775 &[bx.type_ptr(), bx.val_ty(size), bx.type_i8(), bx.type_ptr(), bx.type_ptr()],
1778 &[ptr, size, perms, info.im_layout, info.pin_layout],
1779 )
1780}
1781
1782fn codegen_autodiff<'ll, 'tcx>(
1783 bx: &mut Builder<'_, 'll, 'tcx>,
1784 instance: ty::Instance<'tcx>,
1785 args: &[OperandRef<'tcx, &'ll Value>],
1786 result_layout: ty::layout::TyAndLayout<'tcx>,
1787 result_place: Option<PlaceValue<&'ll Value>>,
1788) -> IntrinsicResult<'tcx, &'ll Value> {
1789 let tcx = bx.tcx;
1790 if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
1791 let _ = tcx.dcx().emit_err(AutoDiffWithoutEnable);
1792 }
1793
1794 let ct = tcx.crate_types();
1795 let lto = tcx.sess.lto();
1796 if ct.len() == 1 && ct.contains(&CrateType::Executable) {
1797 if lto != rustc_session::config::Lto::Fat {
1798 let _ = tcx.dcx().emit_err(AutoDiffWithoutLto);
1799 }
1800 } else {
1801 if lto != rustc_session::config::Lto::Fat && !tcx.sess.opts.cg.linker_plugin_lto.enabled() {
1802 let _ = tcx.dcx().emit_err(AutoDiffWithoutLto);
1803 }
1804 }
1805
1806 let fn_args = instance.args;
1807 let callee_ty = instance.ty(tcx, bx.typing_env());
1808
1809 let sig = callee_ty.fn_sig(tcx).skip_binder();
1810
1811 let ret_ty = sig.output();
1812 let llret_ty = bx.layout_of(ret_ty).llvm_type(bx);
1813
1814 let source_fn_ptr_ty = fn_args.into_type_list(tcx)[0];
1815 let fn_to_diff = args[0].immediate();
1816
1817 let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() {
1818 ty::FnDef(def_id, diff_args) => (def_id, diff_args.no_bound_vars().unwrap()),
1819 _ => ::rustc_span::macros::bug_impl(None, format_args!("invalid args"),
Location::caller())bug!("invalid args"),
1820 };
1821
1822 let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) {
1823 Ok(Some(instance)) => instance,
1824 Ok(None) => ::rustc_span::macros::bug_impl(None,
format_args!("could not resolve ({0:?}, {1:?}) to a specific autodiff instance",
diff_id, diff_args), Location::caller())bug!(
1825 "could not resolve ({:?}, {:?}) to a specific autodiff instance",
1826 diff_id,
1827 diff_args
1828 ),
1829 Err(err) => {
1830 return IntrinsicResult::Err(err);
1832 }
1833 };
1834
1835 let val_arr = get_args_from_tuple(bx, args[2], fn_diff);
1836 let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE);
1837
1838 let Some(Some(mut diff_attrs)) =
1839 {
{
'done:
{
for i in
::rustc_attr_ir::HasAttrs::get_attrs(fn_diff.def_id(), &tcx) {
#[allow(unused_imports)]
use ::rustc_attr_ir::AttributeKind::*;
let i: &::rustc_attr_ir::Attribute = i;
match i {
::rustc_attr_ir::Attribute::Parsed(RustcAutodiff(attr)) => {
break 'done Some(attr.clone());
}
::rustc_attr_ir::Attribute::Tool(..) |
::rustc_attr_ir::Attribute::DeriveHelper(..) =>
{}
#[deny(unreachable_patterns)]
_ => {}
}
}
None
}
}
}find_attr!(tcx, fn_diff.def_id(), RustcAutodiff(attr) => attr.clone())
1840 else {
1841 ::rustc_span::macros::bug_impl(None,
format_args!("could not find autodiff attrs"), Location::caller())bug!("could not find autodiff attrs")
1842 };
1843
1844 adjust_activity_to_abi(
1845 tcx,
1846 source_fn_ptr_ty,
1847 bx.cx.typing_env(),
1848 &mut diff_attrs.input_activity,
1849 );
1850
1851 let fnc_tree = fnc_typetrees(tcx, source_fn_ptr_ty);
1852
1853 generate_enzyme_call(
1855 bx,
1856 fn_to_diff,
1857 &diff_symbol,
1858 llret_ty,
1859 &val_arr,
1860 &diff_attrs,
1861 result_layout,
1862 result_place,
1863 fnc_tree,
1864 )
1865}
1866
1867fn codegen_offload<'ll, 'tcx>(
1872 bx: &mut Builder<'_, 'll, 'tcx>,
1873 tcx: TyCtxt<'tcx>,
1874 instance: ty::Instance<'tcx>,
1875 args: &[OperandRef<'tcx, &'ll Value>],
1876) {
1877 let cx = bx.cx;
1878 let fn_args = instance.args;
1879
1880 let (target_id, target_args) = match fn_args.into_type_list(tcx)[0].kind() {
1881 ty::FnDef(def_id, params) => (def_id, params.no_bound_vars().unwrap()),
1882 _ => ::rustc_span::macros::bug_impl(None,
format_args!("invalid offload intrinsic arg"), Location::caller())bug!("invalid offload intrinsic arg"),
1883 };
1884
1885 let fn_target = match Instance::try_resolve(tcx, cx.typing_env(), *target_id, target_args) {
1886 Ok(Some(instance)) => instance,
1887 Ok(None) => ::rustc_span::macros::bug_impl(None,
format_args!("could not resolve ({0:?}, {1:?}) to a specific offload instance",
target_id, target_args), Location::caller())bug!(
1888 "could not resolve ({:?}, {:?}) to a specific offload instance",
1889 target_id,
1890 target_args
1891 ),
1892 Err(_) => {
1893 return;
1895 }
1896 };
1897
1898 let offload_dims = OffloadKernelDims::from_operands(bx, &args[1], &args[2]);
1899 let dyn_cache = match args[3].val {
1900 OperandValue::Immediate(val) => val,
1901 _ => { ::core::panicking::panic_fmt(format_args!("unparsable")); }panic!("unparsable"),
1902 };
1903 let device_id = match args[4].val {
1904 OperandValue::Immediate(val) => val,
1905 _ => { ::core::panicking::panic_fmt(format_args!("unparsable")); }panic!("unparsable"),
1906 };
1907 let args = get_args_from_tuple(bx, args[5], fn_target);
1908 let target_symbol = mangle_offload_export(tcx, fn_target);
1909
1910 let sig = tcx.fn_sig(fn_target.def_id()).instantiate(tcx, fn_target.args).skip_norm_wip();
1911 let sig = tcx.instantiate_bound_regions_with_erased(sig);
1912 let inputs = sig.inputs();
1913
1914 let fn_abi = cx.fn_abi_of_instance(fn_target, ty::List::empty());
1915
1916 let mut metadata = Vec::new();
1917 let mut types = Vec::new();
1918
1919 for (i, arg_abi) in fn_abi.args.iter().enumerate() {
1920 let ty = inputs[i];
1921 let decomposed = OffloadMetadata::handle_abi(cx, tcx, ty, arg_abi);
1922
1923 for (meta, entry_ty) in decomposed {
1924 metadata.push(meta);
1925 types.push(bx.cx.layout_of(entry_ty).llvm_type(bx.cx));
1926 }
1927 }
1928
1929 let offload_globals_ref = cx.offload_globals.borrow();
1930 let offload_globals = match offload_globals_ref.as_ref() {
1931 Some(globals) => globals,
1932 None => {
1933 return;
1935 }
1936 };
1937 let offload_data =
1938 gpu_offload::gen_define_handling(&cx, &metadata, target_symbol, offload_globals);
1939 gpu_offload::gen_call_handling(
1940 bx,
1941 &offload_data,
1942 &args,
1943 &types,
1944 &metadata,
1945 offload_globals,
1946 &offload_dims,
1947 &dyn_cache,
1948 &device_id,
1949 );
1950}
1951
1952fn get_args_from_tuple<'ll, 'tcx>(
1953 bx: &mut Builder<'_, 'll, 'tcx>,
1954 tuple_op: OperandRef<'tcx, &'ll Value>,
1955 fn_instance: Instance<'tcx>,
1956) -> Vec<&'ll Value> {
1957 let cx = bx.cx;
1958 let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty());
1959
1960 match tuple_op.val {
1961 OperandValue::Immediate(val) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[val]))vec![val],
1962 OperandValue::Pair(v1, v2) => ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
[v1, v2]))vec![v1, v2],
1963 OperandValue::Ref(ptr) => {
1964 let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout };
1965
1966 let mut result = Vec::with_capacity(fn_abi.args.len());
1967 let mut tuple_index = 0;
1968
1969 for arg in &fn_abi.args {
1970 match arg.mode {
1971 PassMode::Ignore => {}
1972 PassMode::Direct(_) | PassMode::Cast { .. } => {
1973 let field = tuple_place.project_field(bx, tuple_index);
1974 let llvm_ty = field.layout.llvm_type(bx.cx);
1975 let val = bx.load(llvm_ty, field.val.llval, field.val.align);
1976 result.push(val);
1977 tuple_index += 1;
1978 }
1979 PassMode::Pair(_, _) => {
1980 let field = tuple_place.project_field(bx, tuple_index);
1981 let llvm_ty = field.layout.llvm_type(bx.cx);
1982 let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align);
1983 result.push(bx.extract_value(pair_val, 0));
1984 result.push(bx.extract_value(pair_val, 1));
1985 tuple_index += 1;
1986 }
1987 PassMode::Indirect { .. } => {
1988 let field = tuple_place.project_field(bx, tuple_index);
1989 result.push(field.val.llval);
1990 tuple_index += 1;
1991 }
1992 PassMode::IndirectUnsized { .. } => {
1993 ::rustc_span::macros::bug_impl(None,
format_args!("autodiff/offload args must not be unsized"),
Location::caller());bug!("autodiff/offload args must not be unsized");
1994 }
1995 }
1996 }
1997
1998 result
1999 }
2000
2001 OperandValue::ZeroSized => ::alloc::vec::Vec::new()vec![],
2002 }
2003}
2004
2005fn generic_simd_intrinsic<'ll, 'tcx>(
2006 bx: &mut Builder<'_, 'll, 'tcx>,
2007 name: Symbol,
2008 fn_args: GenericArgs<'tcx>,
2009 args: &[OperandRef<'tcx, &'ll Value>],
2010 ret_ty: Ty<'tcx>,
2011 llret_ty: &'ll Type,
2012 span: Span,
2013) -> Result<&'ll Value, ErrorGuaranteed> {
2014 macro_rules! return_error {
2015 ($diag: expr) => {{
2016 let err = bx.sess().dcx().emit_err($diag);
2017 return Err(err);
2018 }};
2019 }
2020
2021 macro_rules! require {
2022 ($cond: expr, $diag: expr) => {
2023 if !$cond {
2024 return_error!($diag);
2025 }
2026 };
2027 }
2028
2029 macro_rules! require_simd {
2030 ($ty: expr, $variant:ident) => {{
2031 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
2032 $ty.simd_size_and_type(bx.tcx())
2033 }};
2034 }
2035
2036 macro_rules! require_simd_or_scalable {
2037 ($ty: expr, $variant:ident) => {{
2038 require!(
2039 $ty.is_simd() || $ty.is_scalable_vector(),
2040 InvalidMonomorphization::$variant { span, name, ty: $ty }
2041 );
2042 if $ty.is_simd() {
2043 let (len, ty) = $ty.simd_size_and_type(bx.tcx());
2044 (len, ty, None)
2045 } else {
2046 let (count, ty, num_vecs) =
2047 $ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
2048 (count as u64, ty, Some(num_vecs))
2049 }
2050 }};
2051 }
2052
2053 macro_rules! require_int_or_uint_ty {
2055 ($ty: expr, $diag: expr) => {
2056 match $ty {
2057 ty::Int(i) => {
2058 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2059 }
2060 ty::Uint(i) => {
2061 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
2062 }
2063 _ => {
2064 return_error!($diag);
2065 }
2066 }
2067 };
2068 }
2069
2070 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
2084 bx: &mut Builder<'a, 'll, 'tcx>,
2085 i_xn: &'ll Value,
2086 in_elem_bitwidth: u64,
2087 in_len: u64,
2088 ) -> &'ll Value {
2089 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
2091 let shift_indices = ::alloc::vec::from_elem_global(shift_idx, in_len as _)vec![shift_idx; in_len as _];
2092 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
2093 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
2095 }
2096
2097 if truecfg!(debug_assertions) {
2099 for arg in args {
2100 if arg.layout.ty.is_simd() {
2101 {
match arg.val {
OperandValue::Immediate(_) => {}
ref left_val => {
::core::panicking::assert_matches_failed(left_val,
"OperandValue::Immediate(_)", ::core::option::Option::None);
}
}
};assert_matches!(arg.val, OperandValue::Immediate(_));
2102 }
2103 }
2104 }
2105
2106 if name == sym::simd_select_bitmask {
2107 let (len, _) = {
if !args[1].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(err);
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdArgument);
2108
2109 let expected_int_bits = len.max(8).next_power_of_two();
2110 let expected_bytes = len.div_ceil(8);
2111
2112 let mask_ty = args[0].layout.ty;
2113 let mask = match mask_ty.kind() {
2114 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
2115 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
2116 ty::Array(elem, len)
2117 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2118 && len
2119 .try_to_target_usize(bx.tcx)
2120 .expect("expected monomorphic const in codegen")
2121 == expected_bytes =>
2122 {
2123 let place = PlaceRef::alloca(bx, args[0].layout);
2124 args[0].val.store(bx, place);
2125 let int_ty = bx.type_ix(expected_bytes * 8);
2126 bx.load(int_ty, place.val.llval, Align::ONE)
2127 }
2128 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::InvalidBitmask {
span,
name,
mask_ty,
expected_int_bits,
expected_bytes,
});
return Err(err);
}return_error!(InvalidMonomorphization::InvalidBitmask {
2129 span,
2130 name,
2131 mask_ty,
2132 expected_int_bits,
2133 expected_bytes
2134 }),
2135 };
2136
2137 let i1 = bx.type_i1();
2138 let im = bx.type_ix(len);
2139 let i1xn = bx.type_vector(i1, len);
2140 let m_im = bx.trunc(mask, im);
2141 let m_i1s = bx.bitcast(m_im, i1xn);
2142 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2143 }
2144
2145 if name == sym::simd_splat {
2146 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2147
2148 if !(args[0].layout.ty == out_ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: out_ty,
vector_type: ret_ty,
});
return Err(err);
};
};require!(
2149 args[0].layout.ty == out_ty,
2150 InvalidMonomorphization::ExpectedVectorElementType {
2151 span,
2152 name,
2153 expected_element: out_ty,
2154 vector_type: ret_ty,
2155 }
2156 );
2157
2158 let poison_vec = bx.const_poison(llret_ty);
2160 let idx0 = bx.const_i32(0);
2161 let v0 = bx.insert_element(poison_vec, args[0].immediate(), idx0);
2162
2163 let mask_ty = bx.type_vector(bx.type_i32(), out_len);
2166 let splat = bx.shuffle_vector(v0, poison_vec, bx.const_null(mask_ty));
2167
2168 return Ok(splat);
2169 }
2170
2171 let supports_scalable = match name {
2172 sym::simd_cast | sym::simd_select => true,
2173 _ => false,
2174 };
2175
2176 if !supports_scalable {
2181 let _ = {
if !args[0].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(err);
};
};
args[0].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[0].layout.ty, SimdInput);
2182 }
2183 let (in_len, in_elem, in_num_vecs) = {
if !(args[0].layout.ty.is_simd() ||
args[0].layout.ty.is_scalable_vector()) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdInput {
span,
name,
ty: args[0].layout.ty,
});
return Err(err);
};
};
if args[0].layout.ty.is_simd() {
let (len, ty) = args[0].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[0].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[0].layout.ty, SimdInput);
2184 let in_ty = args[0].layout.ty;
2185
2186 let comparison = match name {
2187 sym::simd_eq => Some(BinOp::Eq),
2188 sym::simd_ne => Some(BinOp::Ne),
2189 sym::simd_lt => Some(BinOp::Lt),
2190 sym::simd_le => Some(BinOp::Le),
2191 sym::simd_gt => Some(BinOp::Gt),
2192 sym::simd_ge => Some(BinOp::Ge),
2193 _ => None,
2194 };
2195
2196 if let Some(cmp_op) = comparison {
2197 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2198
2199 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
2200 in_len == out_len,
2201 InvalidMonomorphization::ReturnLengthInputType {
2202 span,
2203 name,
2204 in_len,
2205 in_ty,
2206 ret_ty,
2207 out_len
2208 }
2209 );
2210 if !bx.vector_element_type(llret_ty).is_some_and(|elem|
bx.int_width(elem).is_some()) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnIntegerType {
span,
name,
ret_ty,
out_ty,
});
return Err(err);
};
};require!(
2211 bx.vector_element_type(llret_ty).is_some_and(|elem| bx.int_width(elem).is_some()),
2212 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
2213 );
2214
2215 return Ok(compare_simd_types(
2216 bx,
2217 args[0].immediate(),
2218 args[1].immediate(),
2219 in_elem,
2220 llret_ty,
2221 cmp_op,
2222 ));
2223 }
2224
2225 if name == sym::simd_shuffle_const_generic {
2226 let idx = fn_args[2].expect_const().to_branch();
2227 let n = idx.len() as u64;
2228
2229 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2230 if !(out_len == n) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
2231 out_len == n,
2232 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2233 );
2234 if !(in_elem == out_ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(err);
};
};require!(
2235 in_elem == out_ty,
2236 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2237 );
2238
2239 let total_len = in_len * 2;
2240
2241 let indices: Option<Vec<_>> = idx
2242 .iter()
2243 .enumerate()
2244 .map(|(arg_idx, val)| {
2245 let idx = val.to_leaf().to_i32();
2246 if idx >= i32::try_from(total_len).unwrap() {
2247 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
2248 span,
2249 name,
2250 arg_idx: arg_idx as u64,
2251 total_len: total_len.into(),
2252 });
2253 None
2254 } else {
2255 Some(bx.const_i32(idx))
2256 }
2257 })
2258 .collect();
2259 let Some(indices) = indices else {
2260 return Ok(bx.const_null(llret_ty));
2261 };
2262
2263 return Ok(bx.shuffle_vector(
2264 args[0].immediate(),
2265 args[1].immediate(),
2266 bx.const_vector(&indices),
2267 ));
2268 }
2269
2270 if name == sym::simd_shuffle {
2271 let idx_ty = args[2].layout.ty;
2273 let n: u64 = if idx_ty.is_simd()
2274 && #[allow(non_exhaustive_omitted_patterns)] match idx_ty.simd_size_and_type(bx.cx.tcx).1.kind()
{
ty::Uint(ty::UintTy::U32) => true,
_ => false,
}matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
2275 {
2276 idx_ty.simd_size_and_type(bx.cx.tcx).0
2277 } else {
2278 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdShuffle {
span,
name,
ty: idx_ty,
});
return Err(err);
}return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
2279 };
2280
2281 let (out_len, out_ty) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
2282 if !(out_len == n) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLength {
span,
name,
in_len: n,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
2283 out_len == n,
2284 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
2285 );
2286 if !(in_elem == out_ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnElement {
span,
name,
in_elem,
in_ty,
ret_ty,
out_ty,
});
return Err(err);
};
};require!(
2287 in_elem == out_ty,
2288 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
2289 );
2290
2291 let total_len = u128::from(in_len) * 2;
2292
2293 let indices = args[2].immediate();
2295 for i in 0..n {
2296 let val = bx.const_get_elt(indices, i as u64);
2297 let idx = bx
2298 .const_to_opt_u128(val, true)
2299 .unwrap_or_else(|| ::rustc_span::macros::bug_impl(None,
format_args!("typeck should have already ensured that these are const"),
Location::caller())bug!("typeck should have already ensured that these are const"));
2300 if idx >= total_len {
2301 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: i,
total_len,
});
return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2302 span,
2303 name,
2304 arg_idx: i,
2305 total_len,
2306 });
2307 }
2308 }
2309
2310 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
2311 }
2312
2313 if name == sym::simd_insert || name == sym::simd_insert_dyn {
2314 if !(in_elem == args[2].layout.ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::InsertedType {
span,
name,
in_elem,
in_ty,
out_ty: args[2].layout.ty,
});
return Err(err);
};
};require!(
2315 in_elem == args[2].layout.ty,
2316 InvalidMonomorphization::InsertedType {
2317 span,
2318 name,
2319 in_elem,
2320 in_ty,
2321 out_ty: args[2].layout.ty
2322 }
2323 );
2324
2325 let index_imm = if name == sym::simd_insert {
2326 let idx = bx
2327 .const_to_opt_u128(args[1].immediate(), false)
2328 .expect("typeck should have ensure that this is a const");
2329 if idx >= in_len.into() {
2330 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2331 span,
2332 name,
2333 arg_idx: 1,
2334 total_len: in_len.into(),
2335 });
2336 }
2337 bx.const_i32(idx as i32)
2338 } else {
2339 args[1].immediate()
2340 };
2341
2342 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
2343 }
2344 if name == sym::simd_extract || name == sym::simd_extract_dyn {
2345 if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};require!(
2346 ret_ty == in_elem,
2347 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2348 );
2349 let index_imm = if name == sym::simd_extract {
2350 let idx = bx
2351 .const_to_opt_u128(args[1].immediate(), false)
2352 .expect("typeck should have ensure that this is a const");
2353 if idx >= in_len.into() {
2354 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
span,
name,
arg_idx: 1,
total_len: in_len.into(),
});
return Err(err);
};return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
2355 span,
2356 name,
2357 arg_idx: 1,
2358 total_len: in_len.into(),
2359 });
2360 }
2361 bx.const_i32(idx as i32)
2362 } else {
2363 args[1].immediate()
2364 };
2365
2366 return Ok(bx.extract_element(args[0].immediate(), index_imm));
2367 }
2368
2369 if name == sym::simd_select {
2370 let m_elem_ty = in_elem;
2371 let m_len = in_len;
2372 let (v_len, _, _) = {
if !(args[1].layout.ty.is_simd() ||
args[1].layout.ty.is_scalable_vector()) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdArgument {
span,
name,
ty: args[1].layout.ty,
});
return Err(err);
};
};
if args[1].layout.ty.is_simd() {
let (len, ty) = args[1].layout.ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
args[1].layout.ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(args[1].layout.ty, SimdArgument);
2373 if !(m_len == v_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MismatchedLengths {
span,
name,
m_len,
v_len,
});
return Err(err);
};
};require!(
2374 m_len == v_len,
2375 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
2376 );
2377
2378 let m_i1s = if args[1].layout.ty.is_scalable_vector() {
2379 match m_elem_ty.kind() {
2380 ty::Bool => {}
2381 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(err);
}return_error!(InvalidMonomorphization::MaskWrongElementType {
2382 span,
2383 name,
2384 ty: m_elem_ty
2385 }),
2386 };
2387 let i1 = bx.type_i1();
2388 let i1xn = bx.type_scalable_vector(i1, m_len as u64);
2389 bx.trunc(args[0].immediate(), i1xn)
2390 } else {
2391 let in_elem_bitwidth = match m_elem_ty.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2392 m_elem_ty.kind(),
2393 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
2394 );
2395 vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
2396 };
2397
2398 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
2399 }
2400
2401 if name == sym::simd_bitmask {
2402 let expected_int_bits = in_len.max(8).next_power_of_two();
2411 let expected_bytes = in_len.div_ceil(8);
2412
2413 let in_elem_bitwidth = match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: in_elem,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2415 in_elem.kind(),
2416 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
2417 );
2418
2419 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
2420 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
2422
2423 match ret_ty.kind() {
2424 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
2425 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
2427 }
2428 ty::Array(elem, len)
2429 if #[allow(non_exhaustive_omitted_patterns)] match elem.kind() {
ty::Uint(ty::UintTy::U8) => true,
_ => false,
}matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
2430 && len
2431 .try_to_target_usize(bx.tcx)
2432 .expect("expected monomorphic const in codegen")
2433 == expected_bytes =>
2434 {
2435 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
2437
2438 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
2440 bx.store(ze, ptr, Align::ONE);
2441 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
2442 return Ok(bx.load(array_ty, ptr, Align::ONE));
2443 }
2444 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::CannotReturn {
span,
name,
ret_ty,
expected_int_bits,
expected_bytes,
});
return Err(err);
}return_error!(InvalidMonomorphization::CannotReturn {
2445 span,
2446 name,
2447 ret_ty,
2448 expected_int_bits,
2449 expected_bytes
2450 }),
2451 }
2452 }
2453
2454 fn simd_simple_float_intrinsic<'ll, 'tcx>(
2455 name: Symbol,
2456 in_elem: Ty<'_>,
2457 in_ty: Ty<'_>,
2458 in_len: u64,
2459 bx: &mut Builder<'_, 'll, 'tcx>,
2460 span: Span,
2461 args: &[OperandRef<'tcx, &'ll Value>],
2462 ) -> Result<&'ll Value, ErrorGuaranteed> {
2463 macro_rules! return_error {
2464 ($diag: expr) => {{
2465 let err = bx.sess().dcx().emit_err($diag);
2466 return Err(err);
2467 }};
2468 }
2469
2470 let ty::Float(f) = in_elem.kind() else {
2471 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
span,
name,
ty: in_ty,
});
return Err(err);
};return_error!(InvalidMonomorphization::BasicFloatType { span, name, ty: in_ty });
2472 };
2473 let elem_ty = bx.cx.type_float_from_ty(*f);
2474
2475 let vec_ty = bx.type_vector(elem_ty, in_len);
2476
2477 let intr_name = match name {
2478 sym::simd_ceil => "llvm.ceil",
2479 sym::simd_fabs => "llvm.fabs",
2480 sym::simd_fcos => "llvm.cos",
2481 sym::simd_fexp2 => "llvm.exp2",
2482 sym::simd_fexp => "llvm.exp",
2483 sym::simd_flog10 => "llvm.log10",
2484 sym::simd_flog2 => "llvm.log2",
2485 sym::simd_flog => "llvm.log",
2486 sym::simd_floor => "llvm.floor",
2487 sym::simd_fma => "llvm.fma",
2488 sym::simd_relaxed_fma => "llvm.fmuladd",
2489 sym::simd_fsin => "llvm.sin",
2490 sym::simd_fsqrt => "llvm.sqrt",
2491 sym::simd_round => "llvm.round",
2492 sym::simd_round_ties_even => "llvm.rint",
2493 sym::simd_trunc => "llvm.trunc",
2494 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnrecognizedIntrinsic {
span,
name,
});
return Err(err);
}return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
2495 };
2496 Ok(bx.call_intrinsic(
2497 intr_name,
2498 &[vec_ty],
2499 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
2500 ))
2501 }
2502
2503 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_ceil | sym::simd_fabs | sym::simd_fcos | sym::simd_fexp2 |
sym::simd_fexp | sym::simd_flog10 | sym::simd_flog2 | sym::simd_flog |
sym::simd_floor | sym::simd_fma | sym::simd_fsin | sym::simd_fsqrt |
sym::simd_relaxed_fma | sym::simd_round | sym::simd_round_ties_even |
sym::simd_trunc => true,
_ => false,
}std::matches!(
2504 name,
2505 sym::simd_ceil
2506 | sym::simd_fabs
2507 | sym::simd_fcos
2508 | sym::simd_fexp2
2509 | sym::simd_fexp
2510 | sym::simd_flog10
2511 | sym::simd_flog2
2512 | sym::simd_flog
2513 | sym::simd_floor
2514 | sym::simd_fma
2515 | sym::simd_fsin
2516 | sym::simd_fsqrt
2517 | sym::simd_relaxed_fma
2518 | sym::simd_round
2519 | sym::simd_round_ties_even
2520 | sym::simd_trunc
2521 ) {
2522 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
2523 }
2524
2525 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
2526 let elem_ty = match *elem_ty.kind() {
2527 ty::Int(v) => cx.type_int_from_ty(v),
2528 ty::Uint(v) => cx.type_uint_from_ty(v),
2529 ty::Float(v) => cx.type_float_from_ty(v),
2530 ty::RawPtr(_, _) => cx.type_ptr(),
2531 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
2532 };
2533 cx.type_vector(elem_ty, vec_len)
2534 }
2535
2536 if name == sym::simd_gather {
2537 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(err);
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2548 let (out_len, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(err);
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2549 let (out_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(err);
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2551 {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2552
2553 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len,
});
return Err(err);
};
};require!(
2555 in_len == out_len,
2556 InvalidMonomorphization::SecondArgumentLength {
2557 span,
2558 name,
2559 in_len,
2560 in_ty,
2561 arg_ty: args[1].layout.ty,
2562 out_len
2563 }
2564 );
2565 if !(in_len == out_len2) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: out_len2,
});
return Err(err);
};
};require!(
2566 in_len == out_len2,
2567 InvalidMonomorphization::ThirdArgumentLength {
2568 span,
2569 name,
2570 in_len,
2571 in_ty,
2572 arg_ty: args[2].layout.ty,
2573 out_len: out_len2
2574 }
2575 );
2576
2577 if !(ret_ty == in_ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty,
ret_ty,
});
return Err(err);
};
};require!(
2579 ret_ty == in_ty,
2580 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
2581 );
2582
2583 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, _) if
p_ty == in_elem && p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(err);
};
};require!(
2584 matches!(
2585 *element_ty1.kind(),
2586 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
2587 ),
2588 InvalidMonomorphization::ExpectedElementType {
2589 span,
2590 name,
2591 expected_element: element_ty1,
2592 second_arg: args[1].layout.ty,
2593 in_elem,
2594 in_ty,
2595 mutability: ExpectedPointerMutability::Not,
2596 }
2597 );
2598
2599 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2600 element_ty2.kind(),
2601 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2602 );
2603
2604 let alignment = bx.align_of(in_elem).bytes();
2606
2607 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2609
2610 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2612
2613 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2615
2616 let call = bx.call_intrinsic(
2617 "llvm.masked.gather",
2618 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2619 &[args[1].immediate(), mask, args[0].immediate()],
2620 );
2621 crate::attributes::apply_to_callsite(
2622 call,
2623 crate::llvm::AttributePlace::Argument(0),
2624 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2625 );
2626 return Ok(call);
2627 }
2628
2629 fn llvm_alignment<'ll, 'tcx>(
2630 bx: &mut Builder<'_, 'll, 'tcx>,
2631 alignment: SimdAlign,
2632 vector_ty: Ty<'tcx>,
2633 element_ty: Ty<'tcx>,
2634 ) -> u64 {
2635 match alignment {
2636 SimdAlign::Unaligned => 1,
2637 SimdAlign::Element => bx.align_of(element_ty).bytes(),
2638 SimdAlign::Vector => bx.align_of(vector_ty).bytes(),
2639 }
2640 }
2641
2642 if name == sym::simd_masked_load {
2643 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2652
2653 let mask_ty = in_ty;
2655 let (mask_len, mask_elem) = (in_len, in_elem);
2656
2657 let pointer_ty = args[1].layout.ty;
2659
2660 let values_ty = args[2].layout.ty;
2662 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(err);
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2663
2664 {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
};require_simd!(ret_ty, SimdReturn);
2665
2666 if !(values_len == mask_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(err);
};
};require!(
2668 values_len == mask_len,
2669 InvalidMonomorphization::ThirdArgumentLength {
2670 span,
2671 name,
2672 in_len: mask_len,
2673 in_ty: mask_ty,
2674 arg_ty: values_ty,
2675 out_len: values_len
2676 }
2677 );
2678
2679 if !(ret_ty == values_ty) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedReturnType {
span,
name,
in_ty: values_ty,
ret_ty,
});
return Err(err);
};
};require!(
2681 ret_ty == values_ty,
2682 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
2683 );
2684
2685 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, _) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() =>
true,
_ => false,
} {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Not,
});
return Err(err);
};
};require!(
2686 matches!(
2687 *pointer_ty.kind(),
2688 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
2689 ),
2690 InvalidMonomorphization::ExpectedElementType {
2691 span,
2692 name,
2693 expected_element: values_elem,
2694 second_arg: pointer_ty,
2695 in_elem: values_elem,
2696 in_ty: values_ty,
2697 mutability: ExpectedPointerMutability::Not,
2698 }
2699 );
2700
2701 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2702 mask_elem.kind(),
2703 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2704 );
2705
2706 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2707
2708 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2710
2711 let llvm_pointer = bx.type_ptr();
2712
2713 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2715
2716 let call = bx.call_intrinsic(
2717 "llvm.masked.load",
2718 &[llvm_elem_vec_ty, llvm_pointer],
2719 &[args[1].immediate(), mask, args[2].immediate()],
2720 );
2721 crate::attributes::apply_to_callsite(
2722 call,
2723 crate::llvm::AttributePlace::Argument(0),
2724 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2725 );
2726 return Ok(call);
2727 }
2728
2729 if name == sym::simd_masked_store {
2730 let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment();
2739
2740 let mask_ty = in_ty;
2742 let (mask_len, mask_elem) = (in_len, in_elem);
2743
2744 let pointer_ty = args[1].layout.ty;
2746
2747 let values_ty = args[2].layout.ty;
2749 let (values_len, values_elem) = {
if !values_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: values_ty,
});
return Err(err);
};
};
values_ty.simd_size_and_type(bx.tcx())
}require_simd!(values_ty, SimdThird);
2750
2751 if !(values_len == mask_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len: mask_len,
in_ty: mask_ty,
arg_ty: values_ty,
out_len: values_len,
});
return Err(err);
};
};require!(
2753 values_len == mask_len,
2754 InvalidMonomorphization::ThirdArgumentLength {
2755 span,
2756 name,
2757 in_len: mask_len,
2758 in_ty: mask_ty,
2759 arg_ty: values_ty,
2760 out_len: values_len
2761 }
2762 );
2763
2764 if !#[allow(non_exhaustive_omitted_patterns)] match *pointer_ty.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == values_elem && p_ty.kind() == values_elem.kind() &&
p_mutbl.is_mut() => true,
_ => false,
} {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: values_elem,
second_arg: pointer_ty,
in_elem: values_elem,
in_ty: values_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(err);
};
};require!(
2766 matches!(
2767 *pointer_ty.kind(),
2768 ty::RawPtr(p_ty, p_mutbl)
2769 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
2770 ),
2771 InvalidMonomorphization::ExpectedElementType {
2772 span,
2773 name,
2774 expected_element: values_elem,
2775 second_arg: pointer_ty,
2776 in_elem: values_elem,
2777 in_ty: values_ty,
2778 mutability: ExpectedPointerMutability::Mut,
2779 }
2780 );
2781
2782 let m_elem_bitwidth = match mask_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: mask_elem,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2783 mask_elem.kind(),
2784 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
2785 );
2786
2787 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
2788
2789 let alignment = llvm_alignment(bx, alignment, values_ty, values_elem);
2791
2792 let llvm_pointer = bx.type_ptr();
2793
2794 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
2796
2797 let call = bx.call_intrinsic(
2798 "llvm.masked.store",
2799 &[llvm_elem_vec_ty, llvm_pointer],
2800 &[args[2].immediate(), args[1].immediate(), mask],
2801 );
2802 crate::attributes::apply_to_callsite(
2803 call,
2804 crate::llvm::AttributePlace::Argument(1),
2805 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2806 );
2807 return Ok(call);
2808 }
2809
2810 if name == sym::simd_scatter {
2811 let (_, element_ty0) = {
if !in_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdFirst {
span,
name,
ty: in_ty,
});
return Err(err);
};
};
in_ty.simd_size_and_type(bx.tcx())
}require_simd!(in_ty, SimdFirst);
2821 let (element_len1, element_ty1) = {
if !args[1].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdSecond {
span,
name,
ty: args[1].layout.ty,
});
return Err(err);
};
};
args[1].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[1].layout.ty, SimdSecond);
2822 let (element_len2, element_ty2) = {
if !args[2].layout.ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdThird {
span,
name,
ty: args[2].layout.ty,
});
return Err(err);
};
};
args[2].layout.ty.simd_size_and_type(bx.tcx())
}require_simd!(args[2].layout.ty, SimdThird);
2823
2824 if !(in_len == element_len1) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SecondArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[1].layout.ty,
out_len: element_len1,
});
return Err(err);
};
};require!(
2826 in_len == element_len1,
2827 InvalidMonomorphization::SecondArgumentLength {
2828 span,
2829 name,
2830 in_len,
2831 in_ty,
2832 arg_ty: args[1].layout.ty,
2833 out_len: element_len1
2834 }
2835 );
2836 if !(in_len == element_len2) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ThirdArgumentLength {
span,
name,
in_len,
in_ty,
arg_ty: args[2].layout.ty,
out_len: element_len2,
});
return Err(err);
};
};require!(
2837 in_len == element_len2,
2838 InvalidMonomorphization::ThirdArgumentLength {
2839 span,
2840 name,
2841 in_len,
2842 in_ty,
2843 arg_ty: args[2].layout.ty,
2844 out_len: element_len2
2845 }
2846 );
2847
2848 if !#[allow(non_exhaustive_omitted_patterns)] match *element_ty1.kind() {
ty::RawPtr(p_ty, p_mutbl) if
p_ty == in_elem && p_mutbl.is_mut() &&
p_ty.kind() == element_ty0.kind() => true,
_ => false,
} {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedElementType {
span,
name,
expected_element: element_ty1,
second_arg: args[1].layout.ty,
in_elem,
in_ty,
mutability: ExpectedPointerMutability::Mut,
});
return Err(err);
};
};require!(
2849 matches!(
2850 *element_ty1.kind(),
2851 ty::RawPtr(p_ty, p_mutbl)
2852 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
2853 ),
2854 InvalidMonomorphization::ExpectedElementType {
2855 span,
2856 name,
2857 expected_element: element_ty1,
2858 second_arg: args[1].layout.ty,
2859 in_elem,
2860 in_ty,
2861 mutability: ExpectedPointerMutability::Mut,
2862 }
2863 );
2864
2865 let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: element_ty2,
});
return Err(err);
};
}
}require_int_or_uint_ty!(
2867 element_ty2.kind(),
2868 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
2869 );
2870
2871 let alignment = bx.align_of(in_elem).bytes();
2873
2874 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
2876
2877 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
2879
2880 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2882
2883 let call = bx.call_intrinsic(
2884 "llvm.masked.scatter",
2885 &[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2886 &[args[0].immediate(), args[1].immediate(), mask],
2887 );
2888 crate::attributes::apply_to_callsite(
2889 call,
2890 crate::llvm::AttributePlace::Argument(1),
2891 &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2892 );
2893 return Ok(call);
2894 }
2895
2896 macro_rules! arith_red {
2897 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2898 $identity:expr) => {
2899 if name == sym::$name {
2900 require!(
2901 ret_ty == in_elem,
2902 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2903 );
2904 return match in_elem.kind() {
2905 ty::Int(_) | ty::Uint(_) => {
2906 let r = bx.$integer_reduce(args[0].immediate());
2907 if $ordered {
2908 Ok(bx.$op(args[1].immediate(), r))
2911 } else {
2912 Ok(bx.$integer_reduce(args[0].immediate()))
2913 }
2914 }
2915 ty::Float(f) => {
2916 let acc = if $ordered {
2917 args[1].immediate()
2919 } else {
2920 match f.bit_width() {
2922 32 => bx.const_real(bx.type_f32(), $identity),
2923 64 => bx.const_real(bx.type_f64(), $identity),
2924 v => return_error!(
2925 InvalidMonomorphization::UnsupportedSymbolOfSize {
2926 span,
2927 name,
2928 symbol: sym::$name,
2929 in_ty,
2930 in_elem,
2931 size: v,
2932 ret_ty
2933 }
2934 ),
2935 }
2936 };
2937 Ok(bx.$float_reduce(acc, args[0].immediate()))
2938 }
2939 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2940 span,
2941 name,
2942 symbol: sym::$name,
2943 in_ty,
2944 in_elem,
2945 ret_ty
2946 }),
2947 };
2948 }
2949 };
2950 }
2951
2952 if name == sym::simd_reduce_add_ordered {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if true {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(err);
}
}
};
Ok(bx.vector_reduce_fadd(acc, args[0].immediate()))
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2953 if name == sym::simd_reduce_mul_ordered {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if true {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if true {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(err);
}
}
};
Ok(bx.vector_reduce_fmul(acc, args[0].immediate()))
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_ordered,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2954 if name == sym::simd_reduce_add_unordered {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_add(args[0].immediate());
if false {
Ok(bx.add(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_add(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), -0.0),
64 => bx.const_real(bx.type_f64(), -0.0),
v => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(err);
}
}
};
Ok(bx.vector_reduce_fadd_reassoc(acc, args[0].immediate()))
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_add_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};arith_red!(
2955 simd_reduce_add_unordered: vector_reduce_add,
2956 vector_reduce_fadd_reassoc,
2957 false,
2958 add,
2959 -0.0
2960 );
2961 if name == sym::simd_reduce_mul_unordered {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_mul(args[0].immediate());
if false {
Ok(bx.mul(args[1].immediate(), r))
} else { Ok(bx.vector_reduce_mul(args[0].immediate())) }
}
ty::Float(f) => {
let acc =
if false {
args[1].immediate()
} else {
match f.bit_width() {
32 => bx.const_real(bx.type_f32(), 1.0),
64 => bx.const_real(bx.type_f64(), 1.0),
v => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbolOfSize {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
size: v,
ret_ty,
});
return Err(err);
}
}
};
Ok(bx.vector_reduce_fmul_reassoc(acc, args[0].immediate()))
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_mul_unordered,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};arith_red!(
2962 simd_reduce_mul_unordered: vector_reduce_mul,
2963 vector_reduce_fmul_reassoc,
2964 false,
2965 mul,
2966 1.0
2967 );
2968
2969 macro_rules! minmax_red {
2970 ($name:ident: $int_red:ident) => {
2971 if name == sym::$name {
2972 require!(
2973 ret_ty == in_elem,
2974 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2975 );
2976 return match in_elem.kind() {
2977 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2978 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2979 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2980 span,
2981 name,
2982 symbol: sym::$name,
2983 in_ty,
2984 in_elem,
2985 ret_ty
2986 }),
2987 };
2988 }
2989 };
2990 }
2991
2992 if name == sym::simd_reduce_min {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_min(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_min(args[0].immediate(), false)),
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_min,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};minmax_red!(simd_reduce_min: vector_reduce_min);
2994 if name == sym::simd_reduce_max {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
return match in_elem.kind() {
ty::Int(_i) =>
Ok(bx.vector_reduce_max(args[0].immediate(), true)),
ty::Uint(_u) =>
Ok(bx.vector_reduce_max(args[0].immediate(), false)),
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_max,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};minmax_red!(simd_reduce_max: vector_reduce_max);
2995
2996 macro_rules! bitwise_red {
2997 ($name:ident : $red:ident, $boolean:expr) => {
2998 if name == sym::$name {
2999 let input = if !$boolean {
3000 require!(
3001 ret_ty == in_elem,
3002 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
3003 );
3004 args[0].immediate()
3005 } else {
3006 let bitwidth = match in_elem.kind() {
3007 ty::Int(i) => {
3008 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
3009 }
3010 ty::Uint(i) => {
3011 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size().bits())
3012 }
3013 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
3014 span,
3015 name,
3016 symbol: sym::$name,
3017 in_ty,
3018 in_elem,
3019 ret_ty
3020 }),
3021 };
3022
3023 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
3024 };
3025 return match in_elem.kind() {
3026 ty::Int(_) | ty::Uint(_) => {
3027 let r = bx.$red(input);
3028 Ok(r)
3029 }
3030 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
3031 span,
3032 name,
3033 symbol: sym::$name,
3034 in_ty,
3035 in_elem,
3036 ret_ty
3037 }),
3038 };
3039 }
3040 };
3041 }
3042
3043 if name == sym::simd_reduce_and {
let input =
if !false {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(r)
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_and,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};bitwise_red!(simd_reduce_and: vector_reduce_and, false);
3044 if name == sym::simd_reduce_or {
let input =
if !false {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(r)
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_or,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};bitwise_red!(simd_reduce_or: vector_reduce_or, false);
3045 if name == sym::simd_reduce_xor {
let input =
if !false {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_xor(input);
Ok(r)
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_xor,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
3046 if name == sym::simd_reduce_all {
let input =
if !true {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_and(input);
Ok(r)
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_all,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};bitwise_red!(simd_reduce_all: vector_reduce_and, true);
3047 if name == sym::simd_reduce_any {
let input =
if !true {
if !(ret_ty == in_elem) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnType {
span,
name,
in_elem,
in_ty,
ret_ty,
});
return Err(err);
};
};
args[0].immediate()
} else {
let bitwidth =
match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(||
bx.data_layout().pointer_size().bits())
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth,
in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_or(input);
Ok(r)
}
_ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
symbol: sym::simd_reduce_any,
in_ty,
in_elem,
ret_ty,
});
return Err(err);
}
};
};bitwise_red!(simd_reduce_any: vector_reduce_or, true);
3048
3049 if name == sym::simd_cast_ptr {
3050 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3051 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
3052 in_len == out_len,
3053 InvalidMonomorphization::ReturnLengthInputType {
3054 span,
3055 name,
3056 in_len,
3057 in_ty,
3058 ret_ty,
3059 out_len
3060 }
3061 );
3062
3063 match in_elem.kind() {
3064 ty::RawPtr(p_ty, _) => {
3065 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
3066 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
3067 });
3068 if !metadata.is_unit() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: in_elem,
});
return Err(err);
};
};require!(
3069 metadata.is_unit(),
3070 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
3071 );
3072 }
3073 _ => {
3074 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
3075 }
3076 }
3077 match out_elem.kind() {
3078 ty::RawPtr(p_ty, _) => {
3079 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
3080 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
3081 });
3082 if !metadata.is_unit() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::CastWidePointer {
span,
name,
ty: out_elem,
});
return Err(err);
};
};require!(
3083 metadata.is_unit(),
3084 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
3085 );
3086 }
3087 _ => {
3088 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3089 }
3090 }
3091
3092 return Ok(args[0].immediate());
3093 }
3094
3095 if name == sym::simd_expose_provenance {
3096 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3097 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
3098 in_len == out_len,
3099 InvalidMonomorphization::ReturnLengthInputType {
3100 span,
3101 name,
3102 in_len,
3103 in_ty,
3104 ret_ty,
3105 out_len
3106 }
3107 );
3108
3109 match in_elem.kind() {
3110 ty::RawPtr(_, _) => {}
3111 _ => {
3112 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: in_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
3113 }
3114 }
3115 match out_elem.kind() {
3116 ty::Uint(ty::UintTy::Usize) => {}
3117 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: out_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
3118 }
3119
3120 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
3121 }
3122
3123 if name == sym::simd_with_exposed_provenance {
3124 let (out_len, out_elem) = {
if !ret_ty.is_simd() {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
ret_ty.simd_size_and_type(bx.tcx())
}require_simd!(ret_ty, SimdReturn);
3125 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
3126 in_len == out_len,
3127 InvalidMonomorphization::ReturnLengthInputType {
3128 span,
3129 name,
3130 in_len,
3131 in_ty,
3132 ret_ty,
3133 out_len
3134 }
3135 );
3136
3137 match in_elem.kind() {
3138 ty::Uint(ty::UintTy::Usize) => {}
3139 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedUsize {
span,
name,
ty: in_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
3140 }
3141 match out_elem.kind() {
3142 ty::RawPtr(_, _) => {}
3143 _ => {
3144 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedPointer {
span,
name,
ty: out_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
3145 }
3146 }
3147
3148 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
3149 }
3150
3151 if name == sym::simd_cast || name == sym::simd_as {
3152 let (out_len, out_elem, out_num_vecs) = {
if !(ret_ty.is_simd() || ret_ty.is_scalable_vector()) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::SimdReturn {
span,
name,
ty: ret_ty,
});
return Err(err);
};
};
if ret_ty.is_simd() {
let (len, ty) = ret_ty.simd_size_and_type(bx.tcx());
(len, ty, None)
} else {
let (count, ty, num_vecs) =
ret_ty.scalable_vector_parts(bx.tcx()).expect("`is_scalable_vector` was wrong");
(count as u64, ty, Some(num_vecs))
}
}require_simd_or_scalable!(ret_ty, SimdReturn);
3153 if !(in_len == out_len) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len,
});
return Err(err);
};
};require!(
3154 in_len == out_len,
3155 InvalidMonomorphization::ReturnLengthInputType {
3156 span,
3157 name,
3158 in_len,
3159 in_ty,
3160 ret_ty,
3161 out_len
3162 }
3163 );
3164 if !(in_num_vecs == out_num_vecs) {
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ReturnNumVecsInputType {
span,
name,
in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
in_ty,
ret_ty,
out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1)),
});
return Err(err);
};
};require!(
3165 in_num_vecs == out_num_vecs,
3166 InvalidMonomorphization::ReturnNumVecsInputType {
3167 span,
3168 name,
3169 in_num_vecs: in_num_vecs.unwrap_or(NumScalableVectors(1)),
3170 in_ty,
3171 ret_ty,
3172 out_num_vecs: out_num_vecs.unwrap_or(NumScalableVectors(1))
3173 }
3174 );
3175
3176 if in_elem == out_elem {
3178 return Ok(args[0].immediate());
3179 }
3180
3181 #[derive(#[automatically_derived]impl ::core::marker::Copy for Sign { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for Sign {
}
#[automatically_derived]impl ::core::clone::Clone for Sign {
#[inline]
fn clone(&self) -> Self { *self }
}Clone)]
3182 enum Sign {
3183 Unsigned,
3184 Signed,
3185 }
3186 use Sign::*;
3187
3188 enum Style {
3189 Float,
3190 Int(Sign),
3191 Unsupported,
3192 }
3193
3194 let (in_style, in_width) = match in_elem.kind() {
3195 ty::Int(i) => (
3198 Style::Int(Signed),
3199 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3200 ),
3201 ty::Uint(u) => (
3202 Style::Int(Unsigned),
3203 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3204 ),
3205 ty::Float(f) => (Style::Float, f.bit_width()),
3206 _ => (Style::Unsupported, 0),
3207 };
3208 let (out_style, out_width) = match out_elem.kind() {
3209 ty::Int(i) => (
3210 Style::Int(Signed),
3211 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3212 ),
3213 ty::Uint(u) => (
3214 Style::Int(Unsigned),
3215 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
3216 ),
3217 ty::Float(f) => (Style::Float, f.bit_width()),
3218 _ => (Style::Unsupported, 0),
3219 };
3220
3221 match (in_style, out_style) {
3222 (Style::Int(sign), Style::Int(_)) => {
3223 return Ok(match in_width.cmp(&out_width) {
3224 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
3225 Ordering::Equal => args[0].immediate(),
3226 Ordering::Less => match sign {
3227 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
3228 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
3229 },
3230 });
3231 }
3232 (Style::Int(Sign::Signed), Style::Float) => {
3233 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
3234 }
3235 (Style::Int(Sign::Unsigned), Style::Float) => {
3236 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
3237 }
3238 (Style::Float, Style::Int(sign)) => {
3239 return Ok(match (sign, name == sym::simd_as) {
3240 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
3241 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
3242 (_, true) => bx.cast_float_to_int(
3243 #[allow(non_exhaustive_omitted_patterns)] match sign {
Sign::Signed => true,
_ => false,
}matches!(sign, Sign::Signed),
3244 args[0].immediate(),
3245 llret_ty,
3246 ),
3247 });
3248 }
3249 (Style::Float, Style::Float) => {
3250 return Ok(match in_width.cmp(&out_width) {
3251 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
3252 Ordering::Equal => args[0].immediate(),
3253 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
3254 });
3255 }
3256 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedCast {
span,
name,
in_ty,
in_elem,
ret_ty,
out_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::UnsupportedCast {
3257 span,
3258 name,
3259 in_ty,
3260 in_elem,
3261 ret_ty,
3262 out_elem
3263 }),
3264 }
3265 }
3266 macro_rules! arith_binary {
3267 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3268 $(if name == sym::$name {
3269 match in_elem.kind() {
3270 $($(ty::$p(_))|* => {
3271 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
3272 })*
3273 _ => {},
3274 }
3275 return_error!(
3276 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3277 );
3278 })*
3279 }
3280 }
3281 if name == sym::simd_add {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.add(args[0].immediate(), args[1].immediate()))
}
ty::Float(_) => {
return Ok(bx.fadd(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_sub {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.sub(args[0].immediate(), args[1].immediate()))
}
ty::Float(_) => {
return Ok(bx.fsub(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_mul {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.mul(args[0].immediate(), args[1].immediate()))
}
ty::Float(_) => {
return Ok(bx.fmul(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_div {
match in_elem.kind() {
ty::Uint(_) => {
return Ok(bx.udiv(args[0].immediate(), args[1].immediate()))
}
ty::Int(_) => {
return Ok(bx.sdiv(args[0].immediate(), args[1].immediate()))
}
ty::Float(_) => {
return Ok(bx.fdiv(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_rem {
match in_elem.kind() {
ty::Uint(_) => {
return Ok(bx.urem(args[0].immediate(), args[1].immediate()))
}
ty::Int(_) => {
return Ok(bx.srem(args[0].immediate(), args[1].immediate()))
}
ty::Float(_) => {
return Ok(bx.frem(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_shl {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.shl(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_shr {
match in_elem.kind() {
ty::Uint(_) => {
return Ok(bx.lshr(args[0].immediate(), args[1].immediate()))
}
ty::Int(_) => {
return Ok(bx.ashr(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_and {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.and(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_or {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.or(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_xor {
match in_elem.kind() {
ty::Uint(_) | ty::Int(_) => {
return Ok(bx.xor(args[0].immediate(), args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_maximum_number_nsz {
match in_elem.kind() {
ty::Float(_) => {
return Ok(bx.maximum_number_nsz(args[0].immediate(),
args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}
if name == sym::simd_minimum_number_nsz {
match in_elem.kind() {
ty::Float(_) => {
return Ok(bx.minimum_number_nsz(args[0].immediate(),
args[1].immediate()))
}
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}arith_binary! {
3282 simd_add: Uint, Int => add, Float => fadd;
3283 simd_sub: Uint, Int => sub, Float => fsub;
3284 simd_mul: Uint, Int => mul, Float => fmul;
3285 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
3286 simd_rem: Uint => urem, Int => srem, Float => frem;
3287 simd_shl: Uint, Int => shl;
3288 simd_shr: Uint => lshr, Int => ashr;
3289 simd_and: Uint, Int => and;
3290 simd_or: Uint, Int => or;
3291 simd_xor: Uint, Int => xor;
3292 simd_maximum_number_nsz: Float => maximum_number_nsz;
3293 simd_minimum_number_nsz: Float => minimum_number_nsz;
3294
3295 }
3296 macro_rules! arith_unary {
3297 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
3298 $(if name == sym::$name {
3299 match in_elem.kind() {
3300 $($(ty::$p(_))|* => {
3301 return Ok(bx.$call(args[0].immediate()))
3302 })*
3303 _ => {},
3304 }
3305 return_error!(
3306 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
3307 );
3308 })*
3309 }
3310 }
3311 if name == sym::simd_neg {
match in_elem.kind() {
ty::Int(_) => { return Ok(bx.neg(args[0].immediate())) }
ty::Float(_) => { return Ok(bx.fneg(args[0].immediate())) }
_ => {}
}
{
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
};
}arith_unary! {
3312 simd_neg: Int => neg, Float => fneg;
3313 }
3314
3315 if #[allow(non_exhaustive_omitted_patterns)] match name {
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop
| sym::simd_cttz | sym::simd_carryless_mul | sym::simd_funnel_shl |
sym::simd_funnel_shr => true,
_ => false,
}matches!(
3317 name,
3318 sym::simd_bswap
3319 | sym::simd_bitreverse
3320 | sym::simd_ctlz
3321 | sym::simd_ctpop
3322 | sym::simd_cttz
3323 | sym::simd_carryless_mul
3324 | sym::simd_funnel_shl
3325 | sym::simd_funnel_shr
3326 ) {
3327 let vec_ty = bx.cx.type_vector(
3328 match *in_elem.kind() {
3329 ty::Int(i) => bx.cx.type_int_from_ty(i),
3330 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
3331 _ => {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::UnsupportedOperation {
span,
name,
in_ty,
in_elem,
});
return Err(err);
}return_error!(InvalidMonomorphization::UnsupportedOperation {
3332 span,
3333 name,
3334 in_ty,
3335 in_elem
3336 }),
3337 },
3338 in_len as u64,
3339 );
3340 let llvm_intrinsic = match name {
3341 sym::simd_bswap => "llvm.bswap",
3342 sym::simd_bitreverse => "llvm.bitreverse",
3343 sym::simd_ctlz => "llvm.ctlz",
3344 sym::simd_ctpop => "llvm.ctpop",
3345 sym::simd_cttz => "llvm.cttz",
3346 sym::simd_funnel_shl => "llvm.fshl",
3347 sym::simd_funnel_shr => "llvm.fshr",
3348 sym::simd_carryless_mul => "llvm.clmul",
3349 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
3350 };
3351 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
3352
3353 return match name {
3354 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
3356 sym::simd_ctlz | sym::simd_cttz => {
3357 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
3359 Ok(bx.call_intrinsic(
3360 llvm_intrinsic,
3361 &[vec_ty],
3362 &[args[0].immediate(), dont_poison_on_zero],
3363 ))
3364 }
3365 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
3366 Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
3368 }
3369 sym::simd_funnel_shl | sym::simd_funnel_shr => Ok(bx.call_intrinsic(
3370 llvm_intrinsic,
3371 &[vec_ty],
3372 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
3373 )),
3374 sym::simd_carryless_mul => Ok(bx.call_intrinsic(
3375 llvm_intrinsic,
3376 &[vec_ty],
3377 &[args[0].immediate(), args[1].immediate()],
3378 )),
3379 _ => ::core::panicking::unreachable_default_msg()unreachable!(),
3380 };
3381 }
3382
3383 if name == sym::simd_arith_offset {
3384 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
3386 ::rustc_span::macros::bug_impl(Some(span),
format_args!("must be called with a vector of pointer types as first argument"),
Location::caller())span_bug!(span, "must be called with a vector of pointer types as first argument")
3387 });
3388 let layout = bx.layout_of(pointee);
3389 let ptrs = args[0].immediate();
3390 let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
3393 if !#[allow(non_exhaustive_omitted_patterns)] match offsets_elem.kind() {
ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize) => true,
_ => false,
}matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
3394 ::rustc_span::macros::bug_impl(Some(span),
format_args!("must be called with a vector of pointer-sized integers as second argument"),
Location::caller());span_bug!(
3395 span,
3396 "must be called with a vector of pointer-sized integers as second argument"
3397 );
3398 }
3399 let offsets = args[1].immediate();
3400
3401 return Ok(bx.gep(layout.size, ptrs, offsets));
3402 }
3403
3404 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
3405 let lhs = args[0].immediate();
3406 let rhs = args[1].immediate();
3407 let is_add = name == sym::simd_saturating_add;
3408 let (signed, elem_ty) = match *in_elem.kind() {
3409 ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
3410 ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
3411 _ => {
3412 {
let err =
bx.sess().dcx().emit_err(InvalidMonomorphization::ExpectedVectorElementType {
span,
name,
expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
vector_type: args[0].layout.ty,
});
return Err(err);
};return_error!(InvalidMonomorphization::ExpectedVectorElementType {
3413 span,
3414 name,
3415 expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
3416 vector_type: args[0].layout.ty
3417 });
3418 }
3419 };
3420 let llvm_intrinsic = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" }))
})format!(
3421 "llvm.{}{}.sat",
3422 if signed { 's' } else { 'u' },
3423 if is_add { "add" } else { "sub" },
3424 );
3425 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
3426
3427 return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
3428 }
3429
3430 ::rustc_span::macros::bug_impl(Some(span),
format_args!("unknown SIMD intrinsic"), Location::caller());span_bug!(span, "unknown SIMD intrinsic");
3431}
3432
3433fn codegen_codeview_annotation<'ll, 'tcx>(
3434 bx: &mut Builder<'_, 'll, 'tcx>,
3435 instance: ty::Instance<'tcx>,
3436 span: Span,
3437) {
3438 const ARGS_CONST_NAME: &str = "`CodeViewAnnotationArgs::ARGS`";
3452
3453 let tcx = bx.tcx();
3454
3455 let Some((args_const_def_id, generic_args)) = tcx
3457 .explicit_clauses_of(instance.def_id())
3458 .instantiate_own(tcx, instance.args)
3459 .filter_map(|(clause, _)| clause.skip_norm_wip().as_trait_clause())
3460 .find_map(|trait_clause| {
3461 let trait_ref = trait_clause.skip_binder().trait_ref;
3462 tcx.associated_items(trait_ref.def_id)
3463 .in_definition_order()
3464 .find(|item| {
3465 item.tag() == ty::AssocTag::Const
3466 && tcx.item_name(item.def_id).as_str() == "ARGS"
3467 })
3468 .map(|item| (item.def_id, trait_ref.args))
3469 })
3470 else {
3471 ::rustc_span::macros::bug_impl(Some(span),
format_args!("could not find {0}", ARGS_CONST_NAME), Location::caller());span_bug!(span, "could not find {ARGS_CONST_NAME}");
3472 };
3473
3474 let args_const_val = match tcx.const_eval_resolve(
3476 bx.typing_env(),
3477 mir::UnevaluatedConst::new(args_const_def_id, generic_args),
3478 span,
3479 ) {
3480 Ok(val) => val,
3481 Err(ErrorHandled::Reported(..)) => return,
3482 Err(ErrorHandled::TooGeneric(_)) => {
3483 ::rustc_span::macros::bug_impl(Some(span),
format_args!("{0} is too generic", ARGS_CONST_NAME), Location::caller())span_bug!(span, "{ARGS_CONST_NAME} is too generic")
3484 }
3485 };
3486
3487 let array_alloc = match args_const_val {
3489 mir::ConstValue::Indirect { alloc_id, offset } => read_slice_pointer(tcx, alloc_id, offset)
3490 .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
format_args!("invalid {0} slice pointer", ARGS_CONST_NAME),
Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} slice pointer")),
3491 _ => ::rustc_span::macros::bug_impl(Some(span),
format_args!("unexpected {0} value: {1:?}", ARGS_CONST_NAME,
args_const_val), Location::caller())span_bug!(span, "unexpected {ARGS_CONST_NAME} value: {args_const_val:?}"),
3492 };
3493
3494 let strings = array_alloc.into_iter().flat_map(|(array_alloc_id, array_offset, array_len)| {
3495 (0..array_len).map(move |index| {
3496 let elem_size = tcx.data_layout.pointer_size().bytes() * 2; let elem_offset = array_offset + Size::from_bytes(index * elem_size);
3498
3499 let (string_alloc_id, string_offset, string_len) =
3500 match read_slice_pointer(tcx, array_alloc_id, elem_offset) {
3501 Ok(Some(slice)) => slice,
3502 Ok(None) => return &[] as &[u8],
3503 Err(()) => ::rustc_span::macros::bug_impl(Some(span),
format_args!("invalid {0} string pointer", ARGS_CONST_NAME),
Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string pointer"),
3504 };
3505
3506 resolve_alloc(tcx, string_alloc_id)
3507 .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
format_args!("invalid {0} string allocation", ARGS_CONST_NAME),
Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string allocation"))
3508 .get_bytes_strip_provenance(
3509 &tcx,
3510 alloc_range(string_offset, Size::from_bytes(string_len)),
3511 )
3512 .unwrap_or_else(|_| ::rustc_span::macros::bug_impl(Some(span),
format_args!("invalid {0} string bytes", ARGS_CONST_NAME),
Location::caller())span_bug!(span, "invalid {ARGS_CONST_NAME} string bytes"))
3513 })
3514 });
3515
3516 let md_strings: Vec<&Metadata> = strings.map(|s| bx.cx.create_metadata(s)).collect();
3518 let md_tuple = bx.cx.md_node_in_context(&md_strings);
3519 let md_value = bx.cx.get_metadata_value(md_tuple);
3520 let (fn_ty, intrinsic_fn) = bx.cx.get_intrinsic("llvm.codeview.annotation".into(), &[]);
3521
3522 bx.call(fn_ty, None, None, intrinsic_fn, ReturnSlot::Direct, &[md_value], None, None);
3523}
3524
3525fn read_slice_pointer<'tcx>(
3529 tcx: TyCtxt<'tcx>,
3530 alloc_id: AllocId,
3531 offset: Size,
3532) -> Result<Option<(AllocId, Size, u64)>, ()> {
3533 let alloc = resolve_alloc(tcx, alloc_id)?;
3534 let pointer_size = tcx.data_layout.pointer_size();
3535
3536 let pointer = alloc
3537 .read_scalar(&tcx, alloc_range(offset, pointer_size), true)
3538 .map_err(|_| ())?
3539 .to_pointer(&tcx);
3540
3541 let len = alloc
3542 .read_scalar(&tcx, alloc_range(offset + pointer_size, pointer_size), false)
3543 .map_err(|_| ())?
3544 .to_target_usize(&tcx)
3545 .discard_err()
3546 .ok_or(())?;
3547
3548 if len == 0 {
3549 return Ok(None);
3550 }
3551
3552 let (provenance, offset) =
3553 pointer.into_pointer_or_addr().map_err(|_| ())?.prov_and_relative_offset();
3554
3555 Ok(Some((provenance.alloc_id(), offset, len)))
3556}
3557
3558fn resolve_alloc<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId) -> Result<&'tcx Allocation, ()> {
3559 let alloc = match tcx.global_alloc(alloc_id) {
3560 GlobalAlloc::Memory(alloc) => alloc,
3561 GlobalAlloc::Static(def_id) => tcx.eval_static_initializer(def_id).map_err(|_| ())?,
3562 _ => return Err(()),
3563 }
3564 .inner();
3565
3566 Ok(alloc)
3567}