1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::bug;
18use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
19use rustc_middle::ty::layout::{
20 FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
21 TyAndLayout,
22};
23use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
24use rustc_sanitizers::{cfi, kcfi};
25use rustc_session::config::OptLevel;
26use rustc_span::Span;
27use rustc_target::callconv::FnAbi;
28use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
29use smallvec::SmallVec;
30use tracing::{debug, instrument};
31
32use crate::abi::FnAbiLlvmExt;
33use crate::attributes;
34use crate::common::Funclet;
35use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
36use crate::llvm::{
37 self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
38};
39use crate::type_::Type;
40use crate::type_of::LayoutLlvmExt;
41use crate::value::Value;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45 pub llbuilder: &'ll mut llvm::Builder<'ll>,
46 pub cx: &'a GenericCx<'ll, CX>,
47}
48
49pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
50pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
51
52impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
53 fn drop(&mut self) {
54 unsafe {
55 llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
56 }
57 }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61 pub(crate) fn call(
62 &mut self,
63 llty: &'ll Type,
64 llfn: &'ll Value,
65 args: &[&'ll Value],
66 funclet: Option<&Funclet<'ll>>,
67 ) -> &'ll Value {
68 debug!("call {:?} with args ({:?})", llfn, args);
69
70 let args = self.check_call("call", llty, llfn, args);
71 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73 if let Some(funclet_bundle) = funclet_bundle {
74 bundles.push(funclet_bundle);
75 }
76
77 let call = unsafe {
78 llvm::LLVMBuildCallWithOperandBundles(
79 self.llbuilder,
80 llty,
81 llfn,
82 args.as_ptr() as *const &llvm::Value,
83 args.len() as c_uint,
84 bundles.as_ptr(),
85 bundles.len() as c_uint,
86 c"".as_ptr(),
87 )
88 };
89 call
90 }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94 fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95 let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
97 GenericBuilder { llbuilder, cx: scx }
98 }
99
100 pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
101 unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
102 }
103
104 pub(crate) fn ret_void(&mut self) {
105 llvm::LLVMBuildRetVoid(self.llbuilder);
106 }
107
108 pub(crate) fn ret(&mut self, v: &'ll Value) {
109 unsafe {
110 llvm::LLVMBuildRet(self.llbuilder, v);
111 }
112 }
113
114 pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
115 let bx = Self::with_cx(cx);
116 unsafe {
117 llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
118 }
119 bx
120 }
121}
122
123pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
127
128impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
129 type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
130 type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
131 type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
132 type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
133 type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
134 type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
135
136 type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
137 type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
138 type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
139}
140
141impl abi::HasDataLayout for Builder<'_, '_, '_> {
142 fn data_layout(&self) -> &abi::TargetDataLayout {
143 self.cx.data_layout()
144 }
145}
146
147impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
148 #[inline]
149 fn tcx(&self) -> TyCtxt<'tcx> {
150 self.cx.tcx
151 }
152}
153
154impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
155 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
156 self.cx.typing_env()
157 }
158}
159
160impl HasTargetSpec for Builder<'_, '_, '_> {
161 #[inline]
162 fn target_spec(&self) -> &Target {
163 self.cx.target_spec()
164 }
165}
166
167impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
168 #[inline]
169 fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
170 self.cx.handle_layout_err(err, span, ty)
171 }
172}
173
174impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
175 #[inline]
176 fn handle_fn_abi_err(
177 &self,
178 err: FnAbiError<'tcx>,
179 span: Span,
180 fn_abi_request: FnAbiRequest<'tcx>,
181 ) -> ! {
182 self.cx.handle_fn_abi_err(err, span, fn_abi_request)
183 }
184}
185
186impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
187 type Target = CodegenCx<'ll, 'tcx>;
188
189 #[inline]
190 fn deref(&self) -> &Self::Target {
191 self.cx
192 }
193}
194
195macro_rules! math_builder_methods {
196 ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
197 $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
198 unsafe {
199 llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
200 }
201 })+
202 }
203}
204
205macro_rules! set_math_builder_methods {
206 ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
207 $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
208 unsafe {
209 let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
210 llvm::$llvm_set_math(instr);
211 instr
212 }
213 })+
214 }
215}
216
217impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
218 type CodegenCx = CodegenCx<'ll, 'tcx>;
219
220 fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
221 let bx = Builder::with_cx(cx);
222 unsafe {
223 llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
224 }
225 bx
226 }
227
228 fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
229 self.cx
230 }
231
232 fn llbb(&self) -> &'ll BasicBlock {
233 unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
234 }
235
236 fn set_span(&mut self, _span: Span) {}
237
238 fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
239 unsafe {
240 let name = SmallCStr::new(name);
241 llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
242 }
243 }
244
245 fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
246 Self::append_block(self.cx, self.llfn(), name)
247 }
248
249 fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
250 *self = Self::build(self.cx, llbb)
251 }
252
253 fn ret_void(&mut self) {
254 llvm::LLVMBuildRetVoid(self.llbuilder);
255 }
256
257 fn ret(&mut self, v: &'ll Value) {
258 unsafe {
259 llvm::LLVMBuildRet(self.llbuilder, v);
260 }
261 }
262
263 fn br(&mut self, dest: &'ll BasicBlock) {
264 unsafe {
265 llvm::LLVMBuildBr(self.llbuilder, dest);
266 }
267 }
268
269 fn cond_br(
270 &mut self,
271 cond: &'ll Value,
272 then_llbb: &'ll BasicBlock,
273 else_llbb: &'ll BasicBlock,
274 ) {
275 unsafe {
276 llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
277 }
278 }
279
280 fn switch(
281 &mut self,
282 v: &'ll Value,
283 else_llbb: &'ll BasicBlock,
284 cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
285 ) {
286 let switch =
287 unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
288 for (on_val, dest) in cases {
289 let on_val = self.const_uint_big(self.val_ty(v), on_val);
290 unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
291 }
292 }
293
294 fn switch_with_weights(
295 &mut self,
296 v: Self::Value,
297 else_llbb: Self::BasicBlock,
298 else_is_cold: bool,
299 cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
300 ) {
301 if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
302 self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
303 return;
304 }
305
306 let id_str = "branch_weights";
307 let id = unsafe {
308 llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
309 };
310
311 let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
316 let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
317 let weight =
318 |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
319
320 let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
321 md.push(id);
322 md.push(weight(else_is_cold));
323
324 let switch =
325 unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
326 for (on_val, dest, is_cold) in cases {
327 let on_val = self.const_uint_big(self.val_ty(v), on_val);
328 unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
329 md.push(weight(is_cold));
330 }
331
332 unsafe {
333 let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
334 self.cx.set_metadata(switch, llvm::MD_prof, md_node);
335 }
336 }
337
338 fn invoke(
339 &mut self,
340 llty: &'ll Type,
341 fn_attrs: Option<&CodegenFnAttrs>,
342 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
343 llfn: &'ll Value,
344 args: &[&'ll Value],
345 then: &'ll BasicBlock,
346 catch: &'ll BasicBlock,
347 funclet: Option<&Funclet<'ll>>,
348 instance: Option<Instance<'tcx>>,
349 ) -> &'ll Value {
350 debug!("invoke {:?} with args ({:?})", llfn, args);
351
352 let args = self.check_call("invoke", llty, llfn, args);
353 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
354 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
355 if let Some(funclet_bundle) = funclet_bundle {
356 bundles.push(funclet_bundle);
357 }
358
359 self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
361
362 let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
364 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
365 bundles.push(kcfi_bundle);
366 }
367
368 let invoke = unsafe {
369 llvm::LLVMBuildInvokeWithOperandBundles(
370 self.llbuilder,
371 llty,
372 llfn,
373 args.as_ptr(),
374 args.len() as c_uint,
375 then,
376 catch,
377 bundles.as_ptr(),
378 bundles.len() as c_uint,
379 UNNAMED,
380 )
381 };
382 if let Some(fn_abi) = fn_abi {
383 fn_abi.apply_attrs_callsite(self, invoke);
384 }
385 invoke
386 }
387
388 fn unreachable(&mut self) {
389 unsafe {
390 llvm::LLVMBuildUnreachable(self.llbuilder);
391 }
392 }
393
394 math_builder_methods! {
395 add(a, b) => LLVMBuildAdd,
396 fadd(a, b) => LLVMBuildFAdd,
397 sub(a, b) => LLVMBuildSub,
398 fsub(a, b) => LLVMBuildFSub,
399 mul(a, b) => LLVMBuildMul,
400 fmul(a, b) => LLVMBuildFMul,
401 udiv(a, b) => LLVMBuildUDiv,
402 exactudiv(a, b) => LLVMBuildExactUDiv,
403 sdiv(a, b) => LLVMBuildSDiv,
404 exactsdiv(a, b) => LLVMBuildExactSDiv,
405 fdiv(a, b) => LLVMBuildFDiv,
406 urem(a, b) => LLVMBuildURem,
407 srem(a, b) => LLVMBuildSRem,
408 frem(a, b) => LLVMBuildFRem,
409 shl(a, b) => LLVMBuildShl,
410 lshr(a, b) => LLVMBuildLShr,
411 ashr(a, b) => LLVMBuildAShr,
412 and(a, b) => LLVMBuildAnd,
413 or(a, b) => LLVMBuildOr,
414 xor(a, b) => LLVMBuildXor,
415 neg(x) => LLVMBuildNeg,
416 fneg(x) => LLVMBuildFNeg,
417 not(x) => LLVMBuildNot,
418 unchecked_sadd(x, y) => LLVMBuildNSWAdd,
419 unchecked_uadd(x, y) => LLVMBuildNUWAdd,
420 unchecked_ssub(x, y) => LLVMBuildNSWSub,
421 unchecked_usub(x, y) => LLVMBuildNUWSub,
422 unchecked_smul(x, y) => LLVMBuildNSWMul,
423 unchecked_umul(x, y) => LLVMBuildNUWMul,
424 }
425
426 fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
427 unsafe {
428 let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
429 if llvm::LLVMIsAInstruction(add).is_some() {
430 llvm::LLVMSetNUW(add, True);
431 llvm::LLVMSetNSW(add, True);
432 }
433 add
434 }
435 }
436 fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
437 unsafe {
438 let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
439 if llvm::LLVMIsAInstruction(sub).is_some() {
440 llvm::LLVMSetNUW(sub, True);
441 llvm::LLVMSetNSW(sub, True);
442 }
443 sub
444 }
445 }
446 fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
447 unsafe {
448 let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
449 if llvm::LLVMIsAInstruction(mul).is_some() {
450 llvm::LLVMSetNUW(mul, True);
451 llvm::LLVMSetNSW(mul, True);
452 }
453 mul
454 }
455 }
456
457 fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
458 unsafe {
459 let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
460
461 if llvm::LLVMIsAInstruction(or).is_some() {
465 llvm::LLVMSetIsDisjoint(or, True);
466 }
467 or
468 }
469 }
470
471 set_math_builder_methods! {
472 fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
473 fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
474 fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
475 fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
476 frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
477 fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
478 fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
479 fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
480 fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
481 frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
482 }
483
484 fn checked_binop(
485 &mut self,
486 oop: OverflowOp,
487 ty: Ty<'_>,
488 lhs: Self::Value,
489 rhs: Self::Value,
490 ) -> (Self::Value, Self::Value) {
491 use rustc_middle::ty::IntTy::*;
492 use rustc_middle::ty::UintTy::*;
493 use rustc_middle::ty::{Int, Uint};
494
495 let new_kind = match ty.kind() {
496 Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
497 Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
498 t @ (Uint(_) | Int(_)) => *t,
499 _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
500 };
501
502 let name = match oop {
503 OverflowOp::Add => match new_kind {
504 Int(I8) => "llvm.sadd.with.overflow.i8",
505 Int(I16) => "llvm.sadd.with.overflow.i16",
506 Int(I32) => "llvm.sadd.with.overflow.i32",
507 Int(I64) => "llvm.sadd.with.overflow.i64",
508 Int(I128) => "llvm.sadd.with.overflow.i128",
509
510 Uint(U8) => "llvm.uadd.with.overflow.i8",
511 Uint(U16) => "llvm.uadd.with.overflow.i16",
512 Uint(U32) => "llvm.uadd.with.overflow.i32",
513 Uint(U64) => "llvm.uadd.with.overflow.i64",
514 Uint(U128) => "llvm.uadd.with.overflow.i128",
515
516 _ => unreachable!(),
517 },
518 OverflowOp::Sub => match new_kind {
519 Int(I8) => "llvm.ssub.with.overflow.i8",
520 Int(I16) => "llvm.ssub.with.overflow.i16",
521 Int(I32) => "llvm.ssub.with.overflow.i32",
522 Int(I64) => "llvm.ssub.with.overflow.i64",
523 Int(I128) => "llvm.ssub.with.overflow.i128",
524
525 Uint(_) => {
526 let sub = self.sub(lhs, rhs);
530 let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
531 return (sub, cmp);
532 }
533
534 _ => unreachable!(),
535 },
536 OverflowOp::Mul => match new_kind {
537 Int(I8) => "llvm.smul.with.overflow.i8",
538 Int(I16) => "llvm.smul.with.overflow.i16",
539 Int(I32) => "llvm.smul.with.overflow.i32",
540 Int(I64) => "llvm.smul.with.overflow.i64",
541 Int(I128) => "llvm.smul.with.overflow.i128",
542
543 Uint(U8) => "llvm.umul.with.overflow.i8",
544 Uint(U16) => "llvm.umul.with.overflow.i16",
545 Uint(U32) => "llvm.umul.with.overflow.i32",
546 Uint(U64) => "llvm.umul.with.overflow.i64",
547 Uint(U128) => "llvm.umul.with.overflow.i128",
548
549 _ => unreachable!(),
550 },
551 };
552
553 let res = self.call_intrinsic(name, &[lhs, rhs]);
554 (self.extract_value(res, 0), self.extract_value(res, 1))
555 }
556
557 fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
558 if self.cx().val_ty(val) == self.cx().type_i1() {
559 self.zext(val, self.cx().type_i8())
560 } else {
561 val
562 }
563 }
564
565 fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
566 if scalar.is_bool() {
567 return self.unchecked_utrunc(val, self.cx().type_i1());
568 }
569 val
570 }
571
572 fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
573 let mut bx = Builder::with_cx(self.cx);
574 bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
575 let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
576 unsafe {
577 let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
578 llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
579 llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
581 }
582 }
583
584 fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
585 unsafe {
586 let alloca =
587 llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
588 llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
589 llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
591 }
592 }
593
594 fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
595 unsafe {
596 let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
597 let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
598 llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
599 load
600 }
601 }
602
603 fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
604 unsafe {
605 let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
606 llvm::LLVMSetVolatile(load, llvm::True);
607 load
608 }
609 }
610
611 fn atomic_load(
612 &mut self,
613 ty: &'ll Type,
614 ptr: &'ll Value,
615 order: rustc_codegen_ssa::common::AtomicOrdering,
616 size: Size,
617 ) -> &'ll Value {
618 unsafe {
619 let load = llvm::LLVMRustBuildAtomicLoad(
620 self.llbuilder,
621 ty,
622 ptr,
623 UNNAMED,
624 AtomicOrdering::from_generic(order),
625 );
626 llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
628 load
629 }
630 }
631
632 #[instrument(level = "trace", skip(self))]
633 fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
634 if place.layout.is_unsized() {
635 let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
636 if matches!(tail.kind(), ty::Foreign(..)) {
637 panic!("unsized locals must not be `extern` types");
641 }
642 }
643 assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
644
645 if place.layout.is_zst() {
646 return OperandRef::zero_sized(place.layout);
647 }
648
649 #[instrument(level = "trace", skip(bx))]
650 fn scalar_load_metadata<'a, 'll, 'tcx>(
651 bx: &mut Builder<'a, 'll, 'tcx>,
652 load: &'ll Value,
653 scalar: abi::Scalar,
654 layout: TyAndLayout<'tcx>,
655 offset: Size,
656 ) {
657 if bx.cx.sess().opts.optimize == OptLevel::No {
658 return;
660 }
661
662 if !scalar.is_uninit_valid() {
663 bx.noundef_metadata(load);
664 }
665
666 match scalar.primitive() {
667 abi::Primitive::Int(..) => {
668 if !scalar.is_always_valid(bx) {
669 bx.range_metadata(load, scalar.valid_range(bx));
670 }
671 }
672 abi::Primitive::Pointer(_) => {
673 if !scalar.valid_range(bx).contains(0) {
674 bx.nonnull_metadata(load);
675 }
676
677 if let Some(pointee) = layout.pointee_info_at(bx, offset) {
678 if let Some(_) = pointee.safe {
679 bx.align_metadata(load, pointee.align);
680 }
681 }
682 }
683 abi::Primitive::Float(_) => {}
684 }
685 }
686
687 let val = if let Some(_) = place.val.llextra {
688 OperandValue::Ref(place.val)
690 } else if place.layout.is_llvm_immediate() {
691 let mut const_llval = None;
692 let llty = place.layout.llvm_type(self);
693 unsafe {
694 if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
695 if llvm::LLVMIsGlobalConstant(global) == llvm::True {
696 if let Some(init) = llvm::LLVMGetInitializer(global) {
697 if self.val_ty(init) == llty {
698 const_llval = Some(init);
699 }
700 }
701 }
702 }
703 }
704 let llval = const_llval.unwrap_or_else(|| {
705 let load = self.load(llty, place.val.llval, place.val.align);
706 if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
707 scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
708 self.to_immediate_scalar(load, scalar)
709 } else {
710 load
711 }
712 });
713 OperandValue::Immediate(llval)
714 } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
715 let b_offset = a.size(self).align_to(b.align(self).abi);
716
717 let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
718 let llptr = if i == 0 {
719 place.val.llval
720 } else {
721 self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
722 };
723 let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
724 let load = self.load(llty, llptr, align);
725 scalar_load_metadata(self, load, scalar, layout, offset);
726 self.to_immediate_scalar(load, scalar)
727 };
728
729 OperandValue::Pair(
730 load(0, a, place.layout, place.val.align, Size::ZERO),
731 load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
732 )
733 } else {
734 OperandValue::Ref(place.val)
735 };
736
737 OperandRef { val, layout: place.layout }
738 }
739
740 fn write_operand_repeatedly(
741 &mut self,
742 cg_elem: OperandRef<'tcx, &'ll Value>,
743 count: u64,
744 dest: PlaceRef<'tcx, &'ll Value>,
745 ) {
746 let zero = self.const_usize(0);
747 let count = self.const_usize(count);
748
749 let header_bb = self.append_sibling_block("repeat_loop_header");
750 let body_bb = self.append_sibling_block("repeat_loop_body");
751 let next_bb = self.append_sibling_block("repeat_loop_next");
752
753 self.br(header_bb);
754
755 let mut header_bx = Self::build(self.cx, header_bb);
756 let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
757
758 let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
759 header_bx.cond_br(keep_going, body_bb, next_bb);
760
761 let mut body_bx = Self::build(self.cx, body_bb);
762 let dest_elem = dest.project_index(&mut body_bx, i);
763 cg_elem.val.store(&mut body_bx, dest_elem);
764
765 let next = body_bx.unchecked_uadd(i, self.const_usize(1));
766 body_bx.br(header_bb);
767 header_bx.add_incoming_to_phi(i, next, body_bb);
768
769 *self = Self::build(self.cx, next_bb);
770 }
771
772 fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
773 if self.cx.sess().opts.optimize == OptLevel::No {
774 return;
776 }
777
778 unsafe {
779 let llty = self.cx.val_ty(load);
780 let md = [
781 llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
782 llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
783 ];
784 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
785 self.set_metadata(load, llvm::MD_range, md);
786 }
787 }
788
789 fn nonnull_metadata(&mut self, load: &'ll Value) {
790 unsafe {
791 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
792 self.set_metadata(load, llvm::MD_nonnull, md);
793 }
794 }
795
796 fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
797 self.store_with_flags(val, ptr, align, MemFlags::empty())
798 }
799
800 fn store_with_flags(
801 &mut self,
802 val: &'ll Value,
803 ptr: &'ll Value,
804 align: Align,
805 flags: MemFlags,
806 ) -> &'ll Value {
807 debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
808 assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
809 unsafe {
810 let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
811 let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
812 let align =
813 if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
814 llvm::LLVMSetAlignment(store, align);
815 if flags.contains(MemFlags::VOLATILE) {
816 llvm::LLVMSetVolatile(store, llvm::True);
817 }
818 if flags.contains(MemFlags::NONTEMPORAL) {
819 const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
832 &["aarch64", "arm", "riscv32", "riscv64"];
833
834 let use_nontemporal =
835 WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
836 if use_nontemporal {
837 let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
842 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
843 self.set_metadata(store, llvm::MD_nontemporal, md);
844 }
845 }
846 store
847 }
848 }
849
850 fn atomic_store(
851 &mut self,
852 val: &'ll Value,
853 ptr: &'ll Value,
854 order: rustc_codegen_ssa::common::AtomicOrdering,
855 size: Size,
856 ) {
857 debug!("Store {:?} -> {:?}", val, ptr);
858 assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
859 unsafe {
860 let store = llvm::LLVMRustBuildAtomicStore(
861 self.llbuilder,
862 val,
863 ptr,
864 AtomicOrdering::from_generic(order),
865 );
866 llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
868 }
869 }
870
871 fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
872 unsafe {
873 llvm::LLVMBuildGEPWithNoWrapFlags(
874 self.llbuilder,
875 ty,
876 ptr,
877 indices.as_ptr(),
878 indices.len() as c_uint,
879 UNNAMED,
880 GEPNoWrapFlags::default(),
881 )
882 }
883 }
884
885 fn inbounds_gep(
886 &mut self,
887 ty: &'ll Type,
888 ptr: &'ll Value,
889 indices: &[&'ll Value],
890 ) -> &'ll Value {
891 unsafe {
892 llvm::LLVMBuildGEPWithNoWrapFlags(
893 self.llbuilder,
894 ty,
895 ptr,
896 indices.as_ptr(),
897 indices.len() as c_uint,
898 UNNAMED,
899 GEPNoWrapFlags::InBounds,
900 )
901 }
902 }
903
904 fn inbounds_nuw_gep(
905 &mut self,
906 ty: &'ll Type,
907 ptr: &'ll Value,
908 indices: &[&'ll Value],
909 ) -> &'ll Value {
910 unsafe {
911 llvm::LLVMBuildGEPWithNoWrapFlags(
912 self.llbuilder,
913 ty,
914 ptr,
915 indices.as_ptr(),
916 indices.len() as c_uint,
917 UNNAMED,
918 GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
919 )
920 }
921 }
922
923 fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
925 unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
926 }
927
928 fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
929 debug_assert_ne!(self.val_ty(val), dest_ty);
930
931 let trunc = self.trunc(val, dest_ty);
932 unsafe {
933 if llvm::LLVMIsAInstruction(trunc).is_some() {
934 llvm::LLVMSetNUW(trunc, True);
935 }
936 }
937 trunc
938 }
939
940 fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
941 debug_assert_ne!(self.val_ty(val), dest_ty);
942
943 let trunc = self.trunc(val, dest_ty);
944 unsafe {
945 if llvm::LLVMIsAInstruction(trunc).is_some() {
946 llvm::LLVMSetNSW(trunc, True);
947 }
948 }
949 trunc
950 }
951
952 fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
953 unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
954 }
955
956 fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
957 self.fptoint_sat(false, val, dest_ty)
958 }
959
960 fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
961 self.fptoint_sat(true, val, dest_ty)
962 }
963
964 fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
965 if self.sess().target.is_like_wasm {
980 let src_ty = self.cx.val_ty(val);
981 if self.cx.type_kind(src_ty) != TypeKind::Vector {
982 let float_width = self.cx.float_width(src_ty);
983 let int_width = self.cx.int_width(dest_ty);
984 let name = match (int_width, float_width) {
985 (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
986 (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
987 (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
988 (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
989 _ => None,
990 };
991 if let Some(name) = name {
992 return self.call_intrinsic(name, &[val]);
993 }
994 }
995 }
996 unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
997 }
998
999 fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1000 if self.sess().target.is_like_wasm {
1002 let src_ty = self.cx.val_ty(val);
1003 if self.cx.type_kind(src_ty) != TypeKind::Vector {
1004 let float_width = self.cx.float_width(src_ty);
1005 let int_width = self.cx.int_width(dest_ty);
1006 let name = match (int_width, float_width) {
1007 (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
1008 (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
1009 (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
1010 (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
1011 _ => None,
1012 };
1013 if let Some(name) = name {
1014 return self.call_intrinsic(name, &[val]);
1015 }
1016 }
1017 }
1018 unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1019 }
1020
1021 fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1022 unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1023 }
1024
1025 fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1026 unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1027 }
1028
1029 fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1030 unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1031 }
1032
1033 fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1034 unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1035 }
1036
1037 fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1038 unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1039 }
1040
1041 fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1042 unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1043 }
1044
1045 fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1046 unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1047 }
1048
1049 fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1050 unsafe {
1051 llvm::LLVMBuildIntCast2(
1052 self.llbuilder,
1053 val,
1054 dest_ty,
1055 if is_signed { True } else { False },
1056 UNNAMED,
1057 )
1058 }
1059 }
1060
1061 fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1062 unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1063 }
1064
1065 fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1067 let op = llvm::IntPredicate::from_generic(op);
1068 unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1069 }
1070
1071 fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1072 let op = llvm::RealPredicate::from_generic(op);
1073 unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1074 }
1075
1076 fn three_way_compare(
1077 &mut self,
1078 ty: Ty<'tcx>,
1079 lhs: Self::Value,
1080 rhs: Self::Value,
1081 ) -> Option<Self::Value> {
1082 if crate::llvm_util::get_version() < (20, 0, 0) {
1084 return None;
1085 }
1086
1087 let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
1088 (true, 8) => "llvm.scmp.i8.i8",
1089 (true, 16) => "llvm.scmp.i8.i16",
1090 (true, 32) => "llvm.scmp.i8.i32",
1091 (true, 64) => "llvm.scmp.i8.i64",
1092 (true, 128) => "llvm.scmp.i8.i128",
1093
1094 (false, 8) => "llvm.ucmp.i8.i8",
1095 (false, 16) => "llvm.ucmp.i8.i16",
1096 (false, 32) => "llvm.ucmp.i8.i32",
1097 (false, 64) => "llvm.ucmp.i8.i64",
1098 (false, 128) => "llvm.ucmp.i8.i128",
1099
1100 _ => bug!("three-way compare unsupported for type {ty:?}"),
1101 };
1102 Some(self.call_intrinsic(name, &[lhs, rhs]))
1103 }
1104
1105 fn memcpy(
1107 &mut self,
1108 dst: &'ll Value,
1109 dst_align: Align,
1110 src: &'ll Value,
1111 src_align: Align,
1112 size: &'ll Value,
1113 flags: MemFlags,
1114 ) {
1115 assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1116 let size = self.intcast(size, self.type_isize(), false);
1117 let is_volatile = flags.contains(MemFlags::VOLATILE);
1118 unsafe {
1119 llvm::LLVMRustBuildMemCpy(
1120 self.llbuilder,
1121 dst,
1122 dst_align.bytes() as c_uint,
1123 src,
1124 src_align.bytes() as c_uint,
1125 size,
1126 is_volatile,
1127 );
1128 }
1129 }
1130
1131 fn memmove(
1132 &mut self,
1133 dst: &'ll Value,
1134 dst_align: Align,
1135 src: &'ll Value,
1136 src_align: Align,
1137 size: &'ll Value,
1138 flags: MemFlags,
1139 ) {
1140 assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1141 let size = self.intcast(size, self.type_isize(), false);
1142 let is_volatile = flags.contains(MemFlags::VOLATILE);
1143 unsafe {
1144 llvm::LLVMRustBuildMemMove(
1145 self.llbuilder,
1146 dst,
1147 dst_align.bytes() as c_uint,
1148 src,
1149 src_align.bytes() as c_uint,
1150 size,
1151 is_volatile,
1152 );
1153 }
1154 }
1155
1156 fn memset(
1157 &mut self,
1158 ptr: &'ll Value,
1159 fill_byte: &'ll Value,
1160 size: &'ll Value,
1161 align: Align,
1162 flags: MemFlags,
1163 ) {
1164 assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1165 let is_volatile = flags.contains(MemFlags::VOLATILE);
1166 unsafe {
1167 llvm::LLVMRustBuildMemSet(
1168 self.llbuilder,
1169 ptr,
1170 align.bytes() as c_uint,
1171 fill_byte,
1172 size,
1173 is_volatile,
1174 );
1175 }
1176 }
1177
1178 fn select(
1179 &mut self,
1180 cond: &'ll Value,
1181 then_val: &'ll Value,
1182 else_val: &'ll Value,
1183 ) -> &'ll Value {
1184 unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1185 }
1186
1187 fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1188 unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1189 }
1190
1191 fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1192 unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1193 }
1194
1195 fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1196 unsafe {
1197 let elt_ty = self.cx.val_ty(elt);
1198 let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1199 let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1200 let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1201 self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1202 }
1203 }
1204
1205 fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1206 assert_eq!(idx as c_uint as u64, idx);
1207 unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1208 }
1209
1210 fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1211 assert_eq!(idx as c_uint as u64, idx);
1212 unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1213 }
1214
1215 fn set_personality_fn(&mut self, personality: &'ll Value) {
1216 unsafe {
1217 llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1218 }
1219 }
1220
1221 fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1222 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1223 let landing_pad = self.landing_pad(ty, pers_fn, 0);
1224 unsafe {
1225 llvm::LLVMSetCleanup(landing_pad, llvm::True);
1226 }
1227 (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1228 }
1229
1230 fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1231 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1232 let landing_pad = self.landing_pad(ty, pers_fn, 1);
1233 self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1234 (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1235 }
1236
1237 fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1238 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1239 let mut exn = self.const_poison(ty);
1240 exn = self.insert_value(exn, exn0, 0);
1241 exn = self.insert_value(exn, exn1, 1);
1242 unsafe {
1243 llvm::LLVMBuildResume(self.llbuilder, exn);
1244 }
1245 }
1246
1247 fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1248 let ret = unsafe {
1249 llvm::LLVMBuildCleanupPad(
1250 self.llbuilder,
1251 parent,
1252 args.as_ptr(),
1253 args.len() as c_uint,
1254 c"cleanuppad".as_ptr(),
1255 )
1256 };
1257 Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1258 }
1259
1260 fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1261 unsafe {
1262 llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1263 .expect("LLVM does not have support for cleanupret");
1264 }
1265 }
1266
1267 fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1268 let ret = unsafe {
1269 llvm::LLVMBuildCatchPad(
1270 self.llbuilder,
1271 parent,
1272 args.as_ptr(),
1273 args.len() as c_uint,
1274 c"catchpad".as_ptr(),
1275 )
1276 };
1277 Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1278 }
1279
1280 fn catch_switch(
1281 &mut self,
1282 parent: Option<&'ll Value>,
1283 unwind: Option<&'ll BasicBlock>,
1284 handlers: &[&'ll BasicBlock],
1285 ) -> &'ll Value {
1286 let ret = unsafe {
1287 llvm::LLVMBuildCatchSwitch(
1288 self.llbuilder,
1289 parent,
1290 unwind,
1291 handlers.len() as c_uint,
1292 c"catchswitch".as_ptr(),
1293 )
1294 };
1295 let ret = ret.expect("LLVM does not have support for catchswitch");
1296 for handler in handlers {
1297 unsafe {
1298 llvm::LLVMAddHandler(ret, handler);
1299 }
1300 }
1301 ret
1302 }
1303
1304 fn atomic_cmpxchg(
1306 &mut self,
1307 dst: &'ll Value,
1308 cmp: &'ll Value,
1309 src: &'ll Value,
1310 order: rustc_codegen_ssa::common::AtomicOrdering,
1311 failure_order: rustc_codegen_ssa::common::AtomicOrdering,
1312 weak: bool,
1313 ) -> (&'ll Value, &'ll Value) {
1314 let weak = if weak { llvm::True } else { llvm::False };
1315 unsafe {
1316 let value = llvm::LLVMBuildAtomicCmpXchg(
1317 self.llbuilder,
1318 dst,
1319 cmp,
1320 src,
1321 AtomicOrdering::from_generic(order),
1322 AtomicOrdering::from_generic(failure_order),
1323 llvm::False, );
1325 llvm::LLVMSetWeak(value, weak);
1326 let val = self.extract_value(value, 0);
1327 let success = self.extract_value(value, 1);
1328 (val, success)
1329 }
1330 }
1331
1332 fn atomic_rmw(
1333 &mut self,
1334 op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1335 dst: &'ll Value,
1336 mut src: &'ll Value,
1337 order: rustc_codegen_ssa::common::AtomicOrdering,
1338 ) -> &'ll Value {
1339 let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1341 && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1342 if requires_cast_to_int {
1343 src = self.ptrtoint(src, self.type_isize());
1344 }
1345 let mut res = unsafe {
1346 llvm::LLVMBuildAtomicRMW(
1347 self.llbuilder,
1348 AtomicRmwBinOp::from_generic(op),
1349 dst,
1350 src,
1351 AtomicOrdering::from_generic(order),
1352 llvm::False, )
1354 };
1355 if requires_cast_to_int {
1356 res = self.inttoptr(res, self.type_ptr());
1357 }
1358 res
1359 }
1360
1361 fn atomic_fence(
1362 &mut self,
1363 order: rustc_codegen_ssa::common::AtomicOrdering,
1364 scope: SynchronizationScope,
1365 ) {
1366 let single_threaded = match scope {
1367 SynchronizationScope::SingleThread => llvm::True,
1368 SynchronizationScope::CrossThread => llvm::False,
1369 };
1370 unsafe {
1371 llvm::LLVMBuildFence(
1372 self.llbuilder,
1373 AtomicOrdering::from_generic(order),
1374 single_threaded,
1375 UNNAMED,
1376 );
1377 }
1378 }
1379
1380 fn set_invariant_load(&mut self, load: &'ll Value) {
1381 unsafe {
1382 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1383 self.set_metadata(load, llvm::MD_invariant_load, md);
1384 }
1385 }
1386
1387 fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1388 self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
1389 }
1390
1391 fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1392 self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
1393 }
1394
1395 fn call(
1396 &mut self,
1397 llty: &'ll Type,
1398 fn_attrs: Option<&CodegenFnAttrs>,
1399 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1400 llfn: &'ll Value,
1401 args: &[&'ll Value],
1402 funclet: Option<&Funclet<'ll>>,
1403 instance: Option<Instance<'tcx>>,
1404 ) -> &'ll Value {
1405 debug!("call {:?} with args ({:?})", llfn, args);
1406
1407 let args = self.check_call("call", llty, llfn, args);
1408 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1409 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1410 if let Some(funclet_bundle) = funclet_bundle {
1411 bundles.push(funclet_bundle);
1412 }
1413
1414 self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1416
1417 let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1419 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1420 bundles.push(kcfi_bundle);
1421 }
1422
1423 let call = unsafe {
1424 llvm::LLVMBuildCallWithOperandBundles(
1425 self.llbuilder,
1426 llty,
1427 llfn,
1428 args.as_ptr() as *const &llvm::Value,
1429 args.len() as c_uint,
1430 bundles.as_ptr(),
1431 bundles.len() as c_uint,
1432 c"".as_ptr(),
1433 )
1434 };
1435 if let Some(fn_abi) = fn_abi {
1436 fn_abi.apply_attrs_callsite(self, call);
1437 }
1438 call
1439 }
1440
1441 fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1442 unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1443 }
1444
1445 fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1446 let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1448 attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1449 }
1450}
1451
1452impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1453 fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1454 let s = self.cx().get_static(def_id);
1456 self.cx().const_pointercast(s, self.type_ptr())
1458 }
1459}
1460
1461impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1462 pub(crate) fn llfn(&self) -> &'ll Value {
1463 unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1464 }
1465}
1466
1467impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1468 fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1469 unsafe {
1470 llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1471 }
1472 }
1473}
1474impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1475 fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1476 unsafe {
1477 let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1478 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1479 self.set_metadata(load, llvm::MD_align, md);
1480 }
1481 }
1482
1483 fn noundef_metadata(&mut self, load: &'ll Value) {
1484 unsafe {
1485 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1486 self.set_metadata(load, llvm::MD_noundef, md);
1487 }
1488 }
1489
1490 pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1491 unsafe {
1492 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1493 self.set_metadata(inst, llvm::MD_unpredictable, md);
1494 }
1495 }
1496}
1497impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1498 pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1499 unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1500 }
1501
1502 pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1503 unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1504 }
1505
1506 pub(crate) fn insert_element(
1507 &mut self,
1508 vec: &'ll Value,
1509 elt: &'ll Value,
1510 idx: &'ll Value,
1511 ) -> &'ll Value {
1512 unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1513 }
1514
1515 pub(crate) fn shuffle_vector(
1516 &mut self,
1517 v1: &'ll Value,
1518 v2: &'ll Value,
1519 mask: &'ll Value,
1520 ) -> &'ll Value {
1521 unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1522 }
1523
1524 pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1525 unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1526 }
1527 pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1528 unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1529 }
1530 pub(crate) fn vector_reduce_fadd_reassoc(
1531 &mut self,
1532 acc: &'ll Value,
1533 src: &'ll Value,
1534 ) -> &'ll Value {
1535 unsafe {
1536 let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1537 llvm::LLVMRustSetAllowReassoc(instr);
1538 instr
1539 }
1540 }
1541 pub(crate) fn vector_reduce_fmul_reassoc(
1542 &mut self,
1543 acc: &'ll Value,
1544 src: &'ll Value,
1545 ) -> &'ll Value {
1546 unsafe {
1547 let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1548 llvm::LLVMRustSetAllowReassoc(instr);
1549 instr
1550 }
1551 }
1552 pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1553 unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1554 }
1555 pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1556 unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1557 }
1558 pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1559 unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1560 }
1561 pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1562 unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1563 }
1564 pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1565 unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1566 }
1567 pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1568 unsafe {
1569 llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, false)
1570 }
1571 }
1572 pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1573 unsafe {
1574 llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, false)
1575 }
1576 }
1577 pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1578 unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1579 }
1580 pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1581 unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1582 }
1583
1584 pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1585 unsafe {
1586 llvm::LLVMAddClause(landing_pad, clause);
1587 }
1588 }
1589
1590 pub(crate) fn catch_ret(
1591 &mut self,
1592 funclet: &Funclet<'ll>,
1593 unwind: &'ll BasicBlock,
1594 ) -> &'ll Value {
1595 let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1596 ret.expect("LLVM does not have support for catchret")
1597 }
1598
1599 fn check_call<'b>(
1600 &mut self,
1601 typ: &str,
1602 fn_ty: &'ll Type,
1603 llfn: &'ll Value,
1604 args: &'b [&'ll Value],
1605 ) -> Cow<'b, [&'ll Value]> {
1606 assert!(
1607 self.cx.type_kind(fn_ty) == TypeKind::Function,
1608 "builder::{typ} not passed a function, but {fn_ty:?}"
1609 );
1610
1611 let param_tys = self.cx.func_params_types(fn_ty);
1612
1613 let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1614 .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1615
1616 if all_args_match {
1617 return Cow::Borrowed(args);
1618 }
1619
1620 let casted_args: Vec<_> = iter::zip(param_tys, args)
1621 .enumerate()
1622 .map(|(i, (expected_ty, &actual_val))| {
1623 let actual_ty = self.cx.val_ty(actual_val);
1624 if expected_ty != actual_ty {
1625 debug!(
1626 "type mismatch in function call of {:?}. \
1627 Expected {:?} for param {}, got {:?}; injecting bitcast",
1628 llfn, expected_ty, i, actual_ty
1629 );
1630 self.bitcast(actual_val, expected_ty)
1631 } else {
1632 actual_val
1633 }
1634 })
1635 .collect();
1636
1637 Cow::Owned(casted_args)
1638 }
1639
1640 pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1641 unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1642 }
1643}
1644
1645impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1646 pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
1647 let (ty, f) = self.cx.get_intrinsic(intrinsic);
1648 self.call(ty, None, None, f, args, None, None)
1649 }
1650
1651 fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1652 let size = size.bytes();
1653 if size == 0 {
1654 return;
1655 }
1656
1657 if !self.cx().sess().emit_lifetime_markers() {
1658 return;
1659 }
1660
1661 self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
1662 }
1663}
1664impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1665 pub(crate) fn phi(
1666 &mut self,
1667 ty: &'ll Type,
1668 vals: &[&'ll Value],
1669 bbs: &[&'ll BasicBlock],
1670 ) -> &'ll Value {
1671 assert_eq!(vals.len(), bbs.len());
1672 let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1673 unsafe {
1674 llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1675 phi
1676 }
1677 }
1678
1679 fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1680 unsafe {
1681 llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1682 }
1683 }
1684}
1685impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1686 fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1687 let src_ty = self.cx.val_ty(val);
1688 let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
1689 assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
1690 (
1691 self.cx.element_type(src_ty),
1692 self.cx.element_type(dest_ty),
1693 Some(self.cx.vector_length(src_ty)),
1694 )
1695 } else {
1696 (src_ty, dest_ty, None)
1697 };
1698 let float_width = self.cx.float_width(float_ty);
1699 let int_width = self.cx.int_width(int_ty);
1700
1701 let instr = if signed { "fptosi" } else { "fptoui" };
1702 let name = if let Some(vector_length) = vector_length {
1703 format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}")
1704 } else {
1705 format!("llvm.{instr}.sat.i{int_width}.f{float_width}")
1706 };
1707 let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
1708 self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None)
1709 }
1710
1711 pub(crate) fn landing_pad(
1712 &mut self,
1713 ty: &'ll Type,
1714 pers_fn: &'ll Value,
1715 num_clauses: usize,
1716 ) -> &'ll Value {
1717 self.set_personality_fn(pers_fn);
1721 unsafe {
1722 llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1723 }
1724 }
1725
1726 pub(crate) fn callbr(
1727 &mut self,
1728 llty: &'ll Type,
1729 fn_attrs: Option<&CodegenFnAttrs>,
1730 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1731 llfn: &'ll Value,
1732 args: &[&'ll Value],
1733 default_dest: &'ll BasicBlock,
1734 indirect_dest: &[&'ll BasicBlock],
1735 funclet: Option<&Funclet<'ll>>,
1736 instance: Option<Instance<'tcx>>,
1737 ) -> &'ll Value {
1738 debug!("invoke {:?} with args ({:?})", llfn, args);
1739
1740 let args = self.check_call("callbr", llty, llfn, args);
1741 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1742 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1743 if let Some(funclet_bundle) = funclet_bundle {
1744 bundles.push(funclet_bundle);
1745 }
1746
1747 self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1749
1750 let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1752 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1753 bundles.push(kcfi_bundle);
1754 }
1755
1756 let callbr = unsafe {
1757 llvm::LLVMBuildCallBr(
1758 self.llbuilder,
1759 llty,
1760 llfn,
1761 default_dest,
1762 indirect_dest.as_ptr(),
1763 indirect_dest.len() as c_uint,
1764 args.as_ptr(),
1765 args.len() as c_uint,
1766 bundles.as_ptr(),
1767 bundles.len() as c_uint,
1768 UNNAMED,
1769 )
1770 };
1771 if let Some(fn_abi) = fn_abi {
1772 fn_abi.apply_attrs_callsite(self, callbr);
1773 }
1774 callbr
1775 }
1776
1777 fn cfi_type_test(
1779 &mut self,
1780 fn_attrs: Option<&CodegenFnAttrs>,
1781 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1782 instance: Option<Instance<'tcx>>,
1783 llfn: &'ll Value,
1784 ) {
1785 let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1786 if self.tcx.sess.is_sanitizer_cfi_enabled()
1787 && let Some(fn_abi) = fn_abi
1788 && is_indirect_call
1789 {
1790 if let Some(fn_attrs) = fn_attrs
1791 && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1792 {
1793 return;
1794 }
1795
1796 let mut options = cfi::TypeIdOptions::empty();
1797 if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1798 options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1799 }
1800 if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1801 options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1802 }
1803
1804 let typeid = if let Some(instance) = instance {
1805 cfi::typeid_for_instance(self.tcx, instance, options)
1806 } else {
1807 cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1808 };
1809 let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1810 let dbg_loc = self.get_dbg_loc();
1811
1812 let cond = self.type_test(llfn, typeid_metadata);
1814 let bb_pass = self.append_sibling_block("type_test.pass");
1815 let bb_fail = self.append_sibling_block("type_test.fail");
1816 self.cond_br(cond, bb_pass, bb_fail);
1817
1818 self.switch_to_block(bb_fail);
1819 if let Some(dbg_loc) = dbg_loc {
1820 self.set_dbg_loc(dbg_loc);
1821 }
1822 self.abort();
1823 self.unreachable();
1824
1825 self.switch_to_block(bb_pass);
1826 if let Some(dbg_loc) = dbg_loc {
1827 self.set_dbg_loc(dbg_loc);
1828 }
1829 }
1830 }
1831
1832 fn kcfi_operand_bundle(
1834 &mut self,
1835 fn_attrs: Option<&CodegenFnAttrs>,
1836 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1837 instance: Option<Instance<'tcx>>,
1838 llfn: &'ll Value,
1839 ) -> Option<llvm::OperandBundleBox<'ll>> {
1840 let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1841 let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1842 && let Some(fn_abi) = fn_abi
1843 && is_indirect_call
1844 {
1845 if let Some(fn_attrs) = fn_attrs
1846 && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1847 {
1848 return None;
1849 }
1850
1851 let mut options = kcfi::TypeIdOptions::empty();
1852 if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1853 options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1854 }
1855 if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1856 options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1857 }
1858
1859 let kcfi_typeid = if let Some(instance) = instance {
1860 kcfi::typeid_for_instance(self.tcx, instance, options)
1861 } else {
1862 kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1863 };
1864
1865 Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1866 } else {
1867 None
1868 };
1869 kcfi_bundle
1870 }
1871
1872 #[instrument(level = "debug", skip(self))]
1874 pub(crate) fn instrprof_increment(
1875 &mut self,
1876 fn_name: &'ll Value,
1877 hash: &'ll Value,
1878 num_counters: &'ll Value,
1879 index: &'ll Value,
1880 ) {
1881 self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
1882 }
1883
1884 #[instrument(level = "debug", skip(self))]
1894 pub(crate) fn mcdc_parameters(
1895 &mut self,
1896 fn_name: &'ll Value,
1897 hash: &'ll Value,
1898 bitmap_bits: &'ll Value,
1899 ) {
1900 self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
1901 }
1902
1903 #[instrument(level = "debug", skip(self))]
1904 pub(crate) fn mcdc_tvbitmap_update(
1905 &mut self,
1906 fn_name: &'ll Value,
1907 hash: &'ll Value,
1908 bitmap_index: &'ll Value,
1909 mcdc_temp: &'ll Value,
1910 ) {
1911 let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1912 self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
1913 }
1914
1915 #[instrument(level = "debug", skip(self))]
1916 pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1917 self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1918 }
1919
1920 #[instrument(level = "debug", skip(self))]
1921 pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1922 let align = self.tcx.data_layout.i32_align.abi;
1923 let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1924 let new_tv_index = self.add(current_tv_index, cond_index);
1925 self.store(new_tv_index, mcdc_temp, align);
1926 }
1927}