1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5use rustc_ast::expand::typetree::FncTree;
6pub(crate) mod autodiff;
7pub(crate) mod gpu_offload;
8
9use libc::{c_char, c_uint};
10use rustc_abi::{self as abi, Align, CanonAbi, Float, Size, WrappingRange};
11use rustc_attr_ir::UnrollAttr;
12use rustc_codegen_ssa::MemFlags;
13use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope};
14use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
15use rustc_codegen_ssa::mir::place::PlaceRef;
16use rustc_codegen_ssa::traits::*;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_hir::def_id::DefId;
19use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
20use rustc_middle::ty::layout::{
21 FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
22 TyAndLayout,
23};
24use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
25use rustc_sanitizers::{cfi, kcfi};
26use rustc_session::config::OptLevel;
27use rustc_span::{Span, bug};
28use rustc_target::callconv::{FnAbi, PassMode};
29use rustc_target::spec::{Arch, HasTargetSpec, SanitizerSet, Target};
30use smallvec::SmallVec;
31use tracing::{debug, instrument};
32
33use crate::abi::FnAbiLlvmExt;
34use crate::attributes;
35use crate::common::Funclet;
36use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
37use crate::llvm::{
38 self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
39 ToLlvmBool, Type, TypeKind, Value,
40};
41use crate::type_of::LayoutLlvmExt;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45 llbuilder_raw: ptr::NonNull<llvm::Builder<'ll>>,
47 pub cx: &'a GenericCx<'ll, CX>,
48 pub span: rustc_span::Span,
49}
50
51pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
52pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
53
54impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
55 fn drop(&mut self) {
56 unsafe { llvm::LLVMDisposeBuilder(self.llbuilder_raw) };
57 }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61 pub(crate) fn call(
62 &mut self,
63 llty: &'ll Type,
64 llfn: &'ll Value,
65 args: &[&'ll Value],
66 funclet: Option<&Funclet<'ll>>,
67 ) -> &'ll Value {
68 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:68",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(68u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call {0:?} with args ({1:?})",
llfn, args) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("call {:?} with args ({:?})", llfn, args);
69
70 let args = self.check_call("call", llty, llfn, args);
71 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73 if let Some(funclet_bundle) = funclet_bundle {
74 bundles.push(funclet_bundle);
75 }
76
77 let call = unsafe {
78 llvm::LLVMBuildCallWithOperandBundles(
79 self.llbuilder(),
80 llty,
81 llfn,
82 args.as_ptr() as *const &llvm::Value,
83 args.len() as c_uint,
84 bundles.as_ptr(),
85 bundles.len() as c_uint,
86 c"".as_ptr(),
87 )
88 };
89 call
90 }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94 fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95 let llbuilder_raw = unsafe { llvm::LLVMCreateBuilderForModule(scx.deref().borrow().llmod) };
97 GenericBuilder { llbuilder_raw, cx: scx, span: rustc_span::DUMMY_SP }
98 }
99
100 pub(crate) fn append_block(
101 cx: &'a GenericCx<'ll, CX>,
102 llfn: &'ll Value,
103 name: &str,
104 ) -> &'ll BasicBlock {
105 unsafe {
106 let name = SmallCStr::new(name);
107 llvm::LLVMAppendBasicBlockInContext(cx.llcx(), llfn, name.as_ptr())
108 }
109 }
110
111 pub(crate) fn llbuilder<'s>(&'s self) -> &'s llvm::Builder<'ll> {
114 unsafe { self.llbuilder_raw.as_ref() }
117 }
118
119 pub(crate) fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
120 unsafe { llvm::LLVMBuildTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
121 }
122
123 pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
124 unsafe { llvm::LLVMBuildBitCast(self.llbuilder(), val, dest_ty, UNNAMED) }
125 }
126
127 pub(crate) fn ret_void(&mut self) {
128 llvm::LLVMBuildRetVoid(self.llbuilder());
129 }
130
131 pub(crate) fn ret(&mut self, v: &'ll Value) {
132 unsafe {
133 llvm::LLVMBuildRet(self.llbuilder(), v);
134 }
135 }
136
137 pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
138 let bx = Self::with_cx(cx);
139 unsafe {
140 llvm::LLVMPositionBuilderAtEnd(bx.llbuilder(), llbb);
141 }
142 bx
143 }
144
145 pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
150 let val = unsafe {
151 let alloca = llvm::LLVMBuildAlloca(self.llbuilder(), ty, UNNAMED);
152 llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
153 llvm::LLVMBuildPointerCast(self.llbuilder(), alloca, self.cx.type_ptr(), UNNAMED)
155 };
156 if name != "" {
157 let name = std::ffi::CString::new(name).unwrap();
158 llvm::set_value_name(val, &name.as_bytes());
159 }
160 val
161 }
162
163 pub(crate) fn typed_inbounds_gep(
166 &mut self,
167 ty: &'ll Type,
168 ptr: &'ll Value,
169 indices: &[&'ll Value],
170 ) -> &'ll Value {
171 unsafe {
172 llvm::LLVMBuildGEPWithNoWrapFlags(
173 self.llbuilder(),
174 ty,
175 ptr,
176 indices.as_ptr(),
177 indices.len() as c_uint,
178 UNNAMED,
179 GEPNoWrapFlags::InBounds,
180 )
181 }
182 }
183
184 fn build_gep(
185 &mut self,
186 stride: Size,
187 ptr: &'ll Value,
188 index: &'ll Value,
189 flags: GEPNoWrapFlags,
190 ) -> &'ll Value {
191 let ty = if stride.bytes() == 1 {
192 self.cx.type_i8()
194 } else {
195 self.cx.type_array(self.cx.type_i8(), stride.bytes())
196 };
197 let indices = &[index];
198 unsafe {
199 llvm::LLVMBuildGEPWithNoWrapFlags(
200 self.llbuilder(),
201 ty,
202 ptr,
203 indices.as_ptr(),
204 indices.len() as c_uint,
205 UNNAMED,
206 flags,
207 )
208 }
209 }
210
211 pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
212 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:212",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(212u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?}",
val, ptr) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("Store {:?} -> {:?}", val, ptr);
213 {
match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
214 unsafe {
215 let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
216 llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
217 store
218 }
219 }
220
221 pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
222 unsafe {
223 let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
224 llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
225 load
226 }
227 }
228}
229
230pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
234
235impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
236 type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
237 type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
238 type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
239
240 type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
241 type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
242 type FunctionSignature = <GenericCx<'ll, CX> as BackendTypes>::FunctionSignature;
243
244 type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
245 type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
246 type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
247}
248
249impl abi::HasDataLayout for Builder<'_, '_, '_> {
250 fn data_layout(&self) -> &abi::TargetDataLayout {
251 self.cx.data_layout()
252 }
253}
254
255impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
256 #[inline]
257 fn tcx(&self) -> TyCtxt<'tcx> {
258 self.cx.tcx
259 }
260}
261
262impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
263 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
264 self.cx.typing_env()
265 }
266}
267
268impl HasTargetSpec for Builder<'_, '_, '_> {
269 #[inline]
270 fn target_spec(&self) -> &Target {
271 self.cx.target_spec()
272 }
273}
274
275impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
276 #[inline]
277 fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
278 self.cx.handle_layout_err(err, span, ty)
279 }
280}
281
282impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
283 #[inline]
284 fn handle_fn_abi_err(
285 &self,
286 err: FnAbiError<'tcx>,
287 span: Span,
288 fn_abi_request: FnAbiRequest<'tcx>,
289 ) -> ! {
290 self.cx.handle_fn_abi_err(err, span, fn_abi_request)
291 }
292}
293
294impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
295 type Target = CodegenCx<'ll, 'tcx>;
296
297 #[inline]
298 fn deref(&self) -> &Self::Target {
299 self.cx
300 }
301}
302
303macro_rules! math_builder_methods {
304 ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
305 $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
306 unsafe {
307 llvm::$llvm_capi(self.llbuilder(), $($arg,)* UNNAMED)
308 }
309 })+
310 }
311}
312
313macro_rules! set_math_builder_methods {
314 ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
315 $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
316 unsafe {
317 let instr = llvm::$llvm_capi(self.llbuilder(), $($arg,)* UNNAMED);
318 llvm::$llvm_set_math(instr);
319 instr
320 }
321 })+
322 }
323}
324
325impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
326 type CodegenCx = CodegenCx<'ll, 'tcx>;
327
328 fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
329 let bx = Builder::with_cx(cx);
330 unsafe {
331 llvm::LLVMPositionBuilderAtEnd(bx.llbuilder(), llbb);
332 }
333 bx
334 }
335
336 fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
337 self.cx
338 }
339
340 fn llbb(&self) -> &'ll BasicBlock {
341 unsafe { llvm::LLVMGetInsertBlock(self.llbuilder()) }
342 }
343
344 fn set_span(&mut self, span: rustc_span::Span) {
345 self.span = span;
346 }
347
348 fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
349 unsafe {
350 let name = SmallCStr::new(name);
351 llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
352 }
353 }
354
355 fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
356 Self::append_block(self.cx, self.llfn(), name)
357 }
358
359 fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
360 *self = Self::build(self.cx, llbb)
361 }
362
363 fn ret_void(&mut self) {
364 llvm::LLVMBuildRetVoid(self.llbuilder());
365 }
366
367 fn ret(&mut self, v: &'ll Value) {
368 unsafe {
369 llvm::LLVMBuildRet(self.llbuilder(), v);
370 }
371 }
372
373 fn br(&mut self, dest: &'ll BasicBlock) {
374 unsafe {
375 llvm::LLVMBuildBr(self.llbuilder(), dest);
376 }
377 }
378
379 fn br_with_attrs(&mut self, dest: &'ll BasicBlock, loop_hint_attrs: &[UnrollAttr]) {
380 unsafe {
381 let val = llvm::LLVMBuildBr(self.llbuilder(), dest);
382
383 let mut nodes = Vec::new();
384
385 for unroll in loop_hint_attrs {
386 let md_node = if let UnrollAttr::Count(count) = unroll {
389 let unroll_meta = self.create_metadata("llvm.loop.unroll.count".as_bytes());
390 let count = llvm::LLVMValueAsMetadata(self.get_const_i32(u64::from(*count)));
391 self.md_node_in_context(&[unroll_meta, count])
392 } else {
393 let metadata_str = match unroll {
394 UnrollAttr::Hint => "llvm.loop.unroll.enable",
395 UnrollAttr::Full => "llvm.loop.unroll.full",
396 UnrollAttr::Never => "llvm.loop.unroll.disable",
397 UnrollAttr::Count(_) => ::core::panicking::unreachable_default_msg()unreachable!(),
398 };
399 let unroll_meta = self.create_metadata(metadata_str.as_bytes());
400 self.md_node_in_context(&[unroll_meta])
401 };
402 nodes.push(md_node);
403 }
404
405 if let [first, ..] = nodes[..] {
406 nodes.insert(0, first);
407
408 let loop_meta_mdnode = self.set_metadata_node(val, llvm::MD_loop, &nodes);
410
411 let loop_meta_val = llvm::LLVMGetMetadata(val, llvm::MD_loop).unwrap();
413
414 llvm::LLVMReplaceMDNodeOperandWith(loop_meta_val, 0, loop_meta_mdnode);
417 }
418 }
419 }
420
421 fn cond_br(
422 &mut self,
423 cond: &'ll Value,
424 then_llbb: &'ll BasicBlock,
425 else_llbb: &'ll BasicBlock,
426 ) {
427 unsafe {
428 llvm::LLVMBuildCondBr(self.llbuilder(), cond, then_llbb, else_llbb);
429 }
430 }
431
432 fn switch(
433 &mut self,
434 v: &'ll Value,
435 else_llbb: &'ll BasicBlock,
436 cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
437 ) {
438 let switch =
439 unsafe { llvm::LLVMBuildSwitch(self.llbuilder(), v, else_llbb, cases.len() as c_uint) };
440 for (on_val, dest) in cases {
441 let on_val = self.const_uint_big(self.val_ty(v), on_val);
442 unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
443 }
444 }
445
446 fn switch_with_weights(
447 &mut self,
448 v: Self::Value,
449 else_llbb: Self::BasicBlock,
450 else_is_cold: bool,
451 cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
452 ) {
453 if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
454 self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
455 return;
456 }
457
458 let id = self.cx.create_metadata(b"branch_weights");
459
460 let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
465 let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
466 let weight =
467 |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
468
469 let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
470 md.push(id);
471 md.push(weight(else_is_cold));
472
473 let switch =
474 unsafe { llvm::LLVMBuildSwitch(self.llbuilder(), v, else_llbb, cases.len() as c_uint) };
475 for (on_val, dest, is_cold) in cases {
476 let on_val = self.const_uint_big(self.val_ty(v), on_val);
477 unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
478 md.push(weight(is_cold));
479 }
480
481 self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
482 }
483
484 fn invoke(
485 &mut self,
486 llty: &'ll Type,
487 fn_attrs: Option<&CodegenFnAttrs>,
488 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
489 llfn: &'ll Value,
490 return_slot: ReturnSlot<&'ll Value>,
491 args: &[&'ll Value],
492 then: &'ll BasicBlock,
493 catch: &'ll BasicBlock,
494 funclet: Option<&Funclet<'ll>>,
495 instance: Option<Instance<'tcx>>,
496 ) -> &'ll Value {
497 let args = match return_slot {
500 ReturnSlot::Direct => args.to_vec(),
501 ReturnSlot::Indirect(sret_ptr) => {
502 let mut args = args.to_vec();
503 args.insert(0, sret_ptr);
504 args
505 }
506 };
507 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:507",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(507u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("invoke {0:?} with args ({1:?})",
llfn, args) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("invoke {:?} with args ({:?})", llfn, args);
508 let args = self.check_call("invoke", llty, llfn, &args);
509 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
510 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
511 if let Some(funclet_bundle) = funclet_bundle {
512 bundles.push(funclet_bundle);
513 }
514
515 self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
517
518 let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
520 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
521 bundles.push(kcfi_bundle);
522 }
523
524 let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
525 if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
526 bundles.push(p);
527 }
528
529 let invoke = unsafe {
530 llvm::LLVMBuildInvokeWithOperandBundles(
531 self.llbuilder(),
532 llty,
533 llfn,
534 args.as_ptr(),
535 args.len() as c_uint,
536 then,
537 catch,
538 bundles.as_ptr(),
539 bundles.len() as c_uint,
540 UNNAMED,
541 )
542 };
543 if let Some(fn_abi) = fn_abi {
544 fn_abi.apply_attrs_callsite(self, invoke);
545 }
546 invoke
547 }
548
549 fn unreachable(&mut self) {
550 unsafe {
551 llvm::LLVMBuildUnreachable(self.llbuilder());
552 }
553 }
554
555 fn add(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildAdd(self.llbuilder(), a, b, UNNAMED) }
}
fn fadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFAdd(self.llbuilder(), a, b, UNNAMED) }
}
fn sub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildSub(self.llbuilder(), a, b, UNNAMED) }
}
fn fsub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFSub(self.llbuilder(), a, b, UNNAMED) }
}
fn mul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildMul(self.llbuilder(), a, b, UNNAMED) }
}
fn fmul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFMul(self.llbuilder(), a, b, UNNAMED) }
}
fn udiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildUDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn exactudiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildExactUDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn sdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildSDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn exactsdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildExactSDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn fdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn urem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildURem(self.llbuilder(), a, b, UNNAMED) }
}
fn srem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildSRem(self.llbuilder(), a, b, UNNAMED) }
}
fn frem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFRem(self.llbuilder(), a, b, UNNAMED) }
}
fn shl(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildShl(self.llbuilder(), a, b, UNNAMED) }
}
fn lshr(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildLShr(self.llbuilder(), a, b, UNNAMED) }
}
fn ashr(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildAShr(self.llbuilder(), a, b, UNNAMED) }
}
fn and(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildAnd(self.llbuilder(), a, b, UNNAMED) }
}
fn or(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildOr(self.llbuilder(), a, b, UNNAMED) }
}
fn xor(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildXor(self.llbuilder(), a, b, UNNAMED) }
}
fn neg(&mut self, x: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNeg(self.llbuilder(), x, UNNAMED) }
}
fn fneg(&mut self, x: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildFNeg(self.llbuilder(), x, UNNAMED) }
}
fn not(&mut self, x: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNot(self.llbuilder(), x, UNNAMED) }
}
fn unchecked_sadd(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNSWAdd(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_uadd(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNUWAdd(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_ssub(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNSWSub(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_usub(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNUWSub(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_smul(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNSWMul(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_umul(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe { llvm::LLVMBuildNUWMul(self.llbuilder(), x, y, UNNAMED) }
}math_builder_methods! {
556 add(a, b) => LLVMBuildAdd,
557 fadd(a, b) => LLVMBuildFAdd,
558 sub(a, b) => LLVMBuildSub,
559 fsub(a, b) => LLVMBuildFSub,
560 mul(a, b) => LLVMBuildMul,
561 fmul(a, b) => LLVMBuildFMul,
562 udiv(a, b) => LLVMBuildUDiv,
563 exactudiv(a, b) => LLVMBuildExactUDiv,
564 sdiv(a, b) => LLVMBuildSDiv,
565 exactsdiv(a, b) => LLVMBuildExactSDiv,
566 fdiv(a, b) => LLVMBuildFDiv,
567 urem(a, b) => LLVMBuildURem,
568 srem(a, b) => LLVMBuildSRem,
569 frem(a, b) => LLVMBuildFRem,
570 shl(a, b) => LLVMBuildShl,
571 lshr(a, b) => LLVMBuildLShr,
572 ashr(a, b) => LLVMBuildAShr,
573 and(a, b) => LLVMBuildAnd,
574 or(a, b) => LLVMBuildOr,
575 xor(a, b) => LLVMBuildXor,
576 neg(x) => LLVMBuildNeg,
577 fneg(x) => LLVMBuildFNeg,
578 not(x) => LLVMBuildNot,
579 unchecked_sadd(x, y) => LLVMBuildNSWAdd,
580 unchecked_uadd(x, y) => LLVMBuildNUWAdd,
581 unchecked_ssub(x, y) => LLVMBuildNSWSub,
582 unchecked_usub(x, y) => LLVMBuildNUWSub,
583 unchecked_smul(x, y) => LLVMBuildNSWMul,
584 unchecked_umul(x, y) => LLVMBuildNUWMul,
585 }
586
587 fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
588 unsafe {
589 let add = llvm::LLVMBuildAdd(self.llbuilder(), a, b, UNNAMED);
590 if llvm::LLVMIsAInstruction(add).is_some() {
591 llvm::LLVMSetNUW(add, TRUE);
592 llvm::LLVMSetNSW(add, TRUE);
593 }
594 add
595 }
596 }
597 fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
598 unsafe {
599 let sub = llvm::LLVMBuildSub(self.llbuilder(), a, b, UNNAMED);
600 if llvm::LLVMIsAInstruction(sub).is_some() {
601 llvm::LLVMSetNUW(sub, TRUE);
602 llvm::LLVMSetNSW(sub, TRUE);
603 }
604 sub
605 }
606 }
607 fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
608 unsafe {
609 let mul = llvm::LLVMBuildMul(self.llbuilder(), a, b, UNNAMED);
610 if llvm::LLVMIsAInstruction(mul).is_some() {
611 llvm::LLVMSetNUW(mul, TRUE);
612 llvm::LLVMSetNSW(mul, TRUE);
613 }
614 mul
615 }
616 }
617
618 fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
619 unsafe {
620 let or = llvm::LLVMBuildOr(self.llbuilder(), a, b, UNNAMED);
621
622 if llvm::LLVMIsAInstruction(or).is_some() {
626 llvm::LLVMSetIsDisjoint(or, TRUE);
627 }
628 or
629 }
630 }
631
632 fn fadd_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFAdd(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
fn fsub_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFSub(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
fn fmul_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFMul(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
fn fdiv_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFDiv(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
fn frem_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFRem(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
fn fadd_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFAdd(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
fn fsub_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFSub(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
fn fmul_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFMul(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
fn fdiv_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFDiv(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}
fn frem_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFRem(self.llbuilder(), x, y, UNNAMED);
llvm::LLVMRustSetAlgebraicMath(instr);
instr
}
}set_math_builder_methods! {
633 fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
634 fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
635 fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
636 fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
637 frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
638 fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
639 fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
640 fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
641 fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
642 frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
643 }
644
645 fn checked_binop(
646 &mut self,
647 oop: OverflowOp,
648 ty: Ty<'tcx>,
649 lhs: Self::Value,
650 rhs: Self::Value,
651 ) -> (Self::Value, Self::Value) {
652 let (size, signed) = ty.int_size_and_signed(self.tcx);
653 let width = size.bits();
654
655 if !signed {
656 match oop {
657 OverflowOp::Sub => {
658 let sub = self.sub(lhs, rhs);
662 let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
663 return (sub, cmp);
664 }
665 OverflowOp::Add => {
666 let add = self.add(lhs, rhs);
669 let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
670 return (add, cmp);
671 }
672 OverflowOp::Mul => {}
673 }
674 }
675
676 let oop_str = match oop {
677 OverflowOp::Add => "add",
678 OverflowOp::Sub => "sub",
679 OverflowOp::Mul => "mul",
680 };
681
682 let name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("llvm.{0}{1}.with.overflow",
if signed { 's' } else { 'u' }, oop_str))
})format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
683
684 let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
685 (self.extract_value(res, 0), self.extract_value(res, 1))
686 }
687
688 fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
689 if self.cx().val_ty(val) == self.cx().type_i1() {
690 self.zext(val, self.cx().type_i8())
691 } else {
692 val
693 }
694 }
695
696 fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
697 if scalar.is_bool() {
698 return self.unchecked_utrunc(val, self.cx().type_i1());
699 }
700 val
701 }
702
703 fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
704 let mut bx = Builder::with_cx(self.cx);
705 bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
706 let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
707 unsafe {
708 let alloca = llvm::LLVMBuildAlloca(bx.llbuilder(), ty, UNNAMED);
709 llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
710 llvm::LLVMBuildPointerCast(bx.llbuilder(), alloca, self.cx().type_ptr(), UNNAMED)
712 }
713 }
714
715 fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value {
716 let mut bx = Builder::with_cx(self.cx);
717 bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
718 let scalable_vector_ty = layout.llvm_type(self.cx);
719
720 unsafe {
721 let alloca = llvm::LLVMBuildAlloca(bx.llbuilder(), scalable_vector_ty, UNNAMED);
722 llvm::LLVMSetAlignment(alloca, layout.align.bytes() as c_uint);
723 alloca
724 }
725 }
726
727 fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
728 unsafe {
729 let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
730 let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
731 llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
732 load
733 }
734 }
735
736 fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
737 unsafe {
738 let load = self.load(ty, ptr, align);
739 llvm::LLVMSetVolatile(load, llvm::TRUE);
740 load
741 }
742 }
743
744 fn atomic_load(
745 &mut self,
746 ty: &'ll Type,
747 ptr: &'ll Value,
748 order: rustc_middle::ty::AtomicOrdering,
749 volatile: bool,
750 size: Size,
751 ) -> &'ll Value {
752 unsafe {
753 let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
754 llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order));
756 if volatile {
757 llvm::LLVMSetVolatile(load, llvm::TRUE);
758 }
759 llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
761 load
762 }
763 }
764
765 {}
#[allow(clippy :: suspicious_else_formatting)]
{
let __tracing_attr_span;
let __tracing_attr_guard;
if ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() ||
{ false } {
__tracing_attr_span =
{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("load_operand",
"rustc_codegen_llvm::builder", ::tracing::Level::TRACE,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(765u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&[{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("place")
}> =
::tracing::__macro_support::FieldName::new("place");
NAME.as_str()
}], ::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::SPAN)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let mut interest = ::tracing::subscriber::Interest::never();
if ::tracing::Level::TRACE <=
::tracing::level_filters::STATIC_MAX_LEVEL &&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() &&
{ interest = __CALLSITE.interest(); !interest.is_never() }
&&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest) {
let meta = __CALLSITE.metadata();
::tracing::Span::new(meta,
&{
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&place)
as &dyn ::tracing::field::Value))])
})
} else {
let span =
::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
{};
span
}
};
__tracing_attr_guard = __tracing_attr_span.enter();
}
#[warn(clippy :: suspicious_else_formatting)]
{
#[allow(unknown_lints, unreachable_code, clippy ::
diverging_sub_expression, clippy :: empty_loop, clippy ::
let_unit_value, clippy :: let_with_type_underscore, clippy ::
needless_return, clippy :: unreachable)]
if false {
let __tracing_attr_fake_return: OperandRef<'tcx, &'ll Value> =
loop {};
return __tracing_attr_fake_return;
}
{
if place.layout.is_unsized() {
let tail =
self.tcx.struct_tail_for_codegen(place.layout.ty,
self.typing_env());
if #[allow(non_exhaustive_omitted_patterns)] match tail.kind()
{
ty::Foreign(..) => true,
_ => false,
} {
{
::core::panicking::panic_fmt(format_args!("unsized locals must not be `extern` types"));
};
}
}
{
match (&place.val.llextra.is_some(),
&place.layout.is_unsized()) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};
if place.layout.is_zst() {
return OperandRef::zero_sized(place.layout);
}
fn scalar_load_metadata<'a, 'll,
'tcx>(bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value,
scalar: abi::Scalar, layout: TyAndLayout<'tcx>,
offset: Size) {
{}
#[allow(clippy :: suspicious_else_formatting)]
{
let __tracing_attr_span;
let __tracing_attr_guard;
if ::tracing::Level::TRACE <=
::tracing::level_filters::STATIC_MAX_LEVEL &&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() ||
{ false } {
__tracing_attr_span =
{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("scalar_load_metadata",
"rustc_codegen_llvm::builder", ::tracing::Level::TRACE,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(782u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&[{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("load")
}> =
::tracing::__macro_support::FieldName::new("load");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("scalar")
}> =
::tracing::__macro_support::FieldName::new("scalar");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("layout")
}> =
::tracing::__macro_support::FieldName::new("layout");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("offset")
}> =
::tracing::__macro_support::FieldName::new("offset");
NAME.as_str()
}], ::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::SPAN)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let mut interest = ::tracing::subscriber::Interest::never();
if ::tracing::Level::TRACE <=
::tracing::level_filters::STATIC_MAX_LEVEL &&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() &&
{ interest = __CALLSITE.interest(); !interest.is_never() }
&&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest) {
let meta = __CALLSITE.metadata();
::tracing::Span::new(meta,
&{
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&load)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&scalar)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&layout)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&offset)
as &dyn ::tracing::field::Value))])
})
} else {
let span =
::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
{};
span
}
};
__tracing_attr_guard = __tracing_attr_span.enter();
}
#[warn(clippy :: suspicious_else_formatting)]
{
#[allow(unknown_lints, unreachable_code, clippy ::
diverging_sub_expression, clippy :: empty_loop, clippy ::
let_unit_value, clippy :: let_with_type_underscore, clippy
:: needless_return, clippy :: unreachable)]
if false {
let __tracing_attr_fake_return: () = loop {};
return __tracing_attr_fake_return;
}
{
if bx.cx.sess().opts.optimize == OptLevel::No { return; }
if !scalar.is_uninit_valid() { bx.noundef_metadata(load); }
match scalar.primitive() {
abi::Primitive::Int(..) => {
if !scalar.is_always_valid(bx) {
bx.range_metadata(load, scalar.valid_range(bx));
}
}
abi::Primitive::Pointer(_) => {
if !scalar.valid_range(bx).contains(0) {
bx.nonnull_metadata(load);
}
if let Some(pointee) = layout.pointee_info_at(bx, offset) &&
pointee.align > Align::ONE {
bx.align_metadata(load, pointee.align);
}
}
abi::Primitive::Float(_) => {}
}
}
}
}
}
let val =
if let Some(_) = place.val.llextra {
OperandValue::Ref(place.val)
} else if place.layout.backend_repr.is_scalar_or_simd() {
let mut const_llval = None;
let llty = place.layout.llvm_type(self);
if let Some(global) =
llvm::LLVMIsAGlobalVariable(place.val.llval) {
if llvm::LLVMIsGlobalConstant(global).is_true() {
if let Some(init) = llvm::LLVMGetInitializer(global) {
if self.val_ty(init) == llty { const_llval = Some(init); }
}
}
}
let llval =
const_llval.unwrap_or_else(||
{
let load =
self.load(llty, place.val.llval, place.val.align);
if let abi::BackendRepr::Scalar(scalar) =
place.layout.backend_repr {
scalar_load_metadata(self, load, scalar, place.layout,
Size::ZERO);
self.to_immediate_scalar(load, scalar)
} else { load }
});
OperandValue::Immediate(llval)
} else if let abi::BackendRepr::ScalarPair { a, b, b_offset }
= place.layout.backend_repr {
let mut load =
|i, scalar: abi::Scalar, layout, align, offset|
{
let llptr =
if i == 0 {
place.val.llval
} else {
self.inbounds_ptradd(place.val.llval,
self.const_usize(b_offset.bytes()))
};
let llty =
place.layout.scalar_pair_element_llvm_type(self, i, false);
let load = self.load(llty, llptr, align);
scalar_load_metadata(self, load, scalar, layout, offset);
self.to_immediate_scalar(load, scalar)
};
OperandValue::Pair(load(0, a, place.layout, place.val.align,
Size::ZERO),
load(1, b, place.layout,
place.val.align.restrict_for_offset(b_offset), b_offset))
} else { OperandValue::Ref(place.val) };
OperandRef { val, layout: place.layout, move_annotation: None }
}
}
}#[instrument(level = "trace", skip(self))]
766 fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
767 if place.layout.is_unsized() {
768 let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
769 if matches!(tail.kind(), ty::Foreign(..)) {
770 panic!("unsized locals must not be `extern` types");
774 }
775 }
776 assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
777
778 if place.layout.is_zst() {
779 return OperandRef::zero_sized(place.layout);
780 }
781
782 #[instrument(level = "trace", skip(bx))]
783 fn scalar_load_metadata<'a, 'll, 'tcx>(
784 bx: &mut Builder<'a, 'll, 'tcx>,
785 load: &'ll Value,
786 scalar: abi::Scalar,
787 layout: TyAndLayout<'tcx>,
788 offset: Size,
789 ) {
790 if bx.cx.sess().opts.optimize == OptLevel::No {
791 return;
793 }
794
795 if !scalar.is_uninit_valid() {
796 bx.noundef_metadata(load);
797 }
798
799 match scalar.primitive() {
800 abi::Primitive::Int(..) => {
801 if !scalar.is_always_valid(bx) {
802 bx.range_metadata(load, scalar.valid_range(bx));
803 }
804 }
805 abi::Primitive::Pointer(_) => {
806 if !scalar.valid_range(bx).contains(0) {
807 bx.nonnull_metadata(load);
808 }
809
810 if let Some(pointee) = layout.pointee_info_at(bx, offset)
811 && pointee.align > Align::ONE
812 {
813 bx.align_metadata(load, pointee.align);
814 }
815 }
816 abi::Primitive::Float(_) => {}
817 }
818 }
819
820 let val = if let Some(_) = place.val.llextra {
821 OperandValue::Ref(place.val)
823 } else if place.layout.backend_repr.is_scalar_or_simd() {
824 let mut const_llval = None;
825 let llty = place.layout.llvm_type(self);
826 if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
827 if llvm::LLVMIsGlobalConstant(global).is_true() {
828 if let Some(init) = llvm::LLVMGetInitializer(global) {
829 if self.val_ty(init) == llty {
830 const_llval = Some(init);
831 }
832 }
833 }
834 }
835
836 let llval = const_llval.unwrap_or_else(|| {
837 let load = self.load(llty, place.val.llval, place.val.align);
838 if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
839 scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
840 self.to_immediate_scalar(load, scalar)
841 } else {
842 load
843 }
844 });
845 OperandValue::Immediate(llval)
846 } else if let abi::BackendRepr::ScalarPair { a, b, b_offset } = place.layout.backend_repr {
847 let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
848 let llptr = if i == 0 {
849 place.val.llval
850 } else {
851 self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
852 };
853 let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
854 let load = self.load(llty, llptr, align);
855 scalar_load_metadata(self, load, scalar, layout, offset);
856 self.to_immediate_scalar(load, scalar)
857 };
858
859 OperandValue::Pair(
860 load(0, a, place.layout, place.val.align, Size::ZERO),
861 load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
862 )
863 } else {
864 OperandValue::Ref(place.val)
865 };
866
867 OperandRef { val, layout: place.layout, move_annotation: None }
868 }
869
870 fn write_operand_repeatedly(
871 &mut self,
872 cg_elem: OperandRef<'tcx, &'ll Value>,
873 count: u64,
874 dest: PlaceRef<'tcx, &'ll Value>,
875 ) {
876 if self.cx.sess().opts.optimize == OptLevel::No {
877 self.write_operand_repeatedly_unoptimized(cg_elem, count, dest);
885 } else {
886 self.write_operand_repeatedly_optimized(cg_elem, count, dest);
887 }
888 }
889
890 fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
891 if self.cx.sess().opts.optimize == OptLevel::No {
892 return;
894 }
895
896 let llty = self.cx.val_ty(load);
897 let md = [
898 llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
899 llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
900 ];
901 self.set_metadata_node(load, llvm::MD_range, &md);
902 }
903
904 fn nonnull_metadata(&mut self, load: &'ll Value) {
905 self.set_metadata_node(load, llvm::MD_nonnull, &[]);
906 }
907
908 fn store_with_flags(
909 &mut self,
910 val: &'ll Value,
911 ptr: &'ll Value,
912 align: Align,
913 flags: MemFlags,
914 ) {
915 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:915",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(915u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?} ({2:?})",
val, ptr, flags) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
916 {
match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
917 unsafe {
918 let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
919 let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
920 let align = align.bytes() as c_uint;
921 llvm::LLVMSetAlignment(store, align);
922 if flags.contains(MemFlags::VOLATILE) {
923 llvm::LLVMSetVolatile(store, llvm::TRUE);
924 }
925 if flags.contains(MemFlags::NONTEMPORAL) {
926 let use_nontemporal = #[allow(non_exhaustive_omitted_patterns)] match self.cx.tcx.sess.target.arch {
Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64 => true,
_ => false,
}matches!(
939 self.cx.tcx.sess.target.arch,
940 Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64
941 );
942 if use_nontemporal {
943 let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
948 self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
949 }
950 }
951 if flags.contains(MemFlags::CAPTURES_READ_ONLY) {
952 if !(self.type_kind(self.val_ty(val)) == TypeKind::Pointer) {
{
::core::panicking::panic_fmt(format_args!("CAPTURED_READ_ONLY is only supported on pointer stores"));
}
};assert!(
953 self.type_kind(self.val_ty(val)) == TypeKind::Pointer,
954 "CAPTURED_READ_ONLY is only supported on pointer stores"
955 );
956 let args = [
957 self.cx.create_metadata(b"address"),
958 self.cx.create_metadata(b"read_provenance"),
959 ];
960 let id = self.get_md_kind_id("captures");
962 let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, args.as_ptr(), args.len());
963 self.set_metadata(store, id, md);
964 }
965 }
966 }
967
968 fn atomic_store(
969 &mut self,
970 val: &'ll Value,
971 ptr: &'ll Value,
972 order: rustc_middle::ty::AtomicOrdering,
973 volatile: bool,
974 size: Size,
975 ) {
976 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:976",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(976u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?}",
val, ptr) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("Store {:?} -> {:?}", val, ptr);
977 {
match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
978 unsafe {
979 let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
980 llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order));
982 if volatile {
983 llvm::LLVMSetVolatile(store, llvm::TRUE);
984 }
985 llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
987 }
988 }
989
990 fn gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
991 self.build_gep(stride, ptr, index, GEPNoWrapFlags::default())
992 }
993
994 fn inbounds_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
995 self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds)
996 }
997
998 fn inbounds_nuw_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
999 self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW)
1000 }
1001
1002 fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1004 unsafe { llvm::LLVMBuildTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
1005 }
1006
1007 fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1008 if true {
{
match (&self.val_ty(val), &dest_ty) {
(left_val, right_val) => {
if *left_val == *right_val {
let kind = ::core::panicking::AssertKind::Ne;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};
};debug_assert_ne!(self.val_ty(val), dest_ty);
1009
1010 let trunc = self.trunc(val, dest_ty);
1011 unsafe {
1012 if llvm::LLVMIsAInstruction(trunc).is_some() {
1013 llvm::LLVMSetNUW(trunc, TRUE);
1014 }
1015 }
1016 trunc
1017 }
1018
1019 fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1020 if true {
{
match (&self.val_ty(val), &dest_ty) {
(left_val, right_val) => {
if *left_val == *right_val {
let kind = ::core::panicking::AssertKind::Ne;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};
};debug_assert_ne!(self.val_ty(val), dest_ty);
1021
1022 let trunc = self.trunc(val, dest_ty);
1023 unsafe {
1024 if llvm::LLVMIsAInstruction(trunc).is_some() {
1025 llvm::LLVMSetNSW(trunc, TRUE);
1026 }
1027 }
1028 trunc
1029 }
1030
1031 fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1032 unsafe { llvm::LLVMBuildSExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1033 }
1034
1035 fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1036 self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
1037 }
1038
1039 fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1040 self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
1041 }
1042
1043 fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1044 if self.sess().target.is_like_wasm {
1059 let src_ty = self.cx.val_ty(val);
1060 if let Some(float_type) = self.cx.float_type(src_ty) {
1061 let int_width = self.cx.int_width(dest_ty).expect("dest was not an integer");
1062 if #[allow(non_exhaustive_omitted_patterns)] match (int_width, float_type) {
(32 | 64, Float::F32 | Float::F64) => true,
_ => false,
}matches!((int_width, float_type), (32 | 64, Float::F32 | Float::F64)) {
1063 return self.call_intrinsic(
1064 "llvm.wasm.trunc.unsigned",
1065 &[dest_ty, src_ty],
1066 &[val],
1067 );
1068 }
1069 }
1070 }
1071 unsafe { llvm::LLVMBuildFPToUI(self.llbuilder(), val, dest_ty, UNNAMED) }
1072 }
1073
1074 fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1075 if self.sess().target.is_like_wasm {
1077 let src_ty = self.cx.val_ty(val);
1078 if let Some(float_type) = self.cx.float_type(src_ty) {
1079 let int_width = self.cx.int_width(dest_ty).expect("dest ty was not an integer");
1080 if #[allow(non_exhaustive_omitted_patterns)] match (int_width, float_type) {
(32 | 64, Float::F32 | Float::F64) => true,
_ => false,
}matches!((int_width, float_type), (32 | 64, Float::F32 | Float::F64)) {
1081 return self.call_intrinsic(
1082 "llvm.wasm.trunc.signed",
1083 &[dest_ty, src_ty],
1084 &[val],
1085 );
1086 }
1087 }
1088 }
1089 unsafe { llvm::LLVMBuildFPToSI(self.llbuilder(), val, dest_ty, UNNAMED) }
1090 }
1091
1092 fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1093 unsafe { llvm::LLVMBuildUIToFP(self.llbuilder(), val, dest_ty, UNNAMED) }
1094 }
1095
1096 fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1097 unsafe { llvm::LLVMBuildSIToFP(self.llbuilder(), val, dest_ty, UNNAMED) }
1098 }
1099
1100 fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1101 unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
1102 }
1103
1104 fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1105 unsafe { llvm::LLVMBuildFPExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1106 }
1107
1108 fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1109 unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder(), val, dest_ty, UNNAMED) }
1110 }
1111
1112 fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1113 unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder(), val, dest_ty, UNNAMED) }
1114 }
1115
1116 fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1117 unsafe { llvm::LLVMBuildBitCast(self.llbuilder(), val, dest_ty, UNNAMED) }
1118 }
1119
1120 fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1121 unsafe {
1122 llvm::LLVMBuildIntCast2(
1123 self.llbuilder(),
1124 val,
1125 dest_ty,
1126 is_signed.to_llvm_bool(),
1127 UNNAMED,
1128 )
1129 }
1130 }
1131
1132 fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1133 unsafe { llvm::LLVMBuildPointerCast(self.llbuilder(), val, dest_ty, UNNAMED) }
1134 }
1135
1136 fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1138 let op = llvm::IntPredicate::from_generic(op);
1139 unsafe { llvm::LLVMBuildICmp(self.llbuilder(), op as c_uint, lhs, rhs, UNNAMED) }
1140 }
1141
1142 fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1143 let op = llvm::RealPredicate::from_generic(op);
1144 unsafe { llvm::LLVMBuildFCmp(self.llbuilder(), op as c_uint, lhs, rhs, UNNAMED) }
1145 }
1146
1147 fn three_way_compare(
1148 &mut self,
1149 ty: Ty<'tcx>,
1150 lhs: Self::Value,
1151 rhs: Self::Value,
1152 ) -> Self::Value {
1153 let size = ty.primitive_size(self.tcx);
1154 let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1155
1156 self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])
1157 }
1158
1159 fn memcpy(
1161 &mut self,
1162 dst: &'ll Value,
1163 dst_align: Align,
1164 src: &'ll Value,
1165 src_align: Align,
1166 size: &'ll Value,
1167 flags: MemFlags,
1168 tt: Option<FncTree>,
1169 ) {
1170 if !!flags.contains(MemFlags::NONTEMPORAL) {
{
::core::panicking::panic_fmt(format_args!("non-temporal memcpy not supported"));
}
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1171 let size = self.intcast(size, self.type_isize(), false);
1172 let is_volatile = flags.contains(MemFlags::VOLATILE);
1173 let memcpy = unsafe {
1174 llvm::LLVMRustBuildMemCpy(
1175 self.llbuilder(),
1176 dst,
1177 dst_align.bytes() as c_uint,
1178 src,
1179 src_align.bytes() as c_uint,
1180 size,
1181 is_volatile,
1182 )
1183 };
1184
1185 if let Some(tt) = tt {
1191 crate::typetree::add_tt(self, memcpy, tt);
1192 }
1193 }
1194
1195 fn memmove(
1196 &mut self,
1197 dst: &'ll Value,
1198 dst_align: Align,
1199 src: &'ll Value,
1200 src_align: Align,
1201 size: &'ll Value,
1202 flags: MemFlags,
1203 ) {
1204 if !!flags.contains(MemFlags::NONTEMPORAL) {
{
::core::panicking::panic_fmt(format_args!("non-temporal memmove not supported"));
}
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1205 let size = self.intcast(size, self.type_isize(), false);
1206 let is_volatile = flags.contains(MemFlags::VOLATILE);
1207 unsafe {
1208 llvm::LLVMRustBuildMemMove(
1209 self.llbuilder(),
1210 dst,
1211 dst_align.bytes() as c_uint,
1212 src,
1213 src_align.bytes() as c_uint,
1214 size,
1215 is_volatile,
1216 );
1217 }
1218 }
1219
1220 fn memset(
1221 &mut self,
1222 ptr: &'ll Value,
1223 fill_byte: &'ll Value,
1224 size: &'ll Value,
1225 align: Align,
1226 flags: MemFlags,
1227 ) {
1228 if !!flags.contains(MemFlags::NONTEMPORAL) {
{
::core::panicking::panic_fmt(format_args!("non-temporal memset not supported"));
}
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1229 let is_volatile = flags.contains(MemFlags::VOLATILE);
1230 unsafe {
1231 llvm::LLVMRustBuildMemSet(
1232 self.llbuilder(),
1233 ptr,
1234 align.bytes() as c_uint,
1235 fill_byte,
1236 size,
1237 is_volatile,
1238 );
1239 }
1240 }
1241
1242 fn vscale(&mut self, ty: &'ll Type) -> &'ll Value {
1243 unsafe { llvm::LLVMRustBuildVScale(self.llbuilder(), ty) }
1244 }
1245
1246 fn select(
1247 &mut self,
1248 cond: &'ll Value,
1249 then_val: &'ll Value,
1250 else_val: &'ll Value,
1251 ) -> &'ll Value {
1252 unsafe { llvm::LLVMBuildSelect(self.llbuilder(), cond, then_val, else_val, UNNAMED) }
1253 }
1254
1255 fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1256 unsafe { llvm::LLVMBuildVAArg(self.llbuilder(), list, ty, UNNAMED) }
1257 }
1258
1259 fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1260 unsafe { llvm::LLVMBuildExtractElement(self.llbuilder(), vec, idx, UNNAMED) }
1261 }
1262
1263 fn vector_splat(&mut self, num_elts: u64, elt: &'ll Value) -> &'ll Value {
1264 unsafe {
1265 let elt_ty = self.cx.val_ty(elt);
1266 let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts));
1267 let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1268 let vec_i32_ty = self.type_vector(self.type_i32(), num_elts);
1269 self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1270 }
1271 }
1272
1273 fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1274 {
match (&(idx as c_uint as u64), &idx) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(idx as c_uint as u64, idx);
1275 unsafe { llvm::LLVMBuildExtractValue(self.llbuilder(), agg_val, idx as c_uint, UNNAMED) }
1276 }
1277
1278 fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1279 {
match (&(idx as c_uint as u64), &idx) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(idx as c_uint as u64, idx);
1280 unsafe {
1281 llvm::LLVMBuildInsertValue(self.llbuilder(), agg_val, elt, idx as c_uint, UNNAMED)
1282 }
1283 }
1284
1285 fn set_personality_fn(&mut self, personality: &'ll Value) {
1286 unsafe {
1287 llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1288 }
1289 }
1290
1291 fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1292 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1293 let landing_pad = self.landing_pad(ty, pers_fn, 0);
1294 unsafe {
1295 llvm::LLVMSetCleanup(landing_pad, llvm::TRUE);
1296 }
1297 (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1298 }
1299
1300 fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1301 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1302 let landing_pad = self.landing_pad(ty, pers_fn, 1);
1303 self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1304 }
1305
1306 fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1307 let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1308 let mut exn = self.const_poison(ty);
1309 exn = self.insert_value(exn, exn0, 0);
1310 exn = self.insert_value(exn, exn1, 1);
1311 unsafe {
1312 llvm::LLVMBuildResume(self.llbuilder(), exn);
1313 }
1314 }
1315
1316 fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1317 let ret = unsafe {
1318 llvm::LLVMBuildCleanupPad(
1319 self.llbuilder(),
1320 parent,
1321 args.as_ptr(),
1322 args.len() as c_uint,
1323 c"cleanuppad".as_ptr(),
1324 )
1325 };
1326 Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1327 }
1328
1329 fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1330 unsafe {
1331 llvm::LLVMBuildCleanupRet(self.llbuilder(), funclet.cleanuppad(), unwind)
1332 .expect("LLVM does not have support for cleanupret");
1333 }
1334 }
1335
1336 fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1337 let ret = unsafe {
1338 llvm::LLVMBuildCatchPad(
1339 self.llbuilder(),
1340 parent,
1341 args.as_ptr(),
1342 args.len() as c_uint,
1343 c"catchpad".as_ptr(),
1344 )
1345 };
1346 Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1347 }
1348
1349 fn catch_switch(
1350 &mut self,
1351 parent: Option<&'ll Value>,
1352 unwind: Option<&'ll BasicBlock>,
1353 handlers: &[&'ll BasicBlock],
1354 ) -> &'ll Value {
1355 let ret = unsafe {
1356 llvm::LLVMBuildCatchSwitch(
1357 self.llbuilder(),
1358 parent,
1359 unwind,
1360 handlers.len() as c_uint,
1361 c"catchswitch".as_ptr(),
1362 )
1363 };
1364 let ret = ret.expect("LLVM does not have support for catchswitch");
1365 for handler in handlers {
1366 unsafe {
1367 llvm::LLVMAddHandler(ret, handler);
1368 }
1369 }
1370 ret
1371 }
1372
1373 fn get_funclet_cleanuppad(&self, funclet: &Funclet<'ll>) -> &'ll Value {
1374 funclet.cleanuppad()
1375 }
1376
1377 fn atomic_cmpxchg(
1379 &mut self,
1380 dst: &'ll Value,
1381 cmp: &'ll Value,
1382 src: &'ll Value,
1383 order: rustc_middle::ty::AtomicOrdering,
1384 failure_order: rustc_middle::ty::AtomicOrdering,
1385 weak: bool,
1386 ) -> (&'ll Value, &'ll Value) {
1387 unsafe {
1388 let value = llvm::LLVMBuildAtomicCmpXchg(
1389 self.llbuilder(),
1390 dst,
1391 cmp,
1392 src,
1393 AtomicOrdering::from_generic(order),
1394 AtomicOrdering::from_generic(failure_order),
1395 llvm::FALSE, );
1397 llvm::LLVMSetWeak(value, weak.to_llvm_bool());
1398 let val = self.extract_value(value, 0);
1399 let success = self.extract_value(value, 1);
1400 (val, success)
1401 }
1402 }
1403
1404 fn atomic_rmw(
1405 &mut self,
1406 op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1407 dst: &'ll Value,
1408 src: &'ll Value,
1409 order: rustc_middle::ty::AtomicOrdering,
1410 ret_ptr: bool,
1411 ) -> &'ll Value {
1412 let mut res = unsafe {
1416 llvm::LLVMBuildAtomicRMW(
1417 self.llbuilder(),
1418 AtomicRmwBinOp::from_generic(op),
1419 dst,
1420 src,
1421 AtomicOrdering::from_generic(order),
1422 llvm::FALSE, )
1424 };
1425 if ret_ptr && self.val_ty(res) != self.type_ptr() {
1426 res = self.inttoptr(res, self.type_ptr());
1427 }
1428 res
1429 }
1430
1431 fn atomic_fence(
1432 &mut self,
1433 order: rustc_middle::ty::AtomicOrdering,
1434 scope: SynchronizationScope,
1435 ) {
1436 let single_threaded = match scope {
1437 SynchronizationScope::SingleThread => true,
1438 SynchronizationScope::CrossThread => false,
1439 };
1440 unsafe {
1441 llvm::LLVMBuildFence(
1442 self.llbuilder(),
1443 AtomicOrdering::from_generic(order),
1444 single_threaded.to_llvm_bool(),
1445 UNNAMED,
1446 );
1447 }
1448 }
1449
1450 fn set_invariant_load(&mut self, load: &'ll Value) {
1451 self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
1452 }
1453
1454 fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1455 self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1456 }
1457
1458 fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1459 self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1460 }
1461
1462 fn call(
1463 &mut self,
1464 llty: &'ll Type,
1465 caller_attrs: Option<&CodegenFnAttrs>,
1466 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1467 llfn: &'ll Value,
1468 return_slot: ReturnSlot<&'ll Value>,
1469 args: &[&'ll Value],
1470 funclet: Option<&Funclet<'ll>>,
1471 callee_instance: Option<Instance<'tcx>>,
1472 ) -> &'ll Value {
1473 let args = match return_slot {
1476 ReturnSlot::Direct => args.to_vec(),
1477 ReturnSlot::Indirect(sret_ptr) => {
1478 let mut args = args.to_vec();
1479 args.insert(0, sret_ptr);
1480 args
1481 }
1482 };
1483 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1483",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(1483u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call {0:?} with args ({1:?})",
llfn, args) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("call {:?} with args ({:?})", llfn, args);
1484 let args = self.check_call("call", llty, llfn, &args);
1485 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1486 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1487 if let Some(funclet_bundle) = funclet_bundle {
1488 bundles.push(funclet_bundle);
1489 }
1490
1491 self.cfi_type_test(caller_attrs, fn_abi, callee_instance, llfn);
1493
1494 let kcfi_bundle = self.kcfi_operand_bundle(caller_attrs, fn_abi, callee_instance, llfn);
1496 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1497 bundles.push(kcfi_bundle);
1498 }
1499
1500 let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
1501 if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
1502 bundles.push(p);
1503 }
1504
1505 let call = unsafe {
1506 llvm::LLVMBuildCallWithOperandBundles(
1507 self.llbuilder(),
1508 llty,
1509 llfn,
1510 args.as_ptr() as *const &llvm::Value,
1511 args.len() as c_uint,
1512 bundles.as_ptr(),
1513 bundles.len() as c_uint,
1514 c"".as_ptr(),
1515 )
1516 };
1517
1518 if let Some(fn_abi) = fn_abi {
1519 fn_abi.apply_attrs_callsite(self, call);
1520 }
1521 call
1522 }
1523
1524 fn tail_call(
1525 &mut self,
1526 llty: Self::Type,
1527 caller_attrs: Option<&CodegenFnAttrs>,
1528 fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1529 llfn: Self::Value,
1530 return_slot: ReturnSlot<Self::Value>,
1531 args: &[Self::Value],
1532 funclet: Option<&Self::Funclet>,
1533 callee_instance: Option<Instance<'tcx>>,
1534 ) {
1535 let call = self.call(
1536 llty,
1537 caller_attrs,
1538 Some(fn_abi),
1539 llfn,
1540 return_slot,
1541 args,
1542 funclet,
1543 callee_instance,
1544 );
1545 llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail);
1546
1547 match &fn_abi.ret.mode {
1548 PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1549 PassMode::Direct(_) | PassMode::Pair { .. } | PassMode::Cast { .. } => self.ret(call),
1550 PassMode::IndirectUnsized { .. } => ::rustc_span::macros::bug_impl(None,
format_args!("unsized returns are not supported"), Location::caller())bug!("unsized returns are not supported"),
1551 }
1552 }
1553
1554 fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1555 unsafe { llvm::LLVMBuildZExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1556 }
1557
1558 fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1559 let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1561 attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1562 }
1563}
1564
1565impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1566 fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1567 let global = self.cx().get_static(def_id);
1569 if self.cx().tcx.is_thread_local_static(def_id) {
1570 let pointer =
1571 self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1572 self.pointercast(pointer, self.type_ptr())
1574 } else {
1575 self.cx().const_pointercast(global, self.type_ptr())
1577 }
1578 }
1579}
1580
1581impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1582 pub(crate) fn llfn(&self) -> &'ll Value {
1583 unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1584 }
1585
1586 fn generate_ubsan_cfi_diag_data(
1587 &mut self,
1588 span: rustc_span::Span,
1589 expected_ty: String,
1590 check_kind: u8,
1591 ) -> &'ll Value {
1592 let cx = self.cx();
1593 let tcx = cx.tcx;
1594
1595 let loc = tcx.sess.source_map().lookup_char_pos(span.lo());
1596
1597 let filename_str = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}\0",
loc.file.name.prefer_local_unconditionally()))
})format!("{}\0", loc.file.name.prefer_local_unconditionally());
1598 let filename_val = cx.const_bytes(filename_str.as_bytes());
1599 let filename_ptr = cx.static_addr_of_impl(filename_val, Align::ONE, None);
1600
1601 let source_location = cx.const_struct(
1603 &[
1604 filename_ptr,
1605 cx.const_u32(loc.line as u32),
1606 cx.const_u32(loc.col.0 as u32 + 1),
1608 ],
1609 false, );
1611
1612 let ty_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}\0", expected_ty))
})format!("{}\0", expected_ty);
1613 let ty_name_val = cx.const_bytes(ty_name.as_bytes());
1614
1615 let type_descriptor =
1617 cx.const_struct(&[cx.const_i16(0xffffu16 as i16), cx.const_i16(0), ty_name_val], false);
1618
1619 let type_descriptor_ptr =
1620 cx.static_addr_of_impl(type_descriptor, Align::from_bytes(2).unwrap(), None);
1621
1622 let cfi_check_fail_data = cx
1624 .const_struct(&[cx.const_u8(check_kind), source_location, type_descriptor_ptr], false);
1625 let align = tcx.data_layout.aggregate_align;
1626
1627 cx.static_addr_of_mut(cfi_check_fail_data, align, Some("__ubsan_cfi_check_fail_data"))
1629 }
1630}
1631
1632impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1633 fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1634 unsafe {
1635 llvm::LLVMRustPositionBuilderAtStart(self.llbuilder(), llbb);
1636 }
1637 }
1638}
1639impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1640 fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1641 let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1642 self.set_metadata_node(load, llvm::MD_align, &md);
1643 }
1644
1645 fn noundef_metadata(&mut self, load: &'ll Value) {
1646 self.set_metadata_node(load, llvm::MD_noundef, &[]);
1647 }
1648
1649 pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1650 self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
1651 }
1652
1653 fn write_operand_repeatedly_optimized(
1654 &mut self,
1655 cg_elem: OperandRef<'tcx, &'ll Value>,
1656 count: u64,
1657 dest: PlaceRef<'tcx, &'ll Value>,
1658 ) {
1659 let zero = self.const_usize(0);
1660 let count = self.const_usize(count);
1661
1662 let header_bb = self.append_sibling_block("repeat_loop_header");
1663 let body_bb = self.append_sibling_block("repeat_loop_body");
1664 let next_bb = self.append_sibling_block("repeat_loop_next");
1665
1666 self.br(header_bb);
1667
1668 let mut header_bx = Self::build(self.cx, header_bb);
1669 let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
1670
1671 let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
1672 header_bx.cond_br(keep_going, body_bb, next_bb);
1673
1674 let mut body_bx = Self::build(self.cx, body_bb);
1675 let dest_elem = dest.project_index(&mut body_bx, i);
1676 cg_elem.val.store(&mut body_bx, dest_elem);
1677
1678 let next = body_bx.unchecked_uadd(i, self.const_usize(1));
1679 body_bx.br(header_bb);
1680 header_bx.add_incoming_to_phi(i, next, body_bb);
1681
1682 *self = Self::build(self.cx, next_bb);
1683 }
1684
1685 fn write_operand_repeatedly_unoptimized(
1686 &mut self,
1687 cg_elem: OperandRef<'tcx, &'ll Value>,
1688 count: u64,
1689 dest: PlaceRef<'tcx, &'ll Value>,
1690 ) {
1691 let zero = self.const_usize(0);
1692 let count = self.const_usize(count);
1693 let start = dest.project_index(self, zero).val.llval;
1694 let end = dest.project_index(self, count).val.llval;
1695
1696 let header_bb = self.append_sibling_block("repeat_loop_header");
1697 let body_bb = self.append_sibling_block("repeat_loop_body");
1698 let next_bb = self.append_sibling_block("repeat_loop_next");
1699
1700 self.br(header_bb);
1701
1702 let mut header_bx = Self::build(self.cx, header_bb);
1703 let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
1704
1705 let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
1706 header_bx.cond_br(keep_going, body_bb, next_bb);
1707
1708 let mut body_bx = Self::build(self.cx, body_bb);
1709 let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
1710 cg_elem
1711 .val
1712 .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
1713
1714 let next = body_bx.inbounds_ptradd(current, self.const_usize(cg_elem.layout.size.bytes()));
1715 body_bx.br(header_bb);
1716 header_bx.add_incoming_to_phi(current, next, body_bb);
1717
1718 *self = Self::build(self.cx, next_bb);
1719 }
1720
1721 pub(crate) fn minimum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1722 let call = self.call_intrinsic("llvm.minimumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
1723 unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
1724 call
1725 }
1726
1727 pub(crate) fn maximum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1728 let call = self.call_intrinsic("llvm.maximumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
1729 unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
1730 call
1731 }
1732
1733 pub(crate) fn insert_element(
1734 &mut self,
1735 vec: &'ll Value,
1736 elt: &'ll Value,
1737 idx: &'ll Value,
1738 ) -> &'ll Value {
1739 unsafe { llvm::LLVMBuildInsertElement(self.llbuilder(), vec, elt, idx, UNNAMED) }
1740 }
1741
1742 pub(crate) fn shuffle_vector(
1743 &mut self,
1744 v1: &'ll Value,
1745 v2: &'ll Value,
1746 mask: &'ll Value,
1747 ) -> &'ll Value {
1748 unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder(), v1, v2, mask, UNNAMED) }
1749 }
1750
1751 pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1752 self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src])
1753 }
1754 pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1755 self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src])
1756 }
1757 pub(crate) fn vector_reduce_fadd_reassoc(
1758 &mut self,
1759 acc: &'ll Value,
1760 src: &'ll Value,
1761 ) -> &'ll Value {
1762 unsafe {
1763 let instr =
1764 self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
1765 llvm::LLVMRustSetAllowReassoc(instr);
1766 instr
1767 }
1768 }
1769 pub(crate) fn vector_reduce_fmul_reassoc(
1770 &mut self,
1771 acc: &'ll Value,
1772 src: &'ll Value,
1773 ) -> &'ll Value {
1774 unsafe {
1775 let instr =
1776 self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
1777 llvm::LLVMRustSetAllowReassoc(instr);
1778 instr
1779 }
1780 }
1781 pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1782 self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
1783 }
1784 pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1785 self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src])
1786 }
1787 pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1788 self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src])
1789 }
1790 pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1791 self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src])
1792 }
1793 pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1794 self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src])
1795 }
1796 pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1797 self.call_intrinsic(
1798 if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" },
1799 &[self.val_ty(src)],
1800 &[src],
1801 )
1802 }
1803 pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1804 self.call_intrinsic(
1805 if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" },
1806 &[self.val_ty(src)],
1807 &[src],
1808 )
1809 }
1810}
1811impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1812 pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1813 unsafe {
1814 llvm::LLVMAddClause(landing_pad, clause);
1815 }
1816 }
1817
1818 pub(crate) fn catch_ret(
1819 &mut self,
1820 funclet: &Funclet<'ll>,
1821 unwind: &'ll BasicBlock,
1822 ) -> &'ll Value {
1823 let ret =
1824 unsafe { llvm::LLVMBuildCatchRet(self.llbuilder(), funclet.cleanuppad(), unwind) };
1825 ret.expect("LLVM does not have support for catchret")
1826 }
1827
1828 pub(crate) fn check_call<'b>(
1829 &mut self,
1830 typ: &str,
1831 fn_ty: &'ll Type,
1832 llfn: &'ll Value,
1833 args: &'b [&'ll Value],
1834 ) -> Cow<'b, [&'ll Value]> {
1835 if !(self.cx.type_kind(fn_ty) == TypeKind::Function) {
{
::core::panicking::panic_fmt(format_args!("builder::{0} not passed a function, but {1:?}",
typ, fn_ty));
}
};assert!(
1836 self.cx.type_kind(fn_ty) == TypeKind::Function,
1837 "builder::{typ} not passed a function, but {fn_ty:?}"
1838 );
1839
1840 let param_tys = self.cx.func_params_types(fn_ty);
1841
1842 let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1843 .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1844
1845 if all_args_match {
1846 return Cow::Borrowed(args);
1847 }
1848
1849 let casted_args: Vec<_> = iter::zip(param_tys, args)
1850 .enumerate()
1851 .map(|(i, (expected_ty, &actual_val))| {
1852 let actual_ty = self.cx.val_ty(actual_val);
1853 if expected_ty != actual_ty {
1854 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1854",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(1854u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("type mismatch in function call of {0:?}. Expected {1:?} for param {2}, got {3:?}; injecting bitcast",
llfn, expected_ty, i, actual_ty) as
&dyn ::tracing::field::Value))])
});
} else { ; }
};debug!(
1855 "type mismatch in function call of {:?}. \
1856 Expected {:?} for param {}, got {:?}; injecting bitcast",
1857 llfn, expected_ty, i, actual_ty
1858 );
1859 self.bitcast(actual_val, expected_ty)
1860 } else {
1861 actual_val
1862 }
1863 })
1864 .collect();
1865
1866 Cow::Owned(casted_args)
1867 }
1868
1869 pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1870 unsafe { llvm::LLVMBuildVAArg(self.llbuilder(), list, ty, UNNAMED) }
1871 }
1872}
1873
1874impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1875 pub(crate) fn call_intrinsic(
1876 &mut self,
1877 base_name: impl Into<Cow<'static, str>>,
1878 type_params: &[&'ll Type],
1879 args: &[&'ll Value],
1880 ) -> &'ll Value {
1881 let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1882 self.call(ty, None, None, f, ReturnSlot::Direct, args, None, None)
1884 }
1885
1886 fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1887 let size = size.bytes();
1888 if size == 0 {
1889 return;
1890 }
1891
1892 if !self.cx().sess().emit_lifetime_markers() {
1893 return;
1894 }
1895
1896 let ptr = unsafe { llvm::LLVMRustStripPointerCasts(ptr) };
1899 self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
1900 }
1901}
1902impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1903 pub(crate) fn phi(
1904 &mut self,
1905 ty: &'ll Type,
1906 vals: &[&'ll Value],
1907 bbs: &[&'ll BasicBlock],
1908 ) -> &'ll Value {
1909 {
match (&vals.len(), &bbs.len()) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(vals.len(), bbs.len());
1910 let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder(), ty, UNNAMED) };
1911 unsafe {
1912 llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1913 phi
1914 }
1915 }
1916
1917 fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1918 unsafe {
1919 llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1920 }
1921 }
1922}
1923impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1924 pub(crate) fn landing_pad(
1925 &mut self,
1926 ty: &'ll Type,
1927 pers_fn: &'ll Value,
1928 num_clauses: usize,
1929 ) -> &'ll Value {
1930 self.set_personality_fn(pers_fn);
1934 unsafe {
1935 llvm::LLVMBuildLandingPad(self.llbuilder(), ty, None, num_clauses as c_uint, UNNAMED)
1936 }
1937 }
1938
1939 pub(crate) fn callbr(
1940 &mut self,
1941 llty: &'ll Type,
1942 fn_attrs: Option<&CodegenFnAttrs>,
1943 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1944 llfn: &'ll Value,
1945 args: &[&'ll Value],
1946 default_dest: &'ll BasicBlock,
1947 indirect_dest: &[&'ll BasicBlock],
1948 funclet: Option<&Funclet<'ll>>,
1949 instance: Option<Instance<'tcx>>,
1950 ) -> &'ll Value {
1951 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1951",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(1951u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("invoke {0:?} with args ({1:?})",
llfn, args) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("invoke {:?} with args ({:?})", llfn, args);
1952
1953 let args = self.check_call("callbr", llty, llfn, args);
1954 let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1955 let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1956 if let Some(funclet_bundle) = funclet_bundle {
1957 bundles.push(funclet_bundle);
1958 }
1959
1960 self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1962
1963 let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1965 if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|bundle| bundle.as_ref()) {
1966 bundles.push(kcfi_bundle);
1967 }
1968
1969 let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
1970 if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
1971 bundles.push(p);
1972 }
1973
1974 let callbr = unsafe {
1975 llvm::LLVMBuildCallBr(
1976 self.llbuilder(),
1977 llty,
1978 llfn,
1979 default_dest,
1980 indirect_dest.as_ptr(),
1981 indirect_dest.len() as c_uint,
1982 args.as_ptr(),
1983 args.len() as c_uint,
1984 bundles.as_ptr(),
1985 bundles.len() as c_uint,
1986 UNNAMED,
1987 )
1988 };
1989 if let Some(fn_abi) = fn_abi {
1990 fn_abi.apply_attrs_callsite(self, callbr);
1991 }
1992 callbr
1993 }
1994
1995 fn cfi_type_test(
1997 &mut self,
1998 fn_attrs: Option<&CodegenFnAttrs>,
1999 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2000 instance: Option<Instance<'tcx>>,
2001 llfn: &'ll Value,
2002 ) {
2003 let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
2004 if self.tcx.sess.is_sanitizer_cfi_enabled()
2005 && let Some(fn_abi) = fn_abi
2006 && is_indirect_call
2007 {
2008 if let Some(fn_attrs) = fn_attrs
2009 && fn_attrs.sanitizers.disabled.contains(SanitizerSet::CFI)
2010 {
2011 return;
2012 }
2013 if crate::llvm::HasStringAttribute(self.llfn(), "no-sanitize-cfi") {
2014 return;
2015 }
2016
2017 let mut options = cfi::TypeIdOptions::empty();
2018 if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
2019 options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
2020 }
2021 if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
2022 options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
2023 }
2024
2025 if self.cx.is_sanitizer_type_ignored(c"cfi", fn_abi) {
2026 return;
2027 }
2028
2029 let typeid = if let Some(instance) = instance {
2030 cfi::typeid_for_instance(self.tcx, instance, options)
2031 } else {
2032 cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
2033 };
2034 let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
2035 let dbg_loc = self.get_dbg_loc();
2036
2037 let typeid = self.get_metadata_value(typeid_metadata);
2041 let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
2042 let bb_pass = self.append_sibling_block("type_test.pass");
2043 let bb_fail = self.append_sibling_block("type_test.fail");
2044 self.cond_br(cond, bb_pass, bb_fail);
2045
2046 self.switch_to_block(bb_fail);
2047 if let Some(dbg_loc) = dbg_loc {
2048 self.set_dbg_loc(dbg_loc);
2049 }
2050
2051 let is_diag = self.tcx.sess.opts.unstable_opts.sanitizer_cfi_diag.unwrap_or(false);
2052 let is_recover =
2053 self.tcx.sess.opts.unstable_opts.sanitizer_cfi_recover.unwrap_or(false);
2054 let is_minimal =
2055 self.tcx.sess.opts.unstable_opts.sanitizer_cfi_minimal_runtime.unwrap_or(false);
2056
2057 if is_diag || is_recover {
2058 let fty = if is_minimal {
2059 self.cx.type_func(&[], self.cx.type_void())
2060 } else {
2061 self.cx.type_func(
2062 &[self.cx.type_ptr(), self.cx.type_isize(), self.cx.type_isize()],
2063 self.cx.type_void(),
2064 )
2065 };
2066 let ubsan_handler = self.declare_cfn(
2067 match (is_minimal, is_recover) {
2068 (true, true) => "__ubsan_handle_cfi_check_fail_minimal",
2069 (true, false) => "__ubsan_handle_cfi_check_fail_minimal_abort",
2070 (false, true) => "__ubsan_handle_cfi_check_fail",
2071 (false, false) => "__ubsan_handle_cfi_check_fail_abort",
2072 },
2073 llvm::UnnamedAddr::Global,
2074 fty,
2075 );
2076
2077 let mut expected_ty = String::from("fn(");
2078 {
let _guard = NoTrimmedGuard::new();
{
for (i, arg) in fn_abi.args.iter().enumerate() {
if i > 0 { expected_ty.push_str(", "); }
use std::fmt::Write;
(&mut expected_ty).write_fmt(format_args!("{0}",
arg.layout.ty)).unwrap();
}
expected_ty.push(')');
if !fn_abi.ret.layout.ty.is_unit() {
use std::fmt::Write;
(&mut expected_ty).write_fmt(format_args!(" -> {0}",
fn_abi.ret.layout.ty)).unwrap();
}
}
};ty::print::with_no_trimmed_paths!({
2079 for (i, arg) in fn_abi.args.iter().enumerate() {
2080 if i > 0 {
2081 expected_ty.push_str(", ");
2082 }
2083 use std::fmt::Write;
2084 write!(&mut expected_ty, "{}", arg.layout.ty).unwrap();
2085 }
2086 expected_ty.push(')');
2087 if !fn_abi.ret.layout.ty.is_unit() {
2088 use std::fmt::Write;
2089 write!(&mut expected_ty, " -> {}", fn_abi.ret.layout.ty).unwrap();
2090 }
2091 });
2092
2093 let check_kind = 4;
2095 let diag_data =
2096 self.generate_ubsan_cfi_diag_data(self.span, expected_ty, check_kind);
2097
2098 let function_address = self.ptrtoint(llfn, self.cx.type_isize());
2099 let arguments: &[_] = if is_minimal {
2100 &[]
2101 } else {
2102 &[diag_data, function_address, self.const_usize(0)]
2103 };
2104 self.call(
2105 fty,
2106 None,
2107 None,
2108 ubsan_handler,
2109 ReturnSlot::Direct,
2110 arguments,
2111 None,
2112 None,
2113 );
2114 if is_recover {
2115 self.br(bb_pass);
2116 } else {
2117 self.unreachable();
2118 }
2119 } else {
2120 self.abort_immediate();
2121 self.unreachable();
2122 }
2123
2124 self.switch_to_block(bb_pass);
2125 if let Some(dbg_loc) = dbg_loc {
2126 self.set_dbg_loc(dbg_loc);
2127 }
2128 }
2129 }
2130
2131 fn kcfi_operand_bundle(
2133 &mut self,
2134 fn_attrs: Option<&CodegenFnAttrs>,
2135 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2136 instance: Option<Instance<'tcx>>,
2137 llfn: &'ll Value,
2138 ) -> Option<llvm::OperandBundleBox<'ll>> {
2139 let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
2140 let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
2141 && let Some(fn_abi) = fn_abi
2142 && is_indirect_call
2143 {
2144 if let Some(fn_attrs) = fn_attrs
2145 && fn_attrs.sanitizers.disabled.contains(SanitizerSet::KCFI)
2146 {
2147 return None;
2148 }
2149 if crate::llvm::HasStringAttribute(self.llfn(), "no-sanitize-kcfi") {
2150 return None;
2151 }
2152
2153 let mut options = kcfi::TypeIdOptions::empty();
2154 if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
2155 options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
2156 }
2157 if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
2158 options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
2159 }
2160
2161 if self.cx.is_sanitizer_type_ignored(c"kcfi", fn_abi) {
2162 return None;
2163 }
2164
2165 let kcfi_typeid = if let Some(instance) = instance {
2166 kcfi::typeid_for_instance(self.tcx, instance, options)
2167 } else {
2168 kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
2169 };
2170
2171 Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
2172 } else {
2173 None
2174 };
2175 kcfi_bundle
2176 }
2177
2178 fn ptrauth_operand_bundle(
2180 &mut self,
2181 llfn: &'ll Value,
2182 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2183 ) -> Option<llvm::OperandBundleBox<'ll>> {
2184 let schema = self.sess().pointer_authentication_functions()?;
2185
2186 if fn_abi?.conv != CanonAbi::C {
2189 return None;
2190 }
2191 if llvm::get_value_name(llfn).starts_with(b"llvm.") {
2193 return None;
2194 }
2195
2196 Some(llvm::OperandBundleBox::new(
2204 "ptrauth",
2205 &[
2206 self.const_u32(schema.key as u32),
2207 self.const_u64(schema.constant_discriminator as u64),
2208 ],
2209 ))
2210 }
2211
2212 {}
#[allow(clippy :: suspicious_else_formatting)]
{
let __tracing_attr_span;
let __tracing_attr_guard;
if ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() ||
{ false } {
__tracing_attr_span =
{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("instrprof_increment",
"rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
::tracing_core::__macro_support::Option::Some(2213u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
::tracing_core::field::FieldSet::new(&[{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("fn_name")
}> =
::tracing::__macro_support::FieldName::new("fn_name");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("hash")
}> =
::tracing::__macro_support::FieldName::new("hash");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("num_counters")
}> =
::tracing::__macro_support::FieldName::new("num_counters");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("index")
}> =
::tracing::__macro_support::FieldName::new("index");
NAME.as_str()
}], ::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::SPAN)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let mut interest = ::tracing::subscriber::Interest::never();
if ::tracing::Level::DEBUG <=
::tracing::level_filters::STATIC_MAX_LEVEL &&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{ interest = __CALLSITE.interest(); !interest.is_never() }
&&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest) {
let meta = __CALLSITE.metadata();
::tracing::Span::new(meta,
&{
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&fn_name)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&hash)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&num_counters)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&index)
as &dyn ::tracing::field::Value))])
})
} else {
let span =
::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
{};
span
}
};
__tracing_attr_guard = __tracing_attr_span.enter();
}
#[warn(clippy :: suspicious_else_formatting)]
{
#[allow(unknown_lints, unreachable_code, clippy ::
diverging_sub_expression, clippy :: empty_loop, clippy ::
let_unit_value, clippy :: let_with_type_underscore, clippy ::
needless_return, clippy :: unreachable)]
if false {
let __tracing_attr_fake_return: () = loop {};
return __tracing_attr_fake_return;
}
{
self.call_intrinsic("llvm.instrprof.increment", &[],
&[fn_name, hash, num_counters, index]);
}
}
}#[instrument(level = "debug", skip(self))]
2214 pub(crate) fn instrprof_increment(
2215 &mut self,
2216 fn_name: &'ll Value,
2217 hash: &'ll Value,
2218 num_counters: &'ll Value,
2219 index: &'ll Value,
2220 ) {
2221 self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
2222 }
2223}