1use std::assert_matches::debug_assert_matches;
4use std::iter;
5use std::ops::{Range, RangeFrom};
6
7use rustc_abi::{ExternAbi, FieldIdx};
8use rustc_hir::attrs::{InlineAttr, OptimizeAttr};
9use rustc_hir::def::DefKind;
10use rustc_hir::def_id::DefId;
11use rustc_index::Idx;
12use rustc_index::bit_set::DenseBitSet;
13use rustc_middle::bug;
14use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
15use rustc_middle::mir::visit::*;
16use rustc_middle::mir::*;
17use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
18use rustc_session::config::{DebugInfo, OptLevel};
19use rustc_span::source_map::Spanned;
20use tracing::{debug, instrument, trace, trace_span};
21
22use crate::cost_checker::{CostChecker, is_call_like};
23use crate::simplify::{UsedInStmtLocals, simplify_cfg};
24use crate::validate::validate_types;
25use crate::{check_inline, util};
26
27pub(crate) mod cycle;
28
29const HISTORY_DEPTH_LIMIT: usize = 20;
30const TOP_DOWN_DEPTH_LIMIT: usize = 5;
31
32#[derive(Clone, Debug)]
33struct CallSite<'tcx> {
34 callee: Instance<'tcx>,
35 fn_sig: ty::PolyFnSig<'tcx>,
36 block: BasicBlock,
37 source_info: SourceInfo,
38}
39
40pub struct Inline;
43
44impl<'tcx> crate::MirPass<'tcx> for Inline {
45 fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
46 if let Some(enabled) = sess.opts.unstable_opts.inline_mir {
47 return enabled;
48 }
49
50 match sess.mir_opt_level() {
51 0 | 1 => false,
52 2 => {
53 (sess.opts.optimize == OptLevel::More || sess.opts.optimize == OptLevel::Aggressive)
54 && sess.opts.incremental == None
55 }
56 _ => true,
57 }
58 }
59
60 fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
61 let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id()));
62 let _guard = span.enter();
63 if inline::<NormalInliner<'tcx>>(tcx, body) {
64 debug!("running simplify cfg on {:?}", body.source);
65 simplify_cfg(tcx, body);
66 }
67 }
68
69 fn is_required(&self) -> bool {
70 false
71 }
72}
73
74pub struct ForceInline;
75
76impl ForceInline {
77 pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
78 matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
79 }
80}
81
82impl<'tcx> crate::MirPass<'tcx> for ForceInline {
83 fn is_enabled(&self, _: &rustc_session::Session) -> bool {
84 true
85 }
86
87 fn can_be_overridden(&self) -> bool {
88 false
89 }
90
91 fn is_required(&self) -> bool {
92 true
93 }
94
95 fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
96 let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id()));
97 let _guard = span.enter();
98 if inline::<ForceInliner<'tcx>>(tcx, body) {
99 debug!("running simplify cfg on {:?}", body.source);
100 simplify_cfg(tcx, body);
101 }
102 }
103}
104
105trait Inliner<'tcx> {
106 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self;
107
108 fn tcx(&self) -> TyCtxt<'tcx>;
109 fn typing_env(&self) -> ty::TypingEnv<'tcx>;
110 fn history(&self) -> &[DefId];
111 fn caller_def_id(&self) -> DefId;
112
113 fn changed(self) -> bool;
115
116 fn should_inline_for_callee(&self, def_id: DefId) -> bool;
118
119 fn check_codegen_attributes_extra(
120 &self,
121 callee_attrs: &CodegenFnAttrs,
122 ) -> Result<(), &'static str>;
123
124 fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool;
125
126 fn check_callee_mir_body(
129 &self,
130 callsite: &CallSite<'tcx>,
131 callee_body: &Body<'tcx>,
132 callee_attrs: &CodegenFnAttrs,
133 ) -> Result<(), &'static str>;
134
135 fn on_inline_success(
137 &mut self,
138 callsite: &CallSite<'tcx>,
139 caller_body: &mut Body<'tcx>,
140 new_blocks: std::ops::Range<BasicBlock>,
141 );
142
143 fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str);
145}
146
147struct ForceInliner<'tcx> {
148 tcx: TyCtxt<'tcx>,
149 typing_env: ty::TypingEnv<'tcx>,
150 def_id: DefId,
152 history: Vec<DefId>,
158 changed: bool,
160}
161
162impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
163 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
164 Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false }
165 }
166
167 fn tcx(&self) -> TyCtxt<'tcx> {
168 self.tcx
169 }
170
171 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
172 self.typing_env
173 }
174
175 fn history(&self) -> &[DefId] {
176 &self.history
177 }
178
179 fn caller_def_id(&self) -> DefId {
180 self.def_id
181 }
182
183 fn changed(self) -> bool {
184 self.changed
185 }
186
187 fn should_inline_for_callee(&self, def_id: DefId) -> bool {
188 ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
189 }
190
191 fn check_codegen_attributes_extra(
192 &self,
193 callee_attrs: &CodegenFnAttrs,
194 ) -> Result<(), &'static str> {
195 debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
196 Ok(())
197 }
198
199 fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool {
200 true
201 }
202
203 #[instrument(level = "debug", skip(self, callee_body))]
204 fn check_callee_mir_body(
205 &self,
206 _: &CallSite<'tcx>,
207 callee_body: &Body<'tcx>,
208 callee_attrs: &CodegenFnAttrs,
209 ) -> Result<(), &'static str> {
210 if callee_body.tainted_by_errors.is_some() {
211 return Err("body has errors");
212 }
213
214 let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id());
215 if callee_attrs.instruction_set != caller_attrs.instruction_set
216 && callee_body
217 .basic_blocks
218 .iter()
219 .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. }))
220 {
221 Err("cannot move inline-asm across instruction sets")
227 } else {
228 Ok(())
229 }
230 }
231
232 fn on_inline_success(
233 &mut self,
234 callsite: &CallSite<'tcx>,
235 caller_body: &mut Body<'tcx>,
236 new_blocks: std::ops::Range<BasicBlock>,
237 ) {
238 self.changed = true;
239
240 self.history.push(callsite.callee.def_id());
241 process_blocks(self, caller_body, new_blocks);
242 self.history.pop();
243 }
244
245 fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
246 let tcx = self.tcx();
247 let InlineAttr::Force { attr_span, reason: justification } =
248 tcx.codegen_instance_attrs(callsite.callee.def).inline
249 else {
250 bug!("called on item without required inlining");
251 };
252
253 let call_span = callsite.source_info.span;
254 tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
255 call_span,
256 attr_span,
257 caller_span: tcx.def_span(self.def_id),
258 caller: tcx.def_path_str(self.def_id),
259 callee_span: tcx.def_span(callsite.callee.def_id()),
260 callee: tcx.def_path_str(callsite.callee.def_id()),
261 reason,
262 justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
263 });
264 }
265}
266
267struct NormalInliner<'tcx> {
268 tcx: TyCtxt<'tcx>,
269 typing_env: ty::TypingEnv<'tcx>,
270 def_id: DefId,
272 history: Vec<DefId>,
278 top_down_counter: usize,
282 changed: bool,
284 caller_is_inline_forwarder: bool,
287}
288
289impl<'tcx> NormalInliner<'tcx> {
290 fn past_depth_limit(&self) -> bool {
291 self.history.len() > HISTORY_DEPTH_LIMIT || self.top_down_counter > TOP_DOWN_DEPTH_LIMIT
292 }
293}
294
295impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
296 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
297 let typing_env = body.typing_env(tcx);
298 let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
299
300 Self {
301 tcx,
302 typing_env,
303 def_id,
304 history: Vec::new(),
305 top_down_counter: 0,
306 changed: false,
307 caller_is_inline_forwarder: matches!(
308 codegen_fn_attrs.inline,
309 InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. }
310 ) && body_is_forwarder(body),
311 }
312 }
313
314 fn tcx(&self) -> TyCtxt<'tcx> {
315 self.tcx
316 }
317
318 fn caller_def_id(&self) -> DefId {
319 self.def_id
320 }
321
322 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
323 self.typing_env
324 }
325
326 fn history(&self) -> &[DefId] {
327 &self.history
328 }
329
330 fn changed(self) -> bool {
331 self.changed
332 }
333
334 fn should_inline_for_callee(&self, _: DefId) -> bool {
335 true
336 }
337
338 fn check_codegen_attributes_extra(
339 &self,
340 callee_attrs: &CodegenFnAttrs,
341 ) -> Result<(), &'static str> {
342 if self.past_depth_limit() && matches!(callee_attrs.inline, InlineAttr::None) {
343 Err("Past depth limit so not inspecting unmarked callee")
344 } else {
345 Ok(())
346 }
347 }
348
349 fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool {
350 if body.coroutine.is_some() {
354 return false;
355 }
356
357 true
358 }
359
360 #[instrument(level = "debug", skip(self, callee_body))]
361 fn check_callee_mir_body(
362 &self,
363 callsite: &CallSite<'tcx>,
364 callee_body: &Body<'tcx>,
365 callee_attrs: &CodegenFnAttrs,
366 ) -> Result<(), &'static str> {
367 let tcx = self.tcx();
368
369 if let Some(_) = callee_body.tainted_by_errors {
370 return Err("body has errors");
371 }
372
373 if self.past_depth_limit() && callee_body.basic_blocks.len() > 1 {
374 return Err("Not inlining multi-block body as we're past a depth limit");
375 }
376
377 let mut threshold = if self.caller_is_inline_forwarder || self.past_depth_limit() {
378 tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
379 } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {
380 tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
381 } else {
382 tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
383 };
384
385 if callee_body.basic_blocks.len() <= 3 {
389 threshold += threshold / 4;
390 }
391 debug!(" final inline threshold = {}", threshold);
392
393 let mut checker =
396 CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body);
397
398 checker.add_function_level_costs();
399
400 let mut work_list = vec![START_BLOCK];
402 let mut visited = DenseBitSet::new_empty(callee_body.basic_blocks.len());
403 while let Some(bb) = work_list.pop() {
404 if !visited.insert(bb.index()) {
405 continue;
406 }
407
408 let blk = &callee_body.basic_blocks[bb];
409 checker.visit_basic_block_data(bb, blk);
410
411 let term = blk.terminator();
412 let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id());
413 if let TerminatorKind::Drop {
414 ref place,
415 target,
416 unwind,
417 replace: _,
418 drop: _,
419 async_fut: _,
420 } = term.kind
421 {
422 work_list.push(target);
423
424 let ty = callsite
426 .callee
427 .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty));
428 if ty.needs_drop(tcx, self.typing_env())
429 && let UnwindAction::Cleanup(unwind) = unwind
430 {
431 work_list.push(unwind);
432 }
433 } else if callee_attrs.instruction_set != caller_attrs.instruction_set
434 && matches!(term.kind, TerminatorKind::InlineAsm { .. })
435 {
436 return Err("cannot move inline-asm across instruction sets");
442 } else if let TerminatorKind::TailCall { .. } = term.kind {
443 return Err("can't inline functions with tail calls");
446 } else {
447 work_list.extend(term.successors())
448 }
449 }
450
451 let cost = checker.cost();
456 if cost <= threshold {
457 debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
458 Ok(())
459 } else {
460 debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
461 Err("cost above threshold")
462 }
463 }
464
465 fn on_inline_success(
466 &mut self,
467 callsite: &CallSite<'tcx>,
468 caller_body: &mut Body<'tcx>,
469 new_blocks: std::ops::Range<BasicBlock>,
470 ) {
471 self.changed = true;
472
473 let new_calls_count = new_blocks
474 .clone()
475 .filter(|&bb| is_call_like(caller_body.basic_blocks[bb].terminator()))
476 .count();
477 if new_calls_count > 1 {
478 self.top_down_counter += 1;
479 }
480
481 self.history.push(callsite.callee.def_id());
482 process_blocks(self, caller_body, new_blocks);
483 self.history.pop();
484
485 if self.history.is_empty() {
486 self.top_down_counter = 0;
487 }
488 }
489
490 fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {}
491}
492
493fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
494 let def_id = body.source.def_id();
495
496 if !tcx.hir_body_owner_kind(def_id).is_fn_or_closure() {
498 return false;
499 }
500
501 let mut inliner = T::new(tcx, def_id, body);
502 if !inliner.check_caller_mir_body(body) {
503 return false;
504 }
505
506 let blocks = START_BLOCK..body.basic_blocks.next_index();
507 process_blocks(&mut inliner, body, blocks);
508 inliner.changed()
509}
510
511fn process_blocks<'tcx, I: Inliner<'tcx>>(
512 inliner: &mut I,
513 caller_body: &mut Body<'tcx>,
514 blocks: Range<BasicBlock>,
515) {
516 for bb in blocks {
517 let bb_data = &caller_body[bb];
518 if bb_data.is_cleanup {
519 continue;
520 }
521
522 let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else {
523 continue;
524 };
525
526 let span = trace_span!("process_blocks", %callsite.callee, ?bb);
527 let _guard = span.enter();
528
529 match try_inlining(inliner, caller_body, &callsite) {
530 Err(reason) => {
531 debug!("not-inlined {} [{}]", callsite.callee, reason);
532 inliner.on_inline_failure(&callsite, reason);
533 }
534 Ok(new_blocks) => {
535 debug!("inlined {}", callsite.callee);
536 inliner.on_inline_success(&callsite, caller_body, new_blocks);
537 }
538 }
539 }
540}
541
542fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
543 inliner: &I,
544 caller_body: &Body<'tcx>,
545 bb: BasicBlock,
546 bb_data: &BasicBlockData<'tcx>,
547) -> Option<CallSite<'tcx>> {
548 let tcx = inliner.tcx();
549 let terminator = bb_data.terminator();
551
552 if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
554 let func_ty = func.ty(caller_body, tcx);
555 if let ty::FnDef(def_id, args) = *func_ty.kind() {
556 if !inliner.should_inline_for_callee(def_id) {
557 debug!("not enabled");
558 return None;
559 }
560
561 let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
563 let callee =
564 Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
565
566 if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
567 return None;
568 }
569
570 if inliner.history().contains(&callee.def_id()) {
571 return None;
572 }
573
574 let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args);
575
576 if let InstanceKind::Item(instance_def_id) = callee.def
579 && tcx.def_kind(instance_def_id) == DefKind::AssocFn
580 && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder()
581 && instance_fn_sig.abi() != fn_sig.abi()
582 {
583 return None;
584 }
585
586 let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
587
588 return Some(CallSite { callee, fn_sig, block: bb, source_info });
589 }
590 }
591
592 None
593}
594
595fn try_inlining<'tcx, I: Inliner<'tcx>>(
599 inliner: &I,
600 caller_body: &mut Body<'tcx>,
601 callsite: &CallSite<'tcx>,
602) -> Result<std::ops::Range<BasicBlock>, &'static str> {
603 let tcx = inliner.tcx();
604 check_mir_is_available(inliner, caller_body, callsite.callee)?;
605
606 let callee_attrs = tcx.codegen_instance_attrs(callsite.callee.def);
607 let callee_attrs = callee_attrs.as_ref();
608 check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;
609 check_codegen_attributes(inliner, callsite, callee_attrs)?;
610 inliner.check_codegen_attributes_extra(callee_attrs)?;
611
612 let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
613 let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
614 let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty;
615 for arg in args {
616 if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) {
617 return Err("call has unsized argument");
620 }
621 }
622
623 let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
624 check_inline::is_inline_valid_on_body(tcx, callee_body)?;
625 inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;
626
627 let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(
628 tcx,
629 inliner.typing_env(),
630 ty::EarlyBinder::bind(callee_body.clone()),
631 ) else {
632 debug!("failed to normalize callee body");
633 return Err("implementation limitation -- could not normalize callee body");
634 };
635
636 if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() {
639 debug!("failed to validate callee body");
640 return Err("implementation limitation -- callee body failed validation");
641 }
642
643 let output_type = callee_body.return_ty();
647 if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) {
648 trace!(?output_type, ?destination_ty);
649 return Err("implementation limitation -- return type mismatch");
650 }
651 if callsite.fn_sig.abi() == ExternAbi::RustCall {
652 let (self_arg, arg_tuple) = match &args[..] {
653 [arg_tuple] => (None, arg_tuple),
654 [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
655 _ => bug!("Expected `rust-call` to have 1 or 2 args"),
656 };
657
658 let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
659
660 let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
661 let arg_tys = if callee_body.spread_arg.is_some() {
662 std::slice::from_ref(&arg_tuple_ty)
663 } else {
664 let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
665 bug!("Closure arguments are not passed as a tuple");
666 };
667 arg_tuple_tys.as_slice()
668 };
669
670 for (arg_ty, input) in
671 self_arg_ty.into_iter().chain(arg_tys.iter().copied()).zip(callee_body.args_iter())
672 {
673 let input_type = callee_body.local_decls[input].ty;
674 if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
675 trace!(?arg_ty, ?input_type);
676 debug!("failed to normalize tuple argument type");
677 return Err("implementation limitation");
678 }
679 }
680 } else {
681 for (arg, input) in args.iter().zip(callee_body.args_iter()) {
682 let input_type = callee_body.local_decls[input].ty;
683 let arg_ty = arg.node.ty(&caller_body.local_decls, tcx);
684 if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
685 trace!(?arg_ty, ?input_type);
686 debug!("failed to normalize argument type");
687 return Err("implementation limitation -- arg mismatch");
688 }
689 }
690 }
691
692 let old_blocks = caller_body.basic_blocks.next_index();
693 inline_call(inliner, caller_body, callsite, callee_body);
694 let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
695
696 Ok(new_blocks)
697}
698
699fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
700 inliner: &I,
701 caller_body: &Body<'tcx>,
702 callee: Instance<'tcx>,
703) -> Result<(), &'static str> {
704 let caller_def_id = caller_body.source.def_id();
705 let callee_def_id = callee.def_id();
706 if callee_def_id == caller_def_id {
707 return Err("self-recursion");
708 }
709
710 match callee.def {
711 InstanceKind::Item(_) => {
712 if !inliner.tcx().is_mir_available(callee_def_id) {
716 debug!("item MIR unavailable");
717 return Err("implementation limitation -- MIR unavailable");
718 }
719 }
720 InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
722 debug!("instance without MIR (intrinsic / virtual)");
723 return Err("implementation limitation -- cannot inline intrinsic");
724 }
725
726 InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
732 debug!("still needs substitution");
733 return Err("implementation limitation -- HACK for dropping polymorphic type");
734 }
735 InstanceKind::AsyncDropGlue(_, ty) | InstanceKind::AsyncDropGlueCtorShim(_, ty) => {
736 return if ty.still_further_specializable() {
737 Err("still needs substitution")
738 } else {
739 Ok(())
740 };
741 }
742 InstanceKind::FutureDropPollShim(_, ty, ty2) => {
743 return if ty.still_further_specializable() || ty2.still_further_specializable() {
744 Err("still needs substitution")
745 } else {
746 Ok(())
747 };
748 }
749
750 InstanceKind::VTableShim(_)
755 | InstanceKind::ReifyShim(..)
756 | InstanceKind::FnPtrShim(..)
757 | InstanceKind::ClosureOnceShim { .. }
758 | InstanceKind::ConstructCoroutineInClosureShim { .. }
759 | InstanceKind::DropGlue(..)
760 | InstanceKind::CloneShim(..)
761 | InstanceKind::ThreadLocalShim(..)
762 | InstanceKind::FnPtrAddrShim(..) => return Ok(()),
763 }
764
765 if inliner.tcx().is_constructor(callee_def_id) {
766 trace!("constructors always have MIR");
767 return Ok(());
769 }
770
771 if let Some(callee_def_id) = callee_def_id.as_local()
772 && !inliner
773 .tcx()
774 .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce)
775 {
776 if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee_def_id)
779 {
780 debug!("query cycle avoidance");
781 return Err("caller might be reachable from callee");
782 }
783
784 Ok(())
785 } else {
786 trace!("functions from other crates always have MIR");
791 Ok(())
792 }
793}
794
795fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(
798 inliner: &I,
799 callsite: &CallSite<'tcx>,
800 callee_attrs: &CodegenFnAttrs,
801) -> Result<(), &'static str> {
802 let tcx = inliner.tcx();
803 if let InlineAttr::Never = callee_attrs.inline {
804 return Err("never inline attribute");
805 }
806
807 if let OptimizeAttr::DoNotOptimize = callee_attrs.optimize {
808 return Err("has DoNotOptimize attribute");
809 }
810
811 inliner.check_codegen_attributes_extra(callee_attrs)?;
812
813 let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();
816 if !is_generic && !tcx.cross_crate_inlinable(callsite.callee.def_id()) {
817 return Err("not exported");
818 }
819
820 let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id());
821 if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {
822 return Err("incompatible sanitizer set");
823 }
824
825 if callee_attrs.instruction_set.is_some()
829 && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set
830 {
831 return Err("incompatible instruction set");
832 }
833
834 let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name);
835 let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name);
836 if callee_feature_names.ne(this_feature_names) {
837 return Err("incompatible target features");
843 }
844
845 Ok(())
846}
847
848fn inline_call<'tcx, I: Inliner<'tcx>>(
849 inliner: &I,
850 caller_body: &mut Body<'tcx>,
851 callsite: &CallSite<'tcx>,
852 mut callee_body: Body<'tcx>,
853) {
854 let tcx = inliner.tcx();
855 let terminator = caller_body[callsite.block].terminator.take().unwrap();
856 let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind
857 else {
858 bug!("unexpected terminator kind {:?}", terminator.kind);
859 };
860
861 let return_block = if let Some(block) = target {
862 let data = BasicBlockData::new(
865 Some(Terminator {
866 source_info: terminator.source_info,
867 kind: TerminatorKind::Goto { target: block },
868 }),
869 caller_body[block].is_cleanup,
870 );
871 Some(caller_body.basic_blocks_mut().push(data))
872 } else {
873 None
874 };
875
876 fn dest_needs_borrow(place: Place<'_>) -> bool {
882 for elem in place.projection.iter() {
883 match elem {
884 ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
885 _ => {}
886 }
887 }
888
889 false
890 }
891
892 let dest = if dest_needs_borrow(destination) {
893 trace!("creating temp for return destination");
894 let dest = Rvalue::Ref(
895 tcx.lifetimes.re_erased,
896 BorrowKind::Mut { kind: MutBorrowKind::Default },
897 destination,
898 );
899 let dest_ty = dest.ty(caller_body, tcx);
900 let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block));
901 caller_body[callsite.block].statements.push(Statement::new(
902 callsite.source_info,
903 StatementKind::Assign(Box::new((temp, dest))),
904 ));
905 tcx.mk_place_deref(temp)
906 } else {
907 destination
908 };
909
910 let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
913 (false, d)
914 } else {
915 (
916 true,
917 new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block),
918 )
919 };
920
921 let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block);
923
924 let mut integrator = Integrator {
925 args: &args,
926 new_locals: caller_body.local_decls.next_index()..,
927 new_scopes: caller_body.source_scopes.next_index()..,
928 new_blocks: caller_body.basic_blocks.next_index()..,
929 destination: destination_local,
930 callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
931 callsite,
932 cleanup_block: unwind,
933 in_cleanup_block: false,
934 return_block,
935 tcx,
936 always_live_locals: UsedInStmtLocals::new(&callee_body).locals,
937 };
938
939 integrator.visit_body(&mut callee_body);
942
943 for local in callee_body.vars_and_temps_iter() {
946 if integrator.always_live_locals.contains(local) {
947 let new_local = integrator.map_local(local);
948 caller_body[callsite.block]
949 .statements
950 .push(Statement::new(callsite.source_info, StatementKind::StorageLive(new_local)));
951 }
952 }
953 if let Some(block) = return_block {
954 let mut n = 0;
957 if remap_destination {
958 caller_body[block].statements.push(Statement::new(
959 callsite.source_info,
960 StatementKind::Assign(Box::new((
961 dest,
962 Rvalue::Use(Operand::Move(destination_local.into())),
963 ))),
964 ));
965 n += 1;
966 }
967 for local in callee_body.vars_and_temps_iter().rev() {
968 if integrator.always_live_locals.contains(local) {
969 let new_local = integrator.map_local(local);
970 caller_body[block].statements.push(Statement::new(
971 callsite.source_info,
972 StatementKind::StorageDead(new_local),
973 ));
974 n += 1;
975 }
976 }
977 caller_body[block].statements.rotate_right(n);
978 }
979
980 caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
982 caller_body.source_scopes.append(&mut callee_body.source_scopes);
983
984 if tcx
986 .sess
987 .opts
988 .unstable_opts
989 .inline_mir_preserve_debug
990 .unwrap_or(tcx.sess.opts.debuginfo == DebugInfo::Full)
991 {
992 caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
996 } else {
997 for bb in callee_body.basic_blocks_mut() {
998 bb.drop_debuginfo();
999 }
1000 }
1001 caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut());
1002
1003 caller_body[callsite.block].terminator = Some(Terminator {
1004 source_info: callsite.source_info,
1005 kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
1006 });
1007
1008 caller_body.required_consts.as_mut().unwrap().extend(
1012 callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()),
1013 );
1014 let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx));
1022 let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap();
1023 if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) {
1024 caller_mentioned_items.remove(idx);
1026 caller_mentioned_items.extend(callee_body.mentioned_items());
1027 } else {
1028 }
1032}
1033
1034fn make_call_args<'tcx, I: Inliner<'tcx>>(
1035 inliner: &I,
1036 args: Box<[Spanned<Operand<'tcx>>]>,
1037 callsite: &CallSite<'tcx>,
1038 caller_body: &mut Body<'tcx>,
1039 callee_body: &Body<'tcx>,
1040 return_block: Option<BasicBlock>,
1041) -> Box<[Local]> {
1042 let tcx = inliner.tcx();
1043
1044 if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() {
1068 let mut args = <_>::into_iter(args);
1070 let self_ = create_temp_if_necessary(
1071 inliner,
1072 args.next().unwrap().node,
1073 callsite,
1074 caller_body,
1075 return_block,
1076 );
1077 let tuple = create_temp_if_necessary(
1078 inliner,
1079 args.next().unwrap().node,
1080 callsite,
1081 caller_body,
1082 return_block,
1083 );
1084 assert!(args.next().is_none());
1085
1086 let tuple = Place::from(tuple);
1087 let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else {
1088 bug!("Closure arguments are not passed as a tuple");
1089 };
1090
1091 let closure_ref_arg = iter::once(self_);
1093
1094 let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
1096 let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
1098
1099 create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block)
1101 });
1102
1103 closure_ref_arg.chain(tuple_tmp_args).collect()
1104 } else {
1105 args.into_iter()
1106 .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
1107 .collect()
1108 }
1109}
1110
1111fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>(
1114 inliner: &I,
1115 arg: Operand<'tcx>,
1116 callsite: &CallSite<'tcx>,
1117 caller_body: &mut Body<'tcx>,
1118 return_block: Option<BasicBlock>,
1119) -> Local {
1120 if let Operand::Move(place) = &arg
1122 && let Some(local) = place.as_local()
1123 && caller_body.local_kind(local) == LocalKind::Temp
1124 {
1125 return local;
1126 }
1127
1128 trace!("creating temp for argument {:?}", arg);
1130 let arg_ty = arg.ty(caller_body, inliner.tcx());
1131 let local = new_call_temp(caller_body, callsite, arg_ty, return_block);
1132 caller_body[callsite.block].statements.push(Statement::new(
1133 callsite.source_info,
1134 StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
1135 ));
1136 local
1137}
1138
1139fn new_call_temp<'tcx>(
1141 caller_body: &mut Body<'tcx>,
1142 callsite: &CallSite<'tcx>,
1143 ty: Ty<'tcx>,
1144 return_block: Option<BasicBlock>,
1145) -> Local {
1146 let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
1147
1148 caller_body[callsite.block]
1149 .statements
1150 .push(Statement::new(callsite.source_info, StatementKind::StorageLive(local)));
1151
1152 if let Some(block) = return_block {
1153 caller_body[block]
1154 .statements
1155 .insert(0, Statement::new(callsite.source_info, StatementKind::StorageDead(local)));
1156 }
1157
1158 local
1159}
1160
1161struct Integrator<'a, 'tcx> {
1169 args: &'a [Local],
1170 new_locals: RangeFrom<Local>,
1171 new_scopes: RangeFrom<SourceScope>,
1172 new_blocks: RangeFrom<BasicBlock>,
1173 destination: Local,
1174 callsite_scope: SourceScopeData<'tcx>,
1175 callsite: &'a CallSite<'tcx>,
1176 cleanup_block: UnwindAction,
1177 in_cleanup_block: bool,
1178 return_block: Option<BasicBlock>,
1179 tcx: TyCtxt<'tcx>,
1180 always_live_locals: DenseBitSet<Local>,
1181}
1182
1183impl Integrator<'_, '_> {
1184 fn map_local(&self, local: Local) -> Local {
1185 let new = if local == RETURN_PLACE {
1186 self.destination
1187 } else {
1188 let idx = local.index() - 1;
1189 if idx < self.args.len() {
1190 self.args[idx]
1191 } else {
1192 self.new_locals.start + (idx - self.args.len())
1193 }
1194 };
1195 trace!("mapping local `{:?}` to `{:?}`", local, new);
1196 new
1197 }
1198
1199 fn map_scope(&self, scope: SourceScope) -> SourceScope {
1200 let new = self.new_scopes.start + scope.index();
1201 trace!("mapping scope `{:?}` to `{:?}`", scope, new);
1202 new
1203 }
1204
1205 fn map_block(&self, block: BasicBlock) -> BasicBlock {
1206 let new = self.new_blocks.start + block.index();
1207 trace!("mapping block `{:?}` to `{:?}`", block, new);
1208 new
1209 }
1210
1211 fn map_unwind(&self, unwind: UnwindAction) -> UnwindAction {
1212 if self.in_cleanup_block {
1213 match unwind {
1214 UnwindAction::Cleanup(_) | UnwindAction::Continue => {
1215 bug!("cleanup on cleanup block");
1216 }
1217 UnwindAction::Unreachable | UnwindAction::Terminate(_) => return unwind,
1218 }
1219 }
1220
1221 match unwind {
1222 UnwindAction::Unreachable | UnwindAction::Terminate(_) => unwind,
1223 UnwindAction::Cleanup(target) => UnwindAction::Cleanup(self.map_block(target)),
1224 UnwindAction::Continue => self.cleanup_block,
1226 }
1227 }
1228}
1229
1230impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
1231 fn tcx(&self) -> TyCtxt<'tcx> {
1232 self.tcx
1233 }
1234
1235 fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) {
1236 *local = self.map_local(*local);
1237 }
1238
1239 fn visit_source_scope_data(&mut self, scope_data: &mut SourceScopeData<'tcx>) {
1240 self.super_source_scope_data(scope_data);
1241 if scope_data.parent_scope.is_none() {
1242 scope_data.parent_scope = Some(self.callsite.source_info.scope);
1245 assert_eq!(scope_data.inlined_parent_scope, None);
1246 scope_data.inlined_parent_scope = if self.callsite_scope.inlined.is_some() {
1247 Some(self.callsite.source_info.scope)
1248 } else {
1249 self.callsite_scope.inlined_parent_scope
1250 };
1251
1252 assert_eq!(scope_data.inlined, None);
1254 scope_data.inlined = Some((self.callsite.callee, self.callsite.source_info.span));
1255 } else if scope_data.inlined_parent_scope.is_none() {
1256 scope_data.inlined_parent_scope = Some(self.map_scope(OUTERMOST_SOURCE_SCOPE));
1258 }
1259 }
1260
1261 fn visit_source_scope(&mut self, scope: &mut SourceScope) {
1262 *scope = self.map_scope(*scope);
1263 }
1264
1265 fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
1266 self.in_cleanup_block = data.is_cleanup;
1267 self.super_basic_block_data(block, data);
1268 self.in_cleanup_block = false;
1269 }
1270
1271 fn visit_retag(&mut self, kind: &mut RetagKind, place: &mut Place<'tcx>, loc: Location) {
1272 self.super_retag(kind, place, loc);
1273
1274 if *kind == RetagKind::FnEntry {
1277 *kind = RetagKind::Default;
1278 }
1279 }
1280
1281 fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
1282 if let StatementKind::StorageLive(local) | StatementKind::StorageDead(local) =
1283 statement.kind
1284 {
1285 self.always_live_locals.remove(local);
1286 }
1287 self.super_statement(statement, location);
1288 }
1289
1290 fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, loc: Location) {
1291 if !matches!(terminator.kind, TerminatorKind::Return) {
1294 self.super_terminator(terminator, loc);
1295 } else {
1296 self.visit_source_info(&mut terminator.source_info);
1297 }
1298
1299 match terminator.kind {
1300 TerminatorKind::CoroutineDrop | TerminatorKind::Yield { .. } => bug!(),
1301 TerminatorKind::Goto { ref mut target } => {
1302 *target = self.map_block(*target);
1303 }
1304 TerminatorKind::SwitchInt { ref mut targets, .. } => {
1305 for tgt in targets.all_targets_mut() {
1306 *tgt = self.map_block(*tgt);
1307 }
1308 }
1309 TerminatorKind::Drop { ref mut target, ref mut unwind, .. } => {
1310 *target = self.map_block(*target);
1311 *unwind = self.map_unwind(*unwind);
1312 }
1313 TerminatorKind::TailCall { .. } => {
1314 unreachable!()
1316 }
1317 TerminatorKind::Call { ref mut target, ref mut unwind, .. } => {
1318 if let Some(ref mut tgt) = *target {
1319 *tgt = self.map_block(*tgt);
1320 }
1321 *unwind = self.map_unwind(*unwind);
1322 }
1323 TerminatorKind::Assert { ref mut target, ref mut unwind, .. } => {
1324 *target = self.map_block(*target);
1325 *unwind = self.map_unwind(*unwind);
1326 }
1327 TerminatorKind::Return => {
1328 terminator.kind = if let Some(tgt) = self.return_block {
1329 TerminatorKind::Goto { target: tgt }
1330 } else {
1331 TerminatorKind::Unreachable
1332 }
1333 }
1334 TerminatorKind::UnwindResume => {
1335 terminator.kind = match self.cleanup_block {
1336 UnwindAction::Cleanup(tgt) => TerminatorKind::Goto { target: tgt },
1337 UnwindAction::Continue => TerminatorKind::UnwindResume,
1338 UnwindAction::Unreachable => TerminatorKind::Unreachable,
1339 UnwindAction::Terminate(reason) => TerminatorKind::UnwindTerminate(reason),
1340 };
1341 }
1342 TerminatorKind::UnwindTerminate(_) => {}
1343 TerminatorKind::Unreachable => {}
1344 TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => {
1345 *real_target = self.map_block(*real_target);
1346 *imaginary_target = self.map_block(*imaginary_target);
1347 }
1348 TerminatorKind::FalseUnwind { real_target: _, unwind: _ } =>
1349 {
1351 bug!("False unwinds should have been removed before inlining")
1352 }
1353 TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => {
1354 for tgt in targets.iter_mut() {
1355 *tgt = self.map_block(*tgt);
1356 }
1357 *unwind = self.map_unwind(*unwind);
1358 }
1359 }
1360 }
1361}
1362
1363#[instrument(skip(tcx), level = "debug")]
1364fn try_instance_mir<'tcx>(
1365 tcx: TyCtxt<'tcx>,
1366 instance: InstanceKind<'tcx>,
1367) -> Result<&'tcx Body<'tcx>, &'static str> {
1368 if let ty::InstanceKind::DropGlue(_, Some(ty)) | ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) =
1369 instance
1370 && let ty::Adt(def, args) = ty.kind()
1371 {
1372 let fields = def.all_fields();
1373 for field in fields {
1374 let field_ty = field.ty(tcx, args);
1375 if field_ty.has_param() && field_ty.has_aliases() {
1376 return Err("cannot build drop shim for polymorphic type");
1377 }
1378 }
1379 }
1380 Ok(tcx.instance_mir(instance))
1381}
1382
1383fn body_is_forwarder(body: &Body<'_>) -> bool {
1384 let TerminatorKind::Call { target, .. } = body.basic_blocks[START_BLOCK].terminator().kind
1385 else {
1386 return false;
1387 };
1388 if let Some(target) = target {
1389 let TerminatorKind::Return = body.basic_blocks[target].terminator().kind else {
1390 return false;
1391 };
1392 }
1393
1394 let max_blocks = if !body.is_polymorphic {
1395 2
1396 } else if target.is_none() {
1397 3
1398 } else {
1399 4
1400 };
1401 if body.basic_blocks.len() > max_blocks {
1402 return false;
1403 }
1404
1405 body.basic_blocks.iter_enumerated().all(|(bb, bb_data)| {
1406 bb == START_BLOCK
1407 || matches!(
1408 bb_data.terminator().kind,
1409 TerminatorKind::Return
1410 | TerminatorKind::Drop { .. }
1411 | TerminatorKind::UnwindResume
1412 | TerminatorKind::UnwindTerminate(_)
1413 )
1414 })
1415}