rustc_mir_transform/
inline.rs

1//! Inlining pass for MIR functions.
2
3use std::assert_matches::debug_assert_matches;
4use std::iter;
5use std::ops::{Range, RangeFrom};
6
7use rustc_abi::{ExternAbi, FieldIdx};
8use rustc_hir::attrs::{InlineAttr, OptimizeAttr};
9use rustc_hir::def::DefKind;
10use rustc_hir::def_id::DefId;
11use rustc_index::Idx;
12use rustc_index::bit_set::DenseBitSet;
13use rustc_middle::bug;
14use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
15use rustc_middle::mir::visit::*;
16use rustc_middle::mir::*;
17use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
18use rustc_session::config::{DebugInfo, OptLevel};
19use rustc_span::source_map::Spanned;
20use tracing::{debug, instrument, trace, trace_span};
21
22use crate::cost_checker::{CostChecker, is_call_like};
23use crate::simplify::{UsedInStmtLocals, simplify_cfg};
24use crate::validate::validate_types;
25use crate::{check_inline, util};
26
27pub(crate) mod cycle;
28
29const HISTORY_DEPTH_LIMIT: usize = 20;
30const TOP_DOWN_DEPTH_LIMIT: usize = 5;
31
32#[derive(Clone, Debug)]
33struct CallSite<'tcx> {
34    callee: Instance<'tcx>,
35    fn_sig: ty::PolyFnSig<'tcx>,
36    block: BasicBlock,
37    source_info: SourceInfo,
38}
39
40// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
41// by custom rustc drivers, running all the steps by themselves. See #114628.
42pub struct Inline;
43
44impl<'tcx> crate::MirPass<'tcx> for Inline {
45    fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
46        if let Some(enabled) = sess.opts.unstable_opts.inline_mir {
47            return enabled;
48        }
49
50        match sess.mir_opt_level() {
51            0 | 1 => false,
52            2 => {
53                (sess.opts.optimize == OptLevel::More || sess.opts.optimize == OptLevel::Aggressive)
54                    && sess.opts.incremental == None
55            }
56            _ => true,
57        }
58    }
59
60    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
61        let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id()));
62        let _guard = span.enter();
63        if inline::<NormalInliner<'tcx>>(tcx, body) {
64            debug!("running simplify cfg on {:?}", body.source);
65            simplify_cfg(tcx, body);
66        }
67    }
68
69    fn is_required(&self) -> bool {
70        false
71    }
72}
73
74pub struct ForceInline;
75
76impl ForceInline {
77    pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
78        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
79    }
80}
81
82impl<'tcx> crate::MirPass<'tcx> for ForceInline {
83    fn is_enabled(&self, _: &rustc_session::Session) -> bool {
84        true
85    }
86
87    fn can_be_overridden(&self) -> bool {
88        false
89    }
90
91    fn is_required(&self) -> bool {
92        true
93    }
94
95    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
96        let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id()));
97        let _guard = span.enter();
98        if inline::<ForceInliner<'tcx>>(tcx, body) {
99            debug!("running simplify cfg on {:?}", body.source);
100            simplify_cfg(tcx, body);
101        }
102    }
103}
104
105trait Inliner<'tcx> {
106    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self;
107
108    fn tcx(&self) -> TyCtxt<'tcx>;
109    fn typing_env(&self) -> ty::TypingEnv<'tcx>;
110    fn history(&self) -> &[DefId];
111    fn caller_def_id(&self) -> DefId;
112
113    /// Has the caller body been changed?
114    fn changed(self) -> bool;
115
116    /// Should inlining happen for a given callee?
117    fn should_inline_for_callee(&self, def_id: DefId) -> bool;
118
119    fn check_codegen_attributes_extra(
120        &self,
121        callee_attrs: &CodegenFnAttrs,
122    ) -> Result<(), &'static str>;
123
124    fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool;
125
126    /// Returns inlining decision that is based on the examination of callee MIR body.
127    /// Assumes that codegen attributes have been checked for compatibility already.
128    fn check_callee_mir_body(
129        &self,
130        callsite: &CallSite<'tcx>,
131        callee_body: &Body<'tcx>,
132        callee_attrs: &CodegenFnAttrs,
133    ) -> Result<(), &'static str>;
134
135    /// Called when inlining succeeds.
136    fn on_inline_success(
137        &mut self,
138        callsite: &CallSite<'tcx>,
139        caller_body: &mut Body<'tcx>,
140        new_blocks: std::ops::Range<BasicBlock>,
141    );
142
143    /// Called when inlining failed or was not performed.
144    fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str);
145}
146
147struct ForceInliner<'tcx> {
148    tcx: TyCtxt<'tcx>,
149    typing_env: ty::TypingEnv<'tcx>,
150    /// `DefId` of caller.
151    def_id: DefId,
152    /// Stack of inlined instances.
153    /// We only check the `DefId` and not the args because we want to
154    /// avoid inlining cases of polymorphic recursion.
155    /// The number of `DefId`s is finite, so checking history is enough
156    /// to ensure that we do not loop endlessly while inlining.
157    history: Vec<DefId>,
158    /// Indicates that the caller body has been modified.
159    changed: bool,
160}
161
162impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
163    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
164        Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false }
165    }
166
167    fn tcx(&self) -> TyCtxt<'tcx> {
168        self.tcx
169    }
170
171    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
172        self.typing_env
173    }
174
175    fn history(&self) -> &[DefId] {
176        &self.history
177    }
178
179    fn caller_def_id(&self) -> DefId {
180        self.def_id
181    }
182
183    fn changed(self) -> bool {
184        self.changed
185    }
186
187    fn should_inline_for_callee(&self, def_id: DefId) -> bool {
188        ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
189    }
190
191    fn check_codegen_attributes_extra(
192        &self,
193        callee_attrs: &CodegenFnAttrs,
194    ) -> Result<(), &'static str> {
195        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
196        Ok(())
197    }
198
199    fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool {
200        true
201    }
202
203    #[instrument(level = "debug", skip(self, callee_body))]
204    fn check_callee_mir_body(
205        &self,
206        _: &CallSite<'tcx>,
207        callee_body: &Body<'tcx>,
208        callee_attrs: &CodegenFnAttrs,
209    ) -> Result<(), &'static str> {
210        if callee_body.tainted_by_errors.is_some() {
211            return Err("body has errors");
212        }
213
214        let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id());
215        if callee_attrs.instruction_set != caller_attrs.instruction_set
216            && callee_body
217                .basic_blocks
218                .iter()
219                .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. }))
220        {
221            // During the attribute checking stage we allow a callee with no
222            // instruction_set assigned to count as compatible with a function that does
223            // assign one. However, during this stage we require an exact match when any
224            // inline-asm is detected. LLVM will still possibly do an inline later on
225            // if the no-attribute function ends up with the same instruction set anyway.
226            Err("cannot move inline-asm across instruction sets")
227        } else {
228            Ok(())
229        }
230    }
231
232    fn on_inline_success(
233        &mut self,
234        callsite: &CallSite<'tcx>,
235        caller_body: &mut Body<'tcx>,
236        new_blocks: std::ops::Range<BasicBlock>,
237    ) {
238        self.changed = true;
239
240        self.history.push(callsite.callee.def_id());
241        process_blocks(self, caller_body, new_blocks);
242        self.history.pop();
243    }
244
245    fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
246        let tcx = self.tcx();
247        let InlineAttr::Force { attr_span, reason: justification } =
248            tcx.codegen_instance_attrs(callsite.callee.def).inline
249        else {
250            bug!("called on item without required inlining");
251        };
252
253        let call_span = callsite.source_info.span;
254        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
255            call_span,
256            attr_span,
257            caller_span: tcx.def_span(self.def_id),
258            caller: tcx.def_path_str(self.def_id),
259            callee_span: tcx.def_span(callsite.callee.def_id()),
260            callee: tcx.def_path_str(callsite.callee.def_id()),
261            reason,
262            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
263        });
264    }
265}
266
267struct NormalInliner<'tcx> {
268    tcx: TyCtxt<'tcx>,
269    typing_env: ty::TypingEnv<'tcx>,
270    /// `DefId` of caller.
271    def_id: DefId,
272    /// Stack of inlined instances.
273    /// We only check the `DefId` and not the args because we want to
274    /// avoid inlining cases of polymorphic recursion.
275    /// The number of `DefId`s is finite, so checking history is enough
276    /// to ensure that we do not loop endlessly while inlining.
277    history: Vec<DefId>,
278    /// How many (multi-call) callsites have we inlined for the top-level call?
279    ///
280    /// We need to limit this in order to prevent super-linear growth in MIR size.
281    top_down_counter: usize,
282    /// Indicates that the caller body has been modified.
283    changed: bool,
284    /// Indicates that the caller is #[inline] and just calls another function,
285    /// and thus we can inline less into it as it'll be inlined itself.
286    caller_is_inline_forwarder: bool,
287}
288
289impl<'tcx> NormalInliner<'tcx> {
290    fn past_depth_limit(&self) -> bool {
291        self.history.len() > HISTORY_DEPTH_LIMIT || self.top_down_counter > TOP_DOWN_DEPTH_LIMIT
292    }
293}
294
295impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
296    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
297        let typing_env = body.typing_env(tcx);
298        let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
299
300        Self {
301            tcx,
302            typing_env,
303            def_id,
304            history: Vec::new(),
305            top_down_counter: 0,
306            changed: false,
307            caller_is_inline_forwarder: matches!(
308                codegen_fn_attrs.inline,
309                InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. }
310            ) && body_is_forwarder(body),
311        }
312    }
313
314    fn tcx(&self) -> TyCtxt<'tcx> {
315        self.tcx
316    }
317
318    fn caller_def_id(&self) -> DefId {
319        self.def_id
320    }
321
322    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
323        self.typing_env
324    }
325
326    fn history(&self) -> &[DefId] {
327        &self.history
328    }
329
330    fn changed(self) -> bool {
331        self.changed
332    }
333
334    fn should_inline_for_callee(&self, _: DefId) -> bool {
335        true
336    }
337
338    fn check_codegen_attributes_extra(
339        &self,
340        callee_attrs: &CodegenFnAttrs,
341    ) -> Result<(), &'static str> {
342        if self.past_depth_limit() && matches!(callee_attrs.inline, InlineAttr::None) {
343            Err("Past depth limit so not inspecting unmarked callee")
344        } else {
345            Ok(())
346        }
347    }
348
349    fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool {
350        // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
351        // which can create a cycle, even when no attempt is made to inline the function in the other
352        // direction.
353        if body.coroutine.is_some() {
354            return false;
355        }
356
357        true
358    }
359
360    #[instrument(level = "debug", skip(self, callee_body))]
361    fn check_callee_mir_body(
362        &self,
363        callsite: &CallSite<'tcx>,
364        callee_body: &Body<'tcx>,
365        callee_attrs: &CodegenFnAttrs,
366    ) -> Result<(), &'static str> {
367        let tcx = self.tcx();
368
369        if let Some(_) = callee_body.tainted_by_errors {
370            return Err("body has errors");
371        }
372
373        if self.past_depth_limit() && callee_body.basic_blocks.len() > 1 {
374            return Err("Not inlining multi-block body as we're past a depth limit");
375        }
376
377        let mut threshold = if self.caller_is_inline_forwarder || self.past_depth_limit() {
378            tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
379        } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {
380            tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
381        } else {
382            tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
383        };
384
385        // Give a bonus functions with a small number of blocks,
386        // We normally have two or three blocks for even
387        // very small functions.
388        if callee_body.basic_blocks.len() <= 3 {
389            threshold += threshold / 4;
390        }
391        debug!("    final inline threshold = {}", threshold);
392
393        // FIXME: Give a bonus to functions with only a single caller
394
395        let mut checker =
396            CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body);
397
398        checker.add_function_level_costs();
399
400        // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
401        let mut work_list = vec![START_BLOCK];
402        let mut visited = DenseBitSet::new_empty(callee_body.basic_blocks.len());
403        while let Some(bb) = work_list.pop() {
404            if !visited.insert(bb.index()) {
405                continue;
406            }
407
408            let blk = &callee_body.basic_blocks[bb];
409            checker.visit_basic_block_data(bb, blk);
410
411            let term = blk.terminator();
412            let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id());
413            if let TerminatorKind::Drop {
414                ref place,
415                target,
416                unwind,
417                replace: _,
418                drop: _,
419                async_fut: _,
420            } = term.kind
421            {
422                work_list.push(target);
423
424                // If the place doesn't actually need dropping, treat it like a regular goto.
425                let ty = callsite
426                    .callee
427                    .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty));
428                if ty.needs_drop(tcx, self.typing_env())
429                    && let UnwindAction::Cleanup(unwind) = unwind
430                {
431                    work_list.push(unwind);
432                }
433            } else if callee_attrs.instruction_set != caller_attrs.instruction_set
434                && matches!(term.kind, TerminatorKind::InlineAsm { .. })
435            {
436                // During the attribute checking stage we allow a callee with no
437                // instruction_set assigned to count as compatible with a function that does
438                // assign one. However, during this stage we require an exact match when any
439                // inline-asm is detected. LLVM will still possibly do an inline later on
440                // if the no-attribute function ends up with the same instruction set anyway.
441                return Err("cannot move inline-asm across instruction sets");
442            } else if let TerminatorKind::TailCall { .. } = term.kind {
443                // FIXME(explicit_tail_calls): figure out how exactly functions containing tail
444                // calls can be inlined (and if they even should)
445                return Err("can't inline functions with tail calls");
446            } else {
447                work_list.extend(term.successors())
448            }
449        }
450
451        // N.B. We still apply our cost threshold to #[inline(always)] functions.
452        // That attribute is often applied to very large functions that exceed LLVM's (very
453        // generous) inlining threshold. Such functions are very poor MIR inlining candidates.
454        // Always inlining #[inline(always)] functions in MIR, on net, slows down the compiler.
455        let cost = checker.cost();
456        if cost <= threshold {
457            debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
458            Ok(())
459        } else {
460            debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
461            Err("cost above threshold")
462        }
463    }
464
465    fn on_inline_success(
466        &mut self,
467        callsite: &CallSite<'tcx>,
468        caller_body: &mut Body<'tcx>,
469        new_blocks: std::ops::Range<BasicBlock>,
470    ) {
471        self.changed = true;
472
473        let new_calls_count = new_blocks
474            .clone()
475            .filter(|&bb| is_call_like(caller_body.basic_blocks[bb].terminator()))
476            .count();
477        if new_calls_count > 1 {
478            self.top_down_counter += 1;
479        }
480
481        self.history.push(callsite.callee.def_id());
482        process_blocks(self, caller_body, new_blocks);
483        self.history.pop();
484
485        if self.history.is_empty() {
486            self.top_down_counter = 0;
487        }
488    }
489
490    fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {}
491}
492
493fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
494    let def_id = body.source.def_id();
495
496    // Only do inlining into fn bodies.
497    if !tcx.hir_body_owner_kind(def_id).is_fn_or_closure() {
498        return false;
499    }
500
501    let mut inliner = T::new(tcx, def_id, body);
502    if !inliner.check_caller_mir_body(body) {
503        return false;
504    }
505
506    let blocks = START_BLOCK..body.basic_blocks.next_index();
507    process_blocks(&mut inliner, body, blocks);
508    inliner.changed()
509}
510
511fn process_blocks<'tcx, I: Inliner<'tcx>>(
512    inliner: &mut I,
513    caller_body: &mut Body<'tcx>,
514    blocks: Range<BasicBlock>,
515) {
516    for bb in blocks {
517        let bb_data = &caller_body[bb];
518        if bb_data.is_cleanup {
519            continue;
520        }
521
522        let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else {
523            continue;
524        };
525
526        let span = trace_span!("process_blocks", %callsite.callee, ?bb);
527        let _guard = span.enter();
528
529        match try_inlining(inliner, caller_body, &callsite) {
530            Err(reason) => {
531                debug!("not-inlined {} [{}]", callsite.callee, reason);
532                inliner.on_inline_failure(&callsite, reason);
533            }
534            Ok(new_blocks) => {
535                debug!("inlined {}", callsite.callee);
536                inliner.on_inline_success(&callsite, caller_body, new_blocks);
537            }
538        }
539    }
540}
541
542fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
543    inliner: &I,
544    caller_body: &Body<'tcx>,
545    bb: BasicBlock,
546    bb_data: &BasicBlockData<'tcx>,
547) -> Option<CallSite<'tcx>> {
548    let tcx = inliner.tcx();
549    // Only consider direct calls to functions
550    let terminator = bb_data.terminator();
551
552    // FIXME(explicit_tail_calls): figure out if we can inline tail calls
553    if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
554        let func_ty = func.ty(caller_body, tcx);
555        if let ty::FnDef(def_id, args) = *func_ty.kind() {
556            if !inliner.should_inline_for_callee(def_id) {
557                debug!("not enabled");
558                return None;
559            }
560
561            // To resolve an instance its args have to be fully normalized.
562            let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
563            let callee =
564                Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
565
566            if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
567                return None;
568            }
569
570            if inliner.history().contains(&callee.def_id()) {
571                return None;
572            }
573
574            let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args);
575
576            // Additionally, check that the body that we're inlining actually agrees
577            // with the ABI of the trait that the item comes from.
578            if let InstanceKind::Item(instance_def_id) = callee.def
579                && tcx.def_kind(instance_def_id) == DefKind::AssocFn
580                && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder()
581                && instance_fn_sig.abi() != fn_sig.abi()
582            {
583                return None;
584            }
585
586            let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
587
588            return Some(CallSite { callee, fn_sig, block: bb, source_info });
589        }
590    }
591
592    None
593}
594
595/// Attempts to inline a callsite into the caller body. When successful returns basic blocks
596/// containing the inlined body. Otherwise returns an error describing why inlining didn't take
597/// place.
598fn try_inlining<'tcx, I: Inliner<'tcx>>(
599    inliner: &I,
600    caller_body: &mut Body<'tcx>,
601    callsite: &CallSite<'tcx>,
602) -> Result<std::ops::Range<BasicBlock>, &'static str> {
603    let tcx = inliner.tcx();
604    check_mir_is_available(inliner, caller_body, callsite.callee)?;
605
606    let callee_attrs = tcx.codegen_instance_attrs(callsite.callee.def);
607    let callee_attrs = callee_attrs.as_ref();
608    check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;
609    check_codegen_attributes(inliner, callsite, callee_attrs)?;
610    inliner.check_codegen_attributes_extra(callee_attrs)?;
611
612    let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
613    let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
614    let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty;
615    for arg in args {
616        if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) {
617            // We do not allow inlining functions with unsized params. Inlining these functions
618            // could create unsized locals, which are unsound and being phased out.
619            return Err("call has unsized argument");
620        }
621    }
622
623    let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
624    check_inline::is_inline_valid_on_body(tcx, callee_body)?;
625    inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;
626
627    let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(
628        tcx,
629        inliner.typing_env(),
630        ty::EarlyBinder::bind(callee_body.clone()),
631    ) else {
632        debug!("failed to normalize callee body");
633        return Err("implementation limitation -- could not normalize callee body");
634    };
635
636    // Normally, this shouldn't be required, but trait normalization failure can create a
637    // validation ICE.
638    if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() {
639        debug!("failed to validate callee body");
640        return Err("implementation limitation -- callee body failed validation");
641    }
642
643    // Check call signature compatibility.
644    // Normally, this shouldn't be required, but trait normalization failure can create a
645    // validation ICE.
646    let output_type = callee_body.return_ty();
647    if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) {
648        trace!(?output_type, ?destination_ty);
649        return Err("implementation limitation -- return type mismatch");
650    }
651    if callsite.fn_sig.abi() == ExternAbi::RustCall {
652        let (self_arg, arg_tuple) = match &args[..] {
653            [arg_tuple] => (None, arg_tuple),
654            [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
655            _ => bug!("Expected `rust-call` to have 1 or 2 args"),
656        };
657
658        let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
659
660        let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
661        let arg_tys = if callee_body.spread_arg.is_some() {
662            std::slice::from_ref(&arg_tuple_ty)
663        } else {
664            let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
665                bug!("Closure arguments are not passed as a tuple");
666            };
667            arg_tuple_tys.as_slice()
668        };
669
670        for (arg_ty, input) in
671            self_arg_ty.into_iter().chain(arg_tys.iter().copied()).zip(callee_body.args_iter())
672        {
673            let input_type = callee_body.local_decls[input].ty;
674            if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
675                trace!(?arg_ty, ?input_type);
676                debug!("failed to normalize tuple argument type");
677                return Err("implementation limitation");
678            }
679        }
680    } else {
681        for (arg, input) in args.iter().zip(callee_body.args_iter()) {
682            let input_type = callee_body.local_decls[input].ty;
683            let arg_ty = arg.node.ty(&caller_body.local_decls, tcx);
684            if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
685                trace!(?arg_ty, ?input_type);
686                debug!("failed to normalize argument type");
687                return Err("implementation limitation -- arg mismatch");
688            }
689        }
690    }
691
692    let old_blocks = caller_body.basic_blocks.next_index();
693    inline_call(inliner, caller_body, callsite, callee_body);
694    let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
695
696    Ok(new_blocks)
697}
698
699fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
700    inliner: &I,
701    caller_body: &Body<'tcx>,
702    callee: Instance<'tcx>,
703) -> Result<(), &'static str> {
704    let caller_def_id = caller_body.source.def_id();
705    let callee_def_id = callee.def_id();
706    if callee_def_id == caller_def_id {
707        return Err("self-recursion");
708    }
709
710    match callee.def {
711        InstanceKind::Item(_) => {
712            // If there is no MIR available (either because it was not in metadata or
713            // because it has no MIR because it's an extern function), then the inliner
714            // won't cause cycles on this.
715            if !inliner.tcx().is_mir_available(callee_def_id) {
716                debug!("item MIR unavailable");
717                return Err("implementation limitation -- MIR unavailable");
718            }
719        }
720        // These have no own callable MIR.
721        InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
722            debug!("instance without MIR (intrinsic / virtual)");
723            return Err("implementation limitation -- cannot inline intrinsic");
724        }
725
726        // FIXME(#127030): `ConstParamHasTy` has bad interactions with
727        // the drop shim builder, which does not evaluate predicates in
728        // the correct param-env for types being dropped. Stall resolving
729        // the MIR for this instance until all of its const params are
730        // substituted.
731        InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
732            debug!("still needs substitution");
733            return Err("implementation limitation -- HACK for dropping polymorphic type");
734        }
735        InstanceKind::AsyncDropGlue(_, ty) | InstanceKind::AsyncDropGlueCtorShim(_, ty) => {
736            return if ty.still_further_specializable() {
737                Err("still needs substitution")
738            } else {
739                Ok(())
740            };
741        }
742        InstanceKind::FutureDropPollShim(_, ty, ty2) => {
743            return if ty.still_further_specializable() || ty2.still_further_specializable() {
744                Err("still needs substitution")
745            } else {
746                Ok(())
747            };
748        }
749
750        // This cannot result in an immediate cycle since the callee MIR is a shim, which does
751        // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we
752        // do not need to catch this here, we can wait until the inliner decides to continue
753        // inlining a second time.
754        InstanceKind::VTableShim(_)
755        | InstanceKind::ReifyShim(..)
756        | InstanceKind::FnPtrShim(..)
757        | InstanceKind::ClosureOnceShim { .. }
758        | InstanceKind::ConstructCoroutineInClosureShim { .. }
759        | InstanceKind::DropGlue(..)
760        | InstanceKind::CloneShim(..)
761        | InstanceKind::ThreadLocalShim(..)
762        | InstanceKind::FnPtrAddrShim(..) => return Ok(()),
763    }
764
765    if inliner.tcx().is_constructor(callee_def_id) {
766        trace!("constructors always have MIR");
767        // Constructor functions cannot cause a query cycle.
768        return Ok(());
769    }
770
771    if let Some(callee_def_id) = callee_def_id.as_local()
772        && !inliner
773            .tcx()
774            .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce)
775    {
776        // If we know for sure that the function we're calling will itself try to
777        // call us, then we avoid inlining that function.
778        if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee_def_id)
779        {
780            debug!("query cycle avoidance");
781            return Err("caller might be reachable from callee");
782        }
783
784        Ok(())
785    } else {
786        // This cannot result in an immediate cycle since the callee MIR is from another crate
787        // and is already optimized. Any subsequent inlining may cause cycles, but we do
788        // not need to catch this here, we can wait until the inliner decides to continue
789        // inlining a second time.
790        trace!("functions from other crates always have MIR");
791        Ok(())
792    }
793}
794
795/// Returns an error if inlining is not possible based on codegen attributes alone. A success
796/// indicates that inlining decision should be based on other criteria.
797fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(
798    inliner: &I,
799    callsite: &CallSite<'tcx>,
800    callee_attrs: &CodegenFnAttrs,
801) -> Result<(), &'static str> {
802    let tcx = inliner.tcx();
803    if let InlineAttr::Never = callee_attrs.inline {
804        return Err("never inline attribute");
805    }
806
807    if let OptimizeAttr::DoNotOptimize = callee_attrs.optimize {
808        return Err("has DoNotOptimize attribute");
809    }
810
811    inliner.check_codegen_attributes_extra(callee_attrs)?;
812
813    // Reachability pass defines which functions are eligible for inlining. Generally inlining
814    // other functions is incorrect because they could reference symbols that aren't exported.
815    let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();
816    if !is_generic && !tcx.cross_crate_inlinable(callsite.callee.def_id()) {
817        return Err("not exported");
818    }
819
820    let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id());
821    if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {
822        return Err("incompatible sanitizer set");
823    }
824
825    // Two functions are compatible if the callee has no attribute (meaning
826    // that it's codegen agnostic), or sets an attribute that is identical
827    // to this function's attribute.
828    if callee_attrs.instruction_set.is_some()
829        && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set
830    {
831        return Err("incompatible instruction set");
832    }
833
834    let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name);
835    let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name);
836    if callee_feature_names.ne(this_feature_names) {
837        // In general it is not correct to inline a callee with target features that are a
838        // subset of the caller. This is because the callee might contain calls, and the ABI of
839        // those calls depends on the target features of the surrounding function. By moving a
840        // `Call` terminator from one MIR body to another with more target features, we might
841        // change the ABI of that call!
842        return Err("incompatible target features");
843    }
844
845    Ok(())
846}
847
848fn inline_call<'tcx, I: Inliner<'tcx>>(
849    inliner: &I,
850    caller_body: &mut Body<'tcx>,
851    callsite: &CallSite<'tcx>,
852    mut callee_body: Body<'tcx>,
853) {
854    let tcx = inliner.tcx();
855    let terminator = caller_body[callsite.block].terminator.take().unwrap();
856    let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind
857    else {
858        bug!("unexpected terminator kind {:?}", terminator.kind);
859    };
860
861    let return_block = if let Some(block) = target {
862        // Prepare a new block for code that should execute when call returns. We don't use
863        // target block directly since it might have other predecessors.
864        let data = BasicBlockData::new(
865            Some(Terminator {
866                source_info: terminator.source_info,
867                kind: TerminatorKind::Goto { target: block },
868            }),
869            caller_body[block].is_cleanup,
870        );
871        Some(caller_body.basic_blocks_mut().push(data))
872    } else {
873        None
874    };
875
876    // If the call is something like `a[*i] = f(i)`, where
877    // `i : &mut usize`, then just duplicating the `a[*i]`
878    // Place could result in two different locations if `f`
879    // writes to `i`. To prevent this we need to create a temporary
880    // borrow of the place and pass the destination as `*temp` instead.
881    fn dest_needs_borrow(place: Place<'_>) -> bool {
882        for elem in place.projection.iter() {
883            match elem {
884                ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
885                _ => {}
886            }
887        }
888
889        false
890    }
891
892    let dest = if dest_needs_borrow(destination) {
893        trace!("creating temp for return destination");
894        let dest = Rvalue::Ref(
895            tcx.lifetimes.re_erased,
896            BorrowKind::Mut { kind: MutBorrowKind::Default },
897            destination,
898        );
899        let dest_ty = dest.ty(caller_body, tcx);
900        let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block));
901        caller_body[callsite.block].statements.push(Statement::new(
902            callsite.source_info,
903            StatementKind::Assign(Box::new((temp, dest))),
904        ));
905        tcx.mk_place_deref(temp)
906    } else {
907        destination
908    };
909
910    // Always create a local to hold the destination, as `RETURN_PLACE` may appear
911    // where a full `Place` is not allowed.
912    let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
913        (false, d)
914    } else {
915        (
916            true,
917            new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block),
918        )
919    };
920
921    // Copy the arguments if needed.
922    let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block);
923
924    let mut integrator = Integrator {
925        args: &args,
926        new_locals: caller_body.local_decls.next_index()..,
927        new_scopes: caller_body.source_scopes.next_index()..,
928        new_blocks: caller_body.basic_blocks.next_index()..,
929        destination: destination_local,
930        callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
931        callsite,
932        cleanup_block: unwind,
933        in_cleanup_block: false,
934        return_block,
935        tcx,
936        always_live_locals: UsedInStmtLocals::new(&callee_body).locals,
937    };
938
939    // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
940    // (or existing ones, in a few special cases) in the caller.
941    integrator.visit_body(&mut callee_body);
942
943    // If there are any locals without storage markers, give them storage only for the
944    // duration of the call.
945    for local in callee_body.vars_and_temps_iter() {
946        if integrator.always_live_locals.contains(local) {
947            let new_local = integrator.map_local(local);
948            caller_body[callsite.block]
949                .statements
950                .push(Statement::new(callsite.source_info, StatementKind::StorageLive(new_local)));
951        }
952    }
953    if let Some(block) = return_block {
954        // To avoid repeated O(n) insert, push any new statements to the end and rotate
955        // the slice once.
956        let mut n = 0;
957        if remap_destination {
958            caller_body[block].statements.push(Statement::new(
959                callsite.source_info,
960                StatementKind::Assign(Box::new((
961                    dest,
962                    Rvalue::Use(Operand::Move(destination_local.into())),
963                ))),
964            ));
965            n += 1;
966        }
967        for local in callee_body.vars_and_temps_iter().rev() {
968            if integrator.always_live_locals.contains(local) {
969                let new_local = integrator.map_local(local);
970                caller_body[block].statements.push(Statement::new(
971                    callsite.source_info,
972                    StatementKind::StorageDead(new_local),
973                ));
974                n += 1;
975            }
976        }
977        caller_body[block].statements.rotate_right(n);
978    }
979
980    // Insert all of the (mapped) parts of the callee body into the caller.
981    caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
982    caller_body.source_scopes.append(&mut callee_body.source_scopes);
983
984    // only "full" debug promises any variable-level information
985    if tcx
986        .sess
987        .opts
988        .unstable_opts
989        .inline_mir_preserve_debug
990        .unwrap_or(tcx.sess.opts.debuginfo == DebugInfo::Full)
991    {
992        // -Zinline-mir-preserve-debug is enabled when building the standard library, so that
993        // people working on rust can build with or without debuginfo while
994        // still getting consistent results from the mir-opt tests.
995        caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
996    } else {
997        for bb in callee_body.basic_blocks_mut() {
998            bb.drop_debuginfo();
999        }
1000    }
1001    caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut());
1002
1003    caller_body[callsite.block].terminator = Some(Terminator {
1004        source_info: callsite.source_info,
1005        kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
1006    });
1007
1008    // Copy required constants from the callee_body into the caller_body. Although we are only
1009    // pushing unevaluated consts to `required_consts`, here they may have been evaluated
1010    // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again.
1011    caller_body.required_consts.as_mut().unwrap().extend(
1012        callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()),
1013    );
1014    // Now that we incorporated the callee's `required_consts`, we can remove the callee from
1015    // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does
1016    // some extra work here to save the monomorphization collector work later. It helps a lot,
1017    // since monomorphization can avoid a lot of work when the "mentioned items" are similar to
1018    // the actually used items. By doing this we can entirely avoid visiting the callee!
1019    // We need to reconstruct the `required_item` for the callee so that we can find and
1020    // remove it.
1021    let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx));
1022    let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap();
1023    if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) {
1024        // We found the callee, so remove it and add its items instead.
1025        caller_mentioned_items.remove(idx);
1026        caller_mentioned_items.extend(callee_body.mentioned_items());
1027    } else {
1028        // If we can't find the callee, there's no point in adding its items. Probably it
1029        // already got removed by being inlined elsewhere in the same function, so we already
1030        // took its items.
1031    }
1032}
1033
1034fn make_call_args<'tcx, I: Inliner<'tcx>>(
1035    inliner: &I,
1036    args: Box<[Spanned<Operand<'tcx>>]>,
1037    callsite: &CallSite<'tcx>,
1038    caller_body: &mut Body<'tcx>,
1039    callee_body: &Body<'tcx>,
1040    return_block: Option<BasicBlock>,
1041) -> Box<[Local]> {
1042    let tcx = inliner.tcx();
1043
1044    // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
1045    // The caller provides the arguments wrapped up in a tuple:
1046    //
1047    //     tuple_tmp = (a, b, c)
1048    //     Fn::call(closure_ref, tuple_tmp)
1049    //
1050    // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`)
1051    // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has
1052    // the job of unpacking this tuple. But here, we are codegen. =) So we want to create
1053    // a vector like
1054    //
1055    //     [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2]
1056    //
1057    // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient
1058    // if we "spill" that into *another* temporary, so that we can map the argument
1059    // variable in the callee MIR directly to an argument variable on our side.
1060    // So we introduce temporaries like:
1061    //
1062    //     tmp0 = tuple_tmp.0
1063    //     tmp1 = tuple_tmp.1
1064    //     tmp2 = tuple_tmp.2
1065    //
1066    // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
1067    if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() {
1068        // FIXME(edition_2024): switch back to a normal method call.
1069        let mut args = <_>::into_iter(args);
1070        let self_ = create_temp_if_necessary(
1071            inliner,
1072            args.next().unwrap().node,
1073            callsite,
1074            caller_body,
1075            return_block,
1076        );
1077        let tuple = create_temp_if_necessary(
1078            inliner,
1079            args.next().unwrap().node,
1080            callsite,
1081            caller_body,
1082            return_block,
1083        );
1084        assert!(args.next().is_none());
1085
1086        let tuple = Place::from(tuple);
1087        let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else {
1088            bug!("Closure arguments are not passed as a tuple");
1089        };
1090
1091        // The `closure_ref` in our example above.
1092        let closure_ref_arg = iter::once(self_);
1093
1094        // The `tmp0`, `tmp1`, and `tmp2` in our example above.
1095        let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
1096            // This is e.g., `tuple_tmp.0` in our example above.
1097            let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
1098
1099            // Spill to a local to make e.g., `tmp0`.
1100            create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block)
1101        });
1102
1103        closure_ref_arg.chain(tuple_tmp_args).collect()
1104    } else {
1105        args.into_iter()
1106            .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
1107            .collect()
1108    }
1109}
1110
1111/// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh temporary `T` and an
1112/// instruction `T = arg`, and returns `T`.
1113fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>(
1114    inliner: &I,
1115    arg: Operand<'tcx>,
1116    callsite: &CallSite<'tcx>,
1117    caller_body: &mut Body<'tcx>,
1118    return_block: Option<BasicBlock>,
1119) -> Local {
1120    // Reuse the operand if it is a moved temporary.
1121    if let Operand::Move(place) = &arg
1122        && let Some(local) = place.as_local()
1123        && caller_body.local_kind(local) == LocalKind::Temp
1124    {
1125        return local;
1126    }
1127
1128    // Otherwise, create a temporary for the argument.
1129    trace!("creating temp for argument {:?}", arg);
1130    let arg_ty = arg.ty(caller_body, inliner.tcx());
1131    let local = new_call_temp(caller_body, callsite, arg_ty, return_block);
1132    caller_body[callsite.block].statements.push(Statement::new(
1133        callsite.source_info,
1134        StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
1135    ));
1136    local
1137}
1138
1139/// Introduces a new temporary into the caller body that is live for the duration of the call.
1140fn new_call_temp<'tcx>(
1141    caller_body: &mut Body<'tcx>,
1142    callsite: &CallSite<'tcx>,
1143    ty: Ty<'tcx>,
1144    return_block: Option<BasicBlock>,
1145) -> Local {
1146    let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
1147
1148    caller_body[callsite.block]
1149        .statements
1150        .push(Statement::new(callsite.source_info, StatementKind::StorageLive(local)));
1151
1152    if let Some(block) = return_block {
1153        caller_body[block]
1154            .statements
1155            .insert(0, Statement::new(callsite.source_info, StatementKind::StorageDead(local)));
1156    }
1157
1158    local
1159}
1160
1161/**
1162 * Integrator.
1163 *
1164 * Integrates blocks from the callee function into the calling function.
1165 * Updates block indices, references to locals and other control flow
1166 * stuff.
1167*/
1168struct Integrator<'a, 'tcx> {
1169    args: &'a [Local],
1170    new_locals: RangeFrom<Local>,
1171    new_scopes: RangeFrom<SourceScope>,
1172    new_blocks: RangeFrom<BasicBlock>,
1173    destination: Local,
1174    callsite_scope: SourceScopeData<'tcx>,
1175    callsite: &'a CallSite<'tcx>,
1176    cleanup_block: UnwindAction,
1177    in_cleanup_block: bool,
1178    return_block: Option<BasicBlock>,
1179    tcx: TyCtxt<'tcx>,
1180    always_live_locals: DenseBitSet<Local>,
1181}
1182
1183impl Integrator<'_, '_> {
1184    fn map_local(&self, local: Local) -> Local {
1185        let new = if local == RETURN_PLACE {
1186            self.destination
1187        } else {
1188            let idx = local.index() - 1;
1189            if idx < self.args.len() {
1190                self.args[idx]
1191            } else {
1192                self.new_locals.start + (idx - self.args.len())
1193            }
1194        };
1195        trace!("mapping local `{:?}` to `{:?}`", local, new);
1196        new
1197    }
1198
1199    fn map_scope(&self, scope: SourceScope) -> SourceScope {
1200        let new = self.new_scopes.start + scope.index();
1201        trace!("mapping scope `{:?}` to `{:?}`", scope, new);
1202        new
1203    }
1204
1205    fn map_block(&self, block: BasicBlock) -> BasicBlock {
1206        let new = self.new_blocks.start + block.index();
1207        trace!("mapping block `{:?}` to `{:?}`", block, new);
1208        new
1209    }
1210
1211    fn map_unwind(&self, unwind: UnwindAction) -> UnwindAction {
1212        if self.in_cleanup_block {
1213            match unwind {
1214                UnwindAction::Cleanup(_) | UnwindAction::Continue => {
1215                    bug!("cleanup on cleanup block");
1216                }
1217                UnwindAction::Unreachable | UnwindAction::Terminate(_) => return unwind,
1218            }
1219        }
1220
1221        match unwind {
1222            UnwindAction::Unreachable | UnwindAction::Terminate(_) => unwind,
1223            UnwindAction::Cleanup(target) => UnwindAction::Cleanup(self.map_block(target)),
1224            // Add an unwind edge to the original call's cleanup block
1225            UnwindAction::Continue => self.cleanup_block,
1226        }
1227    }
1228}
1229
1230impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
1231    fn tcx(&self) -> TyCtxt<'tcx> {
1232        self.tcx
1233    }
1234
1235    fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) {
1236        *local = self.map_local(*local);
1237    }
1238
1239    fn visit_source_scope_data(&mut self, scope_data: &mut SourceScopeData<'tcx>) {
1240        self.super_source_scope_data(scope_data);
1241        if scope_data.parent_scope.is_none() {
1242            // Attach the outermost callee scope as a child of the callsite
1243            // scope, via the `parent_scope` and `inlined_parent_scope` chains.
1244            scope_data.parent_scope = Some(self.callsite.source_info.scope);
1245            assert_eq!(scope_data.inlined_parent_scope, None);
1246            scope_data.inlined_parent_scope = if self.callsite_scope.inlined.is_some() {
1247                Some(self.callsite.source_info.scope)
1248            } else {
1249                self.callsite_scope.inlined_parent_scope
1250            };
1251
1252            // Mark the outermost callee scope as an inlined one.
1253            assert_eq!(scope_data.inlined, None);
1254            scope_data.inlined = Some((self.callsite.callee, self.callsite.source_info.span));
1255        } else if scope_data.inlined_parent_scope.is_none() {
1256            // Make it easy to find the scope with `inlined` set above.
1257            scope_data.inlined_parent_scope = Some(self.map_scope(OUTERMOST_SOURCE_SCOPE));
1258        }
1259    }
1260
1261    fn visit_source_scope(&mut self, scope: &mut SourceScope) {
1262        *scope = self.map_scope(*scope);
1263    }
1264
1265    fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
1266        self.in_cleanup_block = data.is_cleanup;
1267        self.super_basic_block_data(block, data);
1268        self.in_cleanup_block = false;
1269    }
1270
1271    fn visit_retag(&mut self, kind: &mut RetagKind, place: &mut Place<'tcx>, loc: Location) {
1272        self.super_retag(kind, place, loc);
1273
1274        // We have to patch all inlined retags to be aware that they are no longer
1275        // happening on function entry.
1276        if *kind == RetagKind::FnEntry {
1277            *kind = RetagKind::Default;
1278        }
1279    }
1280
1281    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
1282        if let StatementKind::StorageLive(local) | StatementKind::StorageDead(local) =
1283            statement.kind
1284        {
1285            self.always_live_locals.remove(local);
1286        }
1287        self.super_statement(statement, location);
1288    }
1289
1290    fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, loc: Location) {
1291        // Don't try to modify the implicit `_0` access on return (`return` terminators are
1292        // replaced down below anyways).
1293        if !matches!(terminator.kind, TerminatorKind::Return) {
1294            self.super_terminator(terminator, loc);
1295        } else {
1296            self.visit_source_info(&mut terminator.source_info);
1297        }
1298
1299        match terminator.kind {
1300            TerminatorKind::CoroutineDrop | TerminatorKind::Yield { .. } => bug!(),
1301            TerminatorKind::Goto { ref mut target } => {
1302                *target = self.map_block(*target);
1303            }
1304            TerminatorKind::SwitchInt { ref mut targets, .. } => {
1305                for tgt in targets.all_targets_mut() {
1306                    *tgt = self.map_block(*tgt);
1307                }
1308            }
1309            TerminatorKind::Drop { ref mut target, ref mut unwind, .. } => {
1310                *target = self.map_block(*target);
1311                *unwind = self.map_unwind(*unwind);
1312            }
1313            TerminatorKind::TailCall { .. } => {
1314                // check_mir_body forbids tail calls
1315                unreachable!()
1316            }
1317            TerminatorKind::Call { ref mut target, ref mut unwind, .. } => {
1318                if let Some(ref mut tgt) = *target {
1319                    *tgt = self.map_block(*tgt);
1320                }
1321                *unwind = self.map_unwind(*unwind);
1322            }
1323            TerminatorKind::Assert { ref mut target, ref mut unwind, .. } => {
1324                *target = self.map_block(*target);
1325                *unwind = self.map_unwind(*unwind);
1326            }
1327            TerminatorKind::Return => {
1328                terminator.kind = if let Some(tgt) = self.return_block {
1329                    TerminatorKind::Goto { target: tgt }
1330                } else {
1331                    TerminatorKind::Unreachable
1332                }
1333            }
1334            TerminatorKind::UnwindResume => {
1335                terminator.kind = match self.cleanup_block {
1336                    UnwindAction::Cleanup(tgt) => TerminatorKind::Goto { target: tgt },
1337                    UnwindAction::Continue => TerminatorKind::UnwindResume,
1338                    UnwindAction::Unreachable => TerminatorKind::Unreachable,
1339                    UnwindAction::Terminate(reason) => TerminatorKind::UnwindTerminate(reason),
1340                };
1341            }
1342            TerminatorKind::UnwindTerminate(_) => {}
1343            TerminatorKind::Unreachable => {}
1344            TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => {
1345                *real_target = self.map_block(*real_target);
1346                *imaginary_target = self.map_block(*imaginary_target);
1347            }
1348            TerminatorKind::FalseUnwind { real_target: _, unwind: _ } =>
1349            // see the ordering of passes in the optimized_mir query.
1350            {
1351                bug!("False unwinds should have been removed before inlining")
1352            }
1353            TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => {
1354                for tgt in targets.iter_mut() {
1355                    *tgt = self.map_block(*tgt);
1356                }
1357                *unwind = self.map_unwind(*unwind);
1358            }
1359        }
1360    }
1361}
1362
1363#[instrument(skip(tcx), level = "debug")]
1364fn try_instance_mir<'tcx>(
1365    tcx: TyCtxt<'tcx>,
1366    instance: InstanceKind<'tcx>,
1367) -> Result<&'tcx Body<'tcx>, &'static str> {
1368    if let ty::InstanceKind::DropGlue(_, Some(ty)) | ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) =
1369        instance
1370        && let ty::Adt(def, args) = ty.kind()
1371    {
1372        let fields = def.all_fields();
1373        for field in fields {
1374            let field_ty = field.ty(tcx, args);
1375            if field_ty.has_param() && field_ty.has_aliases() {
1376                return Err("cannot build drop shim for polymorphic type");
1377            }
1378        }
1379    }
1380    Ok(tcx.instance_mir(instance))
1381}
1382
1383fn body_is_forwarder(body: &Body<'_>) -> bool {
1384    let TerminatorKind::Call { target, .. } = body.basic_blocks[START_BLOCK].terminator().kind
1385    else {
1386        return false;
1387    };
1388    if let Some(target) = target {
1389        let TerminatorKind::Return = body.basic_blocks[target].terminator().kind else {
1390            return false;
1391        };
1392    }
1393
1394    let max_blocks = if !body.is_polymorphic {
1395        2
1396    } else if target.is_none() {
1397        3
1398    } else {
1399        4
1400    };
1401    if body.basic_blocks.len() > max_blocks {
1402        return false;
1403    }
1404
1405    body.basic_blocks.iter_enumerated().all(|(bb, bb_data)| {
1406        bb == START_BLOCK
1407            || matches!(
1408                bb_data.terminator().kind,
1409                TerminatorKind::Return
1410                    | TerminatorKind::Drop { .. }
1411                    | TerminatorKind::UnwindResume
1412                    | TerminatorKind::UnwindTerminate(_)
1413            )
1414    })
1415}