rustc_mir_transform/
cost_checker.rs

1use rustc_middle::bug;
2use rustc_middle::mir::visit::*;
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, Ty, TyCtxt};
5
6const INSTR_COST: usize = 5;
7const CALL_PENALTY: usize = 25;
8const LANDINGPAD_PENALTY: usize = 50;
9const RESUME_PENALTY: usize = 45;
10const LARGE_SWITCH_PENALTY: usize = 20;
11const CONST_SWITCH_BONUS: usize = 10;
12
13/// Verify that the callee body is compatible with the caller.
14#[derive(Clone)]
15pub(super) struct CostChecker<'b, 'tcx> {
16    tcx: TyCtxt<'tcx>,
17    typing_env: ty::TypingEnv<'tcx>,
18    penalty: usize,
19    bonus: usize,
20    callee_body: &'b Body<'tcx>,
21    instance: Option<ty::Instance<'tcx>>,
22}
23
24impl<'b, 'tcx> CostChecker<'b, 'tcx> {
25    pub(super) fn new(
26        tcx: TyCtxt<'tcx>,
27        typing_env: ty::TypingEnv<'tcx>,
28        instance: Option<ty::Instance<'tcx>>,
29        callee_body: &'b Body<'tcx>,
30    ) -> CostChecker<'b, 'tcx> {
31        CostChecker { tcx, typing_env, callee_body, instance, penalty: 0, bonus: 0 }
32    }
33
34    /// Add function-level costs not well-represented by the block-level costs.
35    ///
36    /// Needed because the `CostChecker` is used sometimes for just blocks,
37    /// and even the full `Inline` doesn't call `visit_body`, so there's nowhere
38    /// to put this logic in the visitor.
39    pub(super) fn add_function_level_costs(&mut self) {
40        // If the only has one Call (or similar), inlining isn't increasing the total
41        // number of calls, so give extra encouragement to inlining that.
42        if self.callee_body.basic_blocks.iter().filter(|bbd| is_call_like(bbd.terminator())).count()
43            == 1
44        {
45            self.bonus += CALL_PENALTY;
46        }
47    }
48
49    pub(super) fn cost(&self) -> usize {
50        usize::saturating_sub(self.penalty, self.bonus)
51    }
52
53    fn instantiate_ty(&self, v: Ty<'tcx>) -> Ty<'tcx> {
54        if let Some(instance) = self.instance {
55            instance.instantiate_mir(self.tcx, ty::EarlyBinder::bind(&v))
56        } else {
57            v
58        }
59    }
60}
61
62impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
63    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
64        // Most costs are in rvalues and terminators, not in statements.
65        match statement.kind {
66            StatementKind::Intrinsic(ref ndi) => {
67                self.penalty += match **ndi {
68                    NonDivergingIntrinsic::Assume(..) => INSTR_COST,
69                    NonDivergingIntrinsic::CopyNonOverlapping(..) => CALL_PENALTY,
70                };
71            }
72            _ => self.super_statement(statement, location),
73        }
74    }
75
76    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, _location: Location) {
77        match rvalue {
78            // FIXME: Should we do the same for `OverflowChecks`?
79            Rvalue::NullaryOp(NullOp::RuntimeChecks(RuntimeChecks::UbChecks), ..)
80                if !self
81                    .tcx
82                    .sess
83                    .opts
84                    .unstable_opts
85                    .inline_mir_preserve_debug
86                    .unwrap_or(self.tcx.sess.ub_checks()) =>
87            {
88                // If this is in optimized MIR it's because it's used later,
89                // so if we don't need UB checks this session, give a bonus
90                // here to offset the cost of the call later.
91                self.bonus += CALL_PENALTY;
92            }
93            // These are essentially constants that didn't end up in an Operand,
94            // so treat them as also being free.
95            Rvalue::NullaryOp(..) => {}
96            _ => self.penalty += INSTR_COST,
97        }
98    }
99
100    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
101        match &terminator.kind {
102            TerminatorKind::Drop { place, unwind, .. } => {
103                // If the place doesn't actually need dropping, treat it like a regular goto.
104                let ty = self.instantiate_ty(place.ty(self.callee_body, self.tcx).ty);
105                if ty.needs_drop(self.tcx, self.typing_env) {
106                    self.penalty += CALL_PENALTY;
107                    if let UnwindAction::Cleanup(_) = unwind {
108                        self.penalty += LANDINGPAD_PENALTY;
109                    }
110                }
111            }
112            TerminatorKind::Call { func, unwind, .. } => {
113                self.penalty += if let Some((def_id, ..)) = func.const_fn_def()
114                    && self.tcx.intrinsic(def_id).is_some()
115                {
116                    // Don't give intrinsics the extra penalty for calls
117                    INSTR_COST
118                } else {
119                    CALL_PENALTY
120                };
121                if let UnwindAction::Cleanup(_) = unwind {
122                    self.penalty += LANDINGPAD_PENALTY;
123                }
124            }
125            TerminatorKind::TailCall { .. } => {
126                self.penalty += CALL_PENALTY;
127            }
128            TerminatorKind::SwitchInt { discr, targets } => {
129                if discr.constant().is_some() {
130                    // Not only will this become a `Goto`, but likely other
131                    // things will be removable as unreachable.
132                    self.bonus += CONST_SWITCH_BONUS;
133                } else if targets.all_targets().len() > 3 {
134                    // More than false/true/unreachable gets extra cost.
135                    self.penalty += LARGE_SWITCH_PENALTY;
136                } else {
137                    self.penalty += INSTR_COST;
138                }
139            }
140            TerminatorKind::Assert { unwind, msg, .. } => {
141                self.penalty += if msg.is_optional_overflow_check()
142                    && !self
143                        .tcx
144                        .sess
145                        .opts
146                        .unstable_opts
147                        .inline_mir_preserve_debug
148                        .unwrap_or(self.tcx.sess.overflow_checks())
149                {
150                    INSTR_COST
151                } else {
152                    CALL_PENALTY
153                };
154                if let UnwindAction::Cleanup(_) = unwind {
155                    self.penalty += LANDINGPAD_PENALTY;
156                }
157            }
158            TerminatorKind::UnwindResume => self.penalty += RESUME_PENALTY,
159            TerminatorKind::InlineAsm { unwind, .. } => {
160                self.penalty += INSTR_COST;
161                if let UnwindAction::Cleanup(_) = unwind {
162                    self.penalty += LANDINGPAD_PENALTY;
163                }
164            }
165            TerminatorKind::Unreachable => {
166                self.bonus += INSTR_COST;
167            }
168            TerminatorKind::Goto { .. } | TerminatorKind::Return => {}
169            TerminatorKind::UnwindTerminate(..) => {}
170            kind @ (TerminatorKind::FalseUnwind { .. }
171            | TerminatorKind::FalseEdge { .. }
172            | TerminatorKind::Yield { .. }
173            | TerminatorKind::CoroutineDrop) => {
174                bug!("{kind:?} should not be in runtime MIR");
175            }
176        }
177    }
178}
179
180/// A terminator that's more call-like (might do a bunch of work, might panic, etc)
181/// than it is goto-/return-like (no side effects, etc).
182///
183/// Used to treat multi-call functions (which could inline exponentially)
184/// different from those that only do one or none of these "complex" things.
185pub(super) fn is_call_like(terminator: &Terminator<'_>) -> bool {
186    use TerminatorKind::*;
187    match terminator.kind {
188        Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true,
189
190        Goto { .. }
191        | SwitchInt { .. }
192        | UnwindResume
193        | UnwindTerminate(_)
194        | Return
195        | Unreachable => false,
196
197        Yield { .. } | CoroutineDrop | FalseEdge { .. } | FalseUnwind { .. } => {
198            unreachable!()
199        }
200    }
201}
rustc_mir_transform/cost_checker.rs

rustc_mir_transform/
cost_checker.rs
