rustc_mir_transform/
add_call_guards.rs

1//! Breaks outgoing critical edges for call terminators in the MIR.
2//!
3//! Critical edges are edges that are neither the only edge leaving a
4//! block, nor the only edge entering one.
5//!
6//! When you want something to happen "along" an edge, you can either
7//! do at the end of the predecessor block, or at the start of the
8//! successor block. Critical edges have to be broken in order to prevent
9//! "edge actions" from affecting other edges. We need this for calls that are
10//! codegened to LLVM invoke instructions, because invoke is a block terminator
11//! in LLVM so we can't insert any code to handle the call's result into the
12//! block that performs the call.
13//!
14//! This function will break those edges by inserting new blocks along them.
15//!
16//! NOTE: Simplify CFG will happily undo most of the work this pass does.
17
18use rustc_index::{Idx, IndexVec};
19use rustc_middle::mir::*;
20use rustc_middle::ty::TyCtxt;
21use tracing::debug;
22
23use crate::PassPolicy;
24
25#[derive(PartialEq)]
26pub(super) enum AddCallGuards {
27    AllCallEdges,
28    CriticalCallEdges,
29}
30pub(super) use self::AddCallGuards::*;
31
32impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
33    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
34        let mut pred_count = IndexVec::from_elem(0u8, &body.basic_blocks);
35        for (_, data) in body.basic_blocks.iter_enumerated() {
36            for succ in data.terminator().successors() {
37                pred_count[succ] = pred_count[succ].saturating_add(1);
38            }
39        }
40
41        enum Action {
42            Call,
43            Asm { target_index: usize },
44        }
45
46        let mut work = Vec::with_capacity(body.basic_blocks.len());
47        for (bb, block) in body.basic_blocks.iter_enumerated() {
48            let term = block.terminator();
49            match term.kind {
50                TerminatorKind::Call { target: Some(destination), unwind, .. }
51                    if pred_count[destination] > 1
52                        && (generates_invoke(unwind) || self == &AllCallEdges) =>
53                {
54                    // It's a critical edge, break it
55                    work.push((bb, Action::Call));
56                }
57                TerminatorKind::InlineAsm {
58                    asm_macro: InlineAsmMacro::Asm,
59                    ref targets,
60                    ref operands,
61                    unwind,
62                    ..
63                } if self == &CriticalCallEdges => {
64                    let has_outputs = operands.iter().any(|op| {
65                        matches!(op, InlineAsmOperand::InOut { .. } | InlineAsmOperand::Out { .. })
66                    });
67                    let has_labels =
68                        operands.iter().any(|op| matches!(op, InlineAsmOperand::Label { .. }));
69                    if has_outputs && (has_labels || generates_invoke(unwind)) {
70                        for (target_index, target) in targets.iter().enumerate() {
71                            if pred_count[*target] > 1 {
72                                work.push((bb, Action::Asm { target_index }));
73                            }
74                        }
75                    }
76                }
77                _ => {}
78            }
79        }
80
81        if work.is_empty() {
82            return;
83        }
84
85        // We need a place to store the new blocks generated
86        let mut new_blocks = Vec::with_capacity(work.len());
87
88        let cur_len = body.basic_blocks.len();
89        let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| {
90            let block = BasicBlockData::new(
91                Some(Terminator { source_info, kind: TerminatorKind::goto(target) }),
92                is_cleanup,
93            );
94            let idx = cur_len + new_blocks.len();
95            new_blocks.push(block);
96            BasicBlock::new(idx)
97        };
98
99        let basic_blocks = body.basic_blocks.as_mut();
100        for (source, action) in work {
101            let block = &mut basic_blocks[source];
102            let is_cleanup = block.is_cleanup;
103            let term = block.terminator_mut();
104            let source_info = term.source_info;
105            let destination = match action {
106                Action::Call => {
107                    let TerminatorKind::Call { target: Some(ref mut destination), .. } = term.kind
108                    else {
109                        unreachable!()
110                    };
111                    destination
112                }
113                Action::Asm { target_index } => {
114                    let TerminatorKind::InlineAsm { ref mut targets, .. } = term.kind else {
115                        unreachable!()
116                    };
117                    &mut targets[target_index]
118                }
119            };
120            *destination = new_block(source_info, is_cleanup, *destination);
121        }
122
123        debug!("Broke {} N edges", new_blocks.len());
124        basic_blocks.extend(new_blocks);
125    }
126
127    fn policy(&self, _ctx: &crate::PassCtx<'_>) -> PassPolicy {
128        // Breaks critical edges so codegen can place edge-specific actions without affecting
129        // other control-flow edges.
130        PassPolicy::Required
131    }
132}
133
134/// Returns true if this unwind action is code generated as an invoke as opposed to a call.
135fn generates_invoke(unwind: UnwindAction) -> bool {
136    match unwind {
137        UnwindAction::Continue | UnwindAction::Unreachable => false,
138        UnwindAction::Cleanup(_) | UnwindAction::Terminate(_) => true,
139    }
140}
rustc_mir_transform/add_call_guards.rs

rustc_mir_transform/
add_call_guards.rs
