rustc_mir_transform/
add_call_guards.rs

1//! Breaks outgoing critical edges for call terminators in the MIR.
2//!
3//! Critical edges are edges that are neither the only edge leaving a
4//! block, nor the only edge entering one.
5//!
6//! When you want something to happen "along" an edge, you can either
7//! do at the end of the predecessor block, or at the start of the
8//! successor block. Critical edges have to be broken in order to prevent
9//! "edge actions" from affecting other edges. We need this for calls that are
10//! codegened to LLVM invoke instructions, because invoke is a block terminator
11//! in LLVM so we can't insert any code to handle the call's result into the
12//! block that performs the call.
13//!
14//! This function will break those edges by inserting new blocks along them.
15//!
16//! NOTE: Simplify CFG will happily undo most of the work this pass does.
17
18use rustc_index::{Idx, IndexVec};
19use rustc_middle::mir::*;
20use rustc_middle::ty::TyCtxt;
21use tracing::debug;
22
23#[derive(PartialEq)]
24pub(super) enum AddCallGuards {
25    AllCallEdges,
26    CriticalCallEdges,
27}
28pub(super) use self::AddCallGuards::*;
29
30impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
31    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
32        let mut pred_count = IndexVec::from_elem(0u8, &body.basic_blocks);
33        for (_, data) in body.basic_blocks.iter_enumerated() {
34            for succ in data.terminator().successors() {
35                pred_count[succ] = pred_count[succ].saturating_add(1);
36            }
37        }
38
39        enum Action {
40            Call,
41            Asm { target_index: usize },
42        }
43
44        let mut work = Vec::with_capacity(body.basic_blocks.len());
45        for (bb, block) in body.basic_blocks.iter_enumerated() {
46            let term = block.terminator();
47            match term.kind {
48                TerminatorKind::Call { target: Some(destination), unwind, .. }
49                    if pred_count[destination] > 1
50                        && (generates_invoke(unwind) || self == &AllCallEdges) =>
51                {
52                    // It's a critical edge, break it
53                    work.push((bb, Action::Call));
54                }
55                TerminatorKind::InlineAsm {
56                    asm_macro: InlineAsmMacro::Asm,
57                    ref targets,
58                    ref operands,
59                    unwind,
60                    ..
61                } if self == &CriticalCallEdges => {
62                    let has_outputs = operands.iter().any(|op| {
63                        matches!(op, InlineAsmOperand::InOut { .. } | InlineAsmOperand::Out { .. })
64                    });
65                    let has_labels =
66                        operands.iter().any(|op| matches!(op, InlineAsmOperand::Label { .. }));
67                    if has_outputs && (has_labels || generates_invoke(unwind)) {
68                        for (target_index, target) in targets.iter().enumerate() {
69                            if pred_count[*target] > 1 {
70                                work.push((bb, Action::Asm { target_index }));
71                            }
72                        }
73                    }
74                }
75                _ => {}
76            }
77        }
78
79        if work.is_empty() {
80            return;
81        }
82
83        // We need a place to store the new blocks generated
84        let mut new_blocks = Vec::with_capacity(work.len());
85
86        let cur_len = body.basic_blocks.len();
87        let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| {
88            let block = BasicBlockData::new(
89                Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }),
90                is_cleanup,
91            );
92            let idx = cur_len + new_blocks.len();
93            new_blocks.push(block);
94            BasicBlock::new(idx)
95        };
96
97        let basic_blocks = body.basic_blocks.as_mut();
98        for (source, action) in work {
99            let block = &mut basic_blocks[source];
100            let is_cleanup = block.is_cleanup;
101            let term = block.terminator_mut();
102            let source_info = term.source_info;
103            let destination = match action {
104                Action::Call => {
105                    let TerminatorKind::Call { target: Some(ref mut destination), .. } = term.kind
106                    else {
107                        unreachable!()
108                    };
109                    destination
110                }
111                Action::Asm { target_index } => {
112                    let TerminatorKind::InlineAsm { ref mut targets, .. } = term.kind else {
113                        unreachable!()
114                    };
115                    &mut targets[target_index]
116                }
117            };
118            *destination = new_block(source_info, is_cleanup, *destination);
119        }
120
121        debug!("Broke {} N edges", new_blocks.len());
122        basic_blocks.extend(new_blocks);
123    }
124
125    fn is_required(&self) -> bool {
126        true
127    }
128}
129
130/// Returns true if this unwind action is code generated as an invoke as opposed to a call.
131fn generates_invoke(unwind: UnwindAction) -> bool {
132    match unwind {
133        UnwindAction::Continue | UnwindAction::Unreachable => false,
134        UnwindAction::Cleanup(_) | UnwindAction::Terminate(_) => true,
135    }
136}
rustc_mir_transform/add_call_guards.rs

rustc_mir_transform/
add_call_guards.rs
