rustc_mir_transform/
add_call_guards.rs

1//! Breaks outgoing critical edges for call terminators in the MIR.
2//!
3//! Critical edges are edges that are neither the only edge leaving a
4//! block, nor the only edge entering one.
5//!
6//! When you want something to happen "along" an edge, you can either
7//! do at the end of the predecessor block, or at the start of the
8//! successor block. Critical edges have to be broken in order to prevent
9//! "edge actions" from affecting other edges. We need this for calls that are
10//! codegened to LLVM invoke instructions, because invoke is a block terminator
11//! in LLVM so we can't insert any code to handle the call's result into the
12//! block that performs the call.
13//!
14//! This function will break those edges by inserting new blocks along them.
15//!
16//! NOTE: Simplify CFG will happily undo most of the work this pass does.
17
18use rustc_data_structures::thin_vec::ThinVec;
19use rustc_index::{Idx, IndexVec};
20use rustc_middle::mir::*;
21use rustc_middle::ty::TyCtxt;
22use tracing::debug;
23
24use crate::PassPolicy;
25
26#[derive(PartialEq)]
27pub(super) enum AddCallGuards {
28    AllCallEdges,
29    CriticalCallEdges,
30}
31pub(super) use self::AddCallGuards::*;
32
33impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
34    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
35        let mut pred_count = IndexVec::from_elem(0u8, &body.basic_blocks);
36        for (_, data) in body.basic_blocks.iter_enumerated() {
37            for succ in data.terminator().successors() {
38                pred_count[succ] = pred_count[succ].saturating_add(1);
39            }
40        }
41
42        enum Action {
43            Call,
44            Asm { target_index: usize },
45        }
46
47        let mut work = Vec::with_capacity(body.basic_blocks.len());
48        for (bb, block) in body.basic_blocks.iter_enumerated() {
49            let term = block.terminator();
50            match term.kind {
51                TerminatorKind::Call { target: Some(destination), unwind, .. }
52                    if pred_count[destination] > 1
53                        && (generates_invoke(unwind) || self == &AllCallEdges) =>
54                {
55                    // It's a critical edge, break it
56                    work.push((bb, Action::Call));
57                }
58                TerminatorKind::InlineAsm {
59                    asm_macro: InlineAsmMacro::Asm,
60                    ref targets,
61                    ref operands,
62                    unwind,
63                    ..
64                } if self == &CriticalCallEdges => {
65                    let has_outputs = operands.iter().any(|op| {
66                        matches!(op, InlineAsmOperand::InOut { .. } | InlineAsmOperand::Out { .. })
67                    });
68                    let has_labels =
69                        operands.iter().any(|op| matches!(op, InlineAsmOperand::Label { .. }));
70                    if has_outputs && (has_labels || generates_invoke(unwind)) {
71                        for (target_index, target) in targets.iter().enumerate() {
72                            if pred_count[*target] > 1 {
73                                work.push((bb, Action::Asm { target_index }));
74                            }
75                        }
76                    }
77                }
78                _ => {}
79            }
80        }
81
82        if work.is_empty() {
83            return;
84        }
85
86        // We need a place to store the new blocks generated
87        let mut new_blocks = Vec::with_capacity(work.len());
88
89        let cur_len = body.basic_blocks.len();
90        let mut new_block = |source_info: SourceInfo, is_cleanup: bool, target: BasicBlock| {
91            let block = BasicBlockData::new(
92                Some(Terminator {
93                    source_info,
94                    kind: TerminatorKind::Goto { target },
95                    attributes: ThinVec::new(),
96                }),
97                is_cleanup,
98            );
99            let idx = cur_len + new_blocks.len();
100            new_blocks.push(block);
101            BasicBlock::new(idx)
102        };
103
104        let basic_blocks = body.basic_blocks.as_mut();
105        for (source, action) in work {
106            let block = &mut basic_blocks[source];
107            let is_cleanup = block.is_cleanup;
108            let term = block.terminator_mut();
109            let source_info = term.source_info;
110            let destination = match action {
111                Action::Call => {
112                    let TerminatorKind::Call { target: Some(ref mut destination), .. } = term.kind
113                    else {
114                        unreachable!()
115                    };
116                    destination
117                }
118                Action::Asm { target_index } => {
119                    let TerminatorKind::InlineAsm { ref mut targets, .. } = term.kind else {
120                        unreachable!()
121                    };
122                    &mut targets[target_index]
123                }
124            };
125            *destination = new_block(source_info, is_cleanup, *destination);
126        }
127
128        debug!("Broke {} N edges", new_blocks.len());
129        basic_blocks.extend(new_blocks);
130    }
131
132    fn policy(&self, _sess: &rustc_session::Session) -> PassPolicy {
133        // Breaks critical edges so codegen can place edge-specific actions without affecting
134        // other control-flow edges.
135        PassPolicy::Required
136    }
137}
138
139/// Returns true if this unwind action is code generated as an invoke as opposed to a call.
140fn generates_invoke(unwind: UnwindAction) -> bool {
141    match unwind {
142        UnwindAction::Continue | UnwindAction::Unreachable => false,
143        UnwindAction::Cleanup(_) | UnwindAction::Terminate(_) => true,
144    }
145}
rustc_mir_transform/add_call_guards.rs

rustc_mir_transform/
add_call_guards.rs
