pub mod query;
mod counters;
mod graph;
mod mappings;
mod spans;
#[cfg(test)]
mod tests;
mod unexpand;
use rustc_hir as hir;
use rustc_hir::intravisit::{walk_expr, Visitor};
use rustc_middle::hir::map::Map;
use rustc_middle::hir::nested_filter;
use rustc_middle::mir::coverage::{
CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, SourceRegion,
};
use rustc_middle::mir::{
self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator,
TerminatorKind,
};
use rustc_middle::ty::TyCtxt;
use rustc_span::def_id::LocalDefId;
use rustc_span::source_map::SourceMap;
use rustc_span::{BytePos, Pos, RelativeBytePos, Span, Symbol};
use tracing::{debug, debug_span, instrument, trace};
use crate::coverage::counters::{CounterIncrementSite, CoverageCounters};
use crate::coverage::graph::CoverageGraph;
use crate::coverage::mappings::ExtractedMappings;
use crate::MirPass;
pub struct InstrumentCoverage;
impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
sess.instrument_coverage()
}
fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
let mir_source = mir_body.source;
assert!(mir_source.promoted.is_none());
let def_id = mir_source.def_id().expect_local();
if !tcx.is_eligible_for_coverage(def_id) {
trace!("InstrumentCoverage skipped for {def_id:?} (not eligible)");
return;
}
match mir_body.basic_blocks[mir::START_BLOCK].terminator().kind {
TerminatorKind::Unreachable => {
trace!("InstrumentCoverage skipped for unreachable `START_BLOCK`");
return;
}
_ => {}
}
instrument_function_for_coverage(tcx, mir_body);
}
}
fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
let def_id = mir_body.source.def_id();
let _span = debug_span!("instrument_function_for_coverage", ?def_id).entered();
let hir_info = extract_hir_info(tcx, def_id.expect_local());
let basic_coverage_blocks = CoverageGraph::from_mir(mir_body);
let extracted_mappings = mappings::extract_all_mapping_info_from_mir(
tcx,
mir_body,
&hir_info,
&basic_coverage_blocks,
);
let bcbs_with_counter_mappings = extracted_mappings.all_bcbs_with_counter_mappings();
if bcbs_with_counter_mappings.is_empty() {
return;
}
let bcb_has_counter_mappings = |bcb| bcbs_with_counter_mappings.contains(bcb);
let coverage_counters =
CoverageCounters::make_bcb_counters(&basic_coverage_blocks, bcb_has_counter_mappings);
let mappings = create_mappings(tcx, &hir_info, &extracted_mappings, &coverage_counters);
if mappings.is_empty() {
debug!("no spans could be converted into valid mappings; skipping");
return;
}
inject_coverage_statements(
mir_body,
&basic_coverage_blocks,
&extracted_mappings,
&coverage_counters,
);
inject_mcdc_statements(mir_body, &basic_coverage_blocks, &extracted_mappings);
let mcdc_num_condition_bitmaps = extracted_mappings
.mcdc_decisions
.iter()
.map(|&mappings::MCDCDecision { decision_depth, .. }| decision_depth)
.max()
.map_or(0, |max| usize::from(max) + 1);
mir_body.function_coverage_info = Some(Box::new(FunctionCoverageInfo {
function_source_hash: hir_info.function_source_hash,
num_counters: coverage_counters.num_counters(),
mcdc_bitmap_bytes: extracted_mappings.mcdc_bitmap_bytes,
expressions: coverage_counters.into_expressions(),
mappings,
mcdc_num_condition_bitmaps,
}));
}
fn create_mappings<'tcx>(
tcx: TyCtxt<'tcx>,
hir_info: &ExtractedHirInfo,
extracted_mappings: &ExtractedMappings,
coverage_counters: &CoverageCounters,
) -> Vec<Mapping> {
let source_map = tcx.sess.source_map();
let body_span = hir_info.body_span;
let source_file = source_map.lookup_source_file(body_span.lo());
use rustc_session::config::RemapPathScopeComponents;
use rustc_session::RemapFileNameExt;
let file_name = Symbol::intern(
&source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(),
);
let term_for_bcb = |bcb| {
coverage_counters
.bcb_counter(bcb)
.expect("all BCBs with spans were given counters")
.as_term()
};
let region_for_span = |span: Span| make_source_region(source_map, file_name, span, body_span);
let ExtractedMappings {
num_bcbs: _,
code_mappings,
branch_pairs,
mcdc_bitmap_bytes: _,
mcdc_branches,
mcdc_decisions,
} = extracted_mappings;
let mut mappings = Vec::new();
mappings.extend(code_mappings.iter().filter_map(
|&mappings::CodeMapping { span, bcb }| {
let source_region = region_for_span(span)?;
let kind = MappingKind::Code(term_for_bcb(bcb));
Some(Mapping { kind, source_region })
},
));
mappings.extend(branch_pairs.iter().filter_map(
|&mappings::BranchPair { span, true_bcb, false_bcb }| {
let true_term = term_for_bcb(true_bcb);
let false_term = term_for_bcb(false_bcb);
let kind = MappingKind::Branch { true_term, false_term };
let source_region = region_for_span(span)?;
Some(Mapping { kind, source_region })
},
));
mappings.extend(mcdc_branches.iter().filter_map(
|&mappings::MCDCBranch { span, true_bcb, false_bcb, condition_info, decision_depth: _ }| {
let source_region = region_for_span(span)?;
let true_term = term_for_bcb(true_bcb);
let false_term = term_for_bcb(false_bcb);
let kind = match condition_info {
Some(mcdc_params) => MappingKind::MCDCBranch { true_term, false_term, mcdc_params },
None => MappingKind::Branch { true_term, false_term },
};
Some(Mapping { kind, source_region })
},
));
mappings.extend(mcdc_decisions.iter().filter_map(
|&mappings::MCDCDecision { span, bitmap_idx, num_conditions, .. }| {
let source_region = region_for_span(span)?;
let kind = MappingKind::MCDCDecision(DecisionInfo { bitmap_idx, num_conditions });
Some(Mapping { kind, source_region })
},
));
mappings
}
fn inject_coverage_statements<'tcx>(
mir_body: &mut mir::Body<'tcx>,
basic_coverage_blocks: &CoverageGraph,
extracted_mappings: &ExtractedMappings,
coverage_counters: &CoverageCounters,
) {
for (id, counter_increment_site) in coverage_counters.counter_increment_sites() {
let target_bb = match *counter_increment_site {
CounterIncrementSite::Node { bcb } => basic_coverage_blocks[bcb].leader_bb(),
CounterIncrementSite::Edge { from_bcb, to_bcb } => {
let from_bb = basic_coverage_blocks[from_bcb].last_bb();
let to_bb = basic_coverage_blocks[to_bcb].leader_bb();
let new_bb = inject_edge_counter_basic_block(mir_body, from_bb, to_bb);
debug!(
"Edge {from_bcb:?} (last {from_bb:?}) -> {to_bcb:?} (leader {to_bb:?}) \
requires a new MIR BasicBlock {new_bb:?} for counter increment {id:?}",
);
new_bb
}
};
inject_statement(mir_body, CoverageKind::CounterIncrement { id }, target_bb);
}
let eligible_bcbs = extracted_mappings.bcbs_with_ordinary_code_mappings();
for (bcb, expression_id) in coverage_counters
.bcb_nodes_with_coverage_expressions()
.filter(|&(bcb, _)| eligible_bcbs.contains(bcb))
{
inject_statement(
mir_body,
CoverageKind::ExpressionUsed { id: expression_id },
basic_coverage_blocks[bcb].leader_bb(),
);
}
}
fn inject_mcdc_statements<'tcx>(
mir_body: &mut mir::Body<'tcx>,
basic_coverage_blocks: &CoverageGraph,
extracted_mappings: &ExtractedMappings,
) {
for &mappings::MCDCDecision {
span: _,
ref end_bcbs,
bitmap_idx,
num_conditions: _,
decision_depth,
} in &extracted_mappings.mcdc_decisions
{
for end in end_bcbs {
let end_bb = basic_coverage_blocks[*end].leader_bb();
inject_statement(
mir_body,
CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth },
end_bb,
);
}
}
for &mappings::MCDCBranch { span: _, true_bcb, false_bcb, condition_info, decision_depth } in
&extracted_mappings.mcdc_branches
{
let Some(condition_info) = condition_info else { continue };
let id = condition_info.condition_id;
let true_bb = basic_coverage_blocks[true_bcb].leader_bb();
inject_statement(
mir_body,
CoverageKind::CondBitmapUpdate { id, value: true, decision_depth },
true_bb,
);
let false_bb = basic_coverage_blocks[false_bcb].leader_bb();
inject_statement(
mir_body,
CoverageKind::CondBitmapUpdate { id, value: false, decision_depth },
false_bb,
);
}
}
fn inject_edge_counter_basic_block(
mir_body: &mut mir::Body<'_>,
from_bb: BasicBlock,
to_bb: BasicBlock,
) -> BasicBlock {
let span = mir_body[from_bb].terminator().source_info.span.shrink_to_hi();
let new_bb = mir_body.basic_blocks_mut().push(BasicBlockData {
statements: vec![], terminator: Some(Terminator {
source_info: SourceInfo::outermost(span),
kind: TerminatorKind::Goto { target: to_bb },
}),
is_cleanup: false,
});
let edge_ref = mir_body[from_bb]
.terminator_mut()
.successors_mut()
.find(|successor| **successor == to_bb)
.expect("from_bb should have a successor for to_bb");
*edge_ref = new_bb;
new_bb
}
fn inject_statement(mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb: BasicBlock) {
debug!(" injecting statement {counter_kind:?} for {bb:?}");
let data = &mut mir_body[bb];
let source_info = data.terminator().source_info;
let statement = Statement { source_info, kind: StatementKind::Coverage(counter_kind) };
data.statements.insert(0, statement);
}
#[instrument(level = "debug", skip(source_map))]
fn make_source_region(
source_map: &SourceMap,
file_name: Symbol,
span: Span,
body_span: Span,
) -> Option<SourceRegion> {
let lo = span.lo();
let hi = span.hi();
let file = source_map.lookup_source_file(lo);
if !file.contains(hi) {
debug!(?span, ?file, ?lo, ?hi, "span crosses multiple files; skipping");
return None;
}
let rpos_and_line_and_byte_column = |pos: BytePos| -> Option<(RelativeBytePos, usize, usize)> {
let rpos = file.relative_position(pos);
let line_index = file.lookup_line(rpos)?;
let line_start = file.lines()[line_index];
Some((rpos, line_index + 1, (rpos - line_start).to_usize() + 1))
};
let (lo_rpos, mut start_line, mut start_col) = rpos_and_line_and_byte_column(lo)?;
let (hi_rpos, mut end_line, mut end_col) = rpos_and_line_and_byte_column(hi)?;
if span.is_empty()
&& body_span.contains(span)
&& let Some(src) = &file.src
{
if hi < body_span.hi() {
let hi_rpos = hi_rpos.to_usize();
let nudge_bytes = src.ceil_char_boundary(hi_rpos + 1) - hi_rpos;
end_col += nudge_bytes;
} else if lo > body_span.lo() {
let lo_rpos = lo_rpos.to_usize();
let nudge_bytes = lo_rpos - src.floor_char_boundary(lo_rpos - 1);
start_col = start_col.saturating_sub(nudge_bytes).max(1);
}
}
start_line = source_map.doctest_offset_line(&file.name, start_line);
end_line = source_map.doctest_offset_line(&file.name, end_line);
check_source_region(SourceRegion {
file_name,
start_line: start_line as u32,
start_col: start_col as u32,
end_line: end_line as u32,
end_col: end_col as u32,
})
}
fn check_source_region(source_region: SourceRegion) -> Option<SourceRegion> {
let SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = source_region;
let all_nonzero = [start_line, start_col, end_line, end_col].into_iter().all(|x| x != 0);
let end_col_has_high_bit_unset = (end_col & (1 << 31)) == 0;
let is_ordered = (start_line, start_col) <= (end_line, end_col);
if all_nonzero && end_col_has_high_bit_unset && is_ordered {
Some(source_region)
} else {
debug!(
?source_region,
?all_nonzero,
?end_col_has_high_bit_unset,
?is_ordered,
"Skipping source region that would be misinterpreted or rejected by LLVM"
);
debug_assert!(false, "Improper source region: {source_region:?}");
None
}
}
#[derive(Debug)]
struct ExtractedHirInfo {
function_source_hash: u64,
is_async_fn: bool,
fn_sig_span_extended: Option<Span>,
body_span: Span,
hole_spans: Vec<Span>,
}
fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHirInfo {
let hir_node = tcx.hir_node_by_def_id(def_id);
let fn_body_id = hir_node.body_id().expect("HIR node is a function with body");
let hir_body = tcx.hir().body(fn_body_id);
let maybe_fn_sig = hir_node.fn_sig();
let is_async_fn = maybe_fn_sig.is_some_and(|fn_sig| fn_sig.header.is_async());
let mut body_span = hir_body.value.span;
use hir::{Closure, Expr, ExprKind, Node};
if let Node::Expr(&Expr { kind: ExprKind::Closure(&Closure { fn_decl_span, .. }), .. }) =
hir_node
{
body_span = body_span.find_ancestor_in_same_ctxt(fn_decl_span).unwrap_or(body_span);
}
let fn_sig_span_extended = maybe_fn_sig
.map(|fn_sig| fn_sig.span)
.filter(|&fn_sig_span| {
let source_map = tcx.sess.source_map();
let file_idx = |span: Span| source_map.lookup_source_file_idx(span.lo());
fn_sig_span.eq_ctxt(body_span)
&& fn_sig_span.hi() <= body_span.lo()
&& file_idx(fn_sig_span) == file_idx(body_span)
})
.map(|fn_sig_span| fn_sig_span.with_hi(body_span.lo()));
let function_source_hash = hash_mir_source(tcx, hir_body);
let hole_spans = extract_hole_spans_from_hir(tcx, body_span, hir_body);
ExtractedHirInfo {
function_source_hash,
is_async_fn,
fn_sig_span_extended,
body_span,
hole_spans,
}
}
fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx hir::Body<'tcx>) -> u64 {
let owner = hir_body.id().hir_id.owner;
tcx.hir_owner_nodes(owner).opt_hash_including_bodies.unwrap().to_smaller_hash().as_u64()
}
fn extract_hole_spans_from_hir<'tcx>(
tcx: TyCtxt<'tcx>,
body_span: Span, hir_body: &hir::Body<'tcx>,
) -> Vec<Span> {
struct HolesVisitor<'hir, F> {
hir: Map<'hir>,
visit_hole_span: F,
}
impl<'hir, F: FnMut(Span)> Visitor<'hir> for HolesVisitor<'hir, F> {
type NestedFilter = nested_filter::All;
fn nested_visit_map(&mut self) -> Self::Map {
self.hir
}
fn visit_item(&mut self, item: &'hir hir::Item<'hir>) {
(self.visit_hole_span)(item.span);
}
fn visit_expr(&mut self, expr: &'hir hir::Expr<'hir>) {
match expr.kind {
hir::ExprKind::Closure(_) | hir::ExprKind::ConstBlock(_) => {
(self.visit_hole_span)(expr.span);
}
_ => walk_expr(self, expr),
}
}
}
let mut hole_spans = vec![];
let mut visitor = HolesVisitor {
hir: tcx.hir(),
visit_hole_span: |hole_span| {
if body_span.contains(hole_span) && body_span.eq_ctxt(hole_span) {
hole_spans.push(hole_span);
}
},
};
visitor.visit_body(hir_body);
hole_spans
}