use std::fmt::{self, Debug};
use rustc_data_structures::captures::Captures;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::graph::DirectedGraph;
use rustc_index::IndexVec;
use rustc_middle::bug;
use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op};
use tracing::{debug, debug_span, instrument};
use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops};
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub(super) enum BcbCounter {
Counter { id: CounterId },
Expression { id: ExpressionId },
}
impl BcbCounter {
pub(super) fn as_term(&self) -> CovTerm {
match *self {
BcbCounter::Counter { id, .. } => CovTerm::Counter(id),
BcbCounter::Expression { id, .. } => CovTerm::Expression(id),
}
}
}
impl Debug for BcbCounter {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Counter { id, .. } => write!(fmt, "Counter({:?})", id.index()),
Self::Expression { id } => write!(fmt, "Expression({:?})", id.index()),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct BcbExpression {
lhs: BcbCounter,
op: Op,
rhs: BcbCounter,
}
#[derive(Debug)]
pub(super) enum CounterIncrementSite {
Node { bcb: BasicCoverageBlock },
Edge { from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock },
}
pub(super) struct CoverageCounters {
counter_increment_sites: IndexVec<CounterId, CounterIncrementSite>,
bcb_counters: IndexVec<BasicCoverageBlock, Option<BcbCounter>>,
bcb_edge_counters: FxHashMap<(BasicCoverageBlock, BasicCoverageBlock), BcbCounter>,
expressions: IndexVec<ExpressionId, BcbExpression>,
expressions_memo: FxHashMap<BcbExpression, BcbCounter>,
}
impl CoverageCounters {
pub(super) fn make_bcb_counters(
basic_coverage_blocks: &CoverageGraph,
bcb_needs_counter: impl Fn(BasicCoverageBlock) -> bool,
) -> Self {
let num_bcbs = basic_coverage_blocks.num_nodes();
let mut this = Self {
counter_increment_sites: IndexVec::new(),
bcb_counters: IndexVec::from_elem_n(None, num_bcbs),
bcb_edge_counters: FxHashMap::default(),
expressions: IndexVec::new(),
expressions_memo: FxHashMap::default(),
};
MakeBcbCounters::new(&mut this, basic_coverage_blocks).make_bcb_counters(bcb_needs_counter);
this
}
fn make_counter(&mut self, site: CounterIncrementSite) -> BcbCounter {
let id = self.counter_increment_sites.push(site);
BcbCounter::Counter { id }
}
fn make_expression(&mut self, lhs: BcbCounter, op: Op, rhs: BcbCounter) -> BcbCounter {
let new_expr = BcbExpression { lhs, op, rhs };
*self
.expressions_memo
.entry(new_expr)
.or_insert_with(|| Self::make_expression_inner(&mut self.expressions, new_expr))
}
fn make_expression_inner(
expressions: &mut IndexVec<ExpressionId, BcbExpression>,
new_expr: BcbExpression,
) -> BcbCounter {
let BcbExpression { lhs, op, rhs } = new_expr;
if let BcbCounter::Expression { id } = lhs {
let lhs_expr = &expressions[id];
if lhs_expr.op == Op::Subtract && op == Op::Add && lhs_expr.rhs == rhs {
return lhs_expr.lhs;
}
if lhs_expr.op == Op::Add && op == Op::Subtract && lhs_expr.rhs == rhs {
return lhs_expr.lhs;
}
if lhs_expr.op == Op::Add && op == Op::Subtract && lhs_expr.lhs == rhs {
return lhs_expr.rhs;
}
}
if let BcbCounter::Expression { id } = rhs {
let rhs_expr = &expressions[id];
if op == Op::Add && rhs_expr.op == Op::Subtract && lhs == rhs_expr.rhs {
return rhs_expr.lhs;
}
if op == Op::Subtract && rhs_expr.op == Op::Subtract && lhs == rhs_expr.lhs {
return rhs_expr.rhs;
}
}
let id = expressions.push(new_expr);
BcbCounter::Expression { id }
}
fn make_sum_expression(&mut self, lhs: Option<BcbCounter>, rhs: BcbCounter) -> BcbCounter {
let Some(lhs) = lhs else { return rhs };
self.make_expression(lhs, Op::Add, rhs)
}
pub(super) fn num_counters(&self) -> usize {
self.counter_increment_sites.len()
}
fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> BcbCounter {
if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) {
bug!(
"attempt to set a BasicCoverageBlock coverage counter more than once; \
{bcb:?} already had counter {replaced:?}",
);
} else {
counter_kind
}
}
fn set_bcb_edge_counter(
&mut self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
counter_kind: BcbCounter,
) -> BcbCounter {
if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) {
bug!(
"attempt to set an edge counter more than once; from_bcb: \
{from_bcb:?} already had counter {replaced:?}",
);
} else {
counter_kind
}
}
pub(super) fn bcb_counter(&self, bcb: BasicCoverageBlock) -> Option<BcbCounter> {
self.bcb_counters[bcb]
}
pub(super) fn counter_increment_sites(
&self,
) -> impl Iterator<Item = (CounterId, &CounterIncrementSite)> {
self.counter_increment_sites.iter_enumerated()
}
pub(super) fn bcb_nodes_with_coverage_expressions(
&self,
) -> impl Iterator<Item = (BasicCoverageBlock, ExpressionId)> + Captures<'_> {
self.bcb_counters.iter_enumerated().filter_map(|(bcb, &counter_kind)| match counter_kind {
Some(BcbCounter::Expression { id }) => Some((bcb, id)),
Some(BcbCounter::Counter { .. }) | None => None,
})
}
pub(super) fn into_expressions(self) -> IndexVec<ExpressionId, Expression> {
let old_len = self.expressions.len();
let expressions = self
.expressions
.into_iter()
.map(|BcbExpression { lhs, op, rhs }| Expression {
lhs: lhs.as_term(),
op,
rhs: rhs.as_term(),
})
.collect::<IndexVec<ExpressionId, _>>();
assert_eq!(old_len, expressions.len());
expressions
}
}
struct MakeBcbCounters<'a> {
coverage_counters: &'a mut CoverageCounters,
basic_coverage_blocks: &'a CoverageGraph,
}
impl<'a> MakeBcbCounters<'a> {
fn new(
coverage_counters: &'a mut CoverageCounters,
basic_coverage_blocks: &'a CoverageGraph,
) -> Self {
Self { coverage_counters, basic_coverage_blocks }
}
fn make_bcb_counters(&mut self, bcb_needs_counter: impl Fn(BasicCoverageBlock) -> bool) {
debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock");
let mut traversal = TraverseCoverageGraphWithLoops::new(self.basic_coverage_blocks);
while let Some(bcb) = traversal.next() {
let _span = debug_span!("traversal", ?bcb).entered();
if bcb_needs_counter(bcb) {
self.make_node_counter_and_out_edge_counters(&traversal, bcb);
}
}
assert!(
traversal.is_complete(),
"`TraverseCoverageGraphWithLoops` missed some `BasicCoverageBlock`s: {:?}",
traversal.unvisited(),
);
}
#[instrument(level = "debug", skip(self, traversal))]
fn make_node_counter_and_out_edge_counters(
&mut self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
from_bcb: BasicCoverageBlock,
) {
let node_counter = self.get_or_make_node_counter(from_bcb);
let successors = self.basic_coverage_blocks.successors[from_bcb].as_slice();
let needs_out_edge_counters = successors.len() > 1
&& successors.iter().any(|&to_bcb| self.edge_has_no_counter(from_bcb, to_bcb));
if !needs_out_edge_counters {
return;
}
if tracing::enabled!(tracing::Level::DEBUG) {
let _span =
debug_span!("node has some out-edges without counters", ?from_bcb).entered();
for &to_bcb in successors {
debug!(?to_bcb, counter=?self.edge_counter(from_bcb, to_bcb));
}
}
let expression_to_bcb = self.choose_out_edge_for_expression(traversal, from_bcb);
let sum_of_all_other_out_edges: BcbCounter = {
let _span = debug_span!("sum_of_all_other_out_edges", ?expression_to_bcb).entered();
successors
.iter()
.copied()
.filter(|&to_bcb| to_bcb != expression_to_bcb)
.fold(None, |accum, to_bcb| {
let _span = debug_span!("to_bcb", ?accum, ?to_bcb).entered();
let edge_counter = self.get_or_make_edge_counter(from_bcb, to_bcb);
Some(self.coverage_counters.make_sum_expression(accum, edge_counter))
})
.expect("there must be at least one other out-edge")
};
let expression = self.coverage_counters.make_expression(
node_counter,
Op::Subtract,
sum_of_all_other_out_edges,
);
debug!("{expression_to_bcb:?} gets an expression: {expression:?}");
if self.basic_coverage_blocks.bcb_has_multiple_in_edges(expression_to_bcb) {
self.coverage_counters.set_bcb_edge_counter(from_bcb, expression_to_bcb, expression);
} else {
self.coverage_counters.set_bcb_counter(expression_to_bcb, expression);
}
}
#[instrument(level = "debug", skip(self))]
fn get_or_make_node_counter(&mut self, bcb: BasicCoverageBlock) -> BcbCounter {
if let Some(counter_kind) = self.coverage_counters.bcb_counters[bcb] {
debug!("{bcb:?} already has a counter: {counter_kind:?}");
return counter_kind;
}
let one_path_to_target = !self.basic_coverage_blocks.bcb_has_multiple_in_edges(bcb);
if one_path_to_target || self.bcb_predecessors(bcb).contains(&bcb) {
let counter_kind =
self.coverage_counters.make_counter(CounterIncrementSite::Node { bcb });
if one_path_to_target {
debug!("{bcb:?} gets a new counter: {counter_kind:?}");
} else {
debug!(
"{bcb:?} has itself as its own predecessor. It can't be part of its own \
Expression sum, so it will get its own new counter: {counter_kind:?}. \
(Note, the compiled code will generate an infinite loop.)",
);
}
return self.coverage_counters.set_bcb_counter(bcb, counter_kind);
}
let sum_of_in_edges: BcbCounter = {
let _span = debug_span!("sum_of_in_edges", ?bcb).entered();
self.basic_coverage_blocks.predecessors[bcb]
.iter()
.copied()
.fold(None, |accum, from_bcb| {
let _span = debug_span!("from_bcb", ?accum, ?from_bcb).entered();
let edge_counter = self.get_or_make_edge_counter(from_bcb, bcb);
Some(self.coverage_counters.make_sum_expression(accum, edge_counter))
})
.expect("there must be at least one in-edge")
};
debug!("{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}");
self.coverage_counters.set_bcb_counter(bcb, sum_of_in_edges)
}
#[instrument(level = "debug", skip(self))]
fn get_or_make_edge_counter(
&mut self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
) -> BcbCounter {
if !self.basic_coverage_blocks.bcb_has_multiple_in_edges(to_bcb) {
assert_eq!([from_bcb].as_slice(), self.basic_coverage_blocks.predecessors[to_bcb]);
return self.get_or_make_node_counter(to_bcb);
}
if self.bcb_successors(from_bcb).len() == 1 {
return self.get_or_make_node_counter(from_bcb);
}
if let Some(&counter_kind) =
self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
{
debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}");
return counter_kind;
}
let counter_kind =
self.coverage_counters.make_counter(CounterIncrementSite::Edge { from_bcb, to_bcb });
debug!("Edge {from_bcb:?}->{to_bcb:?} gets a new counter: {counter_kind:?}");
self.coverage_counters.set_bcb_edge_counter(from_bcb, to_bcb, counter_kind)
}
fn choose_out_edge_for_expression(
&self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
from_bcb: BasicCoverageBlock,
) -> BasicCoverageBlock {
if let Some(reloop_target) = self.find_good_reloop_edge(traversal, from_bcb) {
assert!(self.edge_has_no_counter(from_bcb, reloop_target));
debug!("Selecting reloop target {reloop_target:?} to get an expression");
return reloop_target;
}
let arbitrary_target = self
.bcb_successors(from_bcb)
.iter()
.copied()
.find(|&to_bcb| self.edge_has_no_counter(from_bcb, to_bcb))
.expect("precondition: at least one out-edge without a counter");
debug!(?arbitrary_target, "selecting arbitrary out-edge to get an expression");
arbitrary_target
}
fn find_good_reloop_edge(
&self,
traversal: &TraverseCoverageGraphWithLoops<'_>,
from_bcb: BasicCoverageBlock,
) -> Option<BasicCoverageBlock> {
let successors = self.bcb_successors(from_bcb);
for reloop_bcbs in traversal.reloop_bcbs_per_loop() {
let mut all_edges_exit_this_loop = true;
for &target_bcb in successors {
let is_reloop_edge = reloop_bcbs.iter().any(|&reloop_bcb| {
self.basic_coverage_blocks.dominates(target_bcb, reloop_bcb)
});
if is_reloop_edge {
all_edges_exit_this_loop = false;
if self.edge_has_no_counter(from_bcb, target_bcb) {
return Some(target_bcb);
}
} else {
}
}
if !all_edges_exit_this_loop {
debug!("All reloop edges had counters; skipping the other loops");
return None;
}
}
None
}
#[inline]
fn bcb_predecessors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] {
&self.basic_coverage_blocks.predecessors[bcb]
}
#[inline]
fn bcb_successors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] {
&self.basic_coverage_blocks.successors[bcb]
}
#[inline]
fn edge_has_no_counter(
&self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
) -> bool {
self.edge_counter(from_bcb, to_bcb).is_none()
}
fn edge_counter(
&self,
from_bcb: BasicCoverageBlock,
to_bcb: BasicCoverageBlock,
) -> Option<&BcbCounter> {
if self.basic_coverage_blocks.bcb_has_multiple_in_edges(to_bcb) {
self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
} else {
self.coverage_counters.bcb_counters[to_bcb].as_ref()
}
}
}