1use std::cell::{OnceCell, RefCell};
2use std::ffi::{CStr, CString};
34use rustc_abi::Size;
5use rustc_codegen_ssa::traits::{
6BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods,
7};
8use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
9use rustc_middle::mir::coverage::CoverageKind;
10use rustc_middle::ty::Instance;
11use tracing::{debug, instrument};
1213use crate::builder::Builder;
14use crate::common::CodegenCx;
15use crate::llvm;
1617pub(crate) mod ffi;
18mod llvm_cov;
19mod mapgen;
2021/// Extra per-CGU context/state needed for coverage instrumentation.
22pub(crate) struct CguCoverageContext<'ll, 'tcx> {
23/// Coverage data for each instrumented function identified by DefId.
24pub(crate) instances_used: RefCell<FxIndexSet<Instance<'tcx>>>,
25pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
26pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>,
2728 covfun_section_name: OnceCell<CString>,
29}
3031impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> {
32pub(crate) fn new() -> Self {
33Self {
34 instances_used: RefCell::<FxIndexSet<_>>::default(),
35 pgo_func_name_var_map: Default::default(),
36 mcdc_condition_bitmap_map: Default::default(),
37 covfun_section_name: Default::default(),
38 }
39 }
4041/// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is
42 /// called condition bitmap. In order to handle nested decisions, several condition bitmaps can
43 /// be allocated for a function body. These values are named `mcdc.addr.{i}` and are a 32-bit
44 /// integers. They respectively hold the condition bitmaps for decisions with a depth of `i`.
45fn try_get_mcdc_condition_bitmap(
46&self,
47 instance: &Instance<'tcx>,
48 decision_depth: u16,
49 ) -> Option<&'ll llvm::Value> {
50self.mcdc_condition_bitmap_map
51 .borrow()
52 .get(instance)
53 .and_then(|bitmap_map| bitmap_map.get(decision_depthas usize))
54 .copied() // Dereference Option<&&Value> to Option<&Value>
55}
56}
5758impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
59pub(crate) fn coverageinfo_finalize(&self) {
60mapgen::finalize(self)
61 }
6263/// Returns the section name to use when embedding per-function coverage information
64 /// in the object file, according to the target's object file format. LLVM's coverage
65 /// tools use information from this section when producing coverage reports.
66 ///
67 /// Typical values are:
68 /// - `__llvm_covfun` on Linux
69 /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix)
70 /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix)
71fn covfun_section_name(&self) -> &CStr {
72self.coverage_cx()
73 .covfun_section_name
74 .get_or_init(|| llvm_cov::covfun_section_name(self.llmod))
75 }
7677/// For LLVM codegen, returns a function-specific `Value` for a global
78 /// string, to hold the function name passed to LLVM intrinsic
79 /// `instrprof.increment()`. The `Value` is only created once per instance.
80 /// Multiple invocations with the same instance return the same `Value`.
81fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
82debug!("getting pgo_func_name_var for instance={:?}", instance);
83let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut();
84pgo_func_name_var_map.entry(instance).or_insert_with(|| {
85let llfn = self.get_fn(instance);
86let mangled_fn_name: &str = self.tcx.symbol_name(instance).name;
87llvm_cov::create_pgo_func_name_var(llfn, mangled_fn_name)
88 })
89 }
90}
9192impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
93fn init_coverage(&mut self, instance: Instance<'tcx>) {
94let Some(function_coverage_info) =
95self.tcx.instance_mir(instance.def).function_coverage_info.as_deref()
96else {
97return;
98 };
99100// If there are no MC/DC bitmaps to set up, return immediately.
101if function_coverage_info.mcdc_bitmap_bits == 0 {
102return;
103 }
104105let fn_name = self.get_pgo_func_name_var(instance);
106let hash = self.const_u64(function_coverage_info.function_source_hash);
107let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32);
108self.mcdc_parameters(fn_name, hash, bitmap_bits);
109110// Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps.
111let mut cond_bitmaps = vec![];
112for i in 0..function_coverage_info.mcdc_num_condition_bitmaps {
113// MC/DC intrinsics will perform loads/stores that use the ABI default
114 // alignment for i32, so our variable declaration should match.
115let align = self.tcx.data_layout.i32_align.abi;
116let cond_bitmap = self.alloca(Size::from_bytes(4), align);
117 llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes());
118self.store(self.const_i32(0), cond_bitmap, align);
119 cond_bitmaps.push(cond_bitmap);
120 }
121122self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps);
123 }
124125#[instrument(level = "debug", skip(self))]
126fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) {
127// Our caller should have already taken care of inlining subtleties,
128 // so we can assume that counter/expression IDs in this coverage
129 // statement are meaningful for the given instance.
130 //
131 // (Either the statement was not inlined and directly belongs to this
132 // instance, or it was inlined *from* this instance.)
133134let bx = self;
135136// Due to LocalCopy instantiation or MIR inlining, coverage statements
137 // can end up in a crate that isn't doing coverage instrumentation.
138 // When that happens, we currently just discard those statements, so
139 // the corresponding code will be undercounted.
140 // FIXME(Zalathar): Find a better solution for mixed-coverage builds.
141let Some(coverage_cx) = &bx.cx.coverage_cx else { return };
142143let Some(function_coverage_info) =
144 bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref()
145else {
146debug!("function has a coverage statement but no coverage info");
147return;
148 };
149let Some(ids_info) = bx.tcx.coverage_ids_info(instance.def) else {
150debug!("function has a coverage statement but no IDs info");
151return;
152 };
153154// Mark the instance as used in this CGU, for coverage purposes.
155 // This includes functions that were not partitioned into this CGU,
156 // but were MIR-inlined into one of this CGU's functions.
157coverage_cx.instances_used.borrow_mut().insert(instance);
158159match *kind {
160 CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
161"marker statement {kind:?} should have been removed by CleanupPostBorrowck"
162),
163 CoverageKind::VirtualCounter { bcb }
164if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) =>
165 {
166let fn_name = bx.get_pgo_func_name_var(instance);
167let hash = bx.const_u64(function_coverage_info.function_source_hash);
168let num_counters = bx.const_u32(ids_info.num_counters);
169let index = bx.const_u32(id.as_u32());
170debug!(
171"codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
172 fn_name, hash, num_counters, index,
173 );
174 bx.instrprof_increment(fn_name, hash, num_counters, index);
175 }
176// If a BCB doesn't have an associated physical counter, there's nothing to codegen.
177CoverageKind::VirtualCounter { .. } => {}
178 CoverageKind::CondBitmapUpdate { index, decision_depth } => {
179let cond_bitmap = coverage_cx
180 .try_get_mcdc_condition_bitmap(&instance, decision_depth)
181 .expect("mcdc cond bitmap should have been allocated for updating");
182let cond_index = bx.const_i32(index as i32);
183 bx.mcdc_condbitmap_update(cond_index, cond_bitmap);
184 }
185 CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
186let cond_bitmap =
187 coverage_cx.try_get_mcdc_condition_bitmap(&instance, decision_depth).expect(
188"mcdc cond bitmap should have been allocated for merging \
189 into the global bitmap",
190 );
191assert!(
192 bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits,
193"bitmap index of the decision out of range"
194);
195196let fn_name = bx.get_pgo_func_name_var(instance);
197let hash = bx.const_u64(function_coverage_info.function_source_hash);
198let bitmap_index = bx.const_u32(bitmap_idx);
199 bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap);
200 bx.mcdc_condbitmap_reset(cond_bitmap);
201 }
202 }
203 }
204}