compiletest/runtest/
coverage.rs

1//! Code specific to the coverage test suites.
2
3use std::ffi::OsStr;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6
7use glob::glob;
8
9use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP};
10use crate::runtest::{Emit, ProcRes, TestCx, WillExecute};
11use crate::util::static_regex;
12
13impl<'test> TestCx<'test> {
14    fn coverage_dump_path(&self) -> &Path {
15        self.config
16            .coverage_dump_path
17            .as_deref()
18            .unwrap_or_else(|| self.fatal("missing --coverage-dump"))
19    }
20
21    pub(super) fn run_coverage_map_test(&self) {
22        let coverage_dump_path = self.coverage_dump_path();
23
24        let (proc_res, llvm_ir_path) = self.compile_test_and_save_ir();
25        if !proc_res.status.success() {
26            self.fatal_proc_rec("compilation failed!", &proc_res);
27        }
28        drop(proc_res);
29
30        let mut dump_command = Command::new(coverage_dump_path);
31        dump_command.arg(llvm_ir_path);
32        let proc_res = self.run_command_to_procres(&mut dump_command);
33        if !proc_res.status.success() {
34            self.fatal_proc_rec("coverage-dump failed!", &proc_res);
35        }
36
37        let kind = UI_COVERAGE_MAP;
38
39        let expected_coverage_dump = self.load_expected_output(kind);
40        let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
41
42        let coverage_dump_compare_outcome = self.compare_output(
43            kind,
44            &actual_coverage_dump,
45            &proc_res.stdout,
46            &expected_coverage_dump,
47        );
48
49        if coverage_dump_compare_outcome.should_error() {
50            self.fatal_proc_rec(
51                &format!("an error occurred comparing coverage output."),
52                &proc_res,
53            );
54        }
55    }
56
57    pub(super) fn run_coverage_run_test(&self) {
58        let should_run = self.run_if_enabled();
59        let proc_res = self.compile_test(should_run, Emit::None);
60
61        if !proc_res.status.success() {
62            self.fatal_proc_rec("compilation failed!", &proc_res);
63        }
64        drop(proc_res);
65
66        if let WillExecute::Disabled = should_run {
67            return;
68        }
69
70        let profraw_path = self.output_base_dir().join("default.profraw");
71        let profdata_path = self.output_base_dir().join("default.profdata");
72
73        // Delete any existing profraw/profdata files to rule out unintended
74        // interference between repeated test runs.
75        if profraw_path.exists() {
76            std::fs::remove_file(&profraw_path).unwrap();
77        }
78        if profdata_path.exists() {
79            std::fs::remove_file(&profdata_path).unwrap();
80        }
81
82        let proc_res = self.exec_compiled_test_general(
83            &[("LLVM_PROFILE_FILE", &profraw_path.to_str().unwrap())],
84            false,
85        );
86        if self.props.failure_status.is_some() {
87            self.check_correct_failure_status(&proc_res);
88        } else if !proc_res.status.success() {
89            self.fatal_proc_rec("test run failed!", &proc_res);
90        }
91        drop(proc_res);
92
93        let mut profraw_paths = vec![profraw_path];
94        let mut bin_paths = vec![self.make_exe_name()];
95
96        if self.config.suite == "coverage-run-rustdoc" {
97            self.run_doctests_for_coverage(&mut profraw_paths, &mut bin_paths);
98        }
99
100        // Run `llvm-profdata merge` to index the raw coverage output.
101        let proc_res = self.run_llvm_tool("llvm-profdata", |cmd| {
102            cmd.args(["merge", "--sparse", "--output"]);
103            cmd.arg(&profdata_path);
104            cmd.args(&profraw_paths);
105        });
106        if !proc_res.status.success() {
107            self.fatal_proc_rec("llvm-profdata merge failed!", &proc_res);
108        }
109        drop(proc_res);
110
111        // Run `llvm-cov show` to produce a coverage report in text format.
112        let proc_res = self.run_llvm_tool("llvm-cov", |cmd| {
113            cmd.args(["show", "--format=text", "--show-line-counts-or-regions"]);
114
115            // Specify the demangler binary and its arguments.
116            let coverage_dump_path = self.coverage_dump_path();
117            cmd.arg("--Xdemangler").arg(coverage_dump_path);
118            cmd.arg("--Xdemangler").arg("--demangle");
119
120            cmd.arg("--instr-profile");
121            cmd.arg(&profdata_path);
122
123            for bin in &bin_paths {
124                cmd.arg("--object");
125                cmd.arg(bin);
126            }
127
128            cmd.args(&self.props.llvm_cov_flags);
129        });
130        if !proc_res.status.success() {
131            self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
132        }
133
134        let kind = UI_COVERAGE;
135
136        let expected_coverage = self.load_expected_output(kind);
137        let normalized_actual_coverage =
138            self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {
139                self.fatal_proc_rec(&err, &proc_res);
140            });
141
142        let coverage_dump_compare_outcome = self.compare_output(
143            kind,
144            &normalized_actual_coverage,
145            &proc_res.stdout,
146            &expected_coverage,
147        );
148
149        if coverage_dump_compare_outcome.should_error() {
150            self.fatal_proc_rec(
151                &format!("an error occurred comparing coverage output."),
152                &proc_res,
153            );
154        }
155    }
156
157    /// Run any doctests embedded in this test file, and add any resulting
158    /// `.profraw` files and doctest executables to the given vectors.
159    fn run_doctests_for_coverage(
160        &self,
161        profraw_paths: &mut Vec<PathBuf>,
162        bin_paths: &mut Vec<PathBuf>,
163    ) {
164        // Put .profraw files and doctest executables in dedicated directories,
165        // to make it easier to glob them all later.
166        let profraws_dir = self.output_base_dir().join("doc_profraws");
167        let bins_dir = self.output_base_dir().join("doc_bins");
168
169        // Remove existing directories to prevent cross-run interference.
170        if profraws_dir.try_exists().unwrap() {
171            std::fs::remove_dir_all(&profraws_dir).unwrap();
172        }
173        if bins_dir.try_exists().unwrap() {
174            std::fs::remove_dir_all(&bins_dir).unwrap();
175        }
176
177        let mut rustdoc_cmd =
178            Command::new(self.config.rustdoc_path.as_ref().expect("--rustdoc-path not passed"));
179
180        // In general there will be multiple doctest binaries running, so we
181        // tell the profiler runtime to write their coverage data into separate
182        // profraw files.
183        rustdoc_cmd.env("LLVM_PROFILE_FILE", profraws_dir.join("%p-%m.profraw"));
184
185        rustdoc_cmd.args(["--test", "-Cinstrument-coverage"]);
186
187        // Without this, the doctests complain about not being able to find
188        // their enclosing file's crate for some reason.
189        rustdoc_cmd.args(["--crate-name", "workaround_for_79771"]);
190
191        // Persist the doctest binaries so that `llvm-cov show` can read their
192        // embedded coverage mappings later.
193        rustdoc_cmd.arg("-Zunstable-options");
194        rustdoc_cmd.arg("--persist-doctests");
195        rustdoc_cmd.arg(&bins_dir);
196
197        rustdoc_cmd.arg("-L");
198        rustdoc_cmd.arg(self.aux_output_dir_name());
199
200        rustdoc_cmd.arg(&self.testpaths.file);
201
202        let proc_res = self.compose_and_run_compiler(rustdoc_cmd, None, self.testpaths);
203        if !proc_res.status.success() {
204            self.fatal_proc_rec("rustdoc --test failed!", &proc_res)
205        }
206
207        fn glob_iter(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
208            let path_str = path.as_ref().to_str().unwrap();
209            let iter = glob(path_str).unwrap();
210            iter.map(Result::unwrap)
211        }
212
213        // Find all profraw files in the profraw directory.
214        for p in glob_iter(profraws_dir.join("*.profraw")) {
215            profraw_paths.push(p);
216        }
217        // Find all executables in the `--persist-doctests` directory, while
218        // avoiding other file types (e.g. `.pdb` on Windows). This doesn't
219        // need to be perfect, as long as it can handle the files actually
220        // produced by `rustdoc --test`.
221        for p in glob_iter(bins_dir.join("**/*")) {
222            let is_bin = p.is_file()
223                && match p.extension() {
224                    None => true,
225                    Some(ext) => ext == OsStr::new("exe"),
226                };
227            if is_bin {
228                bin_paths.push(p);
229            }
230        }
231    }
232
233    fn run_llvm_tool(&self, name: &str, configure_cmd_fn: impl FnOnce(&mut Command)) -> ProcRes {
234        let tool_path = self
235            .config
236            .llvm_bin_dir
237            .as_ref()
238            .expect("this test expects the LLVM bin dir to be available")
239            .join(name);
240
241        let mut cmd = Command::new(tool_path);
242        configure_cmd_fn(&mut cmd);
243
244        self.run_command_to_procres(&mut cmd)
245    }
246
247    fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
248        let normalized = self.normalize_output(coverage, &[]);
249        let normalized = Self::anonymize_coverage_line_numbers(&normalized);
250
251        let mut lines = normalized.lines().collect::<Vec<_>>();
252
253        Self::sort_coverage_file_sections(&mut lines)?;
254        Self::sort_coverage_subviews(&mut lines)?;
255
256        let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
257        Ok(joined_lines)
258    }
259
260    /// Replace line numbers in coverage reports with the placeholder `LL`,
261    /// so that the tests are less sensitive to lines being added/removed.
262    fn anonymize_coverage_line_numbers(coverage: &str) -> String {
263        // The coverage reporter prints line numbers at the start of a line.
264        // They are truncated or left-padded to occupy exactly 5 columns.
265        // (`LineNumberColumnWidth` in `SourceCoverageViewText.cpp`.)
266        // A pipe character `|` appears immediately after the final digit.
267        //
268        // Line numbers that appear inside expansion/instantiation subviews
269        // have an additional prefix of `  |` for each nesting level.
270        //
271        // Branch views also include the relevant line number, so we want to
272        // redact those too. (These line numbers don't have padding.)
273        //
274        // Note: The pattern `(?m:^)` matches the start of a line.
275
276        // `    1|` => `   LL|`
277        // `   10|` => `   LL|`
278        // `  100|` => `   LL|`
279        // `  | 1000|`    => `  |   LL|`
280        // `  |  | 1000|` => `  |  |   LL|`
281        let coverage = static_regex!(r"(?m:^)(?<prefix>(?:  \|)*) *[0-9]+\|")
282            .replace_all(&coverage, "${prefix}   LL|");
283
284        // `  |  Branch (1:`     => `  |  Branch (LL:`
285        // `  |  |  Branch (10:` => `  |  |  Branch (LL:`
286        let coverage = static_regex!(r"(?m:^)(?<prefix>(?:  \|)+  Branch \()[0-9]+:")
287            .replace_all(&coverage, "${prefix}LL:");
288
289        // `  |---> MC/DC Decision Region (1:30) to (2:`     => `  |---> MC/DC Decision Region (LL:30) to (LL:`
290        let coverage =
291            static_regex!(r"(?m:^)(?<prefix>(?:  \|)+---> MC/DC Decision Region \()[0-9]+:(?<middle>[0-9]+\) to \()[0-9]+:")
292            .replace_all(&coverage, "${prefix}LL:${middle}LL:");
293
294        // `  |     Condition C1 --> (1:`     => `  |     Condition C1 --> (LL:`
295        let coverage =
296            static_regex!(r"(?m:^)(?<prefix>(?:  \|)+     Condition C[0-9]+ --> \()[0-9]+:")
297                .replace_all(&coverage, "${prefix}LL:");
298
299        coverage.into_owned()
300    }
301
302    /// Coverage reports can describe multiple source files, separated by
303    /// blank lines. The order of these files is unpredictable (since it
304    /// depends on implementation details), so we need to sort the file
305    /// sections into a consistent order before comparing against a snapshot.
306    fn sort_coverage_file_sections(coverage_lines: &mut Vec<&str>) -> Result<(), String> {
307        // Group the lines into file sections, separated by blank lines.
308        let mut sections = coverage_lines.split(|line| line.is_empty()).collect::<Vec<_>>();
309
310        // The last section should be empty, representing an extra trailing blank line.
311        if !sections.last().is_some_and(|last| last.is_empty()) {
312            return Err("coverage report should end with an extra blank line".to_owned());
313        }
314
315        // Sort the file sections (not including the final empty "section").
316        let except_last = sections.len() - 1;
317        (&mut sections[..except_last]).sort();
318
319        // Join the file sections back into a flat list of lines, with
320        // sections separated by blank lines.
321        let joined = sections.join(&[""] as &[_]);
322        assert_eq!(joined.len(), coverage_lines.len());
323        *coverage_lines = joined;
324
325        Ok(())
326    }
327
328    fn sort_coverage_subviews(coverage_lines: &mut Vec<&str>) -> Result<(), String> {
329        let mut output_lines = Vec::new();
330
331        // We accumulate a list of zero or more "subviews", where each
332        // subview is a list of one or more lines.
333        let mut subviews: Vec<Vec<&str>> = Vec::new();
334
335        fn flush<'a>(subviews: &mut Vec<Vec<&'a str>>, output_lines: &mut Vec<&'a str>) {
336            if subviews.is_empty() {
337                return;
338            }
339
340            // Take and clear the list of accumulated subviews.
341            let mut subviews = std::mem::take(subviews);
342
343            // The last "subview" should be just a boundary line on its own,
344            // so exclude it when sorting the other subviews.
345            let except_last = subviews.len() - 1;
346            (&mut subviews[..except_last]).sort();
347
348            for view in subviews {
349                for line in view {
350                    output_lines.push(line);
351                }
352            }
353        }
354
355        for (line, line_num) in coverage_lines.iter().zip(1..) {
356            if line.starts_with("  ------------------") {
357                // This is a subview boundary line, so start a new subview.
358                subviews.push(vec![line]);
359            } else if line.starts_with("  |") {
360                // Add this line to the current subview.
361                subviews
362                    .last_mut()
363                    .ok_or(format!(
364                        "unexpected subview line outside of a subview on line {line_num}"
365                    ))?
366                    .push(line);
367            } else {
368                // This line is not part of a subview, so sort and print any
369                // accumulated subviews, and then print the line as-is.
370                flush(&mut subviews, &mut output_lines);
371                output_lines.push(line);
372            }
373        }
374
375        flush(&mut subviews, &mut output_lines);
376        assert!(subviews.is_empty());
377
378        assert_eq!(output_lines.len(), coverage_lines.len());
379        *coverage_lines = output_lines;
380
381        Ok(())
382    }
383}