compiletest/runtest/
ui.rs

1use std::collections::HashSet;
2use std::fs::OpenOptions;
3use std::io::Write;
4
5use rustfix::{Filter, apply_suggestions, get_suggestions_from_json};
6use tracing::debug;
7
8use super::{
9    AllowUnused, Emit, FailMode, LinkToAux, PassMode, RunFailMode, RunResult, TargetLocation,
10    TestCx, TestOutput, Truncated, UI_FIXED, WillExecute,
11};
12use crate::json;
13
14impl TestCx<'_> {
15    pub(super) fn run_ui_test(&self) {
16        if let Some(FailMode::Build) = self.props.fail_mode {
17            // Make sure a build-fail test cannot fail due to failing analysis (e.g. typeck).
18            let pm = Some(PassMode::Check);
19            let proc_res =
20                self.compile_test_general(WillExecute::No, Emit::Metadata, pm, Vec::new());
21            self.check_if_test_should_compile(self.props.fail_mode, pm, &proc_res);
22        }
23
24        let pm = self.pass_mode();
25        let should_run = self.should_run(pm);
26        let emit_metadata = self.should_emit_metadata(pm);
27        let proc_res = self.compile_test(should_run, emit_metadata);
28        self.check_if_test_should_compile(self.props.fail_mode, pm, &proc_res);
29        if matches!(proc_res.truncated, Truncated::Yes)
30            && !self.props.dont_check_compiler_stdout
31            && !self.props.dont_check_compiler_stderr
32        {
33            self.fatal_proc_rec(
34                "compiler output got truncated, cannot compare with reference file",
35                &proc_res,
36            );
37        }
38
39        // if the user specified a format in the ui test
40        // print the output to the stderr file, otherwise extract
41        // the rendered error messages from json and print them
42        let explicit = self.props.compile_flags.iter().any(|s| s.contains("--error-format"));
43
44        let expected_fixed = self.load_expected_output(UI_FIXED);
45
46        self.check_and_prune_duplicate_outputs(&proc_res, &[], &[]);
47
48        let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit);
49        let rustfix_input = json::rustfix_diagnostics_only(&proc_res.stderr);
50
51        if self.config.compare_mode.is_some() {
52            // don't test rustfix with nll right now
53        } else if self.config.rustfix_coverage {
54            // Find out which tests have `MachineApplicable` suggestions but are missing
55            // `run-rustfix` or `run-rustfix-only-machine-applicable` directives.
56            //
57            // This will return an empty `Vec` in case the executed test file has a
58            // `compile-flags: --error-format=xxxx` directive with a value other than `json`.
59            let suggestions = get_suggestions_from_json(
60                &rustfix_input,
61                &HashSet::new(),
62                Filter::MachineApplicableOnly,
63            )
64            .unwrap_or_default();
65            if !suggestions.is_empty()
66                && !self.props.run_rustfix
67                && !self.props.rustfix_only_machine_applicable
68            {
69                let mut coverage_file_path = self.config.build_test_suite_root.clone();
70                coverage_file_path.push("rustfix_missing_coverage.txt");
71                debug!("coverage_file_path: {}", coverage_file_path);
72
73                let mut file = OpenOptions::new()
74                    .create(true)
75                    .append(true)
76                    .open(coverage_file_path.as_path())
77                    .expect("could not create or open file");
78
79                if let Err(e) = writeln!(file, "{}", self.testpaths.file) {
80                    panic!("couldn't write to {}: {e:?}", coverage_file_path);
81                }
82            }
83        } else if self.props.run_rustfix {
84            // Apply suggestions from rustc to the code itself
85            let unfixed_code = self.load_expected_output_from_path(&self.testpaths.file).unwrap();
86            let suggestions = get_suggestions_from_json(
87                &rustfix_input,
88                &HashSet::new(),
89                if self.props.rustfix_only_machine_applicable {
90                    Filter::MachineApplicableOnly
91                } else {
92                    Filter::Everything
93                },
94            )
95            .unwrap();
96            let fixed_code = apply_suggestions(&unfixed_code, &suggestions).unwrap_or_else(|e| {
97                panic!(
98                    "failed to apply suggestions for {:?} with rustfix: {}",
99                    self.testpaths.file, e
100                )
101            });
102
103            if self
104                .compare_output("fixed", &fixed_code, &fixed_code, &expected_fixed)
105                .should_error()
106            {
107                errors += 1;
108            }
109        } else if !expected_fixed.is_empty() {
110            panic!(
111                "the `//@ run-rustfix` directive wasn't found but a `*.fixed` \
112                 file was found"
113            );
114        }
115
116        if errors > 0 {
117            println!("To update references, rerun the tests and pass the `--bless` flag");
118            let relative_path_to_file =
119                self.testpaths.relative_dir.join(self.testpaths.file.file_name().unwrap());
120            println!(
121                "To only update this specific test, also pass `--test-args {}`",
122                relative_path_to_file,
123            );
124            self.fatal_proc_rec(
125                &format!("{} errors occurred comparing output.", errors),
126                &proc_res,
127            );
128        }
129
130        let output_to_check = if let WillExecute::Yes = should_run {
131            let proc_res = self.exec_compiled_test();
132            let run_output_errors = if self.props.check_run_results {
133                self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
134            } else {
135                0
136            };
137            if run_output_errors > 0 {
138                self.fatal_proc_rec(
139                    &format!("{} errors occurred comparing run output.", run_output_errors),
140                    &proc_res,
141                );
142            }
143            let code = proc_res.status.code();
144            let run_result = if proc_res.status.success() {
145                RunResult::Pass
146            } else if code.is_some_and(|c| c >= 1 && c <= 127) {
147                RunResult::Fail
148            } else {
149                RunResult::Crash
150            };
151            // Help users understand why the test failed by including the actual
152            // exit code and actual run result in the failure message.
153            let pass_hint = format!("code={code:?} so test would pass with `{run_result}`");
154            if self.should_run_successfully(pm) {
155                if run_result != RunResult::Pass {
156                    self.fatal_proc_rec(
157                        &format!("test did not exit with success! {pass_hint}"),
158                        &proc_res,
159                    );
160                }
161            } else if self.props.fail_mode == Some(FailMode::Run(RunFailMode::Fail)) {
162                // If the test is marked as `run-fail` but do not support
163                // unwinding we allow it to crash, since a panic will trigger an
164                // abort (crash) instead of unwind (exit with code 101).
165                let crash_ok = !self.config.can_unwind();
166                if run_result != RunResult::Fail && !(crash_ok && run_result == RunResult::Crash) {
167                    let err = if crash_ok {
168                        format!(
169                            "test did not exit with failure or crash (`{}` can't unwind)! {pass_hint}",
170                            self.config.target
171                        )
172                    } else {
173                        format!("test did not exit with failure! {pass_hint}")
174                    };
175                    self.fatal_proc_rec(&err, &proc_res);
176                }
177            } else if self.props.fail_mode == Some(FailMode::Run(RunFailMode::Crash)) {
178                if run_result != RunResult::Crash {
179                    self.fatal_proc_rec(&format!("test did not crash! {pass_hint}"), &proc_res);
180                }
181            } else if self.props.fail_mode == Some(FailMode::Run(RunFailMode::FailOrCrash)) {
182                if run_result != RunResult::Fail && run_result != RunResult::Crash {
183                    self.fatal_proc_rec(
184                        &format!("test did not exit with failure or crash! {pass_hint}"),
185                        &proc_res,
186                    );
187                }
188            } else {
189                unreachable!("run_ui_test() must not be called if the test should not run");
190            }
191
192            self.get_output(&proc_res)
193        } else {
194            self.get_output(&proc_res)
195        };
196
197        debug!(
198            "run_ui_test: explicit={:?} config.compare_mode={:?} \
199               proc_res.status={:?} props.error_patterns={:?}",
200            explicit, self.config.compare_mode, proc_res.status, self.props.error_patterns
201        );
202
203        self.check_expected_errors(&proc_res);
204        self.check_all_error_patterns(&output_to_check, &proc_res);
205        self.check_forbid_output(&output_to_check, &proc_res);
206
207        if self.props.run_rustfix && self.config.compare_mode.is_none() {
208            // And finally, compile the fixed code and make sure it both
209            // succeeds and has no diagnostics.
210            let mut rustc = self.make_compile_args(
211                &self.expected_output_path(UI_FIXED),
212                TargetLocation::ThisFile(self.make_exe_name()),
213                emit_metadata,
214                AllowUnused::No,
215                LinkToAux::Yes,
216                Vec::new(),
217            );
218
219            // If a test is revisioned, it's fixed source file can be named "a.foo.fixed", which,
220            // well, "a.foo" isn't a valid crate name. So we explicitly mangle the test name
221            // (including the revision) here to avoid the test writer having to manually specify a
222            // `#![crate_name = "..."]` as a workaround. This is okay since we're only checking if
223            // the fixed code is compilable.
224            if self.revision.is_some() {
225                let crate_name =
226                    self.testpaths.file.file_stem().expect("test must have a file stem");
227                // crate name must be alphanumeric or `_`.
228                // replace `a.foo` -> `a__foo` for crate name purposes.
229                // replace `revision-name-with-dashes` -> `revision_name_with_underscore`
230                let crate_name = crate_name.replace('.', "__");
231                let crate_name = crate_name.replace('-', "_");
232                rustc.arg("--crate-name");
233                rustc.arg(crate_name);
234            }
235
236            let res = self.compose_and_run_compiler(rustc, None, self.testpaths);
237            if !res.status.success() {
238                self.fatal_proc_rec("failed to compile fixed code", &res);
239            }
240            if !res.stderr.is_empty()
241                && !self.props.rustfix_only_machine_applicable
242                && !json::rustfix_diagnostics_only(&res.stderr).is_empty()
243            {
244                self.fatal_proc_rec("fixed code is still producing diagnostics", &res);
245            }
246        }
247    }
248}