compiletest/

runtest.rs

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::fs::{self, create_dir_all};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::io::prelude::*;
use std::process::{Child, Command, ExitStatus, Output, Stdio};
use std::{env, fmt, io, iter, str};

use build_helper::fs::remove_and_create_dir_all;
use camino::{Utf8Path, Utf8PathBuf};
use colored::{Color, Colorize};
use regex::{Captures, Regex};
use tracing::*;

use crate::common::{
    CompareMode, Config, Debugger, ForcePassMode, PassFailMode, RunResult, TestMode, TestPaths,
    TestSuite, UI_EXTENSIONS, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT, UI_STDERR, UI_STDOUT, UI_SVG,
    UI_WINDOWS_SVG, expected_output_path, incremental_dir, output_base_dir, output_base_name,
};
use crate::directives::{AuxCrate, TestProps};
use crate::errors::{Error, ErrorKind, load_errors};
use crate::executor::TestVariant;
use crate::output_capture::ConsoleOut;
use crate::read2::{Truncated, read2_abbreviated};
use crate::runtest::compute_diff::{DiffLine, diff_by_lines, make_diff, write_diff};
use crate::util::{ArgFileCommand, Utf8PathBufExt, add_dylib_path, static_regex};
use crate::{json, stamp_file_path};

// Helper modules that implement test running logic for each test suite.
// tidy-alphabetical-start
mod assembly;
mod codegen;
mod codegen_units;
mod coverage;
mod crashes;
mod debuginfo;
mod incremental;
mod js_doc;
mod mir_opt;
mod pretty;
mod run_make;
mod rustdoc;
mod rustdoc_json;
mod ui;
// tidy-alphabetical-end

mod compute_diff;
mod debugger;
#[cfg(test)]
mod tests;

const FAKE_SRC_BASE: &str = "fake-test-src-base";

#[cfg(windows)]
fn disable_error_reporting<F: FnOnce() -> R, R>(f: F) -> R {
    use std::sync::Mutex;

    use windows::Win32::System::Diagnostics::Debug::{
        SEM_FAILCRITICALERRORS, SEM_NOGPFAULTERRORBOX, SetErrorMode,
    };

    static LOCK: Mutex<()> = Mutex::new(());

    // Error mode is a global variable, so lock it so only one thread will change it
    let _lock = LOCK.lock().unwrap();

    // Tell Windows to not show any UI on errors (such as terminating abnormally). This is important
    // for running tests, since some of them use abnormal termination by design. This mode is
    // inherited by all child processes.
    //
    // Note that `run-make` tests require `SEM_FAILCRITICALERRORS` in addition to suppress Windows
    // Error Reporting (WER) error dialogues that come from "critical failures" such as missing
    // DLLs.
    //
    // See <https://github.com/rust-lang/rust/issues/132092> and
    // <https://learn.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-seterrormode?redirectedfrom=MSDN>.
    unsafe {
        // read inherited flags
        let old_mode = SetErrorMode(SEM_NOGPFAULTERRORBOX | SEM_FAILCRITICALERRORS);
        SetErrorMode(old_mode | SEM_NOGPFAULTERRORBOX | SEM_FAILCRITICALERRORS);
        let r = f();
        SetErrorMode(old_mode);
        r
    }
}

#[cfg(not(windows))]
fn disable_error_reporting<F: FnOnce() -> R, R>(f: F) -> R {
    f()
}

/// The platform-specific library name
fn get_lib_name(name: &str, aux_type: AuxType, wasm_proc_macros: bool) -> Option<String> {
    match aux_type {
        AuxType::Bin => None,
        // In some cases (e.g. MUSL), we build a static
        // library, rather than a dynamic library.
        // In this case, the only path we can pass
        // with '--extern-meta' is the '.rlib' file
        AuxType::Lib => Some(format!("lib{name}.rlib")),
        // FIXME maybe use `rustc --print file-names` instead?
        AuxType::ProcMacro if wasm_proc_macros => Some(format!("{name}.wasm")),
        AuxType::Dylib | AuxType::ProcMacro => Some(dylib_name(name)),
    }
}

fn dylib_name(name: &str) -> String {
    format!("{}{name}.{}", std::env::consts::DLL_PREFIX, std::env::consts::DLL_EXTENSION)
}

pub(crate) fn run(
    config: &Config,
    stdout: &dyn ConsoleOut,
    stderr: &dyn ConsoleOut,
    testpaths: &TestPaths,
    variant: &TestVariant,
) {
    match &*config.target {
        "arm-linux-androideabi"
        | "armv7-linux-androideabi"
        | "thumbv7neon-linux-androideabi"
        | "aarch64-linux-android" => {
            if !config.adb_device_status {
                panic!("android device not available");
            }
        }
        _ => {}
    }

    if config.verbose {
        // We're going to be dumping a lot of info. Start on a new line.
        write!(stdout, "\n\n");
    }
    debug!("running {}", testpaths.file);
    let mut props = TestProps::from_file(&testpaths.file, variant.revision(), &config);

    // For non-incremental (i.e. regular UI) tests, the incremental directory
    // takes into account the revision name, since the revisions are independent
    // of each other and can race.
    if props.incremental {
        props.incremental_dir = Some(incremental_dir(&config, testpaths, variant));
    }

    let cx = TestCx { config: &config, stdout, stderr, props: &props, testpaths, variant };

    if let Err(e) = create_dir_all(&cx.output_base_dir()) {
        panic!("failed to create output base directory {}: {e}", cx.output_base_dir());
    }

    if props.incremental {
        cx.init_incremental_test();
    }

    if config.mode == TestMode::Incremental {
        // Incremental tests are special because they cannot be run in
        // parallel.
        assert!(!props.revisions.is_empty(), "Incremental tests require revisions.");
        for revision in &props.revisions {
            let mut revision_props = TestProps::from_file(&testpaths.file, Some(revision), &config);
            revision_props.incremental_dir = props.incremental_dir.clone();
            let rev_cx = TestCx {
                config: &config,
                stdout,
                stderr,
                props: &revision_props,
                testpaths,
                variant: &TestVariant {
                    revision: Some(revision.clone()),
                    debugger: variant.debugger,
                },
            };
            rev_cx.run_revision();
        }
    } else {
        cx.run_revision();
    }

    cx.create_stamp();
}

pub(crate) fn compute_stamp_hash(config: &Config, variant: &TestVariant) -> String {
    let mut hash = DefaultHasher::new();
    config.stage_id.hash(&mut hash);
    config.run.hash(&mut hash);
    config.edition.hash(&mut hash);

    match variant.debugger {
        Some(Debugger::Cdb) => {
            config.cdb.hash(&mut hash);
        }

        Some(Debugger::Gdb) => {
            config.gdb.hash(&mut hash);
            env::var_os("PATH").hash(&mut hash);
            env::var_os("PYTHONPATH").hash(&mut hash);
        }

        Some(Debugger::Lldb) => {
            // LLDB debuginfo tests now use LLDB's embedded Python, with an
            // explicit PYTHONPATH, so they don't depend on `--python` or
            // the ambient PYTHONPATH.
            config.lldb.hash(&mut hash);
            env::var_os("PATH").hash(&mut hash);
        }

        None => {}
    }

    if config.mode == TestMode::Ui {
        config.force_pass_mode.hash(&mut hash);
    }

    format!("{:x}", hash.finish())
}

#[derive(Copy, Clone, Debug)]
struct TestCx<'test> {
    config: &'test Config,
    stdout: &'test dyn ConsoleOut,
    stderr: &'test dyn ConsoleOut,
    props: &'test TestProps,
    testpaths: &'test TestPaths,
    variant: &'test TestVariant,
}

enum ReadFrom {
    Path,
    Stdin(String),
}

enum TestOutput {
    Compile,
    Run,
}

/// Will this test be executed? Should we use `make_exe_name`?
#[derive(Copy, Clone, PartialEq)]
enum WillExecute {
    Yes,
    No,
    Disabled,
}

/// What value should be passed to `--emit`?
#[derive(Copy, Clone)]
enum Emit {
    None,
    Metadata,
    LlvmIr,
    Mir,
    Asm,
    LinkArgsAsm,
}

/// Indicates whether we are using `rustc` or `rustdoc` to compile an input file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CompilerKind {
    Rustc,
    Rustdoc,
}

impl<'test> TestCx<'test> {
    /// Code executed for each revision in turn (or, if there are no
    /// revisions, exactly once, with revision == None).
    fn run_revision(&self) {
        // Run the test multiple times if requested.
        // This is useful for catching flaky tests under the parallel frontend.
        for _ in 0..self.config.iteration_count {
            match self.config.mode {
                TestMode::Pretty => self.run_pretty_test(),
                TestMode::DebugInfo => self.run_debuginfo_test(),
                TestMode::Codegen => self.run_codegen_test(),
                TestMode::RustdocHtml => self.run_rustdoc_html_test(),
                TestMode::RustdocJson => self.run_rustdoc_json_test(),
                TestMode::CodegenUnits => self.run_codegen_units_test(),
                TestMode::Incremental => self.run_incremental_test(),
                TestMode::RunMake => self.run_rmake_test(),
                TestMode::Ui => self.run_ui_test(),
                TestMode::MirOpt => self.run_mir_opt_test(),
                TestMode::Assembly => self.run_assembly_test(),
                TestMode::RustdocJs => self.run_rustdoc_js_test(),
                TestMode::CoverageMap => self.run_coverage_map_test(), // see self::coverage
                TestMode::CoverageRun => self.run_coverage_run_test(), // see self::coverage
                TestMode::Crashes => self.run_crash_test(),
            }
        }
    }

    /// Returns the pass/fail expectation of this UI test
    /// (e.g. `//@ check-pass` or `//@ build-fail`),
    /// possibly modified by an explicit `--pass=check` on the command-line.
    fn effective_pass_fail_mode(&self) -> Option<PassFailMode> {
        assert_eq!(self.config.mode, TestMode::Ui);
        // UI tests always have a pass/fail mode, but their auxiliary crates never have one.
        let declared = self.props.pass_fail_mode?;

        // Specifying `--pass` only overrides `//@ pass-*` modes, and only if
        // the test doesn't opt out with `//@ no-pass-override`.
        if let Some(force_pass_mode) = self.config.force_pass_mode
            && !self.props.no_pass_override
            && declared.is_pass()
        {
            match force_pass_mode {
                ForcePassMode::Check => Some(PassFailMode::CheckPass),
                ForcePassMode::Build => Some(PassFailMode::BuildPass),
                ForcePassMode::Run => Some(PassFailMode::RunPass),
            }
        } else {
            Some(declared)
        }
    }

    fn run_if_enabled(&self) -> WillExecute {
        if self.config.run_enabled() { WillExecute::Yes } else { WillExecute::Disabled }
    }

    fn check_if_test_should_compile(&self, pass_fail: PassFailMode, proc_res: &ProcRes) {
        assert_eq!(self.config.mode, TestMode::Ui);

        let should_compile_successfully = match pass_fail {
            PassFailMode::CheckFail | PassFailMode::BuildFail => false,

            PassFailMode::CheckPass
            | PassFailMode::BuildPass
            | PassFailMode::RunFail
            | PassFailMode::RunCrash
            | PassFailMode::RunFailOrCrash
            | PassFailMode::RunPass => true,
        };

        if should_compile_successfully {
            if !proc_res.status.success() {
                if pass_fail == PassFailMode::CheckPass
                    && self.effective_pass_fail_mode() == Some(PassFailMode::BuildFail)
                {
                    // A `build-fail` test needs to `check-pass`.
                    self.fatal_proc_rec(
                        "`build-fail` test is required to pass check build, but check build failed",
                        proc_res,
                    );
                } else {
                    self.fatal_proc_rec("test compilation failed although it shouldn't!", proc_res);
                }
            }
        } else {
            if proc_res.status.success() {
                let err = &format!("{} test did not emit an error", self.config.mode);
                let extra_note = Some(
                    "note: by default, ui tests are expected not to compile.\nhint: use check-pass, build-pass, or run-pass directive to change this behavior.",
                );
                self.fatal_proc_rec_general(err, extra_note, proc_res, || ());
            }

            if !self.props.dont_check_failure_status {
                self.check_correct_failure_status(proc_res);
            }
        }
    }

    fn get_output(&self, proc_res: &ProcRes) -> String {
        if self.props.check_stdout {
            format!("{}{}", proc_res.stdout, proc_res.stderr)
        } else {
            proc_res.stderr.clone()
        }
    }

    fn check_correct_failure_status(&self, proc_res: &ProcRes) {
        let expected_status = Some(self.props.failure_status.unwrap_or(1));
        let received_status = proc_res.status.code();

        if expected_status != received_status {
            self.fatal_proc_rec(
                &format!(
                    "Error: expected failure status ({:?}) but received status {:?}.",
                    expected_status, received_status
                ),
                proc_res,
            );
        }
    }

    /// Runs a [`ArgFileCommand`] and waits for it to finish, then converts its exit
    /// status and output streams into a [`ProcRes`].
    ///
    /// The command might have succeeded or failed; it is the caller's
    /// responsibility to check the exit status and take appropriate action.
    ///
    /// # Panics
    /// Panics if the command couldn't be executed at all
    /// (e.g. because the executable could not be found).
    #[must_use = "caller should check whether the command succeeded"]
    fn run_command_to_procres(&self, cmd: ArgFileCommand) -> ProcRes {
        let (mut cmd, _arg_file) = cmd.build().unwrap();
        let output = cmd
            .output()
            .unwrap_or_else(|e| self.fatal(&format!("failed to exec `{cmd:?}` because: {e}")));

        let proc_res = ProcRes {
            status: output.status,
            stdout: String::from_utf8(output.stdout).unwrap(),
            stderr: String::from_utf8(output.stderr).unwrap(),
            truncated: Truncated::No,
            cmdline: format!("{cmd:?}"),
        };
        self.dump_output(
            self.config.verbose || !proc_res.status.success(),
            &cmd.get_program().to_string_lossy(),
            &proc_res.stdout,
            &proc_res.stderr,
        );

        proc_res
    }

    fn print_source(&self, read_from: ReadFrom, pretty_type: &str) -> ProcRes {
        let aux_dir = self.aux_output_dir_name();
        let input: &str = match read_from {
            ReadFrom::Stdin(_) => "-",
            ReadFrom::Path => self.testpaths.file.as_str(),
        };

        let mut rustc = Command::new(&self.config.rustc_path);

        self.build_all_auxiliary(&self.aux_output_dir(), &mut rustc);

        rustc
            .arg(input)
            .args(&["-Z", &format!("unpretty={}", pretty_type)])
            .arg("-Zunstable-options")
            .args(&["--target", &self.config.target])
            .arg("-L")
            .arg(&aux_dir)
            .arg("-A")
            .arg("internal_features")
            .args(&self.props.compile_flags)
            .envs(self.props.rustc_env.clone());
        self.maybe_add_external_args(&mut rustc, &self.config.target_rustcflags);

        let src = match read_from {
            ReadFrom::Stdin(src) => Some(src),
            ReadFrom::Path => None,
        };

        self.compose_and_run(
            rustc,
            self.config.host_compile_lib_path.as_path(),
            Some(aux_dir.as_path()),
            src,
        )
    }

    fn compare_source(&self, expected: &str, actual: &str) {
        if expected != actual {
            self.fatal(&format!(
                "pretty-printed source does not match expected source\n\
                 expected:\n\
                 ------------------------------------------\n\
                 {}\n\
                 ------------------------------------------\n\
                 actual:\n\
                 ------------------------------------------\n\
                 {}\n\
                 ------------------------------------------\n\
                 diff:\n\
                 ------------------------------------------\n\
                 {}\n",
                expected,
                actual,
                write_diff(expected, actual, 3),
            ));
        }
    }

    fn set_revision_flags(&self, cmd: &mut Command) {
        // Normalize revisions to be lowercase and replace `-`s with `_`s.
        // Otherwise the `--cfg` flag is not valid.
        let normalize_revision = |revision: &str| revision.to_lowercase().replace("-", "_");

        if let Some(revision) = self.variant.revision() {
            let normalized_revision = normalize_revision(revision);
            let cfg_arg = ["--cfg", &normalized_revision];
            let arg = format!("--cfg={normalized_revision}");
            // Handle if compile_flags is length 1
            let contains_arg =
                self.props.compile_flags.iter().any(|considered_arg| *considered_arg == arg);
            let contains_cfg_arg = self.props.compile_flags.windows(2).any(|args| args == cfg_arg);
            if contains_arg || contains_cfg_arg {
                error!(
                    "redundant cfg argument `{normalized_revision}` is already created by the \
                    revision"
                );
                panic!("redundant cfg argument");
            }
            if self.config.builtin_cfg_names().contains(&normalized_revision) {
                error!("revision `{normalized_revision}` collides with a built-in cfg");
                panic!("revision collides with built-in cfg");
            }
            cmd.args(cfg_arg);
        }

        if !self.props.no_auto_check_cfg {
            let mut check_cfg = String::with_capacity(25);

            // Generate `cfg(FALSE, REV1, ..., REVN)` (for all possible revisions)
            //
            // For compatibility reason we consider the `FALSE` cfg to be expected
            // since it is extensively used in the testsuite, as well as the `test`
            // cfg since we have tests that uses it.
            check_cfg.push_str("cfg(test,FALSE");
            for revision in &self.props.revisions {
                check_cfg.push(',');
                check_cfg.push_str(&normalize_revision(revision));
            }
            check_cfg.push(')');

            cmd.args(&["--check-cfg", &check_cfg]);
        }
    }

    fn typecheck_source(&self, src: String) -> ProcRes {
        let mut rustc = Command::new(&self.config.rustc_path);

        let out_dir = self.output_base_name().with_extension("pretty-out");
        remove_and_create_dir_all(&out_dir).unwrap_or_else(|e| {
            panic!("failed to remove and recreate output directory `{out_dir}`: {e}")
        });

        let target = if self.props.force_host { &*self.config.host } else { &*self.config.target };

        let aux_dir = self.aux_output_dir_name();

        rustc
            .arg("-")
            .arg("-Zno-codegen")
            .arg("-Zunstable-options")
            .arg("--out-dir")
            .arg(&out_dir)
            .arg(&format!("--target={}", target))
            .arg("-L")
            // FIXME(jieyouxu): this search path seems questionable. Is this intended for
            // `rust_test_helpers` in ui tests?
            .arg(&self.config.build_test_suite_root)
            .arg("-L")
            .arg(aux_dir)
            .arg("-A")
            .arg("internal_features");
        self.set_revision_flags(&mut rustc);
        self.maybe_add_external_args(&mut rustc, &self.config.target_rustcflags);
        rustc.args(&self.props.compile_flags);

        self.compose_and_run_compiler(rustc, Some(src))
    }

    fn maybe_add_external_args(&self, cmd: &mut Command, args: &Vec<String>) {
        // Filter out the arguments that should not be added by runtest here.
        //
        // Notable use-cases are: do not add our optimisation flag if
        // `compile-flags: -Copt-level=x` and similar for debug-info level as well.
        const OPT_FLAGS: &[&str] = &["-O", "-Copt-level=", /*-C<space>*/ "opt-level="];
        const DEBUG_FLAGS: &[&str] = &["-g", "-Cdebuginfo=", /*-C<space>*/ "debuginfo="];

        // FIXME: ideally we would "just" check the `cmd` itself, but it does not allow inspecting
        // its arguments. They need to be collected separately. For now I cannot be bothered to
        // implement this the "right" way.
        let have_opt_flag =
            self.props.compile_flags.iter().any(|arg| OPT_FLAGS.iter().any(|f| arg.starts_with(f)));
        let have_debug_flag = self
            .props
            .compile_flags
            .iter()
            .any(|arg| DEBUG_FLAGS.iter().any(|f| arg.starts_with(f)));

        for arg in args {
            if OPT_FLAGS.iter().any(|f| arg.starts_with(f)) && have_opt_flag {
                continue;
            }
            if DEBUG_FLAGS.iter().any(|f| arg.starts_with(f)) && have_debug_flag {
                continue;
            }
            cmd.arg(arg);
        }
    }

    /// Check `error-pattern` and `regex-error-pattern` directives.
    fn check_all_error_patterns(&self, output_to_check: &str, proc_res: &ProcRes) {
        let mut missing_patterns: Vec<String> = Vec::new();
        self.check_error_patterns(output_to_check, &mut missing_patterns);
        self.check_regex_error_patterns(output_to_check, proc_res, &mut missing_patterns);

        if missing_patterns.is_empty() {
            return;
        }

        if missing_patterns.len() == 1 {
            self.fatal_proc_rec(
                &format!("error pattern '{}' not found!", missing_patterns[0]),
                proc_res,
            );
        } else {
            for pattern in missing_patterns {
                writeln!(
                    self.stdout,
                    "\n{prefix}: error pattern '{pattern}' not found!",
                    prefix = self.error_prefix()
                );
            }
            self.fatal_proc_rec("multiple error patterns not found", proc_res);
        }
    }

    fn check_error_patterns(&self, output_to_check: &str, missing_patterns: &mut Vec<String>) {
        debug!("check_error_patterns");
        for pattern in &self.props.error_patterns {
            if output_to_check.contains(pattern.trim()) {
                debug!("found error pattern {}", pattern);
            } else {
                missing_patterns.push(pattern.to_string());
            }
        }
    }

    fn check_regex_error_patterns(
        &self,
        output_to_check: &str,
        proc_res: &ProcRes,
        missing_patterns: &mut Vec<String>,
    ) {
        debug!("check_regex_error_patterns");

        for pattern in &self.props.regex_error_patterns {
            let pattern = pattern.trim();
            let re = match Regex::new(pattern) {
                Ok(re) => re,
                Err(err) => {
                    self.fatal_proc_rec(
                        &format!("invalid regex error pattern '{}': {:?}", pattern, err),
                        proc_res,
                    );
                }
            };
            if re.is_match(output_to_check) {
                debug!("found regex error pattern {}", pattern);
            } else {
                missing_patterns.push(pattern.to_string());
            }
        }
    }

    fn check_forbid_output(&self, output_to_check: &str, proc_res: &ProcRes) {
        for pat in &self.props.forbid_output {
            if output_to_check.contains(pat) {
                self.fatal_proc_rec("forbidden pattern found in compiler output", proc_res);
            }
        }
    }

    /// Check `//~ KIND message` annotations.
    fn check_expected_errors(&self, proc_res: &ProcRes) {
        let expected_errors = load_errors(&self.testpaths.file, self.variant.revision());
        debug!(
            "check_expected_errors: expected_errors={:?} proc_res.status={:?}",
            expected_errors, proc_res.status
        );
        if proc_res.status.success() && expected_errors.iter().any(|x| x.kind == ErrorKind::Error) {
            self.fatal_proc_rec("process did not return an error status", proc_res);
        }

        if self.props.known_bug {
            if !expected_errors.is_empty() {
                self.fatal_proc_rec(
                    "`known_bug` tests should not have an expected error",
                    proc_res,
                );
            }
            return;
        }

        // On Windows, keep all '\' path separators to match the paths reported in the JSON output
        // from the compiler
        let diagnostic_file_name = if self.props.remap_src_base {
            let mut p = Utf8PathBuf::from(FAKE_SRC_BASE);
            p.push(&self.testpaths.relative_dir);
            p.push(self.testpaths.file.file_name().unwrap());
            p.to_string()
        } else {
            self.testpaths.file.to_string()
        };

        // Errors and warnings are always expected, other diagnostics are only expected
        // if one of them actually occurs in the test.
        let expected_kinds: HashSet<_> = [ErrorKind::Error, ErrorKind::Warning]
            .into_iter()
            .chain(expected_errors.iter().map(|e| e.kind))
            .collect();

        // Parse the JSON output from the compiler and extract out the messages.
        let actual_errors = json::parse_output(&diagnostic_file_name, &self.get_output(proc_res))
            .into_iter()
            .map(|e| Error { msg: self.normalize_output(&e.msg, &[]), ..e });

        let mut unexpected = Vec::new();
        let mut unimportant = Vec::new();
        let mut found = vec![false; expected_errors.len()];
        for actual_error in actual_errors {
            for pattern in &self.props.error_patterns {
                let pattern = pattern.trim();
                if actual_error.msg.contains(pattern) {
                    let q = if actual_error.line_num.is_none() { "?" } else { "" };
                    self.fatal(&format!(
                        "error pattern '{pattern}' is found in structured \
                         diagnostics, use `//~{q} {} {pattern}` instead",
                        actual_error.kind,
                    ));
                }
            }

            let opt_index =
                expected_errors.iter().enumerate().position(|(index, expected_error)| {
                    !found[index]
                        && actual_error.line_num == expected_error.line_num
                        && actual_error.kind == expected_error.kind
                        && actual_error.msg.contains(&expected_error.msg)
                });

            match opt_index {
                Some(index) => {
                    // found a match, everybody is happy
                    assert!(!found[index]);
                    found[index] = true;
                }

                None => {
                    if actual_error.require_annotation
                        && expected_kinds.contains(&actual_error.kind)
                        && !self.props.dont_require_annotations.contains(&actual_error.kind)
                    {
                        unexpected.push(actual_error);
                    } else {
                        unimportant.push(actual_error);
                    }
                }
            }
        }

        unexpected.sort_by_key(|e| (e.line_num, e.column_num));
        unimportant.sort_by_key(|e| (e.line_num, e.column_num));

        // `not_found` are sorted because `expected_errors` are sorted as they are read from file
        // line by line.
        let mut not_found = Vec::new();
        // anything not yet found is a problem
        for (index, expected_error) in expected_errors.iter().enumerate() {
            if !found[index] {
                not_found.push(expected_error);
            }
        }

        if !unexpected.is_empty() || !not_found.is_empty() {
            // Emit locations in a format that is short (relative paths) but "clickable" in editors.
            // Also normalize path separators to `/`.
            let file_name = self
                .testpaths
                .file
                .strip_prefix(self.config.src_root.as_str())
                .unwrap_or(&self.testpaths.file)
                .to_string()
                .replace(r"\", "/");
            let line_str = |e: &Error| {
                let line_num = e.line_num.map_or("?".to_string(), |line_num| line_num.to_string());
                // `file:?:NUM` may be confusing to editors and unclickable.
                let opt_col_num = match e.column_num {
                    Some(col_num) if line_num != "?" => format!(":{col_num}"),
                    _ => "".to_string(),
                };
                format!("{file_name}:{line_num}{opt_col_num}")
            };
            let print_error =
                |e| writeln!(self.stdout, "{}: {}: {}", line_str(e), e.kind, e.msg.cyan());
            let push_suggestion =
                |suggestions: &mut Vec<_>, e: &Error, kind, line, msg, color, rank| {
                    let mut ret = String::new();
                    if kind {
                        ret += &format!("{} {}", "with different kind:".color(color), e.kind);
                    }
                    if line {
                        if !ret.is_empty() {
                            ret.push(' ');
                        }
                        ret += &format!("{} {}", "on different line:".color(color), line_str(e));
                    }
                    if msg {
                        if !ret.is_empty() {
                            ret.push(' ');
                        }
                        ret +=
                            &format!("{} {}", "with different message:".color(color), e.msg.cyan());
                    }
                    suggestions.push((ret, rank));
                };
            let show_suggestions = |mut suggestions: Vec<_>, prefix: &str, color| {
                // Only show suggestions with the highest rank.
                suggestions.sort_by_key(|(_, rank)| *rank);
                if let Some(&(_, top_rank)) = suggestions.first() {
                    for (suggestion, rank) in suggestions {
                        if rank == top_rank {
                            writeln!(self.stdout, "  {} {suggestion}", prefix.color(color));
                        }
                    }
                }
            };

            // Fuzzy matching quality:
            // - message and line / message and kind - great, suggested
            // - only message - good, suggested
            // - known line and kind - ok, suggested
            // - only known line - meh, but suggested
            // - others are not worth suggesting
            if !unexpected.is_empty() {
                writeln!(
                    self.stdout,
                    "\n{prefix}: {n} diagnostics reported in rustc output but not expected in test file",
                    prefix = self.error_prefix(),
                    n = unexpected.len(),
                );
                for error in &unexpected {
                    print_error(error);
                    let mut suggestions = Vec::new();
                    for candidate in &not_found {
                        let kind_mismatch = candidate.kind != error.kind;
                        let mut push_red_suggestion = |line, msg, rank| {
                            push_suggestion(
                                &mut suggestions,
                                candidate,
                                kind_mismatch,
                                line,
                                msg,
                                Color::Red,
                                rank,
                            )
                        };
                        if error.msg.contains(&candidate.msg) {
                            push_red_suggestion(candidate.line_num != error.line_num, false, 0);
                        } else if candidate.line_num.is_some()
                            && candidate.line_num == error.line_num
                        {
                            push_red_suggestion(false, true, if kind_mismatch { 2 } else { 1 });
                        }
                    }

                    show_suggestions(suggestions, "expected", Color::Red);
                }
            }
            if !not_found.is_empty() {
                writeln!(
                    self.stdout,
                    "\n{prefix}: {n} diagnostics expected in test file but not reported in rustc output",
                    prefix = self.error_prefix(),
                    n = not_found.len(),
                );

                // FIXME: Ideally, we should check this at the place where we actually parse error annotations.
                // it's better to use (negated) heuristic inside normalize_output if possible
                if let Some(human_format) = self.props.compile_flags.iter().find(|flag| {
                    // `human`, `human-unicode`, `short` will not generate JSON output
                    flag.contains("error-format")
                        && (flag.contains("short") || flag.contains("human"))
                }) {
                    let msg = format!(
                        "tests with compile flag `{}` should not have error annotations such as `//~ ERROR`",
                        human_format
                    ).color(Color::Red);
                    writeln!(self.stdout, "{}", msg);
                }

                for error in &not_found {
                    print_error(error);
                    let mut suggestions = Vec::new();
                    for candidate in unexpected.iter().chain(&unimportant) {
                        let kind_mismatch = candidate.kind != error.kind;
                        let mut push_green_suggestion = |line, msg, rank| {
                            push_suggestion(
                                &mut suggestions,
                                candidate,
                                kind_mismatch,
                                line,
                                msg,
                                Color::Green,
                                rank,
                            )
                        };
                        if candidate.msg.contains(&error.msg) {
                            push_green_suggestion(candidate.line_num != error.line_num, false, 0);
                        } else if candidate.line_num.is_some()
                            && candidate.line_num == error.line_num
                        {
                            push_green_suggestion(false, true, if kind_mismatch { 2 } else { 1 });
                        }
                    }

                    show_suggestions(suggestions, "reported", Color::Green);
                }
            }
            panic!(
                "errors differ from expected\nstatus: {}\ncommand: {}\n",
                proc_res.status, proc_res.cmdline
            );
        }
    }

    fn compile_test(&self, will_execute: WillExecute, emit: Emit) -> ProcRes {
        self.compile_test_general(will_execute, emit, Vec::new())
    }

    fn compile_test_general(
        &self,
        will_execute: WillExecute,
        emit: Emit,
        passes: Vec<String>,
    ) -> ProcRes {
        let compiler_kind = self.compiler_kind_for_non_aux();

        // Only use `make_exe_name` when the test ends up being executed.
        let output_file = match will_execute {
            WillExecute::Yes => TargetLocation::ThisFile(self.make_exe_name()),
            WillExecute::No | WillExecute::Disabled => {
                TargetLocation::ThisDirectory(self.output_base_dir())
            }
        };

        let allow_unused = match self.config.mode {
            TestMode::Ui => {
                // UI tests tend to have tons of unused code as
                // it's just testing various pieces of the compile, but we don't
                // want to actually assert warnings about all this code. Instead
                // let's just ignore unused code warnings by defaults and tests
                // can turn it back on if needed.
                if compiler_kind == CompilerKind::Rustc
                    // Note that we use the declared pass mode here as we don't want
                    // to set unused to allow if we've overridden the pass mode
                    // via command line flags.
                    // FIXME(Zalathar): We should probably also warn in run-fail/crash
                    // tests, but that requires changes to some existing tests.
                    && self.props.pass_fail_mode != Some(PassFailMode::RunPass)
                {
                    AllowUnused::Yes
                } else {
                    AllowUnused::No
                }
            }
            TestMode::Incremental => AllowUnused::Yes,
            _ => AllowUnused::No,
        };

        let rustc = self.make_compile_args(
            compiler_kind,
            &self.testpaths.file,
            output_file,
            emit,
            allow_unused,
            LinkToAux::Yes,
            passes,
        );

        self.compose_and_run_compiler(rustc, None)
    }

    /// `root_out_dir` and `root_testpaths` refer to the parameters of the actual test being run.
    /// Auxiliaries, no matter how deep, have the same root_out_dir and root_testpaths.
    fn document(&self, root_out_dir: &Utf8Path, kind: DocKind) -> ProcRes {
        self.document_inner(&self.testpaths.file, root_out_dir, kind)
    }

    /// Like `document`, but takes an explicit `file_to_doc` argument so that
    /// it can also be used for documenting auxiliaries, in addition to
    /// documenting the main test file.
    fn document_inner(
        &self,
        file_to_doc: &Utf8Path,
        root_out_dir: &Utf8Path,
        kind: DocKind,
    ) -> ProcRes {
        if self.props.build_aux_docs {
            assert_eq!(kind, DocKind::Html, "build-aux-docs only make sense for html output");

            for rel_ab in &self.props.aux.builds {
                let aux_path = self.resolve_aux_path(rel_ab);
                let props_for_aux =
                    self.props.from_aux_file(&aux_path, self.variant.revision(), self.config);
                let aux_cx = TestCx {
                    config: self.config,
                    stdout: self.stdout,
                    stderr: self.stderr,
                    props: &props_for_aux,
                    testpaths: self.testpaths,
                    variant: self.variant,
                };
                // Create the directory for the stdout/stderr files.
                create_dir_all(aux_cx.output_base_dir()).unwrap();
                let auxres = aux_cx.document_inner(&aux_path, &root_out_dir, kind);
                if !auxres.status.success() {
                    return auxres;
                }
            }
        }

        let aux_dir = self.aux_output_dir_name();

        let rustdoc_path = self.config.rustdoc_path.as_ref().expect("--rustdoc-path not passed");

        // actual --out-dir given to the auxiliary or test, as opposed to the root out dir for the entire
        // test
        let out_dir: Cow<'_, Utf8Path> = if self.props.unique_doc_out_dir {
            let file_name = file_to_doc.file_stem().expect("file name should not be empty");
            let out_dir = Utf8PathBuf::from_iter([
                root_out_dir,
                Utf8Path::new("docs"),
                Utf8Path::new(file_name),
                Utf8Path::new("doc"),
            ]);
            create_dir_all(&out_dir).unwrap();
            Cow::Owned(out_dir)
        } else {
            Cow::Borrowed(root_out_dir)
        };

        let mut rustdoc = Command::new(rustdoc_path);
        let current_dir = self.output_base_dir();
        rustdoc.current_dir(current_dir);
        rustdoc
            .arg("-L")
            .arg(self.config.target_run_lib_path.as_path())
            .arg("-L")
            .arg(aux_dir)
            .arg("-o")
            .arg(out_dir.as_ref())
            .arg("--deny")
            .arg("warnings")
            .arg(file_to_doc)
            .arg("-A")
            .arg("internal_features")
            // FIXME(#160895): While the new solver is enabled by default on nightly,
            // we don't want to use it in our tests for now.
            .arg("-Znext-solver=coherence")
            .args(&self.props.compile_flags)
            .args(&self.props.doc_flags);

        match kind {
            DocKind::Html => {}
            DocKind::Json => {
                rustdoc.arg("--output-format").arg("json");
            }
        }

        // Both JSON output and `--disable-minification` are unstable rustdoc options.
        if matches!(kind, DocKind::Json) || self.config.disable_minification {
            rustdoc.arg("-Zunstable-options");
        }
        if self.config.disable_minification {
            rustdoc.arg("--disable-minification");
        }

        if let Some(ref linker) = self.config.target_linker {
            rustdoc.arg(format!("-Clinker={}", linker));
        }

        self.compose_and_run_compiler(rustdoc, None)
    }

    fn exec_compiled_test(&self) -> ProcRes {
        self.exec_compiled_test_general(&[], true)
    }

    fn exec_compiled_test_general(
        &self,
        env_extra: &[(&str, &str)],
        delete_after_success: bool,
    ) -> ProcRes {
        let prepare_env = |cmd: &mut Command| {
            for (key, val) in &self.props.exec_env {
                cmd.env(key, val);
            }
            for (key, val) in env_extra {
                cmd.env(key, val);
            }

            for key in &self.props.unset_exec_env {
                cmd.env_remove(key);
            }
        };

        let proc_res = match &*self.config.target {
            // This is pretty similar to below, we're transforming:
            //
            // ```text
            // program arg1 arg2
            // ```
            //
            // into
            //
            // ```text
            // remote-test-client run program 2 support-lib.so support-lib2.so arg1 arg2
            // ```
            //
            // The test-client program will upload `program` to the emulator along with all other
            // support libraries listed (in this case `support-lib.so` and `support-lib2.so`. It
            // will then execute the program on the emulator with the arguments specified (in the
            // environment we give the process) and then report back the same result.
            _ if self.config.remote_test_client.is_some() => {
                let aux_dir = self.aux_output_dir_name();
                let ProcArgs { prog, args } = self.make_run_args();
                let mut support_libs = Vec::new();
                if let Ok(entries) = aux_dir.read_dir() {
                    for entry in entries {
                        let entry = entry.unwrap();
                        if !entry.path().is_file() {
                            continue;
                        }
                        support_libs.push(entry.path());
                    }
                }
                let mut test_client =
                    Command::new(self.config.remote_test_client.as_ref().unwrap());
                test_client
                    .args(&["run", &support_libs.len().to_string()])
                    .arg(&prog)
                    .args(support_libs)
                    .args(args);

                prepare_env(&mut test_client);

                self.compose_and_run(
                    test_client,
                    self.config.target_run_lib_path.as_path(),
                    Some(aux_dir.as_path()),
                    None,
                )
            }
            _ if self.config.target.contains("vxworks") => {
                let aux_dir = self.aux_output_dir_name();
                let ProcArgs { prog, args } = self.make_run_args();
                let mut wr_run = Command::new("wr-run");
                wr_run.args(&[&prog]).args(args);

                prepare_env(&mut wr_run);

                self.compose_and_run(
                    wr_run,
                    self.config.target_run_lib_path.as_path(),
                    Some(aux_dir.as_path()),
                    None,
                )
            }
            _ => {
                let aux_dir = self.aux_output_dir_name();
                let ProcArgs { prog, args } = self.make_run_args();
                let mut program = Command::new(&prog);
                program.args(args).current_dir(&self.output_base_dir());

                prepare_env(&mut program);

                self.compose_and_run(
                    program,
                    self.config.target_run_lib_path.as_path(),
                    Some(aux_dir.as_path()),
                    None,
                )
            }
        };

        if delete_after_success && proc_res.status.success() {
            // delete the executable after running it to save space.
            // it is ok if the deletion failed.
            let _ = fs::remove_file(self.make_exe_name());
        }

        proc_res
    }

    /// For each `aux-build: foo/bar` annotation, we check to find the file in an `auxiliary`
    /// directory relative to the test itself (not any intermediate auxiliaries).
    fn resolve_aux_path(&self, relative_aux_path: &str) -> Utf8PathBuf {
        let aux_path = self
            .testpaths
            .file
            .parent()
            .expect("test file path has no parent")
            .join("auxiliary")
            .join(relative_aux_path);
        if !aux_path.exists() {
            self.fatal(&format!(
                "auxiliary source file `{relative_aux_path}` not found at `{aux_path}`"
            ));
        }

        aux_path
    }

    fn is_vxworks_pure_static(&self) -> bool {
        if self.config.target.contains("vxworks") {
            match env::var("RUST_VXWORKS_TEST_DYLINK") {
                Ok(s) => s != "1",
                _ => true,
            }
        } else {
            false
        }
    }

    fn is_vxworks_pure_dynamic(&self) -> bool {
        self.config.target.contains("vxworks") && !self.is_vxworks_pure_static()
    }

    fn has_aux_dir(&self) -> bool {
        !self.props.aux.builds.is_empty()
            || !self.props.aux.crates.is_empty()
            || !self.props.aux.proc_macros.is_empty()
    }

    fn aux_output_dir(&self) -> Utf8PathBuf {
        let aux_dir = self.aux_output_dir_name();

        if !self.props.aux.builds.is_empty() {
            remove_and_create_dir_all(&aux_dir).unwrap_or_else(|e| {
                panic!("failed to remove and recreate output directory `{aux_dir}`: {e}")
            });
        }

        if !self.props.aux.bins.is_empty() {
            let aux_bin_dir = self.aux_bin_output_dir_name();
            remove_and_create_dir_all(&aux_dir).unwrap_or_else(|e| {
                panic!("failed to remove and recreate output directory `{aux_dir}`: {e}")
            });
            remove_and_create_dir_all(&aux_bin_dir).unwrap_or_else(|e| {
                panic!("failed to remove and recreate output directory `{aux_bin_dir}`: {e}")
            });
        }

        aux_dir
    }

    fn build_all_auxiliary(&self, aux_dir: &Utf8Path, rustc: &mut Command) {
        for rel_ab in &self.props.aux.builds {
            self.build_auxiliary(rel_ab, &aux_dir, None);
        }

        for rel_ab in &self.props.aux.bins {
            self.build_auxiliary(rel_ab, &aux_dir, Some(AuxType::Bin));
        }

        let path_to_crate_name = |path: &str| -> String {
            path.rsplit_once('/')
                .map_or(path, |(_, tail)| tail)
                .trim_end_matches(".rs")
                .replace('-', "_")
        };

        let add_extern = |rustc: &mut Command,
                          extern_modifiers: Option<&str>,
                          aux_name: &str,
                          aux_path: &str,
                          aux_type: AuxType| {
            let lib_name =
                get_lib_name(&path_to_crate_name(aux_path), aux_type, self.config.wasm_proc_macros);
            if let Some(lib_name) = lib_name {
                let modifiers_and_name = match extern_modifiers {
                    Some(modifiers) => format!("{modifiers}:{aux_name}"),
                    None => aux_name.to_string(),
                };
                rustc.arg("--extern").arg(format!("{modifiers_and_name}={aux_dir}/{lib_name}"));
            }
        };

        for AuxCrate { extern_modifiers, name, path } in &self.props.aux.crates {
            let aux_type = self.build_auxiliary(&path, &aux_dir, None);
            add_extern(rustc, extern_modifiers.as_deref(), name, path, aux_type);
        }

        for proc_macro in &self.props.aux.proc_macros {
            self.build_auxiliary(&proc_macro.path, &aux_dir, Some(AuxType::ProcMacro));
            let crate_name = path_to_crate_name(&proc_macro.path);
            add_extern(
                rustc,
                proc_macro.extern_modifiers.as_deref(),
                &crate_name,
                &proc_macro.path,
                AuxType::ProcMacro,
            );
        }

        // Build any `//@ aux-codegen-backend`, and pass the resulting library
        // to `-Zcodegen-backend` when compiling the test file.
        if let Some(aux_file) = &self.props.aux.codegen_backend {
            let aux_type = self.build_auxiliary(aux_file, aux_dir, None);
            if let Some(lib_name) = get_lib_name(
                aux_file.trim_end_matches(".rs"),
                aux_type,
                self.config.wasm_proc_macros,
            ) {
                let lib_path = aux_dir.join(&lib_name);
                rustc.arg(format!("-Zcodegen-backend={}", lib_path));
            }
        }
    }

    /// `root_testpaths` refers to the path of the original test. the auxiliary and the test with an
    /// aux-build have the same `root_testpaths`.
    fn compose_and_run_compiler(&self, mut rustc: Command, input: Option<String>) -> ProcRes {
        if self.props.add_minicore {
            let minicore_path = self.build_minicore();
            rustc.arg("--extern");
            rustc.arg(&format!("minicore={}", minicore_path));
        }

        let aux_dir = self.aux_output_dir();
        self.build_all_auxiliary(&aux_dir, &mut rustc);

        rustc.envs(self.props.rustc_env.clone());
        self.props.unset_rustc_env.iter().fold(&mut rustc, Command::env_remove);
        self.compose_and_run(
            rustc,
            self.config.host_compile_lib_path.as_path(),
            Some(aux_dir.as_path()),
            input,
        )
    }

    /// Builds `minicore`. Returns the path to the minicore rlib within the base test output
    /// directory.
    fn build_minicore(&self) -> Utf8PathBuf {
        let output_file_path = self.output_base_dir().join("libminicore.rlib");
        let mut rustc = self.make_compile_args(
            CompilerKind::Rustc,
            &self.config.minicore_path,
            TargetLocation::ThisFile(output_file_path.clone()),
            Emit::None,
            AllowUnused::Yes,
            LinkToAux::No,
            vec![],
        );

        rustc.args(&["--crate-type", "rlib"]);
        rustc.arg("-Cpanic=abort");
        rustc.args(self.props.minicore_compile_flags.clone());

        let res =
            self.compose_and_run(rustc, self.config.host_compile_lib_path.as_path(), None, None);
        if !res.status.success() {
            self.fatal_proc_rec(
                &format!("auxiliary build of {} failed to compile: ", self.config.minicore_path),
                &res,
            );
        }

        output_file_path
    }

    /// Builds an aux dependency.
    ///
    /// If `aux_type` is `None`, then this will determine the aux-type automatically.
    fn build_auxiliary(
        &self,
        source_path: &str,
        aux_dir: &Utf8Path,
        aux_type: Option<AuxType>,
    ) -> AuxType {
        let aux_path = self.resolve_aux_path(source_path);
        let mut aux_props =
            self.props.from_aux_file(&aux_path, self.variant.revision(), self.config);
        if aux_type == Some(AuxType::ProcMacro) {
            if self.config.wasm_proc_macros {
                aux_props.compile_flags.push("--target=wasm32-wasip2".to_owned());
                // Override any earlier linkers for now, otherwise we fail to build since compiletest
                // thinks we're building for a different target and passes its linker (if one is
                // configured).
                //
                // wasm32-wasip2 should in principle always be able to link with wasm-component-ld +
                // wasm-ld. This does mean that rust.lld needs to be enabled to build wasm-ld wrapper
                // around rust-lld.
                aux_props.compile_flags.push("-Clinker=wasm-component-ld".to_owned());
                aux_props.compile_flags.push(format!(
                    "-Clink-arg=--wasm-ld-path={}",
                    self.config
                        .sysroot_base
                        .join("lib/rustlib")
                        .join(&self.config.host)
                        .join("bin/gcc-ld/wasm-ld")
                ));
            } else {
                aux_props.force_host = true;
            }
        }
        let mut aux_dir = aux_dir.to_path_buf();
        if aux_type == Some(AuxType::Bin) {
            // On unix, the binary of `auxiliary/foo.rs` will be named
            // `auxiliary/foo` which clashes with the _dir_ `auxiliary/foo`, so
            // put bins in a `bin` subfolder.
            aux_dir.push("bin");
        }
        let aux_output = TargetLocation::ThisDirectory(aux_dir.clone());
        let aux_cx = TestCx {
            config: self.config,
            stdout: self.stdout,
            stderr: self.stderr,
            props: &aux_props,
            testpaths: self.testpaths,
            variant: self.variant,
        };
        // Create the directory for the stdout/stderr files.
        create_dir_all(aux_cx.output_base_dir()).unwrap();
        let mut aux_rustc = aux_cx.make_compile_args(
            // Always use `rustc` for aux crates, even in rustdoc tests.
            CompilerKind::Rustc,
            &aux_path,
            aux_output,
            Emit::None,
            AllowUnused::No,
            LinkToAux::No,
            Vec::new(),
        );
        aux_cx.build_all_auxiliary(&aux_dir, &mut aux_rustc);

        aux_rustc.envs(aux_props.rustc_env.clone());
        for key in &aux_props.unset_rustc_env {
            aux_rustc.env_remove(key);
        }

        let (aux_type, crate_type) = if aux_type == Some(AuxType::Bin) {
            (AuxType::Bin, Some("bin"))
        } else if aux_type == Some(AuxType::ProcMacro) {
            (AuxType::ProcMacro, Some("proc-macro"))
        } else if aux_type.is_some() {
            panic!("aux_type {aux_type:?} not expected");
        } else if aux_props.no_prefer_dynamic {
            (AuxType::Lib, None)
        } else if self.config.target.contains("emscripten")
            || (self.config.target.contains("musl")
                && !aux_props.force_host
                && !self.config.host.contains("musl"))
            || self.config.target.contains("wasm32")
            || self.config.target.contains("nvptx")
            || self.is_vxworks_pure_static()
            || self.config.target.contains("bpf")
            || !self.config.target_cfg().dynamic_linking
            || matches!(self.config.mode, TestMode::CoverageMap | TestMode::CoverageRun)
        {
            // We primarily compile all auxiliary libraries as dynamic libraries
            // to avoid code size bloat and large binaries as much as possible
            // for the test suite (otherwise including libstd statically in all
            // executables takes up quite a bit of space).
            //
            // For targets like MUSL or Emscripten, however, there is no support for
            // dynamic libraries so we just go back to building a normal library. Note,
            // however, that for MUSL if the library is built with `force_host` then
            // it's ok to be a dylib as the host should always support dylibs.
            //
            // Coverage tests want static linking by default so that coverage
            // mappings in auxiliary libraries can be merged into the final
            // executable.
            (AuxType::Lib, Some("lib"))
        } else {
            (AuxType::Dylib, Some("dylib"))
        };

        if let Some(crate_type) = crate_type {
            aux_rustc.args(&["--crate-type", crate_type]);
        }

        if aux_type == AuxType::ProcMacro {
            // For convenience, but this only works on 2018.
            aux_rustc.args(&["--extern", "proc_macro"]);
        }

        aux_rustc.arg("-L").arg(&aux_dir);

        if aux_props.add_minicore {
            let minicore_path = self.build_minicore();
            aux_rustc.arg("--extern");
            aux_rustc.arg(&format!("minicore={}", minicore_path));
        }

        let auxres = aux_cx.compose_and_run(
            aux_rustc,
            aux_cx.config.host_compile_lib_path.as_path(),
            Some(aux_dir.as_path()),
            None,
        );
        if !auxres.status.success() {
            self.fatal_proc_rec(
                &format!("auxiliary build of {aux_path} failed to compile: "),
                &auxres,
            );
        }
        aux_type
    }

    fn read2_abbreviated(&self, child: Child) -> (Output, Truncated) {
        let mut filter_paths_from_len = Vec::new();
        let mut add_path = |path: &Utf8Path| {
            let path = path.to_string();
            let windows = path.replace("\\", "\\\\");
            if windows != path {
                filter_paths_from_len.push(windows);
            }
            filter_paths_from_len.push(path);
        };

        // List of paths that will not be measured when determining whether the output is larger
        // than the output truncation threshold.
        //
        // Note: avoid adding a subdirectory of an already filtered directory here, otherwise the
        // same slice of text will be double counted and the truncation might not happen.
        add_path(&self.config.src_test_suite_root);
        add_path(&self.config.build_test_suite_root);

        read2_abbreviated(child, &filter_paths_from_len).expect("failed to read output")
    }

    fn compose_and_run(
        &self,
        mut command: Command,
        lib_path: &Utf8Path,
        aux_path: Option<&Utf8Path>,
        input: Option<String>,
    ) -> ProcRes {
        let cmdline = {
            let cmdline = self.make_cmdline(&command, lib_path);
            self.logv(format_args!("executing {cmdline}"));
            cmdline
        };

        command.stdout(Stdio::piped()).stderr(Stdio::piped()).stdin(Stdio::piped());

        // Need to be sure to put both the lib_path and the aux path in the dylib
        // search path for the child.
        add_dylib_path(&mut command, iter::once(lib_path).chain(aux_path));

        let mut child = disable_error_reporting(|| command.spawn())
            .unwrap_or_else(|e| panic!("failed to exec `{command:?}`: {e:?}"));
        if let Some(input) = input {
            child.stdin.as_mut().unwrap().write_all(input.as_bytes()).unwrap();
        }

        let (Output { status, stdout, stderr }, truncated) = self.read2_abbreviated(child);

        let result = ProcRes {
            status,
            stdout: String::from_utf8_lossy(&stdout).into_owned(),
            stderr: String::from_utf8_lossy(&stderr).into_owned(),
            truncated,
            cmdline,
        };

        self.dump_output(
            self.config.verbose || (!result.status.success() && self.config.mode != TestMode::Ui),
            &command.get_program().to_string_lossy(),
            &result.stdout,
            &result.stderr,
        );

        result
    }

    /// Choose a compiler kind (rustc or rustdoc) for compiling test files,
    /// based on the test suite being tested.
    fn compiler_kind_for_non_aux(&self) -> CompilerKind {
        match self.config.suite {
            TestSuite::RustdocJs | TestSuite::RustdocJson | TestSuite::RustdocUi => {
                CompilerKind::Rustdoc
            }

            // Exhaustively match all other suites.
            // Note that some suites never actually use this method, so the
            // return value for those suites is not necessarily meaningful.
            TestSuite::AssemblyLlvm
            | TestSuite::BuildStd
            | TestSuite::CodegenLlvm
            | TestSuite::CodegenUnits
            | TestSuite::Coverage
            | TestSuite::CoverageRunRustdoc
            | TestSuite::Crashes
            | TestSuite::Debuginfo
            | TestSuite::Incremental
            | TestSuite::MirOpt
            | TestSuite::Pretty
            | TestSuite::RunMake
            | TestSuite::RunMakeCargo
            | TestSuite::RustdocGui
            | TestSuite::RustdocHtml
            | TestSuite::RustdocJsStd
            | TestSuite::Ui
            | TestSuite::UiFullDeps => CompilerKind::Rustc,
        }
    }

    fn make_compile_args(
        &self,
        compiler_kind: CompilerKind,
        input_file: &Utf8Path,
        output_file: TargetLocation,
        emit: Emit,
        allow_unused: AllowUnused,
        link_to_aux: LinkToAux,
        passes: Vec<String>, // Vec of passes under mir-opt test to be dumped
    ) -> Command {
        // FIXME(Zalathar): We should have a cleaner distinction between
        // `rustc` flags, `rustdoc` flags, and flags shared by both.
        let mut compiler = match compiler_kind {
            CompilerKind::Rustc => Command::new(&self.config.rustc_path),
            CompilerKind::Rustdoc => {
                Command::new(&self.config.rustdoc_path.clone().expect("no rustdoc built yet"))
            }
        };
        compiler.arg(input_file);

        // Enable wasm proc macros.
        if self.config.wasm_proc_macros {
            compiler.arg("-Zwasm-proc-macros");
        }

        // `--disable-minification` is an unstable rustdoc option. Rustdoc UI tests intentionally
        // exercise diagnostics for unstable options, so don't enable them for that suite.
        if compiler_kind == CompilerKind::Rustdoc
            && self.config.disable_minification
            && self.config.mode != TestMode::Ui
        {
            compiler.arg("-Zunstable-options").arg("--disable-minification");
        }

        // Hide libstd sources from ui tests to make sure we generate the stderr
        // output that users will see.
        // Without this, we may be producing good diagnostics in-tree but users
        // will not see half the information.
        //
        // This also has the benefit of more effectively normalizing output between different
        // compilers, so that we don't have to know the `/rustc/$sha` output to normalize after the
        // fact.
        compiler.arg("-Zsimulate-remapped-rust-src-base=/rustc/FAKE_PREFIX");
        compiler.arg("-Ztranslate-remapped-path-to-local-path=no");

        // Hide Cargo dependency sources from ui tests to make sure the error message doesn't
        // change depending on whether $CARGO_HOME is remapped or not. If this is not present,
        // when $CARGO_HOME is remapped the source won't be shown, and when it's not remapped the
        // source will be shown, causing a blessing hell.
        compiler.arg("-Z").arg(format!(
            "ignore-directory-in-diagnostics-source-blocks={}",
            home::cargo_home().expect("failed to find cargo home").to_str().unwrap()
        ));
        // Similarly, vendored sources shouldn't be shown when running from a dist tarball.
        compiler.arg("-Z").arg(format!(
            "ignore-directory-in-diagnostics-source-blocks={}",
            self.config.src_root.join("vendor"),
        ));

        // Optionally prevent default --sysroot if specified in test compile-flags.
        //
        // FIXME: I feel like this logic is fairly sus.
        if !self.props.compile_flags.iter().any(|flag| flag.starts_with("--sysroot"))
            && !self.config.host_rustcflags.iter().any(|flag| flag == "--sysroot")
        {
            // In stage 0, make sure we use `stage0-sysroot` instead of the bootstrap sysroot.
            compiler.arg("--sysroot").arg(&self.config.sysroot_base);
        }

        // If the provided codegen backend is not LLVM, we need to pass it.
        if let Some(ref backend) = self.config.override_codegen_backend {
            compiler.arg(format!("-Zcodegen-backend={}", backend));
        }

        // Optionally prevent default --target if specified in test compile-flags.
        let custom_target = self.props.compile_flags.iter().any(|x| x.starts_with("--target"));

        if !custom_target {
            let target =
                if self.props.force_host { &*self.config.host } else { &*self.config.target };

            compiler.arg(&format!("--target={}", target));
            if target.ends_with(".json") {
                // `-Zunstable-options` is necessary when compiletest is running with custom targets
                // (such as synthetic targets used to bless mir-opt tests).
                compiler.arg("-Zunstable-options");
            }
        }
        self.set_revision_flags(&mut compiler);

        if compiler_kind == CompilerKind::Rustc {
            if let Some(ref incremental_dir) = self.props.incremental_dir {
                compiler.args(&["-C", &format!("incremental={}", incremental_dir)]);
                compiler.args(&["-Z", "incremental-verify-ich"]);
            }

            if self.config.mode == TestMode::CodegenUnits {
                compiler.args(&["-Z", "human_readable_cgu_names"]);
            }

            if self.config.mode == TestMode::DebugInfo && cfg!(target_os = "windows") {
                // Prevent debugger processes from creating new console windows.
                compiler.args(&["-Z", r#"crate-attr=windows_subsystem="windows""#]);
            }
        }

        if self.config.optimize_tests && compiler_kind == CompilerKind::Rustc {
            match self.config.mode {
                TestMode::Ui => {
                    // If optimize-tests is true we still only want to optimize tests that actually get
                    // executed and that don't specify their own optimization levels.
                    // Note: aux libs don't have a pass/fail mode, so they won't get optimized
                    // unless compile-flags are set in the aux file.
                    // FIXME(Zalathar): We could also optimize run-fail/run-crash tests,
                    // but it's unclear whether that would be helpful or a waste of time.
                    if self.effective_pass_fail_mode() == Some(PassFailMode::RunPass)
                        && !self
                            .props
                            .compile_flags
                            .iter()
                            .any(|arg| arg == "-O" || arg.contains("opt-level"))
                    {
                        compiler.arg("-O");
                    }
                }
                TestMode::DebugInfo => { /* debuginfo tests must be unoptimized */ }
                TestMode::CoverageMap | TestMode::CoverageRun => {
                    // Coverage mappings and coverage reports are affected by
                    // optimization level, so they ignore the optimize-tests
                    // setting and set an optimization level in their mode's
                    // compile flags (below) or in per-test `compile-flags`.
                }
                _ => {
                    compiler.arg("-O");
                }
            }
        }

        let set_mir_dump_dir = |rustc: &mut Command| {
            let mir_dump_dir = self.output_base_dir();
            let mut dir_opt = "-Zdump-mir-dir=".to_string();
            dir_opt.push_str(mir_dump_dir.as_str());
            debug!("dir_opt: {:?}", dir_opt);
            rustc.arg(dir_opt);
        };

        match self.config.mode {
            TestMode::Incremental => {
                // If we are extracting and matching errors in the new
                // fashion, then you want JSON mode. Old-skool error
                // patterns still match the raw compiler output.
                if self.props.error_patterns.is_empty()
                    && self.props.regex_error_patterns.is_empty()
                {
                    compiler.args(&["--error-format", "json"]);
                    compiler.args(&["--json", "future-incompat"]);
                }
                compiler.arg("-Zui-testing");
                compiler.arg("-Zdeduplicate-diagnostics=no");
            }
            TestMode::Ui => {
                if !self.props.compile_flags.iter().any(|s| s.starts_with("--error-format")) {
                    compiler.args(&["--error-format", "json"]);
                    compiler.args(&["--json", "future-incompat"]);
                }
                compiler.arg("-Ccodegen-units=1");
                // Hide line numbers to reduce churn
                compiler.arg("-Zui-testing");
                compiler.arg("-Zdeduplicate-diagnostics=no");
                compiler.arg("-Zwrite-long-types-to-disk=no");
                // FIXME: use this for other modes too, for perf?
                compiler.arg("-Cstrip=debuginfo");

                if self.config.parallel_frontend_enabled() {
                    // Currently, we only use multiple threads for the UI test suite,
                    // because UI tests can effectively verify the parallel frontend and
                    // require minimal modification. The option will later be extended to
                    // other test suites.
                    compiler.arg(&format!("-Zthreads={}", self.config.parallel_frontend_threads));
                }
            }
            TestMode::MirOpt => {
                // We check passes under test to minimize the mir-opt test dump
                // if files_for_miropt_test parses the passes, we dump only those passes
                // otherwise we conservatively pass -Zdump-mir=all
                let zdump_arg = if !passes.is_empty() {
                    format!("-Zdump-mir={}", passes.join(" | "))
                } else {
                    "-Zdump-mir=all".to_string()
                };

                compiler.args(&[
                    "-Copt-level=1",
                    &zdump_arg,
                    "-Zvalidate-mir",
                    "-Zlint-mir",
                    "-Zdump-mir-exclude-pass-number",
                    "-Zmir-include-spans=false", // remove span comments from NLL MIR dumps
                    "--crate-type=rlib",
                ]);
                if let Some(pass) = &self.props.mir_unit_test {
                    compiler
                        .args(&["-Zmir-opt-level=0", &format!("-Zmir-enable-passes=+{}", pass)]);
                } else {
                    compiler.args(&[
                        "-Zmir-opt-level=4",
                        "-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals",
                    ]);
                }

                set_mir_dump_dir(&mut compiler);
            }
            TestMode::CoverageMap => {
                compiler.arg("-Cinstrument-coverage");
                // These tests only compile to LLVM IR, so they don't need the
                // profiler runtime to be present.
                compiler.arg("-Zno-profiler-runtime");
                // Coverage mappings are sensitive to MIR optimizations, and
                // the current snapshots assume `opt-level=2` unless overridden
                // by `compile-flags`.
                compiler.arg("-Copt-level=2");
            }
            TestMode::CoverageRun => {
                compiler.arg("-Cinstrument-coverage");
                // Coverage reports are sometimes sensitive to optimizations,
                // and the current snapshots assume `opt-level=2` unless
                // overridden by `compile-flags`.
                compiler.arg("-Copt-level=2");
            }
            TestMode::Assembly | TestMode::Codegen => {
                compiler.arg("-Cdebug-assertions=no");
                // For assembly and codegen tests, we want to use the same order
                // of the items of a codegen unit as the source order, so that
                // we can compare the output with the source code through filecheck.
                compiler.arg("-Zcodegen-source-order");
            }
            TestMode::Crashes => {
                set_mir_dump_dir(&mut compiler);
            }
            TestMode::CodegenUnits => {
                compiler.arg("-Zprint-mono-items");
            }
            TestMode::Pretty
            | TestMode::DebugInfo
            | TestMode::RustdocHtml
            | TestMode::RustdocJson
            | TestMode::RunMake
            | TestMode::RustdocJs => {
                // do not use JSON output
            }
        }

        if self.props.remap_src_base {
            compiler.arg(format!(
                "--remap-path-prefix={}={}",
                self.config.src_test_suite_root, FAKE_SRC_BASE,
            ));
        }

        if compiler_kind == CompilerKind::Rustc {
            match emit {
                Emit::None => {}
                Emit::Metadata => {
                    compiler.args(&["--emit", "metadata"]);
                }
                Emit::LlvmIr => {
                    compiler.args(&["--emit", "llvm-ir"]);
                }
                Emit::Mir => {
                    compiler.args(&["--emit", "mir"]);
                }
                Emit::Asm => {
                    compiler.args(&["--emit", "asm"]);
                }
                Emit::LinkArgsAsm => {
                    compiler.args(&["-Clink-args=--emit=asm"]);
                }
            }
        }

        if compiler_kind == CompilerKind::Rustc {
            if self.config.target == "wasm32-unknown-unknown" || self.is_vxworks_pure_static() {
                // rustc.arg("-g"); // get any backtrace at all on errors
            } else if !self.props.no_prefer_dynamic {
                compiler.args(&["-C", "prefer-dynamic"]);
            }
        }

        match output_file {
            // If the test's compile flags specify an output path with `-o`,
            // avoid a compiler warning about `--out-dir` being ignored.
            _ if self.props.compile_flags.iter().any(|flag| flag == "-o") => {}
            TargetLocation::ThisFile(path) => {
                compiler.arg("-o").arg(path);
            }
            TargetLocation::ThisDirectory(path) => match compiler_kind {
                CompilerKind::Rustdoc => {
                    // `rustdoc` uses `-o` for the output directory.
                    compiler.arg("-o").arg(path);
                }
                CompilerKind::Rustc => {
                    compiler.arg("--out-dir").arg(path);
                }
            },
        }

        // FIXME(#160895): While the new solver is enabled by default on nightly,
        // we don't want to use it in our tests for now.
        compiler.args(["-Znext-solver=coherence"]);

        match self.config.compare_mode {
            Some(CompareMode::Polonius) => {
                compiler.args(&["-Zpolonius=next"]);
            }
            Some(CompareMode::NextSolver) => {
                compiler.args(&["-Znext-solver"]);
            }
            Some(CompareMode::NextSolverCoherence) => {
                compiler.args(&["-Znext-solver=coherence"]);
            }
            Some(CompareMode::SplitDwarf) if self.config.target.contains("windows") => {
                compiler.args(&["-Csplit-debuginfo=unpacked", "-Zunstable-options"]);
            }
            Some(CompareMode::SplitDwarf) => {
                compiler.args(&["-Csplit-debuginfo=unpacked"]);
            }
            Some(CompareMode::SplitDwarfSingle) => {
                compiler.args(&["-Csplit-debuginfo=packed"]);
            }
            None => {}
        }

        // Add `-A unused` before `config` flags and in-test (`props`) flags, so that they can
        // overwrite this.
        // Don't allow `unused_attributes` since these are usually actual mistakes, rather than just unused code.
        if let AllowUnused::Yes = allow_unused {
            compiler.args(&["-A", "unused", "-W", "unused_attributes"]);
        }

        // Allow tests to use internal and incomplete features.
        compiler.args(&["-A", "internal_features"]);
        compiler.args(&["-A", "incomplete_features"]);

        // Allow tests to have unused parens and braces.
        // Add #![deny(unused_parens, unused_braces)] to the test file if you want to
        // test that these lints are working.
        compiler.args(&["-A", "unused_parens"]);
        compiler.args(&["-A", "unused_braces"]);

        if self.props.force_host {
            self.maybe_add_external_args(&mut compiler, &self.config.host_rustcflags);
            if compiler_kind == CompilerKind::Rustc
                && let Some(ref linker) = self.config.host_linker
            {
                compiler.arg(format!("-Clinker={linker}"));
            }
        } else {
            self.maybe_add_external_args(&mut compiler, &self.config.target_rustcflags);
            if compiler_kind == CompilerKind::Rustc
                && let Some(ref linker) = self.config.target_linker
            {
                compiler.arg(format!("-Clinker={linker}"));
            }
        }

        // Use dynamic musl for tests because static doesn't allow creating dylibs
        if self.config.host.contains("musl") || self.is_vxworks_pure_dynamic() {
            compiler.arg("-Ctarget-feature=-crt-static");
        }

        if let LinkToAux::Yes = link_to_aux {
            // if we pass an `-L` argument to a directory that doesn't exist,
            // macOS ld emits warnings which disrupt the .stderr files
            if self.has_aux_dir() {
                compiler.arg("-L").arg(self.aux_output_dir_name());
            }
        }

        // FIXME(jieyouxu): we should report a fatal error or warning if user wrote `-Cpanic=` with
        // something that's not `abort` and `-Cforce-unwind-tables` with a value that is not `yes`.
        //
        // We could apply these last and override any provided flags. That would ensure that the
        // build works, but some tests want to exercise that mixing panic modes in specific ways is
        // rejected. So we enable aborting panics and unwind tables before adding flags, just to
        // change the default.
        //
        // `minicore` requires `#![no_std]` and `#![no_core]`, which means no unwinding panics.
        if self.props.add_minicore {
            compiler.arg("-Cpanic=abort");
            compiler.arg("-Cforce-unwind-tables=yes");
        }

        compiler.args(&self.props.compile_flags);

        compiler
    }

    fn make_exe_name(&self) -> Utf8PathBuf {
        // Using a single letter here to keep the path length down for
        // Windows.  Some test names get very long.  rustc creates `rcgu`
        // files with the module name appended to it which can more than
        // double the length.
        let mut f = self.output_base_dir().join("a");
        // FIXME: This is using the host architecture exe suffix, not target!
        if self.config.target.contains("emscripten") {
            f = f.with_extra_extension("js");
        } else if self.config.target.starts_with("wasm") {
            f = f.with_extra_extension("wasm");
        } else if self.config.target.contains("spirv") {
            f = f.with_extra_extension("spv");
        } else if !env::consts::EXE_SUFFIX.is_empty() {
            f = f.with_extra_extension(env::consts::EXE_SUFFIX);
        }
        f
    }

    fn make_run_args(&self) -> ProcArgs {
        // If we've got another tool to run under (valgrind),
        // then split apart its command
        let mut args = self.split_maybe_args(&self.config.runner);

        let exe_file = self.make_exe_name();

        args.push(exe_file.into_os_string());

        // Add the arguments in the run_flags directive
        args.extend(self.props.run_flags.iter().map(OsString::from));

        let prog = args.remove(0);
        ProcArgs { prog, args }
    }

    fn split_maybe_args(&self, argstr: &Option<String>) -> Vec<OsString> {
        match *argstr {
            Some(ref s) => s
                .split(' ')
                .filter_map(|s| {
                    if s.chars().all(|c| c.is_whitespace()) {
                        None
                    } else {
                        Some(OsString::from(s))
                    }
                })
                .collect(),
            None => Vec::new(),
        }
    }

    fn make_cmdline(&self, command: &Command, libpath: &Utf8Path) -> String {
        use crate::util;

        // Linux and mac don't require adjusting the library search path
        if cfg!(unix) {
            format!("{:?}", command)
        } else {
            // Build the LD_LIBRARY_PATH variable as it would be seen on the command line
            // for diagnostic purposes
            fn lib_path_cmd_prefix(path: &str) -> String {
                format!("{}=\"{}\"", util::lib_path_env_var(), util::make_new_path(path))
            }

            format!("{} {:?}", lib_path_cmd_prefix(libpath.as_str()), command)
        }
    }

    fn dump_output(&self, print_output: bool, proc_name: &str, out: &str, err: &str) {
        let revision =
            if let Some(r) = self.variant.revision() { format!("{}.", r) } else { String::new() };

        self.dump_output_file(out, &format!("{}out", revision));
        self.dump_output_file(err, &format!("{}err", revision));

        if !print_output {
            return;
        }

        let path = Utf8Path::new(proc_name);
        let proc_name = if path.file_stem().is_some_and(|p| p == "rmake") {
            String::from_iter(
                path.parent()
                    .unwrap()
                    .file_name()
                    .into_iter()
                    .chain(Some("/"))
                    .chain(path.file_name()),
            )
        } else {
            path.file_name().unwrap().into()
        };
        writeln!(self.stdout, "------{proc_name} stdout------------------------------");
        writeln!(self.stdout, "{}", out);
        writeln!(self.stdout, "------{proc_name} stderr------------------------------");
        writeln!(self.stdout, "{}", err);
        writeln!(self.stdout, "------------------------------------------");
    }

    fn dump_output_file(&self, out: &str, extension: &str) {
        let outfile = self.make_out_name(extension);
        fs::write(outfile.as_std_path(), out)
            .unwrap_or_else(|err| panic!("failed to write {outfile}: {err:?}"));
    }

    /// Creates a filename for output with the given extension.
    /// E.g., `/.../testname.revision.mode/testname.extension`.
    fn make_out_name(&self, extension: &str) -> Utf8PathBuf {
        self.output_base_name().with_extension(extension)
    }

    /// Gets the directory where auxiliary files are written.
    /// E.g., `/.../testname.revision.mode/auxiliary/`.
    fn aux_output_dir_name(&self) -> Utf8PathBuf {
        self.output_base_dir()
            .join("auxiliary")
            .with_extra_extension(self.config.mode.aux_dir_disambiguator())
    }

    /// Gets the directory where auxiliary binaries are written.
    /// E.g., `/.../testname.revision.mode/auxiliary/bin`.
    fn aux_bin_output_dir_name(&self) -> Utf8PathBuf {
        self.aux_output_dir_name().join("bin")
    }

    /// The revision, ignored for incremental compilation since it wants all revisions in
    /// the same directory.
    fn variant_with_safe_revision(&self) -> TestVariant {
        if self.config.mode == TestMode::Incremental {
            TestVariant { revision: None, debugger: self.variant.debugger }
        } else {
            self.variant.clone()
        }
    }

    /// Gets the absolute path to the directory where all output for the given
    /// test/revision should reside.
    /// E.g., `/path/to/build/host-tuple/test/ui/relative/testname.revision.mode/`.
    fn output_base_dir(&self) -> Utf8PathBuf {
        output_base_dir(self.config, self.testpaths, &self.variant_with_safe_revision())
    }

    /// Gets the absolute path to the base filename used as output for the given
    /// test/revision.
    /// E.g., `/.../relative/testname.revision.mode/testname`.
    fn output_base_name(&self) -> Utf8PathBuf {
        output_base_name(self.config, self.testpaths, &self.variant_with_safe_revision())
    }

    /// Prints a message to (captured) stdout if `config.verbose` is true.
    /// The message is also logged to `tracing::debug!` regardless of verbosity.
    ///
    /// Use `format_args!` as the argument to perform formatting if required.
    fn logv(&self, message: impl fmt::Display) {
        debug!("{message}");
        if self.config.verbose {
            // Note: `./x test ... --verbose --no-capture` is needed to see this print.
            writeln!(self.stdout, "{message}");
        }
    }

    /// Prefix to print before error messages. Normally just `error`, but also
    /// includes the revision name for tests that use revisions.
    #[must_use]
    fn error_prefix(&self) -> String {
        match self.variant.revision() {
            Some(rev) => format!("error in revision `{rev}`"),
            None => format!("error"),
        }
    }

    #[track_caller]
    fn fatal(&self, err: &str) -> ! {
        writeln!(self.stdout, "\n{prefix}: {err}", prefix = self.error_prefix());
        error!("fatal error, panic: {:?}", err);
        panic!("fatal error");
    }

    fn fatal_proc_rec(&self, err: &str, proc_res: &ProcRes) -> ! {
        self.fatal_proc_rec_general(err, None, proc_res, || ());
    }

    /// Underlying implementation of [`Self::fatal_proc_rec`], providing some
    /// extra capabilities not needed by most callers.
    fn fatal_proc_rec_general(
        &self,
        err: &str,
        extra_note: Option<&str>,
        proc_res: &ProcRes,
        callback_before_unwind: impl FnOnce(),
    ) -> ! {
        writeln!(self.stdout, "\n{prefix}: {err}", prefix = self.error_prefix());

        // Some callers want to print additional notes after the main error message.
        if let Some(note) = extra_note {
            writeln!(self.stdout, "{note}");
        }

        // Print the details and output of the subprocess that caused this test to fail.
        writeln!(self.stdout, "{}", proc_res.format_info());

        // Some callers want print more context or show a custom diff before the unwind occurs.
        callback_before_unwind();

        // Use resume_unwind instead of panic!() to prevent a panic message + backtrace from
        // compiletest, which is unnecessary noise.
        std::panic::resume_unwind(Box::new(()));
    }

    // codegen tests (using FileCheck)

    fn compile_test_and_save_ir(&self) -> (ProcRes, Utf8PathBuf) {
        let output_path = self.output_base_name().with_extension("ll");
        let input_file = &self.testpaths.file;
        let rustc = self.make_compile_args(
            CompilerKind::Rustc,
            input_file,
            TargetLocation::ThisFile(output_path.clone()),
            Emit::LlvmIr,
            AllowUnused::No,
            LinkToAux::Yes,
            Vec::new(),
        );

        let proc_res = self.compose_and_run_compiler(rustc, None);
        (proc_res, output_path)
    }

    fn verify_with_filecheck(&self, output: &Utf8Path) -> ProcRes {
        let mut filecheck = Command::new(self.config.llvm_filecheck.as_ref().unwrap());
        filecheck.arg("--input-file").arg(output).arg(&self.testpaths.file);

        // Because we use custom prefixes, we also have to register the default prefix.
        filecheck.arg("--check-prefix=CHECK");

        // FIXME(#134510): auto-registering revision names as check prefix is a bit sketchy, and
        // that having to pass `--allow-unused-prefix` is an unfortunate side-effect of not knowing
        // whether the test author actually wanted revision-specific check prefixes or not.
        //
        // TL;DR We may not want to conflate `compiletest` revisions and `FileCheck` prefixes.

        // HACK: tests are allowed to use a revision name as a check prefix.
        if let Some(rev) = self.variant.revision() {
            filecheck.arg("--check-prefix").arg(rev);
        }

        // HACK: the filecheck tool normally fails if a prefix is defined but not used. However,
        // sometimes revisions are used to specify *compiletest* directives which are not FileCheck
        // concerns.
        filecheck.arg("--allow-unused-prefixes");

        // Provide more context on failures.
        filecheck.args(&["--dump-input-context", "100"]);

        // Add custom flags supplied by the `filecheck-flags:` test directive.
        filecheck.args(&self.props.filecheck_flags);

        // FIXME(jieyouxu): don't pass an empty Path
        self.compose_and_run(filecheck, Utf8Path::new(""), None, None)
    }

    fn charset() -> &'static str {
        // FreeBSD 10.1 defaults to GDB 6.1.1 which doesn't support "auto" charset
        if cfg!(target_os = "freebsd") { "ISO-8859-1" } else { "UTF-8" }
    }

    fn get_lines(&self, path: &Utf8Path, mut other_files: Option<&mut Vec<String>>) -> Vec<usize> {
        let content = fs::read_to_string(path.as_std_path()).unwrap();
        let mut ignore = false;
        content
            .lines()
            .enumerate()
            .filter_map(|(line_nb, line)| {
                if (line.trim_start().starts_with("pub mod ")
                    || line.trim_start().starts_with("mod "))
                    && line.ends_with(';')
                {
                    if let Some(ref mut other_files) = other_files {
                        other_files.push(line.rsplit("mod ").next().unwrap().replace(';', ""));
                    }
                    None
                } else {
                    let sline = line.rsplit("///").next().unwrap();
                    let line = sline.trim_start();
                    if line.starts_with("```") {
                        if ignore {
                            ignore = false;
                            None
                        } else {
                            ignore = true;
                            Some(line_nb + 1)
                        }
                    } else {
                        None
                    }
                }
            })
            .collect()
    }

    /// This method is used for `//@ check-test-line-numbers-match`.
    ///
    /// It checks that doctests line in the displayed doctest "name" matches where they are
    /// defined in source code.
    fn check_rustdoc_test_option(&self, res: ProcRes) {
        let mut other_files = Vec::new();
        let mut files: HashMap<String, Vec<usize>> = HashMap::new();
        let normalized = fs::canonicalize(&self.testpaths.file).expect("failed to canonicalize");
        let normalized = normalized.to_str().unwrap().replace('\\', "/");
        files.insert(normalized, self.get_lines(&self.testpaths.file, Some(&mut other_files)));
        for other_file in other_files {
            let mut path = self.testpaths.file.clone();
            path.set_file_name(&format!("{}.rs", other_file));
            let path = path.canonicalize_utf8().expect("failed to canonicalize");
            let normalized = path.as_str().replace('\\', "/");
            files.insert(normalized, self.get_lines(&path, None));
        }

        let mut tested = 0;
        for _ in res.stdout.split('\n').filter(|s| s.starts_with("test ")).inspect(|s| {
            if let Some((left, right)) = s.split_once(" - ") {
                let path = left.rsplit("test ").next().unwrap();
                let path = fs::canonicalize(&path).expect("failed to canonicalize");
                let path = path.to_str().unwrap().replace('\\', "/");
                if let Some(ref mut v) = files.get_mut(&path) {
                    tested += 1;
                    let mut iter = right.split("(line ");
                    iter.next();
                    let line = iter
                        .next()
                        .unwrap_or(")")
                        .split(')')
                        .next()
                        .unwrap_or("0")
                        .parse()
                        .unwrap_or(0);
                    if let Ok(pos) = v.binary_search(&line) {
                        v.remove(pos);
                    } else {
                        self.fatal_proc_rec(
                            &format!("Not found doc test: \"{}\" in \"{}\":{:?}", s, path, v),
                            &res,
                        );
                    }
                }
            }
        }) {}
        if tested == 0 {
            self.fatal_proc_rec(&format!("No test has been found... {:?}", files), &res);
        } else {
            for (entry, v) in &files {
                if !v.is_empty() {
                    self.fatal_proc_rec(
                        &format!(
                            "Not found test at line{} \"{}\":{:?}",
                            if v.len() > 1 { "s" } else { "" },
                            entry,
                            v
                        ),
                        &res,
                    );
                }
            }
        }
    }

    fn force_color_svg(&self) -> bool {
        self.props.compile_flags.iter().any(|s| s.contains("--color=always"))
    }

    /// Returns the lines for the by-lines comparison, normalized for the
    /// parallel front-end: for SVG output, strip the header line and `y`
    /// offsets; otherwise, filter out padded empty code lines (a single `|`).
    fn lines_for_comparison(&self, output: &str) -> Vec<String> {
        if self.force_color_svg() {
            let strip_y = static_regex!(r#"y="\d+px""#);
            output
                .lines()
                // anstyle_svg causes environment-dependent width parameter
                .skip(1)
                .map(|line| strip_y.replace_all(line, r#"y="0px""#).into_owned())
                .collect()
        } else {
            output.lines().filter(|l| l.trim() != "|").map(str::to_owned).collect()
        }
    }

    fn load_compare_outputs(
        &self,
        proc_res: &ProcRes,
        output_kind: TestOutput,
        explicit_format: bool,
    ) -> usize {
        let stderr_bits = format!("{}bit.stderr", self.config.get_pointer_width());
        let (stderr_kind, stdout_kind) = match output_kind {
            TestOutput::Compile => (
                if self.force_color_svg() {
                    if self.config.target.contains("windows") {
                        // We single out Windows here because some of the CLI coloring is
                        // specifically changed for Windows.
                        UI_WINDOWS_SVG
                    } else {
                        UI_SVG
                    }
                } else if self.props.stderr_per_bitwidth {
                    &stderr_bits
                } else {
                    UI_STDERR
                },
                UI_STDOUT,
            ),
            TestOutput::Run => (UI_RUN_STDERR, UI_RUN_STDOUT),
        };

        let expected_stderr = self.load_expected_output(stderr_kind);
        let expected_stdout = self.load_expected_output(stdout_kind);

        let mut normalized_stdout =
            self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout);
        match output_kind {
            TestOutput::Run if self.config.remote_test_client.is_some() => {
                // When tests are run using the remote-test-client, the string
                // 'uploaded "$TEST_BUILD_DIR/<test_executable>, waiting for result"'
                // is printed to stdout by the client and then captured in the ProcRes,
                // so it needs to be removed when comparing the run-pass test execution output.
                normalized_stdout = static_regex!(
                    "^uploaded \"\\$TEST_BUILD_DIR(/[[:alnum:]_\\-.]+)+\", waiting for result\n"
                )
                .replace(&normalized_stdout, "")
                .to_string();
                // When there is a panic, the remote-test-client also prints "died due to signal";
                // that needs to be removed as well.
                normalized_stdout = static_regex!("^died due to signal [0-9]+\n")
                    .replace(&normalized_stdout, "")
                    .to_string();
                // FIXME: it would be much nicer if we could just tell the remote-test-client to not
                // print these things.
            }
            _ => {}
        };

        let stderr;
        let normalized_stderr;

        if self.force_color_svg() {
            let normalized = self.normalize_output(&proc_res.stderr, &self.props.normalize_stderr);
            stderr = anstyle_svg::Term::new().render_svg(&normalized);
            normalized_stderr = stderr.clone();
        } else {
            stderr = if explicit_format {
                proc_res.stderr.clone()
            } else {
                json::extract_rendered(&proc_res.stderr)
            };
            normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr);
        }

        let mut errors = 0;
        match output_kind {
            TestOutput::Compile => {
                if !self.props.dont_check_compiler_stdout {
                    if self
                        .compare_output(
                            stdout_kind,
                            &normalized_stdout,
                            &proc_res.stdout,
                            &expected_stdout,
                        )
                        .should_error()
                    {
                        errors += 1;
                    }
                }
                if !self.props.dont_check_compiler_stderr {
                    if self
                        .compare_output(stderr_kind, &normalized_stderr, &stderr, &expected_stderr)
                        .should_error()
                    {
                        errors += 1;
                    }
                }
            }
            TestOutput::Run => {
                if self
                    .compare_output(
                        stdout_kind,
                        &normalized_stdout,
                        &proc_res.stdout,
                        &expected_stdout,
                    )
                    .should_error()
                {
                    errors += 1;
                }

                if self
                    .compare_output(stderr_kind, &normalized_stderr, &stderr, &expected_stderr)
                    .should_error()
                {
                    errors += 1;
                }
            }
        }
        errors
    }

    fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> String {
        // Crude heuristic to detect when the output should have JSON-specific
        // normalization steps applied.
        let rflags = self.props.run_flags.join(" ");
        let cflags = self.props.compile_flags.join(" ");
        let json = rflags.contains("--format json")
            || rflags.contains("--format=json")
            || cflags.contains("--error-format json")
            || cflags.contains("--error-format pretty-json")
            || cflags.contains("--error-format=json")
            || cflags.contains("--error-format=pretty-json")
            || cflags.contains("--output-format json")
            || cflags.contains("--output-format=json");

        let mut normalized = output.to_string();

        let mut normalize_path = |from: &Utf8Path, to: &str| {
            let from = if json { &from.as_str().replace("\\", "\\\\") } else { from.as_str() };

            normalized = normalized.replace(from, to);
        };

        let parent_dir = self.testpaths.file.parent().unwrap();
        normalize_path(parent_dir, "$DIR");

        if self.props.remap_src_base {
            let mut remapped_parent_dir = Utf8PathBuf::from(FAKE_SRC_BASE);
            if self.testpaths.relative_dir != Utf8Path::new("") {
                remapped_parent_dir.push(&self.testpaths.relative_dir);
            }
            normalize_path(&remapped_parent_dir, "$DIR");
        }

        let base_dir = Utf8Path::new("/rustc/FAKE_PREFIX");
        // Fake paths into the libstd/libcore
        normalize_path(&base_dir.join("library"), "$SRC_DIR");
        // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
        // `rustc_macros`
        // eg. /home/user/rust/compiler
        normalize_path(&base_dir.join("compiler"), "$COMPILER_DIR");

        // Real paths into the libstd/libcore
        let rust_src_dir = &self.config.sysroot_base.join("lib/rustlib/src/rust");
        rust_src_dir.try_exists().expect(&*format!("{} should exists", rust_src_dir));
        let rust_src_dir =
            rust_src_dir.read_link_utf8().unwrap_or_else(|_| rust_src_dir.to_path_buf());
        normalize_path(&rust_src_dir.join("library"), "$SRC_DIR_REAL");

        // Real paths into the compiler
        let rustc_src_dir = &self.config.sysroot_base.join("lib/rustlib/rustc-src/rust");
        rustc_src_dir.try_exists().expect(&*format!("{} should exists", rustc_src_dir));
        let rustc_src_dir = rustc_src_dir.read_link_utf8().unwrap_or(rustc_src_dir.to_path_buf());
        normalize_path(&rustc_src_dir.join("compiler"), "$COMPILER_DIR_REAL");

        // eg.
        // /home/user/rust/build/x86_64-unknown-linux-gnu/test/ui/<test_dir>/$name.$revision.$mode/
        normalize_path(&self.output_base_dir(), "$TEST_BUILD_DIR");
        // Same as above, but with a canonicalized path.
        // This is required because some tests print canonical paths inside test build directory,
        // so if the build directory is a symlink, normalization doesn't help.
        //
        // NOTE: There are also tests which print the non-canonical name, so we need both this and
        // the above normalizations.
        normalize_path(&self.output_base_dir().canonicalize_utf8().unwrap(), "$TEST_BUILD_DIR");
        // eg. /home/user/rust/build
        normalize_path(&self.config.build_root, "$BUILD_DIR");

        if json {
            // escaped newlines in json strings should be readable
            // in the stderr files. There's no point in being correct,
            // since only humans process the stderr files.
            // Thus we just turn escaped newlines back into newlines.
            normalized = normalized.replace("\\n", "\n");
        }

        // If there are `$SRC_DIR` normalizations with line and column numbers, then replace them
        // with placeholders as we do not want tests needing updated when compiler source code
        // changes.
        // eg. $SRC_DIR/libcore/mem.rs:323:14 becomes $SRC_DIR/libcore/mem.rs:LL:COL
        normalized = static_regex!("SRC_DIR(.+):\\d+:\\d+(: \\d+:\\d+)?")
            .replace_all(&normalized, "SRC_DIR$1:LL:COL")
            .into_owned();

        normalized = Self::normalize_platform_differences(&normalized);

        // Normalize long type name hash.
        normalized =
            static_regex!(r"\$TEST_BUILD_DIR/(?P<filename>[^\.]+).long-type-(?P<hash>\d+).txt")
                .replace_all(&normalized, |caps: &Captures<'_>| {
                    format!(
                        "$TEST_BUILD_DIR/{filename}.long-type-$LONG_TYPE_HASH.txt",
                        filename = &caps["filename"]
                    )
                })
                .into_owned();

        // Normalize thread IDs in panic messages
        normalized = static_regex!(r"thread '(?P<name>.*?)' \((rtid )?\d+\) panicked")
            .replace_all(&normalized, "thread '$name' ($$TID) panicked")
            .into_owned();

        normalized = normalized.replace("\t", "\\t"); // makes tabs visible

        // Remove test annotations like `//~ ERROR text` from the output,
        // since they duplicate actual errors and make the output hard to read.
        // This mirrors the regex in src/tools/tidy/src/style.rs, please update
        // both if either are changed.
        normalized =
            static_regex!("\\s*//(\\[.*\\])?~.*").replace_all(&normalized, "").into_owned();

        // This code normalizes various hashes in v0 symbol mangling that is
        // emitted in the ui and mir-opt tests.
        let v0_crate_hash_prefix_re = static_regex!(r"_R.*?Cs[0-9a-zA-Z]+_");
        let v0_crate_hash_re = static_regex!(r"Cs[0-9a-zA-Z]+_");

        const V0_CRATE_HASH_PLACEHOLDER: &str = r"CsCRATE_HASH_";
        if v0_crate_hash_prefix_re.is_match(&normalized) {
            // Normalize crate hash
            normalized =
                v0_crate_hash_re.replace_all(&normalized, V0_CRATE_HASH_PLACEHOLDER).into_owned();
        }

        let v0_back_ref_prefix_re = static_regex!(r"\(_R.*?B[0-9a-zA-Z]_");
        let v0_back_ref_re = static_regex!(r"B[0-9a-zA-Z]_");

        const V0_BACK_REF_PLACEHOLDER: &str = r"B<REF>_";
        if v0_back_ref_prefix_re.is_match(&normalized) {
            // Normalize back references (see RFC 2603)
            normalized =
                v0_back_ref_re.replace_all(&normalized, V0_BACK_REF_PLACEHOLDER).into_owned();
        }

        // AllocId are numbered globally in a compilation session. This can lead to changes
        // depending on the exact compilation flags and host architecture. Meanwhile, we want
        // to keep them numbered, to see if the same id appears multiple times.
        // So we remap to deterministic numbers that only depend on the subset of allocations
        // that actually appear in the output.
        // We use uppercase ALLOC to distinguish from the non-normalized version.
        {
            match self.config.mode {
                // Unfortunately, due to parallel frontend assigning alloc-ids
                // nondeterministically we resort to dropping ids altogether for now
                // in ui tests
                TestMode::Ui => {
                    // The alloc-id appears in pretty-printed allocations.
                    normalized = static_regex!(
                        r"╾─*(a(lloc)?|A(LLOC)?)\d+(\+0x[0-9a-f]+)?(<imm>)?( ?\(\d+ ptr bytes\))?─*╼"
                    )
                    .replace_all(&normalized, |_: &Captures<'_>| "╾ALLOC$ID╼".to_string())
                    .into_owned();

                    // The alloc-id appears in a sentence.
                    normalized = static_regex!(r"\b(alloc|ALLOC)\d+\b")
                        .replace_all(&normalized, |_: &Captures<'_>| "ALLOC$ID".to_string())
                        .into_owned();
                }
                // use consistent `AllocId`s in other test modes, where parallel frontend
                // should not (theoretically) be an issue
                _ => {
                    let mut seen_allocs = indexmap::IndexSet::new();
                    // The alloc-id appears in pretty-printed allocations.
                    normalized = static_regex!(
                        r"╾─*a(lloc)?([0-9]+)(\+0x[0-9a-f]+)?(<imm>)?( \([0-9]+ ptr bytes\))?─*╼"
                    )
                    .replace_all(&normalized, |caps: &Captures<'_>| {
                        // Renumber the captured index.
                        let index = caps.get(2).unwrap().as_str().to_string();
                        let (index, _) = seen_allocs.insert_full(index);
                        let offset = caps.get(3).map_or("", |c| c.as_str());
                        let imm = caps.get(4).map_or("", |c| c.as_str());
                        // Do not bother keeping it pretty, just make it deterministic.
                        format!("╾ALLOC{index}{offset}{imm}╼")
                    })
                    .into_owned();

                    // The alloc-id appears in a sentence.
                    normalized = static_regex!(r"\balloc([0-9]+)\b")
                        .replace_all(&normalized, |caps: &Captures<'_>| {
                            let index = caps.get(1).unwrap().as_str().to_string();
                            let (index, _) = seen_allocs.insert_full(index);
                            format!("ALLOC{index}")
                        })
                        .into_owned();
                }
            }
        }

        // Custom normalization rules
        for rule in custom_rules {
            let re = Regex::new(&rule.0).expect("bad regex in custom normalization rule");
            normalized = re.replace_all(&normalized, &rule.1[..]).into_owned();
        }
        normalized
    }

    /// Normalize output differences across platforms. Generally changes Windows output to be more
    /// Unix-like.
    ///
    /// Replaces backslashes in paths with forward slashes, and replaces CRLF line endings
    /// with LF.
    fn normalize_platform_differences(output: &str) -> String {
        let output = output.replace(r"\\", r"\");

        // Used to find Windows paths.
        //
        // It's not possible to detect paths in the error messages generally, but this is a
        // decent enough heuristic.
        let re = static_regex!(
            r#"(?x)
                (?:
                  # Match paths that don't include spaces.
                  (?:\\[\pL\pN\.\-_']+)+\.\pL+
                |
                  # If the path starts with a well-known root, then allow spaces and no file extension.
                  \$(?:DIR|SRC_DIR|TEST_BUILD_DIR|BUILD_DIR|LIB_DIR)(?:\\[\pL\pN\.\-_'\ ]+)+
                )"#
        );
        re.replace_all(&output, |caps: &Captures<'_>| caps[0].replace(r"\", "/"))
            .replace("\r\n", "\n")
    }

    fn expected_output_path(&self, kind: &str) -> Utf8PathBuf {
        let mut path = expected_output_path(
            &self.testpaths,
            self.variant.revision(),
            &self.config.compare_mode,
            kind,
        );

        if !path.exists() {
            if let Some(CompareMode::Polonius) = self.config.compare_mode {
                path = expected_output_path(&self.testpaths, self.variant.revision(), &None, kind);
            }
        }

        if !path.exists() {
            path = expected_output_path(&self.testpaths, self.variant.revision(), &None, kind);
        }

        path
    }

    fn load_expected_output(&self, kind: &str) -> String {
        let path = self.expected_output_path(kind);
        if path.exists() {
            match self.load_expected_output_from_path(&path) {
                Ok(x) => x,
                Err(x) => self.fatal(&x),
            }
        } else {
            String::new()
        }
    }

    fn load_expected_output_from_path(&self, path: &Utf8Path) -> Result<String, String> {
        fs::read_to_string(path)
            .map_err(|err| format!("failed to load expected output from `{}`: {}", path, err))
    }

    /// Attempts to delete a file, succeeding if the file does not exist.
    fn delete_file(&self, file: &Utf8Path) {
        if let Err(e) = fs::remove_file(file.as_std_path())
            && e.kind() != io::ErrorKind::NotFound
        {
            self.fatal(&format!("failed to delete `{}`: {}", file, e,));
        }
    }

    fn compare_output(
        &self,
        stream: &str,
        actual: &str,
        actual_unnormalized: &str,
        expected: &str,
    ) -> CompareOutcome {
        let expected_path = expected_output_path(
            self.testpaths,
            self.variant.revision(),
            &self.config.compare_mode,
            stream,
        );

        if self.config.bless && actual.is_empty() && expected_path.exists() {
            self.delete_file(&expected_path);
        }

        let are_different = match (self.force_color_svg(), expected.find('\n'), actual.find('\n')) {
            // FIXME: We ignore the first line of SVG files
            // because the width parameter is non-deterministic.
            (true, Some(nl_e), Some(nl_a)) => expected[nl_e..] != actual[nl_a..],
            _ => expected != actual,
        };
        if !are_different {
            return CompareOutcome::Same;
        }

        // Wrapper tools set by `runner` might provide extra output on failure,
        // for example a WebAssembly runtime might print the stack trace of an
        // `unreachable` instruction by default.
        let compare_output_by_lines_subset = self.config.runner.is_some();

        // Also, some tests like `ui/parallel-rustc` have non-deterministic
        // orders of output, so we need to compare by lines.
        let compare_output_by_lines = self.props.compare_output_by_lines;

        let tmp;
        let (expected, actual): (&str, &str) = if compare_output_by_lines_subset {
            let actual_lines: HashSet<_> = actual.lines().collect();
            let expected_lines: Vec<_> = expected.lines().collect();
            let mut used = expected_lines.clone();
            used.retain(|line| actual_lines.contains(line));

            // check if `expected` contains a subset of the lines of `actual`
            if used.len() == expected_lines.len() && (expected.is_empty() == actual.is_empty()) {
                return CompareOutcome::Same;
            }
            if expected_lines.is_empty() {
                // if we have no lines to check, force a full overwrite
                ("", actual)
            } else {
                // this prints/blesses the subset, not the actual
                tmp = (expected_lines.join("\n"), used.join("\n"));
                (&tmp.0, &tmp.1)
            }
        } else if compare_output_by_lines {
            let mut actual_lines = self.lines_for_comparison(actual);
            let mut expected_lines = self.lines_for_comparison(expected);
            actual_lines.sort_unstable();
            expected_lines.sort_unstable();
            if actual_lines == expected_lines {
                return CompareOutcome::Same;
            } else {
                (expected, actual)
            }
        } else {
            (expected, actual)
        };

        // Write the actual output to a file in build directory.
        let actual_path = self
            .output_base_name()
            .with_extra_extension(self.variant.revision().unwrap_or(""))
            .with_extra_extension(
                self.config.compare_mode.as_ref().map(|cm| cm.to_str()).unwrap_or(""),
            )
            .with_extra_extension(stream);

        if let Err(err) = fs::write(&actual_path, &actual) {
            self.fatal(&format!("failed to write {stream} to `{actual_path}`: {err}",));
        }
        writeln!(self.stdout, "Saved the actual {stream} to `{actual_path}`");

        if !self.config.bless {
            if expected.is_empty() {
                writeln!(self.stdout, "normalized {}:\n{}\n", stream, actual);
            } else {
                self.show_diff(
                    stream,
                    &expected_path,
                    &actual_path,
                    expected,
                    actual,
                    actual_unnormalized,
                    compare_output_by_lines || compare_output_by_lines_subset,
                );
            }
        } else {
            // Delete non-revision .stderr/.stdout file if revisions are used.
            // Without this, we'd just generate the new files and leave the old files around.
            if self.variant.revision().is_some() {
                let old =
                    expected_output_path(self.testpaths, None, &self.config.compare_mode, stream);
                self.delete_file(&old);
            }

            if !actual.is_empty() {
                if let Err(err) = fs::write(&expected_path, &actual) {
                    self.fatal(&format!("failed to write {stream} to `{expected_path}`: {err}"));
                }
                writeln!(
                    self.stdout,
                    "Blessing the {stream} of `{test_name}` as `{expected_path}`",
                    test_name = self.testpaths.file
                );
            }
        }

        writeln!(self.stdout, "\nThe actual {stream} differed from the expected {stream}");

        if self.config.bless { CompareOutcome::Blessed } else { CompareOutcome::Differed }
    }

    /// Returns whether to show the full stderr/stdout.
    fn show_diff(
        &self,
        stream: &str,
        expected_path: &Utf8Path,
        actual_path: &Utf8Path,
        expected: &str,
        actual: &str,
        actual_unnormalized: &str,
        show_diff_by_lines: bool,
    ) {
        writeln!(self.stderr, "diff of {stream}:\n");
        if let Some(diff_command) = self.config.diff_command.as_deref() {
            let mut args = diff_command.split_whitespace();
            let name = args.next().unwrap();
            match Command::new(name).args(args).args([expected_path, actual_path]).output() {
                Err(err) => {
                    self.fatal(&format!(
                        "failed to call custom diff command `{diff_command}`: {err}"
                    ));
                }
                Ok(output) => {
                    let output = String::from_utf8_lossy(&output.stdout);
                    write!(self.stderr, "{output}");
                }
            }
        } else {
            write!(self.stderr, "{}", write_diff(expected, actual, 3));
        }

        // NOTE: argument order is important, we need `actual` to be on the left so the line number match up when we compare it to `actual_unnormalized` below.
        let diff_results = make_diff(actual, expected, 0);

        let (mut mismatches_normalized, mut mismatch_line_nos) = (String::new(), vec![]);
        for hunk in diff_results {
            let mut line_no = hunk.line_number;
            for line in hunk.lines {
                // NOTE: `Expected` is actually correct here, the argument order is reversed so our line numbers match up
                if let DiffLine::Expected(normalized) = line {
                    mismatches_normalized += &normalized;
                    mismatches_normalized += "\n";
                    mismatch_line_nos.push(line_no);
                    line_no += 1;
                }
            }
        }
        let mut mismatches_unnormalized = String::new();
        let diff_normalized = make_diff(actual, actual_unnormalized, 0);
        for hunk in diff_normalized {
            if mismatch_line_nos.contains(&hunk.line_number) {
                for line in hunk.lines {
                    if let DiffLine::Resulting(unnormalized) = line {
                        mismatches_unnormalized += &unnormalized;
                        mismatches_unnormalized += "\n";
                    }
                }
            }
        }

        let normalized_diff = make_diff(&mismatches_normalized, &mismatches_unnormalized, 0);
        // HACK: instead of checking if each hunk is empty, this only checks if the whole input is empty. we should be smarter about this so we don't treat added or removed output as normalized.
        if !normalized_diff.is_empty()
            && !mismatches_unnormalized.is_empty()
            && !mismatches_normalized.is_empty()
        {
            writeln!(
                self.stderr,
                "Note: some mismatched output was normalized before being compared"
            );
            // FIXME: respect diff_command
            write!(
                self.stderr,
                "{}",
                write_diff(&mismatches_unnormalized, &mismatches_normalized, 0)
            );
        }

        if show_diff_by_lines {
            let expected_lines = self.lines_for_comparison(expected);
            let actual_lines = self.lines_for_comparison(actual);
            write!(self.stderr, "{}", diff_by_lines(&expected_lines, &actual_lines));
        }
    }

    fn check_and_prune_duplicate_outputs(
        &self,
        proc_res: &ProcRes,
        modes: &[CompareMode],
        require_same_modes: &[CompareMode],
    ) {
        for kind in UI_EXTENSIONS {
            let canon_comparison_path =
                expected_output_path(&self.testpaths, self.variant.revision(), &None, kind);

            let canon = match self.load_expected_output_from_path(&canon_comparison_path) {
                Ok(canon) => canon,
                _ => continue,
            };
            let bless = self.config.bless;
            let check_and_prune_duplicate_outputs = |mode: &CompareMode, require_same: bool| {
                let examined_path = expected_output_path(
                    &self.testpaths,
                    self.variant.revision(),
                    &Some(mode.clone()),
                    kind,
                );

                // If there is no output, there is nothing to do
                let examined_content = match self.load_expected_output_from_path(&examined_path) {
                    Ok(content) => content,
                    _ => return,
                };

                let is_duplicate = canon == examined_content;

                match (bless, require_same, is_duplicate) {
                    // If we're blessing and the output is the same, then delete the file.
                    (true, _, true) => {
                        self.delete_file(&examined_path);
                    }
                    // If we want them to be the same, but they are different, then error.
                    // We do this whether we bless or not
                    (_, true, false) => {
                        self.fatal_proc_rec(
                            &format!("`{}` should not have different output from base test!", kind),
                            proc_res,
                        );
                    }
                    _ => {}
                }
            };
            for mode in modes {
                check_and_prune_duplicate_outputs(mode, false);
            }
            for mode in require_same_modes {
                check_and_prune_duplicate_outputs(mode, true);
            }
        }
    }

    fn create_stamp(&self) {
        let stamp_file_path = stamp_file_path(&self.config, self.testpaths, self.variant);
        fs::write(&stamp_file_path, compute_stamp_hash(&self.config, self.variant)).unwrap();
    }

    fn init_incremental_test(&self) {
        // (See `run_incremental_test` for an overview of how incremental tests work.)

        // Before any of the revisions have executed, create the
        // incremental workproduct directory.  Delete any old
        // incremental work products that may be there from prior
        // runs.
        let incremental_dir = self.props.incremental_dir.as_ref().unwrap();
        if incremental_dir.exists() {
            // Canonicalizing the path will convert it to the //?/ format
            // on Windows, which enables paths longer than 260 character
            let canonicalized = incremental_dir.canonicalize().unwrap();
            fs::remove_dir_all(canonicalized).unwrap();
        }
        fs::create_dir_all(&incremental_dir).unwrap();

        if self.config.verbose {
            writeln!(self.stdout, "init_incremental_test: incremental_dir={incremental_dir}");
        }
    }
}

struct ProcArgs {
    prog: OsString,
    args: Vec<OsString>,
}

#[derive(Debug)]
pub(crate) struct ProcRes {
    status: ExitStatus,
    stdout: String,
    stderr: String,
    truncated: Truncated,
    cmdline: String,
}

impl ProcRes {
    #[must_use]
    pub(crate) fn format_info(&self) -> String {
        fn render(name: &str, contents: &str) -> String {
            let contents = json::extract_rendered(contents);
            let contents = contents.trim_end();
            if contents.is_empty() {
                format!("{name}: none")
            } else {
                format!(
                    "\
                     --- {name} -------------------------------\n\
                     {contents}\n\
                     ------------------------------------------",
                )
            }
        }

        format!(
            "status: {}\ncommand: {}\n{}\n{}\n",
            self.status,
            self.cmdline,
            render("stdout", &self.stdout),
            render("stderr", &self.stderr),
        )
    }
}

#[derive(Debug)]
enum TargetLocation {
    ThisFile(Utf8PathBuf),
    ThisDirectory(Utf8PathBuf),
}

enum AllowUnused {
    Yes,
    No,
}

enum LinkToAux {
    Yes,
    No,
}

#[derive(Debug, PartialEq)]
enum AuxType {
    Bin,
    Lib,
    Dylib,
    ProcMacro,
}

/// Outcome of comparing a stream to a blessed file,
/// e.g. `.stderr` and `.fixed`.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum CompareOutcome {
    /// Expected and actual outputs are the same
    Same,
    /// Outputs differed but were blessed
    Blessed,
    /// Outputs differed and an error should be emitted
    Differed,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum DocKind {
    Html,
    Json,
}

impl CompareOutcome {
    fn should_error(&self) -> bool {
        matches!(self, CompareOutcome::Differed)
    }
}