test/
cli.rs

1//! Module converting command-line arguments into test configuration.
2
3use std::env;
4use std::io::{self, IsTerminal, Write};
5use std::path::PathBuf;
6
7use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
8use super::time::TestTimeOptions;
9
10#[derive(Debug)]
11pub struct TestOpts {
12    pub list: bool,
13    pub filters: Vec<String>,
14    pub filter_exact: bool,
15    pub force_run_in_process: bool,
16    pub exclude_should_panic: bool,
17    pub run_ignored: RunIgnored,
18    pub run_tests: bool,
19    pub bench_benchmarks: bool,
20    pub logfile: Option<PathBuf>,
21    pub nocapture: bool,
22    pub color: ColorConfig,
23    pub format: OutputFormat,
24    pub shuffle: bool,
25    pub shuffle_seed: Option<u64>,
26    pub test_threads: Option<usize>,
27    pub skip: Vec<String>,
28    pub time_options: Option<TestTimeOptions>,
29    /// Stop at first failing test.
30    /// May run a few more tests due to threading, but will
31    /// abort as soon as possible.
32    pub fail_fast: bool,
33    pub options: Options,
34}
35
36impl TestOpts {
37    pub fn use_color(&self) -> bool {
38        match self.color {
39            ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
40            ColorConfig::AlwaysColor => true,
41            ColorConfig::NeverColor => false,
42        }
43    }
44}
45
46/// Result of parsing the options.
47pub(crate) type OptRes = Result<TestOpts, String>;
48/// Result of parsing the option part.
49type OptPartRes<T> = Result<T, String>;
50
51fn optgroups() -> getopts::Options {
52    let mut opts = getopts::Options::new();
53    opts.optflag("", "include-ignored", "Run ignored and not ignored tests")
54        .optflag("", "ignored", "Run only ignored tests")
55        .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort")
56        .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic")
57        .optflag("", "test", "Run tests and not benchmarks")
58        .optflag("", "bench", "Run benchmarks instead of tests")
59        .optflag("", "list", "List all tests and benchmarks")
60        .optflag("h", "help", "Display this message")
61        .optopt("", "logfile", "Write logs to the specified file (deprecated)", "PATH")
62        .optflag(
63            "",
64            "nocapture",
65            "don't capture stdout/stderr of each \
66             task, allow printing directly",
67        )
68        .optopt(
69            "",
70            "test-threads",
71            "Number of threads used for running tests \
72             in parallel",
73            "n_threads",
74        )
75        .optmulti(
76            "",
77            "skip",
78            "Skip tests whose names contain FILTER (this flag can \
79             be used multiple times)",
80            "FILTER",
81        )
82        .optflag(
83            "q",
84            "quiet",
85            "Display one character per test instead of one line. \
86             Alias to --format=terse",
87        )
88        .optflag("", "exact", "Exactly match filters rather than by substring")
89        .optopt(
90            "",
91            "color",
92            "Configure coloring of output:
93            auto   = colorize if stdout is a tty and tests are run on serially (default);
94            always = always colorize output;
95            never  = never colorize output;",
96            "auto|always|never",
97        )
98        .optopt(
99            "",
100            "format",
101            "Configure formatting of output:
102            pretty = Print verbose output;
103            terse  = Display one character per test;
104            json   = Output a json document;
105            junit  = Output a JUnit document",
106            "pretty|terse|json|junit",
107        )
108        .optflag("", "show-output", "Show captured stdout of successful tests")
109        .optopt(
110            "Z",
111            "",
112            "Enable nightly-only flags:
113            unstable-options = Allow use of experimental features",
114            "unstable-options",
115        )
116        .optflag(
117            "",
118            "report-time",
119            "Show execution time of each test.
120
121            Threshold values for colorized output can be configured via
122            `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
123            `RUST_TEST_TIME_DOCTEST` environment variables.
124
125            Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
126            Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time
127            is 0.5 seconds, and the critical time is 2 seconds.
128
129            Not available for --format=terse",
130        )
131        .optflag(
132            "",
133            "ensure-time",
134            "Treat excess of the test execution time limit as error.
135
136            Threshold values for this option can be configured via
137            `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
138            `RUST_TEST_TIME_DOCTEST` environment variables.
139
140            Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
141
142            `CRITICAL_TIME` here means the limit that should not be exceeded by test.
143            ",
144        )
145        .optflag("", "shuffle", "Run tests in random order")
146        .optopt(
147            "",
148            "shuffle-seed",
149            "Run tests in random order; seed the random number generator with SEED",
150            "SEED",
151        );
152    opts
153}
154
155fn usage(binary: &str, options: &getopts::Options) {
156    let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]");
157    println!(
158        r#"{usage}
159
160The FILTER string is tested against the name of all tests, and only those
161tests whose names contain the filter are run. Multiple filter strings may
162be passed, which will run all tests matching any of the filters.
163
164By default, all tests are run in parallel. This can be altered with the
165--test-threads flag or the RUST_TEST_THREADS environment variable when running
166tests (set it to 1).
167
168By default, the tests are run in alphabetical order. Use --shuffle or set
169RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated
170"shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the
171tests in the same order again. Note that --shuffle and --shuffle-seed do not
172affect whether the tests are run in parallel.
173
174All tests have their standard output and standard error captured by default.
175This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
176environment variable to a value other than "0". Logging is not captured by default.
177
178Test Attributes:
179
180    `#[test]`        - Indicates a function is a test to be run. This function
181                       takes no arguments.
182    `#[bench]`       - Indicates a function is a benchmark to be run. This
183                       function takes one argument (test::Bencher).
184    `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if
185                        the code causes a panic (an assertion failure or panic!)
186                        A message may be provided, which the failure string must
187                        contain: #[should_panic(expected = "foo")].
188    `#[ignore]`       - When applied to a function which is already attributed as a
189                        test, then the test runner will ignore these tests during
190                        normal test runs. Running with --ignored or --include-ignored will run
191                        these tests."#,
192        usage = options.usage(&message)
193    );
194}
195
196/// Parses command line arguments into test options.
197/// Returns `None` if help was requested (since we only show help message and don't run tests),
198/// returns `Some(Err(..))` if provided arguments are incorrect,
199/// otherwise creates a `TestOpts` object and returns it.
200pub fn parse_opts(args: &[String]) -> Option<OptRes> {
201    // Parse matches.
202    let opts = optgroups();
203    let binary = args.first().map(|c| &**c).unwrap_or("...");
204    let args = args.get(1..).unwrap_or(args);
205    let matches = match opts.parse(args) {
206        Ok(m) => m,
207        Err(f) => return Some(Err(f.to_string())),
208    };
209
210    // Check if help was requested.
211    if matches.opt_present("h") {
212        // Show help and do nothing more.
213        usage(binary, &opts);
214        return None;
215    }
216
217    // Actually parse the opts.
218    let opts_result = parse_opts_impl(matches);
219
220    Some(opts_result)
221}
222
223// Gets the option value and checks if unstable features are enabled.
224macro_rules! unstable_optflag {
225    ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
226        let opt = $matches.opt_present($option_name);
227        if !$allow_unstable && opt {
228            return Err(format!(
229                "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options",
230                $option_name
231            ));
232        }
233
234        opt
235    }};
236}
237
238// Gets the option value and checks if unstable features are enabled.
239macro_rules! unstable_optopt {
240    ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
241        let opt = $matches.opt_str($option_name);
242        if !$allow_unstable && opt.is_some() {
243            return Err(format!(
244                "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options",
245                $option_name
246            ));
247        }
248
249        opt
250    }};
251}
252
253// Implementation of `parse_opts` that doesn't care about help message
254// and returns a `Result`.
255fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
256    let allow_unstable = get_allow_unstable(&matches)?;
257
258    // Unstable flags
259    let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process");
260    let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic");
261    let time_options = get_time_options(&matches, allow_unstable)?;
262    let shuffle = get_shuffle(&matches, allow_unstable)?;
263    let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?;
264
265    let include_ignored = matches.opt_present("include-ignored");
266    let quiet = matches.opt_present("quiet");
267    let exact = matches.opt_present("exact");
268    let list = matches.opt_present("list");
269    let skip = matches.opt_strs("skip");
270
271    let bench_benchmarks = matches.opt_present("bench");
272    let run_tests = !bench_benchmarks || matches.opt_present("test");
273
274    let logfile = get_log_file(&matches)?;
275    let run_ignored = get_run_ignored(&matches, include_ignored)?;
276    let filters = matches.free.clone();
277    let nocapture = get_nocapture(&matches)?;
278    let test_threads = get_test_threads(&matches)?;
279    let color = get_color_config(&matches)?;
280    let format = get_format(&matches, quiet, allow_unstable)?;
281
282    let options = Options::new().display_output(matches.opt_present("show-output"));
283
284    if logfile.is_some() {
285        let _ = write!(io::stderr(), "warning: `--logfile` is deprecated");
286    }
287
288    let test_opts = TestOpts {
289        list,
290        filters,
291        filter_exact: exact,
292        force_run_in_process,
293        exclude_should_panic,
294        run_ignored,
295        run_tests,
296        bench_benchmarks,
297        logfile,
298        nocapture,
299        color,
300        format,
301        shuffle,
302        shuffle_seed,
303        test_threads,
304        skip,
305        time_options,
306        options,
307        fail_fast: false,
308    };
309
310    Ok(test_opts)
311}
312
313// FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
314fn is_nightly() -> bool {
315    // Whether this is a feature-staged build, i.e., on the beta or stable channel
316    let disable_unstable_features =
317        option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(false);
318    // Whether we should enable unstable features for bootstrapping
319    let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
320
321    bootstrap || !disable_unstable_features
322}
323
324// Gets the CLI options associated with `report-time` feature.
325fn get_time_options(
326    matches: &getopts::Matches,
327    allow_unstable: bool,
328) -> OptPartRes<Option<TestTimeOptions>> {
329    let report_time = unstable_optflag!(matches, allow_unstable, "report-time");
330    let ensure_test_time = unstable_optflag!(matches, allow_unstable, "ensure-time");
331
332    // If `ensure-test-time` option is provided, time output is enforced,
333    // so user won't be confused if any of tests will silently fail.
334    let options = if report_time || ensure_test_time {
335        Some(TestTimeOptions::new_from_env(ensure_test_time))
336    } else {
337        None
338    };
339
340    Ok(options)
341}
342
343fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> {
344    let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle");
345    if !shuffle && allow_unstable {
346        shuffle = match env::var("RUST_TEST_SHUFFLE") {
347            Ok(val) => &val != "0",
348            Err(_) => false,
349        };
350    }
351
352    Ok(shuffle)
353}
354
355fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> {
356    let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") {
357        Some(n_str) => match n_str.parse::<u64>() {
358            Ok(n) => Some(n),
359            Err(e) => {
360                return Err(format!(
361                    "argument for --shuffle-seed must be a number \
362                     (error: {e})"
363                ));
364            }
365        },
366        None => None,
367    };
368
369    if shuffle_seed.is_none() && allow_unstable {
370        shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") {
371            Ok(val) => match val.parse::<u64>() {
372                Ok(n) => Some(n),
373                Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."),
374            },
375            Err(_) => None,
376        };
377    }
378
379    Ok(shuffle_seed)
380}
381
382fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> {
383    let test_threads = match matches.opt_str("test-threads") {
384        Some(n_str) => match n_str.parse::<usize>() {
385            Ok(0) => return Err("argument for --test-threads must not be 0".to_string()),
386            Ok(n) => Some(n),
387            Err(e) => {
388                return Err(format!(
389                    "argument for --test-threads must be a number > 0 \
390                     (error: {e})"
391                ));
392            }
393        },
394        None => None,
395    };
396
397    Ok(test_threads)
398}
399
400fn get_format(
401    matches: &getopts::Matches,
402    quiet: bool,
403    allow_unstable: bool,
404) -> OptPartRes<OutputFormat> {
405    let format = match matches.opt_str("format").as_deref() {
406        None if quiet => OutputFormat::Terse,
407        Some("pretty") | None => OutputFormat::Pretty,
408        Some("terse") => OutputFormat::Terse,
409        Some("json") => {
410            if !allow_unstable {
411                return Err("The \"json\" format is only accepted on the nightly compiler with -Z unstable-options".into());
412            }
413            OutputFormat::Json
414        }
415        Some("junit") => {
416            if !allow_unstable {
417                return Err("The \"junit\" format is only accepted on the nightly compiler with -Z unstable-options".into());
418            }
419            OutputFormat::Junit
420        }
421        Some(v) => {
422            return Err(format!(
423                "argument for --format must be pretty, terse, json or junit (was \
424                 {v})"
425            ));
426        }
427    };
428
429    Ok(format)
430}
431
432fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> {
433    let color = match matches.opt_str("color").as_deref() {
434        Some("auto") | None => ColorConfig::AutoColor,
435        Some("always") => ColorConfig::AlwaysColor,
436        Some("never") => ColorConfig::NeverColor,
437
438        Some(v) => {
439            return Err(format!(
440                "argument for --color must be auto, always, or never (was \
441                 {v})"
442            ));
443        }
444    };
445
446    Ok(color)
447}
448
449fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> {
450    let mut nocapture = matches.opt_present("nocapture");
451    if !nocapture {
452        nocapture = match env::var("RUST_TEST_NOCAPTURE") {
453            Ok(val) => &val != "0",
454            Err(_) => false,
455        };
456    }
457
458    Ok(nocapture)
459}
460
461fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> {
462    let run_ignored = match (include_ignored, matches.opt_present("ignored")) {
463        (true, true) => {
464            return Err("the options --include-ignored and --ignored are mutually exclusive".into());
465        }
466        (true, false) => RunIgnored::Yes,
467        (false, true) => RunIgnored::Only,
468        (false, false) => RunIgnored::No,
469    };
470
471    Ok(run_ignored)
472}
473
474fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> {
475    let mut allow_unstable = false;
476
477    if let Some(opt) = matches.opt_str("Z") {
478        if !is_nightly() {
479            return Err("the option `Z` is only accepted on the nightly compiler".into());
480        }
481
482        match &*opt {
483            "unstable-options" => {
484                allow_unstable = true;
485            }
486            _ => {
487                return Err("Unrecognized option to `Z`".into());
488            }
489        }
490    };
491
492    Ok(allow_unstable)
493}
494
495fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
496    let logfile = matches.opt_str("logfile").map(|s| PathBuf::from(&s));
497
498    Ok(logfile)
499}