test/
cli.rs

1//! Module converting command-line arguments into test configuration.
2
3use std::env;
4use std::io::{self, IsTerminal, Write};
5use std::path::PathBuf;
6
7use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
8use super::time::TestTimeOptions;
9
10#[derive(Debug)]
11pub struct TestOpts {
12    pub list: bool,
13    pub filters: Vec<String>,
14    pub filter_exact: bool,
15    pub force_run_in_process: bool,
16    pub exclude_should_panic: bool,
17    pub run_ignored: RunIgnored,
18    pub run_tests: bool,
19    pub bench_benchmarks: bool,
20    pub logfile: Option<PathBuf>,
21    pub nocapture: bool,
22    pub color: ColorConfig,
23    pub format: OutputFormat,
24    pub shuffle: bool,
25    pub shuffle_seed: Option<u64>,
26    pub test_threads: Option<usize>,
27    pub skip: Vec<String>,
28    pub time_options: Option<TestTimeOptions>,
29    /// Stop at first failing test.
30    /// May run a few more tests due to threading, but will
31    /// abort as soon as possible.
32    pub fail_fast: bool,
33    pub options: Options,
34}
35
36impl TestOpts {
37    pub fn use_color(&self) -> bool {
38        match self.color {
39            ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
40            ColorConfig::AlwaysColor => true,
41            ColorConfig::NeverColor => false,
42        }
43    }
44}
45
46/// Result of parsing the options.
47pub(crate) type OptRes = Result<TestOpts, String>;
48/// Result of parsing the option part.
49type OptPartRes<T> = Result<T, String>;
50
51fn optgroups() -> getopts::Options {
52    let mut opts = getopts::Options::new();
53    opts.optflag("", "include-ignored", "Run ignored and not ignored tests")
54        .optflag("", "ignored", "Run only ignored tests")
55        .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort")
56        .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic")
57        .optflag("", "test", "Run tests and not benchmarks")
58        .optflag("", "bench", "Run benchmarks instead of tests")
59        .optflag("", "list", "List all tests and benchmarks")
60        .optflag("", "fail-fast", "Don't start new tests after the first failure")
61        .optflag("h", "help", "Display this message")
62        .optopt("", "logfile", "Write logs to the specified file (deprecated)", "PATH")
63        .optflag(
64            "",
65            "no-capture",
66            "don't capture stdout/stderr of each \
67             task, allow printing directly",
68        )
69        .optopt(
70            "",
71            "test-threads",
72            "Number of threads used for running tests \
73             in parallel",
74            "n_threads",
75        )
76        .optmulti(
77            "",
78            "skip",
79            "Skip tests whose names contain FILTER (this flag can \
80             be used multiple times)",
81            "FILTER",
82        )
83        .optflag(
84            "q",
85            "quiet",
86            "Display one character per test instead of one line. \
87             Alias to --format=terse",
88        )
89        .optflag("", "exact", "Exactly match filters rather than by substring")
90        .optopt(
91            "",
92            "color",
93            "Configure coloring of output:
94            auto   = colorize if stdout is a tty and tests are run on serially (default);
95            always = always colorize output;
96            never  = never colorize output;",
97            "auto|always|never",
98        )
99        .optopt(
100            "",
101            "format",
102            "Configure formatting of output:
103            pretty = Print verbose output;
104            terse  = Display one character per test;
105            json   = Output a json document;
106            junit  = Output a JUnit document",
107            "pretty|terse|json|junit",
108        )
109        .optflag("", "show-output", "Show captured stdout of successful tests")
110        .optopt(
111            "Z",
112            "",
113            "Enable nightly-only flags:
114            unstable-options = Allow use of experimental features",
115            "unstable-options",
116        )
117        .optflag(
118            "",
119            "report-time",
120            "Show execution time of each test.
121
122            Threshold values for colorized output can be configured via
123            `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
124            `RUST_TEST_TIME_DOCTEST` environment variables.
125
126            Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
127            Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time
128            is 0.5 seconds, and the critical time is 2 seconds.
129
130            Not available for --format=terse",
131        )
132        .optflag(
133            "",
134            "ensure-time",
135            "Treat excess of the test execution time limit as error.
136
137            Threshold values for this option can be configured via
138            `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
139            `RUST_TEST_TIME_DOCTEST` environment variables.
140
141            Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
142
143            `CRITICAL_TIME` here means the limit that should not be exceeded by test.
144            ",
145        )
146        .optflag("", "shuffle", "Run tests in random order")
147        .optopt(
148            "",
149            "shuffle-seed",
150            "Run tests in random order; seed the random number generator with SEED",
151            "SEED",
152        );
153    opts
154}
155
156fn usage(binary: &str, options: &getopts::Options) {
157    let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]");
158    println!(
159        r#"{usage}
160
161The FILTER string is tested against the name of all tests, and only those
162tests whose names contain the filter are run. Multiple filter strings may
163be passed, which will run all tests matching any of the filters.
164
165By default, all tests are run in parallel. This can be altered with the
166--test-threads flag when running tests (set it to 1).
167
168By default, the tests are run in alphabetical order. Use --shuffle to run
169the tests in random order. Pass the generated "shuffle seed" to
170--shuffle-seed to run the tests in the same order again. Note that
171--shuffle and --shuffle-seed do not affect whether the tests are run in
172parallel.
173
174All tests have their standard output and standard error captured by default.
175This can be overridden with the --no-capture flag to a value other than "0".
176Logging is not captured by default.
177
178Test Attributes:
179
180    `#[test]`        - Indicates a function is a test to be run. This function
181                       takes no arguments.
182    `#[bench]`       - Indicates a function is a benchmark to be run. This
183                       function takes one argument (test::Bencher).
184    `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if
185                        the code causes a panic (an assertion failure or panic!)
186                        A message may be provided, which the failure string must
187                        contain: #[should_panic(expected = "foo")].
188    `#[ignore]`       - When applied to a function which is already attributed as a
189                        test, then the test runner will ignore these tests during
190                        normal test runs. Running with --ignored or --include-ignored will run
191                        these tests."#,
192        usage = options.usage(&message)
193    );
194}
195
196/// Parses command line arguments into test options.
197/// Returns `None` if help was requested (since we only show help message and don't run tests),
198/// returns `Some(Err(..))` if provided arguments are incorrect,
199/// otherwise creates a `TestOpts` object and returns it.
200pub fn parse_opts(args: &[String]) -> Option<OptRes> {
201    // Parse matches.
202    let mut opts = optgroups();
203    // Flags hidden from `usage`
204    opts.optflag("", "nocapture", "Deprecated, use `--no-capture`");
205
206    let binary = args.first().map(|c| &**c).unwrap_or("...");
207    let args = args.get(1..).unwrap_or(args);
208    let matches = match opts.parse(args) {
209        Ok(m) => m,
210        Err(f) => return Some(Err(f.to_string())),
211    };
212
213    // Check if help was requested.
214    if matches.opt_present("h") {
215        // Show help and do nothing more.
216        usage(binary, &optgroups());
217        return None;
218    }
219
220    // Actually parse the opts.
221    let opts_result = parse_opts_impl(matches);
222
223    Some(opts_result)
224}
225
226// Gets the option value and checks if unstable features are enabled.
227macro_rules! unstable_optflag {
228    ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
229        let opt = $matches.opt_present($option_name);
230        if !$allow_unstable && opt {
231            return Err(format!(
232                "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options",
233                $option_name
234            ));
235        }
236
237        opt
238    }};
239}
240
241// Gets the option value and checks if unstable features are enabled.
242macro_rules! unstable_optopt {
243    ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
244        let opt = $matches.opt_str($option_name);
245        if !$allow_unstable && opt.is_some() {
246            return Err(format!(
247                "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options",
248                $option_name
249            ));
250        }
251
252        opt
253    }};
254}
255
256// Implementation of `parse_opts` that doesn't care about help message
257// and returns a `Result`.
258fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
259    let allow_unstable = get_allow_unstable(&matches)?;
260
261    // Unstable flags
262    let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process");
263    let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic");
264    let fail_fast = unstable_optflag!(matches, allow_unstable, "fail-fast");
265    let time_options = get_time_options(&matches, allow_unstable)?;
266    let shuffle = get_shuffle(&matches, allow_unstable)?;
267    let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?;
268
269    let include_ignored = matches.opt_present("include-ignored");
270    let quiet = matches.opt_present("quiet");
271    let exact = matches.opt_present("exact");
272    let list = matches.opt_present("list");
273    let skip = matches.opt_strs("skip");
274
275    let bench_benchmarks = matches.opt_present("bench");
276    let run_tests = !bench_benchmarks || matches.opt_present("test");
277
278    let logfile = get_log_file(&matches)?;
279    let run_ignored = get_run_ignored(&matches, include_ignored)?;
280    let filters = matches.free.clone();
281    let nocapture = get_nocapture(&matches)?;
282    let test_threads = get_test_threads(&matches)?;
283    let color = get_color_config(&matches)?;
284    let format = get_format(&matches, quiet, allow_unstable)?;
285
286    let options = Options::new().display_output(matches.opt_present("show-output"));
287
288    if logfile.is_some() {
289        let _ = write!(io::stderr(), "warning: `--logfile` is deprecated");
290    }
291
292    let test_opts = TestOpts {
293        list,
294        filters,
295        filter_exact: exact,
296        force_run_in_process,
297        exclude_should_panic,
298        run_ignored,
299        run_tests,
300        bench_benchmarks,
301        logfile,
302        nocapture,
303        color,
304        format,
305        shuffle,
306        shuffle_seed,
307        test_threads,
308        skip,
309        time_options,
310        options,
311        fail_fast,
312    };
313
314    Ok(test_opts)
315}
316
317// FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
318fn is_nightly() -> bool {
319    // Whether this is a feature-staged build, i.e., on the beta or stable channel
320    let disable_unstable_features =
321        option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(false);
322    // Whether we should enable unstable features for bootstrapping
323    let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
324
325    bootstrap || !disable_unstable_features
326}
327
328// Gets the CLI options associated with `report-time` feature.
329fn get_time_options(
330    matches: &getopts::Matches,
331    allow_unstable: bool,
332) -> OptPartRes<Option<TestTimeOptions>> {
333    let report_time = unstable_optflag!(matches, allow_unstable, "report-time");
334    let ensure_test_time = unstable_optflag!(matches, allow_unstable, "ensure-time");
335
336    // If `ensure-test-time` option is provided, time output is enforced,
337    // so user won't be confused if any of tests will silently fail.
338    let options = if report_time || ensure_test_time {
339        Some(TestTimeOptions::new_from_env(ensure_test_time))
340    } else {
341        None
342    };
343
344    Ok(options)
345}
346
347fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> {
348    let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle");
349    if !shuffle && allow_unstable {
350        shuffle = match env::var("RUST_TEST_SHUFFLE") {
351            Ok(val) => &val != "0",
352            Err(_) => false,
353        };
354    }
355
356    Ok(shuffle)
357}
358
359fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> {
360    let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") {
361        Some(n_str) => match n_str.parse::<u64>() {
362            Ok(n) => Some(n),
363            Err(e) => {
364                return Err(format!(
365                    "argument for --shuffle-seed must be a number \
366                     (error: {e})"
367                ));
368            }
369        },
370        None => None,
371    };
372
373    if shuffle_seed.is_none() && allow_unstable {
374        shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") {
375            Ok(val) => match val.parse::<u64>() {
376                Ok(n) => Some(n),
377                Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."),
378            },
379            Err(_) => None,
380        };
381    }
382
383    Ok(shuffle_seed)
384}
385
386fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> {
387    let test_threads = match matches.opt_str("test-threads") {
388        Some(n_str) => match n_str.parse::<usize>() {
389            Ok(0) => return Err("argument for --test-threads must not be 0".to_string()),
390            Ok(n) => Some(n),
391            Err(e) => {
392                return Err(format!(
393                    "argument for --test-threads must be a number > 0 \
394                     (error: {e})"
395                ));
396            }
397        },
398        None => None,
399    };
400
401    Ok(test_threads)
402}
403
404fn get_format(
405    matches: &getopts::Matches,
406    quiet: bool,
407    allow_unstable: bool,
408) -> OptPartRes<OutputFormat> {
409    let format = match matches.opt_str("format").as_deref() {
410        None if quiet => OutputFormat::Terse,
411        Some("pretty") | None => OutputFormat::Pretty,
412        Some("terse") => OutputFormat::Terse,
413        Some("json") => {
414            if !allow_unstable {
415                return Err("The \"json\" format is only accepted on the nightly compiler with -Z unstable-options".into());
416            }
417            OutputFormat::Json
418        }
419        Some("junit") => {
420            if !allow_unstable {
421                return Err("The \"junit\" format is only accepted on the nightly compiler with -Z unstable-options".into());
422            }
423            OutputFormat::Junit
424        }
425        Some(v) => {
426            return Err(format!(
427                "argument for --format must be pretty, terse, json or junit (was \
428                 {v})"
429            ));
430        }
431    };
432
433    Ok(format)
434}
435
436fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> {
437    let color = match matches.opt_str("color").as_deref() {
438        Some("auto") | None => ColorConfig::AutoColor,
439        Some("always") => ColorConfig::AlwaysColor,
440        Some("never") => ColorConfig::NeverColor,
441
442        Some(v) => {
443            return Err(format!(
444                "argument for --color must be auto, always, or never (was \
445                 {v})"
446            ));
447        }
448    };
449
450    Ok(color)
451}
452
453fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> {
454    let mut nocapture = matches.opt_present("nocapture") || matches.opt_present("no-capture");
455    if !nocapture {
456        nocapture = match env::var("RUST_TEST_NOCAPTURE") {
457            Ok(val) => &val != "0",
458            Err(_) => false,
459        };
460    }
461
462    Ok(nocapture)
463}
464
465fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> {
466    let run_ignored = match (include_ignored, matches.opt_present("ignored")) {
467        (true, true) => {
468            return Err("the options --include-ignored and --ignored are mutually exclusive".into());
469        }
470        (true, false) => RunIgnored::Yes,
471        (false, true) => RunIgnored::Only,
472        (false, false) => RunIgnored::No,
473    };
474
475    Ok(run_ignored)
476}
477
478fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> {
479    let mut allow_unstable = false;
480
481    if let Some(opt) = matches.opt_str("Z") {
482        if !is_nightly() {
483            return Err("the option `Z` is only accepted on the nightly compiler".into());
484        }
485
486        match &*opt {
487            "unstable-options" => {
488                allow_unstable = true;
489            }
490            _ => {
491                return Err("Unrecognized option to `Z`".into());
492            }
493        }
494    };
495
496    Ok(allow_unstable)
497}
498
499fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
500    let logfile = matches.opt_str("logfile").map(|s| PathBuf::from(&s));
501
502    Ok(logfile)
503}