rustdoc/
doctest.rs

1mod extracted;
2mod make;
3mod markdown;
4mod runner;
5mod rust;
6
7use std::fs::File;
8use std::hash::{Hash, Hasher};
9use std::io::{self, Write};
10use std::path::{Path, PathBuf};
11use std::process::{self, Command, Stdio};
12use std::sync::atomic::{AtomicUsize, Ordering};
13use std::sync::{Arc, Mutex};
14use std::time::{Duration, Instant};
15use std::{fmt, panic, str};
16
17pub(crate) use make::{BuildDocTestBuilder, DocTestBuilder};
18pub(crate) use markdown::test as test_markdown;
19use rustc_data_structures::fx::{FxHashMap, FxHasher, FxIndexMap, FxIndexSet};
20use rustc_errors::emitter::HumanReadableErrorType;
21use rustc_errors::{ColorConfig, DiagCtxtHandle};
22use rustc_hir as hir;
23use rustc_hir::CRATE_HIR_ID;
24use rustc_hir::def_id::LOCAL_CRATE;
25use rustc_interface::interface;
26use rustc_session::config::{self, CrateType, ErrorOutputType, Input};
27use rustc_session::lint;
28use rustc_span::edition::Edition;
29use rustc_span::symbol::sym;
30use rustc_span::{FileName, Span};
31use rustc_target::spec::{Target, TargetTuple};
32use tempfile::{Builder as TempFileBuilder, TempDir};
33use tracing::debug;
34
35use self::rust::HirCollector;
36use crate::config::{Options as RustdocOptions, OutputFormat};
37use crate::html::markdown::{ErrorCodes, Ignore, LangString, MdRelLine};
38use crate::lint::init_lints;
39
40/// Type used to display times (compilation and total) information for merged doctests.
41struct MergedDoctestTimes {
42    total_time: Instant,
43    /// Total time spent compiling all merged doctests.
44    compilation_time: Duration,
45    /// This field is used to keep track of how many merged doctests we (tried to) compile.
46    added_compilation_times: usize,
47}
48
49impl MergedDoctestTimes {
50    fn new() -> Self {
51        Self {
52            total_time: Instant::now(),
53            compilation_time: Duration::default(),
54            added_compilation_times: 0,
55        }
56    }
57
58    fn add_compilation_time(&mut self, duration: Duration) {
59        self.compilation_time += duration;
60        self.added_compilation_times += 1;
61    }
62
63    fn display_times(&self) {
64        // If no merged doctest was compiled, then there is nothing to display since the numbers
65        // displayed by `libtest` for standalone tests are already accurate (they include both
66        // compilation and runtime).
67        if self.added_compilation_times > 0 {
68            println!("{self}");
69        }
70    }
71}
72
73impl fmt::Display for MergedDoctestTimes {
74    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75        write!(
76            f,
77            "all doctests ran in {:.2}s; merged doctests compilation took {:.2}s",
78            self.total_time.elapsed().as_secs_f64(),
79            self.compilation_time.as_secs_f64(),
80        )
81    }
82}
83
84/// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`).
85#[derive(Clone)]
86pub(crate) struct GlobalTestOptions {
87    /// Name of the crate (for regular `rustdoc`) or Markdown file (for `rustdoc foo.md`).
88    pub(crate) crate_name: String,
89    /// Whether to disable the default `extern crate my_crate;` when creating doctests.
90    pub(crate) no_crate_inject: bool,
91    /// Whether inserting extra indent spaces in code block,
92    /// default is `false`, only `true` for generating code link of Rust playground
93    pub(crate) insert_indent_space: bool,
94    /// Path to file containing arguments for the invocation of rustc.
95    pub(crate) args_file: PathBuf,
96}
97
98pub(crate) fn generate_args_file(file_path: &Path, options: &RustdocOptions) -> Result<(), String> {
99    let mut file = File::create(file_path)
100        .map_err(|error| format!("failed to create args file: {error:?}"))?;
101
102    // We now put the common arguments into the file we created.
103    let mut content = vec![];
104
105    for cfg in &options.cfgs {
106        content.push(format!("--cfg={cfg}"));
107    }
108    for check_cfg in &options.check_cfgs {
109        content.push(format!("--check-cfg={check_cfg}"));
110    }
111
112    for lib_str in &options.lib_strs {
113        content.push(format!("-L{lib_str}"));
114    }
115    for extern_str in &options.extern_strs {
116        content.push(format!("--extern={extern_str}"));
117    }
118    content.push("-Ccodegen-units=1".to_string());
119    for codegen_options_str in &options.codegen_options_strs {
120        content.push(format!("-C{codegen_options_str}"));
121    }
122    for unstable_option_str in &options.unstable_opts_strs {
123        content.push(format!("-Z{unstable_option_str}"));
124    }
125
126    content.extend(options.doctest_build_args.clone());
127
128    let content = content.join("\n");
129
130    file.write_all(content.as_bytes())
131        .map_err(|error| format!("failed to write arguments to temporary file: {error:?}"))?;
132    Ok(())
133}
134
135fn get_doctest_dir() -> io::Result<TempDir> {
136    TempFileBuilder::new().prefix("rustdoctest").tempdir()
137}
138
139pub(crate) fn run(dcx: DiagCtxtHandle<'_>, input: Input, options: RustdocOptions) {
140    let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
141
142    // See core::create_config for what's going on here.
143    let allowed_lints = vec![
144        invalid_codeblock_attributes_name.to_owned(),
145        lint::builtin::UNKNOWN_LINTS.name.to_owned(),
146        lint::builtin::RENAMED_AND_REMOVED_LINTS.name.to_owned(),
147    ];
148
149    let (lint_opts, lint_caps) = init_lints(allowed_lints, options.lint_opts.clone(), |lint| {
150        if lint.name == invalid_codeblock_attributes_name {
151            None
152        } else {
153            Some((lint.name_lower(), lint::Allow))
154        }
155    });
156
157    debug!(?lint_opts);
158
159    let crate_types =
160        if options.proc_macro_crate { vec![CrateType::ProcMacro] } else { vec![CrateType::Rlib] };
161
162    let sessopts = config::Options {
163        sysroot: options.sysroot.clone(),
164        search_paths: options.libs.clone(),
165        crate_types,
166        lint_opts,
167        lint_cap: Some(options.lint_cap.unwrap_or(lint::Forbid)),
168        cg: options.codegen_options.clone(),
169        externs: options.externs.clone(),
170        unstable_features: options.unstable_features,
171        actually_rustdoc: true,
172        edition: options.edition,
173        target_triple: options.target.clone(),
174        crate_name: options.crate_name.clone(),
175        remap_path_prefix: options.remap_path_prefix.clone(),
176        ..config::Options::default()
177    };
178
179    let mut cfgs = options.cfgs.clone();
180    cfgs.push("doc".to_owned());
181    cfgs.push("doctest".to_owned());
182    let config = interface::Config {
183        opts: sessopts,
184        crate_cfg: cfgs,
185        crate_check_cfg: options.check_cfgs.clone(),
186        input: input.clone(),
187        output_file: None,
188        output_dir: None,
189        file_loader: None,
190        locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
191        lint_caps,
192        psess_created: None,
193        hash_untracked_state: None,
194        register_lints: Some(Box::new(crate::lint::register_lints)),
195        override_queries: None,
196        extra_symbols: Vec::new(),
197        make_codegen_backend: None,
198        registry: rustc_driver::diagnostics_registry(),
199        ice_file: None,
200        using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES,
201        expanded_args: options.expanded_args.clone(),
202    };
203
204    let externs = options.externs.clone();
205    let json_unused_externs = options.json_unused_externs;
206
207    let temp_dir = match get_doctest_dir()
208        .map_err(|error| format!("failed to create temporary directory: {error:?}"))
209    {
210        Ok(temp_dir) => temp_dir,
211        Err(error) => return crate::wrap_return(dcx, Err(error)),
212    };
213    let args_path = temp_dir.path().join("rustdoc-cfgs");
214    crate::wrap_return(dcx, generate_args_file(&args_path, &options));
215
216    let extract_doctests = options.output_format == OutputFormat::Doctest;
217    let result = interface::run_compiler(config, |compiler| {
218        let krate = rustc_interface::passes::parse(&compiler.sess);
219
220        let collector = rustc_interface::create_and_enter_global_ctxt(compiler, krate, |tcx| {
221            let crate_name = tcx.crate_name(LOCAL_CRATE).to_string();
222            let crate_attrs = tcx.hir_attrs(CRATE_HIR_ID);
223            let opts = scrape_test_config(crate_name, crate_attrs, args_path);
224
225            let hir_collector = HirCollector::new(
226                ErrorCodes::from(compiler.sess.opts.unstable_features.is_nightly_build()),
227                tcx,
228            );
229            let tests = hir_collector.collect_crate();
230            if extract_doctests {
231                let mut collector = extracted::ExtractedDocTests::new();
232                tests.into_iter().for_each(|t| collector.add_test(t, &opts, &options));
233
234                let stdout = std::io::stdout();
235                let mut stdout = stdout.lock();
236                if let Err(error) = serde_json::ser::to_writer(&mut stdout, &collector) {
237                    eprintln!();
238                    Err(format!("Failed to generate JSON output for doctests: {error:?}"))
239                } else {
240                    Ok(None)
241                }
242            } else {
243                let mut collector = CreateRunnableDocTests::new(options, opts);
244                tests.into_iter().for_each(|t| collector.add_test(t, Some(compiler.sess.dcx())));
245
246                Ok(Some(collector))
247            }
248        });
249        compiler.sess.dcx().abort_if_errors();
250
251        collector
252    });
253
254    let CreateRunnableDocTests {
255        standalone_tests,
256        mergeable_tests,
257        rustdoc_options,
258        opts,
259        unused_extern_reports,
260        compiling_test_count,
261        ..
262    } = match result {
263        Ok(Some(collector)) => collector,
264        Ok(None) => return,
265        Err(error) => {
266            eprintln!("{error}");
267            // Since some files in the temporary folder are still owned and alive, we need
268            // to manually remove the folder.
269            let _ = std::fs::remove_dir_all(temp_dir.path());
270            std::process::exit(1);
271        }
272    };
273
274    run_tests(
275        opts,
276        &rustdoc_options,
277        &unused_extern_reports,
278        standalone_tests,
279        mergeable_tests,
280        Some(temp_dir),
281    );
282
283    let compiling_test_count = compiling_test_count.load(Ordering::SeqCst);
284
285    // Collect and warn about unused externs, but only if we've gotten
286    // reports for each doctest
287    if json_unused_externs.is_enabled() {
288        let unused_extern_reports: Vec<_> =
289            std::mem::take(&mut unused_extern_reports.lock().unwrap());
290        if unused_extern_reports.len() == compiling_test_count {
291            let extern_names =
292                externs.iter().map(|(name, _)| name).collect::<FxIndexSet<&String>>();
293            let mut unused_extern_names = unused_extern_reports
294                .iter()
295                .map(|uexts| uexts.unused_extern_names.iter().collect::<FxIndexSet<&String>>())
296                .fold(extern_names, |uextsa, uextsb| {
297                    uextsa.intersection(&uextsb).copied().collect::<FxIndexSet<&String>>()
298                })
299                .iter()
300                .map(|v| (*v).clone())
301                .collect::<Vec<String>>();
302            unused_extern_names.sort();
303            // Take the most severe lint level
304            let lint_level = unused_extern_reports
305                .iter()
306                .map(|uexts| uexts.lint_level.as_str())
307                .max_by_key(|v| match *v {
308                    "warn" => 1,
309                    "deny" => 2,
310                    "forbid" => 3,
311                    // The allow lint level is not expected,
312                    // as if allow is specified, no message
313                    // is to be emitted.
314                    v => unreachable!("Invalid lint level '{v}'"),
315                })
316                .unwrap_or("warn")
317                .to_string();
318            let uext = UnusedExterns { lint_level, unused_extern_names };
319            let unused_extern_json = serde_json::to_string(&uext).unwrap();
320            eprintln!("{unused_extern_json}");
321        }
322    }
323}
324
325pub(crate) fn run_tests(
326    opts: GlobalTestOptions,
327    rustdoc_options: &Arc<RustdocOptions>,
328    unused_extern_reports: &Arc<Mutex<Vec<UnusedExterns>>>,
329    mut standalone_tests: Vec<test::TestDescAndFn>,
330    mergeable_tests: FxIndexMap<MergeableTestKey, Vec<(DocTestBuilder, ScrapedDocTest)>>,
331    // We pass this argument so we can drop it manually before using `exit`.
332    mut temp_dir: Option<TempDir>,
333) {
334    let mut test_args = Vec::with_capacity(rustdoc_options.test_args.len() + 1);
335    test_args.insert(0, "rustdoctest".to_string());
336    test_args.extend_from_slice(&rustdoc_options.test_args);
337    if rustdoc_options.nocapture {
338        test_args.push("--nocapture".to_string());
339    }
340
341    let mut nb_errors = 0;
342    let mut ran_edition_tests = 0;
343    let mut times = MergedDoctestTimes::new();
344    let target_str = rustdoc_options.target.to_string();
345
346    for (MergeableTestKey { edition, global_crate_attrs_hash }, mut doctests) in mergeable_tests {
347        if doctests.is_empty() {
348            continue;
349        }
350        doctests.sort_by(|(_, a), (_, b)| a.name.cmp(&b.name));
351
352        let mut tests_runner = runner::DocTestRunner::new();
353
354        let rustdoc_test_options = IndividualTestOptions::new(
355            rustdoc_options,
356            &Some(format!("merged_doctest_{edition}_{global_crate_attrs_hash}")),
357            PathBuf::from(format!("doctest_{edition}_{global_crate_attrs_hash}.rs")),
358        );
359
360        for (doctest, scraped_test) in &doctests {
361            tests_runner.add_test(doctest, scraped_test, &target_str);
362        }
363        let (duration, ret) = tests_runner.run_merged_tests(
364            rustdoc_test_options,
365            edition,
366            &opts,
367            &test_args,
368            rustdoc_options,
369        );
370        times.add_compilation_time(duration);
371        if let Ok(success) = ret {
372            ran_edition_tests += 1;
373            if !success {
374                nb_errors += 1;
375            }
376            continue;
377        }
378        // We failed to compile all compatible tests as one so we push them into the
379        // `standalone_tests` doctests.
380        debug!("Failed to compile compatible doctests for edition {} all at once", edition);
381        for (doctest, scraped_test) in doctests {
382            doctest.generate_unique_doctest(
383                &scraped_test.text,
384                scraped_test.langstr.test_harness,
385                &opts,
386                Some(&opts.crate_name),
387            );
388            standalone_tests.push(generate_test_desc_and_fn(
389                doctest,
390                scraped_test,
391                opts.clone(),
392                Arc::clone(rustdoc_options),
393                unused_extern_reports.clone(),
394            ));
395        }
396    }
397
398    // We need to call `test_main` even if there is no doctest to run to get the output
399    // `running 0 tests...`.
400    if ran_edition_tests == 0 || !standalone_tests.is_empty() {
401        standalone_tests.sort_by(|a, b| a.desc.name.as_slice().cmp(b.desc.name.as_slice()));
402        test::test_main_with_exit_callback(&test_args, standalone_tests, None, || {
403            // We ensure temp dir destructor is called.
404            std::mem::drop(temp_dir.take());
405            times.display_times();
406        });
407    }
408    if nb_errors != 0 {
409        // We ensure temp dir destructor is called.
410        std::mem::drop(temp_dir);
411        times.display_times();
412        // FIXME(GuillaumeGomez): Uncomment the next line once #144297 has been merged.
413        // std::process::exit(test::ERROR_EXIT_CODE);
414        std::process::exit(101);
415    }
416}
417
418// Look for `#![doc(test(no_crate_inject))]`, used by crates in the std facade.
419fn scrape_test_config(
420    crate_name: String,
421    attrs: &[hir::Attribute],
422    args_file: PathBuf,
423) -> GlobalTestOptions {
424    let mut opts = GlobalTestOptions {
425        crate_name,
426        no_crate_inject: false,
427        insert_indent_space: false,
428        args_file,
429    };
430
431    let test_attrs: Vec<_> = attrs
432        .iter()
433        .filter(|a| a.has_name(sym::doc))
434        .flat_map(|a| a.meta_item_list().unwrap_or_default())
435        .filter(|a| a.has_name(sym::test))
436        .collect();
437    let attrs = test_attrs.iter().flat_map(|a| a.meta_item_list().unwrap_or(&[]));
438
439    for attr in attrs {
440        if attr.has_name(sym::no_crate_inject) {
441            opts.no_crate_inject = true;
442        }
443        // NOTE: `test(attr(..))` is handled when discovering the individual tests
444    }
445
446    opts
447}
448
449/// Documentation test failure modes.
450enum TestFailure {
451    /// The test failed to compile.
452    CompileError,
453    /// The test is marked `compile_fail` but compiled successfully.
454    UnexpectedCompilePass,
455    /// The test failed to compile (as expected) but the compiler output did not contain all
456    /// expected error codes.
457    MissingErrorCodes(Vec<String>),
458    /// The test binary was unable to be executed.
459    ExecutionError(io::Error),
460    /// The test binary exited with a non-zero exit code.
461    ///
462    /// This typically means an assertion in the test failed or another form of panic occurred.
463    ExecutionFailure(process::Output),
464    /// The test is marked `should_panic` but the test binary executed successfully.
465    UnexpectedRunPass,
466}
467
468enum DirState {
469    Temp(TempDir),
470    Perm(PathBuf),
471}
472
473impl DirState {
474    fn path(&self) -> &std::path::Path {
475        match self {
476            DirState::Temp(t) => t.path(),
477            DirState::Perm(p) => p.as_path(),
478        }
479    }
480}
481
482// NOTE: Keep this in sync with the equivalent structs in rustc
483// and cargo.
484// We could unify this struct the one in rustc but they have different
485// ownership semantics, so doing so would create wasteful allocations.
486#[derive(serde::Serialize, serde::Deserialize)]
487pub(crate) struct UnusedExterns {
488    /// Lint level of the unused_crate_dependencies lint
489    lint_level: String,
490    /// List of unused externs by their names.
491    unused_extern_names: Vec<String>,
492}
493
494fn add_exe_suffix(input: String, target: &TargetTuple) -> String {
495    let exe_suffix = match target {
496        TargetTuple::TargetTuple(_) => Target::expect_builtin(target).options.exe_suffix,
497        TargetTuple::TargetJson { contents, .. } => {
498            Target::from_json(contents).unwrap().0.options.exe_suffix
499        }
500    };
501    input + &exe_suffix
502}
503
504fn wrapped_rustc_command(rustc_wrappers: &[PathBuf], rustc_binary: &Path) -> Command {
505    let mut args = rustc_wrappers.iter().map(PathBuf::as_path).chain([rustc_binary]);
506
507    let exe = args.next().expect("unable to create rustc command");
508    let mut command = Command::new(exe);
509    for arg in args {
510        command.arg(arg);
511    }
512
513    command
514}
515
516/// Information needed for running a bundle of doctests.
517///
518/// This data structure contains the "full" test code, including the wrappers
519/// (if multiple doctests are merged), `main` function,
520/// and everything needed to calculate the compiler's command-line arguments.
521/// The `# ` prefix on boring lines has also been stripped.
522pub(crate) struct RunnableDocTest {
523    full_test_code: String,
524    full_test_line_offset: usize,
525    test_opts: IndividualTestOptions,
526    global_opts: GlobalTestOptions,
527    langstr: LangString,
528    line: usize,
529    edition: Edition,
530    no_run: bool,
531    merged_test_code: Option<String>,
532}
533
534impl RunnableDocTest {
535    fn path_for_merged_doctest_bundle(&self) -> PathBuf {
536        self.test_opts.outdir.path().join(format!("doctest_bundle_{}.rs", self.edition))
537    }
538    fn path_for_merged_doctest_runner(&self) -> PathBuf {
539        self.test_opts.outdir.path().join(format!("doctest_runner_{}.rs", self.edition))
540    }
541    fn is_multiple_tests(&self) -> bool {
542        self.merged_test_code.is_some()
543    }
544}
545
546/// Execute a `RunnableDoctest`.
547///
548/// This is the function that calculates the compiler command line, invokes the compiler, then
549/// invokes the test or tests in a separate executable (if applicable).
550///
551/// Returns a tuple containing the `Duration` of the compilation and the `Result` of the test.
552fn run_test(
553    doctest: RunnableDocTest,
554    rustdoc_options: &RustdocOptions,
555    supports_color: bool,
556    report_unused_externs: impl Fn(UnusedExterns),
557) -> (Duration, Result<(), TestFailure>) {
558    let langstr = &doctest.langstr;
559    // Make sure we emit well-formed executable names for our target.
560    let rust_out = add_exe_suffix("rust_out".to_owned(), &rustdoc_options.target);
561    let output_file = doctest.test_opts.outdir.path().join(rust_out);
562    let instant = Instant::now();
563
564    // Common arguments used for compiling the doctest runner.
565    // On merged doctests, the compiler is invoked twice: once for the test code itself,
566    // and once for the runner wrapper (which needs to use `#![feature]` on stable).
567    let mut compiler_args = vec![];
568
569    compiler_args.push(format!("@{}", doctest.global_opts.args_file.display()));
570
571    let sysroot = &rustdoc_options.sysroot;
572    if let Some(explicit_sysroot) = &sysroot.explicit {
573        compiler_args.push(format!("--sysroot={}", explicit_sysroot.display()));
574    }
575
576    compiler_args.extend_from_slice(&["--edition".to_owned(), doctest.edition.to_string()]);
577    if langstr.test_harness {
578        compiler_args.push("--test".to_owned());
579    }
580    if rustdoc_options.json_unused_externs.is_enabled() && !langstr.compile_fail {
581        compiler_args.push("--error-format=json".to_owned());
582        compiler_args.extend_from_slice(&["--json".to_owned(), "unused-externs".to_owned()]);
583        compiler_args.extend_from_slice(&["-W".to_owned(), "unused_crate_dependencies".to_owned()]);
584        compiler_args.extend_from_slice(&["-Z".to_owned(), "unstable-options".to_owned()]);
585    }
586
587    if doctest.no_run && !langstr.compile_fail && rustdoc_options.persist_doctests.is_none() {
588        // FIXME: why does this code check if it *shouldn't* persist doctests
589        //        -- shouldn't it be the negation?
590        compiler_args.push("--emit=metadata".to_owned());
591    }
592    compiler_args.extend_from_slice(&[
593        "--target".to_owned(),
594        match &rustdoc_options.target {
595            TargetTuple::TargetTuple(s) => s.clone(),
596            TargetTuple::TargetJson { path_for_rustdoc, .. } => {
597                path_for_rustdoc.to_str().expect("target path must be valid unicode").to_owned()
598            }
599        },
600    ]);
601    if let ErrorOutputType::HumanReadable { kind, color_config } = rustdoc_options.error_format {
602        let short = kind.short();
603        let unicode = kind == HumanReadableErrorType::Unicode;
604
605        if short {
606            compiler_args.extend_from_slice(&["--error-format".to_owned(), "short".to_owned()]);
607        }
608        if unicode {
609            compiler_args
610                .extend_from_slice(&["--error-format".to_owned(), "human-unicode".to_owned()]);
611        }
612
613        match color_config {
614            ColorConfig::Never => {
615                compiler_args.extend_from_slice(&["--color".to_owned(), "never".to_owned()]);
616            }
617            ColorConfig::Always => {
618                compiler_args.extend_from_slice(&["--color".to_owned(), "always".to_owned()]);
619            }
620            ColorConfig::Auto => {
621                compiler_args.extend_from_slice(&[
622                    "--color".to_owned(),
623                    if supports_color { "always" } else { "never" }.to_owned(),
624                ]);
625            }
626        }
627    }
628
629    let rustc_binary = rustdoc_options
630        .test_builder
631        .as_deref()
632        .unwrap_or_else(|| rustc_interface::util::rustc_path(sysroot).expect("found rustc"));
633    let mut compiler = wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
634
635    compiler.args(&compiler_args);
636
637    // If this is a merged doctest, we need to write it into a file instead of using stdin
638    // because if the size of the merged doctests is too big, it'll simply break stdin.
639    if doctest.is_multiple_tests() {
640        // It makes the compilation failure much faster if it is for a combined doctest.
641        compiler.arg("--error-format=short");
642        let input_file = doctest.path_for_merged_doctest_bundle();
643        if std::fs::write(&input_file, &doctest.full_test_code).is_err() {
644            // If we cannot write this file for any reason, we leave. All combined tests will be
645            // tested as standalone tests.
646            return (Duration::default(), Err(TestFailure::CompileError));
647        }
648        if !rustdoc_options.nocapture {
649            // If `nocapture` is disabled, then we don't display rustc's output when compiling
650            // the merged doctests.
651            compiler.stderr(Stdio::null());
652        }
653        // bundled tests are an rlib, loaded by a separate runner executable
654        compiler
655            .arg("--crate-type=lib")
656            .arg("--out-dir")
657            .arg(doctest.test_opts.outdir.path())
658            .arg(input_file);
659    } else {
660        compiler.arg("--crate-type=bin").arg("-o").arg(&output_file);
661        // Setting these environment variables is unneeded if this is a merged doctest.
662        compiler.env("UNSTABLE_RUSTDOC_TEST_PATH", &doctest.test_opts.path);
663        compiler.env(
664            "UNSTABLE_RUSTDOC_TEST_LINE",
665            format!("{}", doctest.line as isize - doctest.full_test_line_offset as isize),
666        );
667        compiler.arg("-");
668        compiler.stdin(Stdio::piped());
669        compiler.stderr(Stdio::piped());
670    }
671
672    debug!("compiler invocation for doctest: {compiler:?}");
673
674    let mut child = compiler.spawn().expect("Failed to spawn rustc process");
675    let output = if let Some(merged_test_code) = &doctest.merged_test_code {
676        // compile-fail tests never get merged, so this should always pass
677        let status = child.wait().expect("Failed to wait");
678
679        // the actual test runner is a separate component, built with nightly-only features;
680        // build it now
681        let runner_input_file = doctest.path_for_merged_doctest_runner();
682
683        let mut runner_compiler =
684            wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
685        // the test runner does not contain any user-written code, so this doesn't allow
686        // the user to exploit nightly-only features on stable
687        runner_compiler.env("RUSTC_BOOTSTRAP", "1");
688        runner_compiler.args(compiler_args);
689        runner_compiler.args(["--crate-type=bin", "-o"]).arg(&output_file);
690        let mut extern_path = std::ffi::OsString::from(format!(
691            "--extern=doctest_bundle_{edition}=",
692            edition = doctest.edition
693        ));
694        for extern_str in &rustdoc_options.extern_strs {
695            if let Some((_cratename, path)) = extern_str.split_once('=') {
696                // Direct dependencies of the tests themselves are
697                // indirect dependencies of the test runner.
698                // They need to be in the library search path.
699                let dir = Path::new(path)
700                    .parent()
701                    .filter(|x| x.components().count() > 0)
702                    .unwrap_or(Path::new("."));
703                runner_compiler.arg("-L").arg(dir);
704            }
705        }
706        let output_bundle_file = doctest
707            .test_opts
708            .outdir
709            .path()
710            .join(format!("libdoctest_bundle_{edition}.rlib", edition = doctest.edition));
711        extern_path.push(&output_bundle_file);
712        runner_compiler.arg(extern_path);
713        runner_compiler.arg(&runner_input_file);
714        if std::fs::write(&runner_input_file, merged_test_code).is_err() {
715            // If we cannot write this file for any reason, we leave. All combined tests will be
716            // tested as standalone tests.
717            return (instant.elapsed(), Err(TestFailure::CompileError));
718        }
719        if !rustdoc_options.nocapture {
720            // If `nocapture` is disabled, then we don't display rustc's output when compiling
721            // the merged doctests.
722            runner_compiler.stderr(Stdio::null());
723        }
724        runner_compiler.arg("--error-format=short");
725        debug!("compiler invocation for doctest runner: {runner_compiler:?}");
726
727        let status = if !status.success() {
728            status
729        } else {
730            let mut child_runner = runner_compiler.spawn().expect("Failed to spawn rustc process");
731            child_runner.wait().expect("Failed to wait")
732        };
733
734        process::Output { status, stdout: Vec::new(), stderr: Vec::new() }
735    } else {
736        let stdin = child.stdin.as_mut().expect("Failed to open stdin");
737        stdin.write_all(doctest.full_test_code.as_bytes()).expect("could write out test sources");
738        child.wait_with_output().expect("Failed to read stdout")
739    };
740
741    struct Bomb<'a>(&'a str);
742    impl Drop for Bomb<'_> {
743        fn drop(&mut self) {
744            eprint!("{}", self.0);
745        }
746    }
747    let mut out = str::from_utf8(&output.stderr)
748        .unwrap()
749        .lines()
750        .filter(|l| {
751            if let Ok(uext) = serde_json::from_str::<UnusedExterns>(l) {
752                report_unused_externs(uext);
753                false
754            } else {
755                true
756            }
757        })
758        .intersperse_with(|| "\n")
759        .collect::<String>();
760
761    // Add a \n to the end to properly terminate the last line,
762    // but only if there was output to be printed
763    if !out.is_empty() {
764        out.push('\n');
765    }
766
767    let _bomb = Bomb(&out);
768    match (output.status.success(), langstr.compile_fail) {
769        (true, true) => {
770            return (instant.elapsed(), Err(TestFailure::UnexpectedCompilePass));
771        }
772        (true, false) => {}
773        (false, true) => {
774            if !langstr.error_codes.is_empty() {
775                // We used to check if the output contained "error[{}]: " but since we added the
776                // colored output, we can't anymore because of the color escape characters before
777                // the ":".
778                let missing_codes: Vec<String> = langstr
779                    .error_codes
780                    .iter()
781                    .filter(|err| !out.contains(&format!("error[{err}]")))
782                    .cloned()
783                    .collect();
784
785                if !missing_codes.is_empty() {
786                    return (instant.elapsed(), Err(TestFailure::MissingErrorCodes(missing_codes)));
787                }
788            }
789        }
790        (false, false) => {
791            return (instant.elapsed(), Err(TestFailure::CompileError));
792        }
793    }
794
795    let duration = instant.elapsed();
796    if doctest.no_run {
797        return (duration, Ok(()));
798    }
799
800    // Run the code!
801    let mut cmd;
802
803    let output_file = make_maybe_absolute_path(output_file);
804    if let Some(tool) = &rustdoc_options.test_runtool {
805        let tool = make_maybe_absolute_path(tool.into());
806        cmd = Command::new(tool);
807        cmd.args(&rustdoc_options.test_runtool_args);
808        cmd.arg(&output_file);
809    } else {
810        cmd = Command::new(&output_file);
811        if doctest.is_multiple_tests() {
812            cmd.env("RUSTDOC_DOCTEST_BIN_PATH", &output_file);
813        }
814    }
815    if let Some(run_directory) = &rustdoc_options.test_run_directory {
816        cmd.current_dir(run_directory);
817    }
818
819    let result = if doctest.is_multiple_tests() || rustdoc_options.nocapture {
820        cmd.status().map(|status| process::Output {
821            status,
822            stdout: Vec::new(),
823            stderr: Vec::new(),
824        })
825    } else {
826        cmd.output()
827    };
828    match result {
829        Err(e) => return (duration, Err(TestFailure::ExecutionError(e))),
830        Ok(out) => {
831            if langstr.should_panic && out.status.success() {
832                return (duration, Err(TestFailure::UnexpectedRunPass));
833            } else if !langstr.should_panic && !out.status.success() {
834                return (duration, Err(TestFailure::ExecutionFailure(out)));
835            }
836        }
837    }
838
839    (duration, Ok(()))
840}
841
842/// Converts a path intended to use as a command to absolute if it is
843/// relative, and not a single component.
844///
845/// This is needed to deal with relative paths interacting with
846/// `Command::current_dir` in a platform-specific way.
847fn make_maybe_absolute_path(path: PathBuf) -> PathBuf {
848    if path.components().count() == 1 {
849        // Look up process via PATH.
850        path
851    } else {
852        std::env::current_dir().map(|c| c.join(&path)).unwrap_or_else(|_| path)
853    }
854}
855struct IndividualTestOptions {
856    outdir: DirState,
857    path: PathBuf,
858}
859
860impl IndividualTestOptions {
861    fn new(options: &RustdocOptions, test_id: &Option<String>, test_path: PathBuf) -> Self {
862        let outdir = if let Some(ref path) = options.persist_doctests {
863            let mut path = path.clone();
864            path.push(test_id.as_deref().unwrap_or("<doctest>"));
865
866            if let Err(err) = std::fs::create_dir_all(&path) {
867                eprintln!("Couldn't create directory for doctest executables: {err}");
868                panic::resume_unwind(Box::new(()));
869            }
870
871            DirState::Perm(path)
872        } else {
873            DirState::Temp(get_doctest_dir().expect("rustdoc needs a tempdir"))
874        };
875
876        Self { outdir, path: test_path }
877    }
878}
879
880/// A doctest scraped from the code, ready to be turned into a runnable test.
881///
882/// The pipeline goes: [`clean`] AST -> `ScrapedDoctest` -> `RunnableDoctest`.
883/// [`run_merged_tests`] converts a bunch of scraped doctests to a single runnable doctest,
884/// while [`generate_unique_doctest`] does the standalones.
885///
886/// [`clean`]: crate::clean
887/// [`run_merged_tests`]: crate::doctest::runner::DocTestRunner::run_merged_tests
888/// [`generate_unique_doctest`]: crate::doctest::make::DocTestBuilder::generate_unique_doctest
889#[derive(Debug)]
890pub(crate) struct ScrapedDocTest {
891    filename: FileName,
892    line: usize,
893    langstr: LangString,
894    text: String,
895    name: String,
896    span: Span,
897    global_crate_attrs: Vec<String>,
898}
899
900impl ScrapedDocTest {
901    fn new(
902        filename: FileName,
903        line: usize,
904        logical_path: Vec<String>,
905        langstr: LangString,
906        text: String,
907        span: Span,
908        global_crate_attrs: Vec<String>,
909    ) -> Self {
910        let mut item_path = logical_path.join("::");
911        item_path.retain(|c| c != ' ');
912        if !item_path.is_empty() {
913            item_path.push(' ');
914        }
915        let name =
916            format!("{} - {item_path}(line {line})", filename.prefer_remapped_unconditionally());
917
918        Self { filename, line, langstr, text, name, span, global_crate_attrs }
919    }
920    fn edition(&self, opts: &RustdocOptions) -> Edition {
921        self.langstr.edition.unwrap_or(opts.edition)
922    }
923
924    fn no_run(&self, opts: &RustdocOptions) -> bool {
925        self.langstr.no_run || opts.no_run
926    }
927    fn path(&self) -> PathBuf {
928        match &self.filename {
929            FileName::Real(path) => {
930                if let Some(local_path) = path.local_path() {
931                    local_path.to_path_buf()
932                } else {
933                    // Somehow we got the filename from the metadata of another crate, should never happen
934                    unreachable!("doctest from a different crate");
935                }
936            }
937            _ => PathBuf::from(r"doctest.rs"),
938        }
939    }
940}
941
942pub(crate) trait DocTestVisitor {
943    fn visit_test(&mut self, test: String, config: LangString, rel_line: MdRelLine);
944    fn visit_header(&mut self, _name: &str, _level: u32) {}
945}
946
947#[derive(Clone, Debug, Hash, Eq, PartialEq)]
948pub(crate) struct MergeableTestKey {
949    edition: Edition,
950    global_crate_attrs_hash: u64,
951}
952
953struct CreateRunnableDocTests {
954    standalone_tests: Vec<test::TestDescAndFn>,
955    mergeable_tests: FxIndexMap<MergeableTestKey, Vec<(DocTestBuilder, ScrapedDocTest)>>,
956
957    rustdoc_options: Arc<RustdocOptions>,
958    opts: GlobalTestOptions,
959    visited_tests: FxHashMap<(String, usize), usize>,
960    unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>,
961    compiling_test_count: AtomicUsize,
962    can_merge_doctests: bool,
963}
964
965impl CreateRunnableDocTests {
966    fn new(rustdoc_options: RustdocOptions, opts: GlobalTestOptions) -> CreateRunnableDocTests {
967        let can_merge_doctests = rustdoc_options.edition >= Edition::Edition2024;
968        CreateRunnableDocTests {
969            standalone_tests: Vec::new(),
970            mergeable_tests: FxIndexMap::default(),
971            rustdoc_options: Arc::new(rustdoc_options),
972            opts,
973            visited_tests: FxHashMap::default(),
974            unused_extern_reports: Default::default(),
975            compiling_test_count: AtomicUsize::new(0),
976            can_merge_doctests,
977        }
978    }
979
980    fn add_test(&mut self, scraped_test: ScrapedDocTest, dcx: Option<DiagCtxtHandle<'_>>) {
981        // For example `module/file.rs` would become `module_file_rs`
982        let file = scraped_test
983            .filename
984            .prefer_local()
985            .to_string_lossy()
986            .chars()
987            .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
988            .collect::<String>();
989        let test_id = format!(
990            "{file}_{line}_{number}",
991            file = file,
992            line = scraped_test.line,
993            number = {
994                // Increases the current test number, if this file already
995                // exists or it creates a new entry with a test number of 0.
996                self.visited_tests
997                    .entry((file.clone(), scraped_test.line))
998                    .and_modify(|v| *v += 1)
999                    .or_insert(0)
1000            },
1001        );
1002
1003        let edition = scraped_test.edition(&self.rustdoc_options);
1004        let doctest = BuildDocTestBuilder::new(&scraped_test.text)
1005            .crate_name(&self.opts.crate_name)
1006            .global_crate_attrs(scraped_test.global_crate_attrs.clone())
1007            .edition(edition)
1008            .can_merge_doctests(self.can_merge_doctests)
1009            .test_id(test_id)
1010            .lang_str(&scraped_test.langstr)
1011            .span(scraped_test.span)
1012            .build(dcx);
1013        let is_standalone = !doctest.can_be_merged
1014            || scraped_test.langstr.compile_fail
1015            || scraped_test.langstr.test_harness
1016            || scraped_test.langstr.standalone_crate
1017            || self.rustdoc_options.nocapture
1018            || self.rustdoc_options.test_args.iter().any(|arg| arg == "--show-output");
1019        if is_standalone {
1020            let test_desc = self.generate_test_desc_and_fn(doctest, scraped_test);
1021            self.standalone_tests.push(test_desc);
1022        } else {
1023            self.mergeable_tests
1024                .entry(MergeableTestKey {
1025                    edition,
1026                    global_crate_attrs_hash: {
1027                        let mut hasher = FxHasher::default();
1028                        scraped_test.global_crate_attrs.hash(&mut hasher);
1029                        hasher.finish()
1030                    },
1031                })
1032                .or_default()
1033                .push((doctest, scraped_test));
1034        }
1035    }
1036
1037    fn generate_test_desc_and_fn(
1038        &mut self,
1039        test: DocTestBuilder,
1040        scraped_test: ScrapedDocTest,
1041    ) -> test::TestDescAndFn {
1042        if !scraped_test.langstr.compile_fail {
1043            self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
1044        }
1045
1046        generate_test_desc_and_fn(
1047            test,
1048            scraped_test,
1049            self.opts.clone(),
1050            Arc::clone(&self.rustdoc_options),
1051            self.unused_extern_reports.clone(),
1052        )
1053    }
1054}
1055
1056fn generate_test_desc_and_fn(
1057    test: DocTestBuilder,
1058    scraped_test: ScrapedDocTest,
1059    opts: GlobalTestOptions,
1060    rustdoc_options: Arc<RustdocOptions>,
1061    unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
1062) -> test::TestDescAndFn {
1063    let target_str = rustdoc_options.target.to_string();
1064    let rustdoc_test_options =
1065        IndividualTestOptions::new(&rustdoc_options, &test.test_id, scraped_test.path());
1066
1067    debug!("creating test {}: {}", scraped_test.name, scraped_test.text);
1068    test::TestDescAndFn {
1069        desc: test::TestDesc {
1070            name: test::DynTestName(scraped_test.name.clone()),
1071            ignore: match scraped_test.langstr.ignore {
1072                Ignore::All => true,
1073                Ignore::None => false,
1074                Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
1075            },
1076            ignore_message: None,
1077            source_file: "",
1078            start_line: 0,
1079            start_col: 0,
1080            end_line: 0,
1081            end_col: 0,
1082            // compiler failures are test failures
1083            should_panic: test::ShouldPanic::No,
1084            compile_fail: scraped_test.langstr.compile_fail,
1085            no_run: scraped_test.no_run(&rustdoc_options),
1086            test_type: test::TestType::DocTest,
1087        },
1088        testfn: test::DynTestFn(Box::new(move || {
1089            doctest_run_fn(
1090                rustdoc_test_options,
1091                opts,
1092                test,
1093                scraped_test,
1094                rustdoc_options,
1095                unused_externs,
1096            )
1097        })),
1098    }
1099}
1100
1101fn doctest_run_fn(
1102    test_opts: IndividualTestOptions,
1103    global_opts: GlobalTestOptions,
1104    doctest: DocTestBuilder,
1105    scraped_test: ScrapedDocTest,
1106    rustdoc_options: Arc<RustdocOptions>,
1107    unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
1108) -> Result<(), String> {
1109    let report_unused_externs = |uext| {
1110        unused_externs.lock().unwrap().push(uext);
1111    };
1112    let (wrapped, full_test_line_offset) = doctest.generate_unique_doctest(
1113        &scraped_test.text,
1114        scraped_test.langstr.test_harness,
1115        &global_opts,
1116        Some(&global_opts.crate_name),
1117    );
1118    let runnable_test = RunnableDocTest {
1119        full_test_code: wrapped.to_string(),
1120        full_test_line_offset,
1121        test_opts,
1122        global_opts,
1123        langstr: scraped_test.langstr.clone(),
1124        line: scraped_test.line,
1125        edition: scraped_test.edition(&rustdoc_options),
1126        no_run: scraped_test.no_run(&rustdoc_options),
1127        merged_test_code: None,
1128    };
1129    let (_, res) =
1130        run_test(runnable_test, &rustdoc_options, doctest.supports_color, report_unused_externs);
1131
1132    if let Err(err) = res {
1133        match err {
1134            TestFailure::CompileError => {
1135                eprint!("Couldn't compile the test.");
1136            }
1137            TestFailure::UnexpectedCompilePass => {
1138                eprint!("Test compiled successfully, but it's marked `compile_fail`.");
1139            }
1140            TestFailure::UnexpectedRunPass => {
1141                eprint!("Test executable succeeded, but it's marked `should_panic`.");
1142            }
1143            TestFailure::MissingErrorCodes(codes) => {
1144                eprint!("Some expected error codes were not found: {codes:?}");
1145            }
1146            TestFailure::ExecutionError(err) => {
1147                eprint!("Couldn't run the test: {err}");
1148                if err.kind() == io::ErrorKind::PermissionDenied {
1149                    eprint!(" - maybe your tempdir is mounted with noexec?");
1150                }
1151            }
1152            TestFailure::ExecutionFailure(out) => {
1153                eprintln!("Test executable failed ({reason}).", reason = out.status);
1154
1155                // FIXME(#12309): An unfortunate side-effect of capturing the test
1156                // executable's output is that the relative ordering between the test's
1157                // stdout and stderr is lost. However, this is better than the
1158                // alternative: if the test executable inherited the parent's I/O
1159                // handles the output wouldn't be captured at all, even on success.
1160                //
1161                // The ordering could be preserved if the test process' stderr was
1162                // redirected to stdout, but that functionality does not exist in the
1163                // standard library, so it may not be portable enough.
1164                let stdout = str::from_utf8(&out.stdout).unwrap_or_default();
1165                let stderr = str::from_utf8(&out.stderr).unwrap_or_default();
1166
1167                if !stdout.is_empty() || !stderr.is_empty() {
1168                    eprintln!();
1169
1170                    if !stdout.is_empty() {
1171                        eprintln!("stdout:\n{stdout}");
1172                    }
1173
1174                    if !stderr.is_empty() {
1175                        eprintln!("stderr:\n{stderr}");
1176                    }
1177                }
1178            }
1179        }
1180
1181        panic::resume_unwind(Box::new(()));
1182    }
1183    Ok(())
1184}
1185
1186#[cfg(test)] // used in tests
1187impl DocTestVisitor for Vec<usize> {
1188    fn visit_test(&mut self, _test: String, _config: LangString, rel_line: MdRelLine) {
1189        self.push(1 + rel_line.offset());
1190    }
1191}
1192
1193#[cfg(test)]
1194mod tests;