bootstrap/core/build_steps/
format.rs

1//! Runs rustfmt on the repository.
2
3use std::collections::VecDeque;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::sync::Mutex;
7use std::sync::mpsc::SyncSender;
8
9use build_helper::git::get_git_modified_files;
10use ignore::WalkBuilder;
11
12use crate::core::builder::{Builder, Kind, Step};
13use crate::core::download::maybe_download_rustfmt;
14use crate::utils::build_stamp::BuildStamp;
15use crate::utils::exec::command;
16use crate::utils::helpers::{self, t};
17
18#[must_use]
19enum RustfmtStatus {
20    InProgress,
21    Ok,
22    Failed,
23}
24
25fn rustfmt(
26    src: &Path,
27    rustfmt: &Path,
28    paths: &[PathBuf],
29    check: bool,
30) -> impl FnMut(bool) -> RustfmtStatus + use<> {
31    let mut cmd = Command::new(rustfmt);
32    // Avoid the submodule config paths from coming into play. We only allow a single global config
33    // for the workspace for now.
34    cmd.arg("--config-path").arg(src.canonicalize().unwrap());
35    cmd.arg("--edition").arg("2024");
36    cmd.arg("--unstable-features");
37    cmd.arg("--skip-children");
38    if check {
39        cmd.arg("--check");
40    }
41    cmd.args(paths);
42    let mut cmd = cmd.spawn().expect("running rustfmt");
43    // Poor man's async: return a closure that might wait for rustfmt's completion (depending on
44    // the value of the `block` argument).
45    move |block: bool| -> RustfmtStatus {
46        let status = if !block {
47            match cmd.try_wait() {
48                Ok(Some(status)) => Ok(status),
49                Ok(None) => return RustfmtStatus::InProgress,
50                Err(err) => Err(err),
51            }
52        } else {
53            cmd.wait()
54        };
55        if status.unwrap().success() { RustfmtStatus::Ok } else { RustfmtStatus::Failed }
56    }
57}
58
59fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, BuildStamp)> {
60    let stamp_file = BuildStamp::new(&build.out).with_prefix("rustfmt");
61
62    let rustfmt = build.ensure(InternalRustfmt);
63    let mut cmd = command(rustfmt.as_ref()?);
64    cmd.arg("--version");
65
66    let output = cmd.allow_failure().run_capture(build);
67    if output.is_failure() {
68        return None;
69    }
70    Some((output.stdout(), stamp_file))
71}
72
73/// Return whether the format cache can be reused.
74fn verify_rustfmt_version(build: &Builder<'_>) -> bool {
75    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
76        return false;
77    };
78    stamp_file.add_stamp(version).is_up_to_date()
79}
80
81/// Updates the last rustfmt version used.
82fn update_rustfmt_version(build: &Builder<'_>) {
83    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
84        return;
85    };
86
87    t!(stamp_file.add_stamp(version).write());
88}
89
90/// Returns the Rust files modified between the last merge commit and what is now on the disk.
91/// Does not include removed files.
92///
93/// Returns `None` if all files should be formatted.
94fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, String> {
95    // In CI `get_git_modified_files` returns something different to normal environment.
96    // This shouldn't be called in CI anyway.
97    assert!(!build.config.is_running_on_ci());
98
99    if !verify_rustfmt_version(build) {
100        return Ok(None);
101    }
102
103    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]).map(Some)
104}
105
106/// Rustfmt set via the config, or downloaded from CI, used to format local Rust code.
107///
108/// We never ship this rustfmt, it is designed only for internal usage.
109#[derive(Clone, Debug, PartialEq, Eq, Hash)]
110pub struct InternalRustfmt;
111
112impl Step for InternalRustfmt {
113    type Output = Option<PathBuf>;
114
115    fn run(self, builder: &Builder<'_>) -> Self::Output {
116        // Rustfmt configured through the config
117        if let Some(initial_rustfmt) = &builder.config.external_rustfmt {
118            return Some(initial_rustfmt.clone());
119        }
120        // No rustfmt was configured, try to download it
121        maybe_download_rustfmt(&builder.config, &builder.config.out)
122    }
123}
124
125#[derive(serde_derive::Deserialize)]
126struct RustfmtConfig {
127    ignore: Vec<String>,
128}
129
130// Prints output describing a collection of paths, with lines such as "formatted modified file
131// foo/bar/baz" or "skipped 20 untracked files".
132fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) {
133    let len = paths.len();
134    let adjective =
135        if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() };
136    if len <= 10 {
137        for path in paths {
138            println!("fmt: {verb} {adjective}file {path}");
139        }
140    } else {
141        println!("fmt: {verb} {len} {adjective}files");
142    }
143}
144
145pub fn format(
146    build: &Builder<'_>,
147    rustfmt_path: PathBuf,
148    check: bool,
149    all: bool,
150    paths: &[PathBuf],
151) {
152    if build.kind == Kind::Format && build.top_stage != 0 {
153        eprintln!("ERROR: `x fmt` only supports stage 0.");
154        eprintln!("HELP: Use `x run rustfmt` to run in-tree rustfmt.");
155        crate::exit!(1);
156    }
157
158    if !paths.is_empty() {
159        eprintln!(
160            "fmt error: path arguments are no longer accepted; use `--all` to format everything"
161        );
162        crate::exit!(1);
163    };
164    if build.config.dry_run() {
165        return;
166    }
167
168    // By default, we only check modified files locally to speed up runtime. Exceptions are if
169    // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in
170    // `get_modified_rs_files` letting regressions slip through; we also care about CI time less
171    // since this is still very fast compared to building the compiler.
172    let all = all || build.config.is_running_on_ci();
173
174    let mut builder = ignore::types::TypesBuilder::new();
175    builder.add_defaults();
176    builder.select("rust");
177    let matcher = builder.build().unwrap();
178    let rustfmt_config = build.src.join("rustfmt.toml");
179    if !rustfmt_config.exists() {
180        eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist.");
181        eprintln!("fmt error: This may happen in distributed tarballs.");
182        return;
183    }
184    let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
185    let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
186    let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src);
187    for ignore in rustfmt_config.ignore {
188        if ignore.starts_with('!') {
189            // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`,
190            // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an
191            // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse
192            // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine
193            // explicit whitelisted entries and traversal of unmentioned files, but for now just
194            // forbid such entries.
195            eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry");
196            crate::exit!(1);
197        } else {
198            override_builder.add(&format!("!{ignore}")).expect(&ignore);
199        }
200    }
201    let git_available =
202        helpers::git(None).allow_failure().arg("--version").run_capture(build).is_success();
203
204    let mut adjective = None;
205    if git_available {
206        let in_working_tree = helpers::git(Some(&build.src))
207            .allow_failure()
208            .arg("rev-parse")
209            .arg("--is-inside-work-tree")
210            .run_capture(build)
211            .is_success();
212        if in_working_tree {
213            let untracked_paths_output = helpers::git(Some(&build.src))
214                .arg("status")
215                .arg("--porcelain")
216                .arg("-z")
217                .arg("--untracked-files=normal")
218                .run_capture_stdout(build)
219                .stdout();
220            let untracked_paths: Vec<_> = untracked_paths_output
221                .split_terminator('\0')
222                .filter_map(
223                    |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match
224                )
225                .map(|x| x.to_string())
226                .collect();
227            print_paths("skipped", Some("untracked"), &untracked_paths);
228
229            for untracked_path in untracked_paths {
230                // The leading `/` makes it an exact match against the
231                // repository root, rather than a glob. Without that, if you
232                // have `foo.rs` in the repository root it will also match
233                // against anything like `compiler/rustc_foo/src/foo.rs`,
234                // preventing the latter from being formatted.
235                override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path);
236            }
237            if !all {
238                adjective = Some("modified");
239                match get_modified_rs_files(build) {
240                    Ok(Some(files)) => {
241                        if files.is_empty() {
242                            println!("fmt info: No modified files detected for formatting.");
243                            return;
244                        }
245
246                        for file in files {
247                            override_builder.add(&format!("/{file}")).expect(&file);
248                        }
249                    }
250                    Ok(None) => {
251                        // NOTE: `Ok(None)` signifies that we need to format all files.
252                        // The tricky part here is that if `override_builder` isn't given any white
253                        // list files (i.e. files to be formatted, added without leading `!`), it
254                        // will instead look for *all* files. So, by doing nothing here, we are
255                        // actually making it so we format all files.
256                    }
257                    Err(err) => {
258                        eprintln!("fmt warning: Something went wrong running git commands:");
259                        eprintln!("fmt warning: {err}");
260                        eprintln!("fmt warning: Falling back to formatting all files.");
261                    }
262                }
263            }
264        } else {
265            eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks");
266        }
267    } else {
268        eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks");
269    }
270
271    let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword
272
273    assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
274    let src = build.src.clone();
275    let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
276    let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel();
277
278    // There is a lot of blocking involved in spawning a child process and reading files to format.
279    // Spawn more processes than available concurrency to keep the CPU busy.
280    let max_processes = build.jobs() as usize * 2;
281
282    // Spawn child processes on a separate thread so we can batch entries we have received from
283    // ignore.
284    let thread = std::thread::spawn(move || {
285        let mut result = Ok(());
286
287        let mut children = VecDeque::new();
288        while let Ok(path) = rx.recv() {
289            // Try getting more paths from the channel to amortize the overhead of spawning
290            // processes.
291            let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect();
292
293            let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
294            children.push_back(child);
295
296            // Poll completion before waiting.
297            for i in (0..children.len()).rev() {
298                match children[i](false) {
299                    RustfmtStatus::InProgress => {}
300                    RustfmtStatus::Failed => {
301                        result = Err(());
302                        children.swap_remove_back(i);
303                        break;
304                    }
305                    RustfmtStatus::Ok => {
306                        children.swap_remove_back(i);
307                        break;
308                    }
309                }
310            }
311
312            if children.len() >= max_processes {
313                // Await oldest child.
314                match children.pop_front().unwrap()(true) {
315                    RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
316                    RustfmtStatus::Failed => result = Err(()),
317                }
318            }
319        }
320
321        // Await remaining children.
322        for mut child in children {
323            match child(true) {
324                RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
325                RustfmtStatus::Failed => result = Err(()),
326            }
327        }
328
329        result
330    });
331
332    let formatted_paths = Mutex::new(Vec::new());
333    let formatted_paths_ref = &formatted_paths;
334    walker.run(|| {
335        let tx = tx.clone();
336        Box::new(move |entry| {
337            let cwd = std::env::current_dir();
338            let entry = t!(entry);
339            if entry.file_type().is_some_and(|t| t.is_file()) {
340                formatted_paths_ref.lock().unwrap().push({
341                    // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter
342                    // relative path.
343                    let mut path = entry.clone().into_path();
344                    if let Ok(cwd) = cwd
345                        && let Ok(path2) = path.strip_prefix(cwd)
346                    {
347                        path = path2.to_path_buf();
348                    }
349                    path.display().to_string()
350                });
351                t!(tx.send(entry.into_path()));
352            }
353            ignore::WalkState::Continue
354        })
355    });
356    let mut paths = formatted_paths.into_inner().unwrap();
357    paths.sort();
358    print_paths(if check { "checked" } else { "formatted" }, adjective, &paths);
359
360    drop(tx);
361
362    let result = thread.join().unwrap();
363
364    if result.is_err() {
365        crate::exit!(1);
366    }
367
368    // Update `build/.rustfmt-stamp`, allowing this code to ignore files which have not been changed
369    // since last merge.
370    //
371    // NOTE: Because of the exit above, this is only reachable if formatting / format checking
372    // succeeded. So we are not committing the version if formatting was not good.
373    update_rustfmt_version(build);
374}
bootstrap/core/build_steps/format.rs

bootstrap/core/build_steps/
format.rs
