bootstrap/core/build_steps/
format.rs

1//! Runs rustfmt on the repository.
2
3use std::collections::VecDeque;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::sync::Mutex;
7use std::sync::mpsc::SyncSender;
8
9use build_helper::git::get_git_modified_files;
10use ignore::WalkBuilder;
11
12use crate::core::builder::{Builder, Kind, Step};
13use crate::core::download::maybe_download_rustfmt;
14use crate::utils::build_stamp::BuildStamp;
15use crate::utils::exec::command;
16use crate::utils::helpers::{self, t};
17
18#[must_use]
19enum RustfmtStatus {
20    InProgress,
21    Ok,
22    Failed,
23}
24
25fn rustfmt(
26    src: &Path,
27    rustfmt: &Path,
28    paths: &[PathBuf],
29    check: bool,
30) -> impl FnMut(bool) -> RustfmtStatus + use<> {
31    let mut cmd = Command::new(rustfmt);
32    // Avoid the submodule config paths from coming into play. We only allow a single global config
33    // for the workspace for now.
34    cmd.arg("--config-path").arg(src.canonicalize().unwrap());
35    cmd.arg("--edition").arg("2024");
36    cmd.arg("--unstable-features");
37    cmd.arg("--skip-children");
38    if check {
39        cmd.arg("--check");
40    }
41    cmd.args(paths);
42    let mut cmd = cmd.spawn().expect("running rustfmt");
43    // Poor man's async: return a closure that might wait for rustfmt's completion (depending on
44    // the value of the `block` argument).
45    move |block: bool| -> RustfmtStatus {
46        let status = if !block {
47            match cmd.try_wait() {
48                Ok(Some(status)) => Ok(status),
49                Ok(None) => return RustfmtStatus::InProgress,
50                Err(err) => Err(err),
51            }
52        } else {
53            cmd.wait()
54        };
55        if status.unwrap().success() { RustfmtStatus::Ok } else { RustfmtStatus::Failed }
56    }
57}
58
59fn get_rustfmt_version(builder: &Builder<'_>) -> Option<(String, BuildStamp)> {
60    let stamp_file = BuildStamp::new(&builder.out).with_prefix("rustfmt");
61
62    let rustfmt = builder.ensure(InternalRustfmt);
63    let mut cmd = command(rustfmt.as_ref()?);
64    cmd.arg("--version");
65
66    let output = cmd.allow_failure().run_capture(builder);
67    if output.is_failure() {
68        return None;
69    }
70    Some((output.stdout(), stamp_file))
71}
72
73/// Return whether the format cache can be reused.
74fn verify_rustfmt_version(builder: &Builder<'_>) -> bool {
75    let Some((version, stamp_file)) = get_rustfmt_version(builder) else {
76        return false;
77    };
78    stamp_file.add_stamp(version).is_up_to_date()
79}
80
81/// Updates the last rustfmt version used.
82fn update_rustfmt_version(builder: &Builder<'_>) {
83    let Some((version, stamp_file)) = get_rustfmt_version(builder) else {
84        return;
85    };
86
87    t!(stamp_file.add_stamp(version).write());
88}
89
90/// Returns the Rust files modified between the last merge commit and what is now on the disk.
91/// Does not include removed files.
92///
93/// Returns `None` if all files should be formatted.
94fn get_modified_rs_files(builder: &Builder<'_>) -> Result<Option<Vec<String>>, String> {
95    // In CI `get_git_modified_files` returns something different to normal environment.
96    // This shouldn't be called in CI anyway.
97    assert!(!builder.config.is_running_on_ci());
98
99    if !verify_rustfmt_version(builder) {
100        return Ok(None);
101    }
102
103    get_git_modified_files(&builder.config.git_config(), Some(&builder.config.src), &["rs"])
104        .map(Some)
105}
106
107/// Rustfmt set via the config, or downloaded from CI, used to format local Rust code.
108///
109/// We never ship this rustfmt, it is designed only for internal usage.
110#[derive(Clone, Debug, PartialEq, Eq, Hash)]
111pub struct InternalRustfmt;
112
113impl Step for InternalRustfmt {
114    type Output = Option<PathBuf>;
115
116    fn run(self, builder: &Builder<'_>) -> Self::Output {
117        // Rustfmt configured through the config
118        if let Some(initial_rustfmt) = &builder.config.external_rustfmt {
119            return Some(initial_rustfmt.clone());
120        }
121        // No rustfmt was configured, try to download it
122        maybe_download_rustfmt(&builder.config, &builder.config.out)
123    }
124}
125
126#[derive(serde_derive::Deserialize)]
127struct RustfmtConfig {
128    ignore: Vec<String>,
129}
130
131// Prints output describing a collection of paths, with lines such as "formatted modified file
132// foo/bar/baz" or "skipped 20 untracked files".
133fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) {
134    let len = paths.len();
135    let adjective =
136        if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() };
137    if len <= 10 {
138        for path in paths {
139            println!("fmt: {verb} {adjective}file {path}");
140        }
141    } else {
142        println!("fmt: {verb} {len} {adjective}files");
143    }
144}
145
146pub fn format(
147    builder: &Builder<'_>,
148    rustfmt_path: PathBuf,
149    check: bool,
150    all: bool,
151    paths: &[PathBuf],
152) {
153    if builder.kind == Kind::Format && builder.top_stage != 0 {
154        eprintln!("ERROR: `x fmt` only supports stage 0.");
155        eprintln!("HELP: Use `x run rustfmt` to run in-tree rustfmt.");
156        helpers::exit_process(1);
157    }
158
159    if !paths.is_empty() {
160        eprintln!(
161            "fmt error: path arguments are no longer accepted; use `--all` to format everything"
162        );
163        helpers::exit_process(1);
164    };
165    if builder.config.dry_run() {
166        return;
167    }
168
169    // By default, we only check modified files locally to speed up runtime. Exceptions are if
170    // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in
171    // `get_modified_rs_files` letting regressions slip through; we also care about CI time less
172    // since this is still very fast compared to building the compiler.
173    let all = all || builder.config.is_running_on_ci();
174
175    let matcher = {
176        let mut types = ignore::types::TypesBuilder::new();
177        types.add_defaults();
178        types.select("rust");
179        types.build().unwrap()
180    };
181    let rustfmt_config = builder.src.join("rustfmt.toml");
182    if !rustfmt_config.exists() {
183        eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist.");
184        eprintln!("fmt error: This may happen in distributed tarballs.");
185        return;
186    }
187    let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
188    let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
189    let mut override_builder = ignore::overrides::OverrideBuilder::new(&builder.src);
190    for ignore in rustfmt_config.ignore {
191        if ignore.starts_with('!') {
192            // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`,
193            // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an
194            // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse
195            // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine
196            // explicit whitelisted entries and traversal of unmentioned files, but for now just
197            // forbid such entries.
198            eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry");
199            helpers::exit_process(1);
200        } else {
201            override_builder.add(&format!("!{ignore}")).expect(&ignore);
202        }
203    }
204    let git_available =
205        helpers::git(None).allow_failure().arg("--version").run_capture(builder).is_success();
206
207    let mut adjective = None;
208    if git_available {
209        let in_working_tree = helpers::git(Some(&builder.src))
210            .allow_failure()
211            .arg("rev-parse")
212            .arg("--is-inside-work-tree")
213            .run_capture(builder)
214            .is_success();
215        if in_working_tree {
216            let untracked_paths_output = helpers::git(Some(&builder.src))
217                .arg("status")
218                .arg("--porcelain")
219                .arg("-z")
220                .arg("--untracked-files=normal")
221                .run_capture_stdout(builder)
222                .stdout();
223            let untracked_paths: Vec<_> = untracked_paths_output
224                .split_terminator('\0')
225                .filter_map(
226                    |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match
227                )
228                .map(|x| x.to_string())
229                .collect();
230            print_paths("skipped", Some("untracked"), &untracked_paths);
231
232            for untracked_path in untracked_paths {
233                // The leading `/` makes it an exact match against the
234                // repository root, rather than a glob. Without that, if you
235                // have `foo.rs` in the repository root it will also match
236                // against anything like `compiler/rustc_foo/src/foo.rs`,
237                // preventing the latter from being formatted.
238                override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path);
239            }
240            if !all {
241                adjective = Some("modified");
242                match get_modified_rs_files(builder) {
243                    Ok(Some(files)) => {
244                        if files.is_empty() {
245                            println!("fmt info: No modified files detected for formatting.");
246                            return;
247                        }
248
249                        for file in files {
250                            override_builder.add(&format!("/{file}")).expect(&file);
251                        }
252                    }
253                    Ok(None) => {
254                        // NOTE: `Ok(None)` signifies that we need to format all files.
255                        // The tricky part here is that if `override_builder` isn't given any white
256                        // list files (i.e. files to be formatted, added without leading `!`), it
257                        // will instead look for *all* files. So, by doing nothing here, we are
258                        // actually making it so we format all files.
259                    }
260                    Err(err) => {
261                        eprintln!("fmt warning: Something went wrong running git commands:");
262                        eprintln!("fmt warning: {err}");
263                        eprintln!("fmt warning: Falling back to formatting all files.");
264                    }
265                }
266            }
267        } else {
268            eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks");
269        }
270    } else {
271        eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks");
272    }
273
274    let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword
275
276    assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
277    let src = builder.src.clone();
278    let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
279    let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel();
280
281    // There is a lot of blocking involved in spawning a child process and reading files to format.
282    // Spawn more processes than available concurrency to keep the CPU busy.
283    let max_processes = builder.jobs() as usize * 2;
284
285    // Spawn child processes on a separate thread so we can batch entries we have received from
286    // ignore.
287    let thread = std::thread::spawn(move || {
288        let mut result = Ok(());
289
290        let mut children = VecDeque::new();
291        while let Ok(path) = rx.recv() {
292            // Try getting more paths from the channel to amortize the overhead of spawning
293            // processes.
294            let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect();
295
296            let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
297            children.push_back(child);
298
299            // Poll completion before waiting.
300            for i in (0..children.len()).rev() {
301                match children[i](false) {
302                    RustfmtStatus::InProgress => {}
303                    RustfmtStatus::Failed => {
304                        result = Err(());
305                        children.swap_remove_back(i);
306                        break;
307                    }
308                    RustfmtStatus::Ok => {
309                        children.swap_remove_back(i);
310                        break;
311                    }
312                }
313            }
314
315            if children.len() >= max_processes {
316                // Await oldest child.
317                match children.pop_front().unwrap()(true) {
318                    RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
319                    RustfmtStatus::Failed => result = Err(()),
320                }
321            }
322        }
323
324        // Await remaining children.
325        for mut child in children {
326            match child(true) {
327                RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
328                RustfmtStatus::Failed => result = Err(()),
329            }
330        }
331
332        result
333    });
334
335    let formatted_paths = Mutex::new(Vec::new());
336    let formatted_paths_ref = &formatted_paths;
337    walker.run(|| {
338        let tx = tx.clone();
339        Box::new(move |entry| {
340            let cwd = std::env::current_dir();
341            let entry = t!(entry);
342            if entry.file_type().is_some_and(|t| t.is_file()) {
343                formatted_paths_ref.lock().unwrap().push({
344                    // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter
345                    // relative path.
346                    let mut path = entry.clone().into_path();
347                    if let Ok(cwd) = cwd
348                        && let Ok(path2) = path.strip_prefix(cwd)
349                    {
350                        path = path2.to_path_buf();
351                    }
352                    path.display().to_string()
353                });
354                t!(tx.send(entry.into_path()));
355            }
356            ignore::WalkState::Continue
357        })
358    });
359    let mut paths = formatted_paths.into_inner().unwrap();
360    paths.sort();
361    print_paths(if check { "checked" } else { "formatted" }, adjective, &paths);
362
363    drop(tx);
364
365    let result = thread.join().unwrap();
366
367    if result.is_err() {
368        helpers::exit_process(1);
369    }
370
371    // Update `build/.rustfmt-stamp`, allowing this code to ignore files which have not been changed
372    // since last merge.
373    //
374    // NOTE: Because of the exit above, this is only reachable if formatting / format checking
375    // succeeded. So we are not committing the version if formatting was not good.
376    update_rustfmt_version(builder);
377}
bootstrap/core/build_steps/format.rs

bootstrap/core/build_steps/
format.rs
