bootstrap/core/build_steps/
format.rs

Help
1//! Runs rustfmt on the repository.
2
3use std::collections::VecDeque;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::sync::Mutex;
7use std::sync::mpsc::SyncSender;
8
9use build_helper::git::get_git_modified_files;
10use ignore::WalkBuilder;
11
12use crate::core::builder::Builder;
13use crate::utils::build_stamp::BuildStamp;
14use crate::utils::exec::command;
15use crate::utils::helpers::{self, t};
16
17#[must_use]
18enum RustfmtStatus {
19    InProgress,
20    Ok,
21    Failed,
22}
23
24fn rustfmt(
25    src: &Path,
26    rustfmt: &Path,
27    paths: &[PathBuf],
28    check: bool,
29) -> impl FnMut(bool) -> RustfmtStatus + use<> {
30    let mut cmd = Command::new(rustfmt);
31    // Avoid the submodule config paths from coming into play. We only allow a single global config
32    // for the workspace for now.
33    cmd.arg("--config-path").arg(src.canonicalize().unwrap());
34    cmd.arg("--edition").arg("2021");
35    cmd.arg("--unstable-features");
36    cmd.arg("--skip-children");
37    if check {
38        cmd.arg("--check");
39    }
40    cmd.args(paths);
41    let mut cmd = cmd.spawn().expect("running rustfmt");
42    // Poor man's async: return a closure that might wait for rustfmt's completion (depending on
43    // the value of the `block` argument).
44    move |block: bool| -> RustfmtStatus {
45        let status = if !block {
46            match cmd.try_wait() {
47                Ok(Some(status)) => Ok(status),
48                Ok(None) => return RustfmtStatus::InProgress,
49                Err(err) => Err(err),
50            }
51        } else {
52            cmd.wait()
53        };
54        if status.unwrap().success() { RustfmtStatus::Ok } else { RustfmtStatus::Failed }
55    }
56}
57
58fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, BuildStamp)> {
59    let stamp_file = BuildStamp::new(&build.out).with_prefix("rustfmt");
60
61    let mut cmd = command(build.initial_rustfmt()?);
62    cmd.arg("--version");
63
64    let output = cmd.allow_failure().run_capture(build);
65    if output.is_failure() {
66        return None;
67    }
68    Some((output.stdout(), stamp_file))
69}
70
71/// Return whether the format cache can be reused.
72fn verify_rustfmt_version(build: &Builder<'_>) -> bool {
73    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
74        return false;
75    };
76    stamp_file.add_stamp(version).is_up_to_date()
77}
78
79/// Updates the last rustfmt version used.
80fn update_rustfmt_version(build: &Builder<'_>) {
81    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
82        return;
83    };
84    t!(std::fs::write(stamp_file.path(), version))
85}
86
87/// Returns the Rust files modified between the `merge-base` of HEAD and
88/// rust-lang/master and what is now on the disk. Does not include removed files.
89///
90/// Returns `None` if all files should be formatted.
91fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, String> {
92    if !verify_rustfmt_version(build) {
93        return Ok(None);
94    }
95
96    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]).map(Some)
97}
98
99#[derive(serde_derive::Deserialize)]
100struct RustfmtConfig {
101    ignore: Vec<String>,
102}
103
104// Prints output describing a collection of paths, with lines such as "formatted modified file
105// foo/bar/baz" or "skipped 20 untracked files".
106fn print_paths(build: &Builder<'_>, verb: &str, adjective: Option<&str>, paths: &[String]) {
107    let len = paths.len();
108    let adjective =
109        if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() };
110    if len <= 10 {
111        for path in paths {
112            println!("fmt: {verb} {adjective}file {path}");
113        }
114    } else {
115        println!("fmt: {verb} {len} {adjective}files");
116    }
117    if len > 1000 && !build.config.is_running_on_ci {
118        println!("hint: if this number seems too high, try running `git fetch origin master`");
119    }
120}
121
122pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) {
123    if !paths.is_empty() {
124        eprintln!(
125            "fmt error: path arguments are no longer accepted; use `--all` to format everything"
126        );
127        crate::exit!(1);
128    };
129    if build.config.dry_run() {
130        return;
131    }
132
133    // By default, we only check modified files locally to speed up runtime. Exceptions are if
134    // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in
135    // `get_modified_rs_files` letting regressions slip through; we also care about CI time less
136    // since this is still very fast compared to building the compiler.
137    let all = all || build.config.is_running_on_ci;
138
139    let mut builder = ignore::types::TypesBuilder::new();
140    builder.add_defaults();
141    builder.select("rust");
142    let matcher = builder.build().unwrap();
143    let rustfmt_config = build.src.join("rustfmt.toml");
144    if !rustfmt_config.exists() {
145        eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist.");
146        eprintln!("fmt error: This may happen in distributed tarballs.");
147        return;
148    }
149    let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
150    let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
151    let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src);
152    for ignore in rustfmt_config.ignore {
153        if ignore.starts_with('!') {
154            // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`,
155            // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an
156            // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse
157            // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine
158            // explicit whitelisted entries and traversal of unmentioned files, but for now just
159            // forbid such entries.
160            eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry");
161            crate::exit!(1);
162        } else {
163            override_builder.add(&format!("!{ignore}")).expect(&ignore);
164        }
165    }
166    let git_available =
167        helpers::git(None).allow_failure().arg("--version").run_capture(build).is_success();
168
169    let mut adjective = None;
170    if git_available {
171        let in_working_tree = helpers::git(Some(&build.src))
172            .allow_failure()
173            .arg("rev-parse")
174            .arg("--is-inside-work-tree")
175            .run_capture(build)
176            .is_success();
177        if in_working_tree {
178            let untracked_paths_output = helpers::git(Some(&build.src))
179                .arg("status")
180                .arg("--porcelain")
181                .arg("-z")
182                .arg("--untracked-files=normal")
183                .run_capture_stdout(build)
184                .stdout();
185            let untracked_paths: Vec<_> = untracked_paths_output
186                .split_terminator('\0')
187                .filter_map(
188                    |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match
189                )
190                .map(|x| x.to_string())
191                .collect();
192            print_paths(build, "skipped", Some("untracked"), &untracked_paths);
193
194            for untracked_path in untracked_paths {
195                // The leading `/` makes it an exact match against the
196                // repository root, rather than a glob. Without that, if you
197                // have `foo.rs` in the repository root it will also match
198                // against anything like `compiler/rustc_foo/src/foo.rs`,
199                // preventing the latter from being formatted.
200                override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path);
201            }
202            if !all {
203                adjective = Some("modified");
204                match get_modified_rs_files(build) {
205                    Ok(Some(files)) => {
206                        if files.is_empty() {
207                            println!("fmt info: No modified files detected for formatting.");
208                            return;
209                        }
210
211                        for file in files {
212                            override_builder.add(&format!("/{file}")).expect(&file);
213                        }
214                    }
215                    Ok(None) => {}
216                    Err(err) => {
217                        eprintln!("fmt warning: Something went wrong running git commands:");
218                        eprintln!("fmt warning: {err}");
219                        eprintln!("fmt warning: Falling back to formatting all files.");
220                    }
221                }
222            }
223        } else {
224            eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks");
225        }
226    } else {
227        eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks");
228    }
229
230    let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword
231
232    let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| {
233        eprintln!("fmt error: `x fmt` is not supported on this channel");
234        crate::exit!(1);
235    });
236    assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
237    let src = build.src.clone();
238    let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
239    let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel();
240
241    // There is a lot of blocking involved in spawning a child process and reading files to format.
242    // Spawn more processes than available concurrency to keep the CPU busy.
243    let max_processes = build.jobs() as usize * 2;
244
245    // Spawn child processes on a separate thread so we can batch entries we have received from
246    // ignore.
247    let thread = std::thread::spawn(move || {
248        let mut result = Ok(());
249
250        let mut children = VecDeque::new();
251        while let Ok(path) = rx.recv() {
252            // Try getting more paths from the channel to amortize the overhead of spawning
253            // processes.
254            let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect();
255
256            let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
257            children.push_back(child);
258
259            // Poll completion before waiting.
260            for i in (0..children.len()).rev() {
261                match children[i](false) {
262                    RustfmtStatus::InProgress => {}
263                    RustfmtStatus::Failed => {
264                        result = Err(());
265                        children.swap_remove_back(i);
266                        break;
267                    }
268                    RustfmtStatus::Ok => {
269                        children.swap_remove_back(i);
270                        break;
271                    }
272                }
273            }
274
275            if children.len() >= max_processes {
276                // Await oldest child.
277                match children.pop_front().unwrap()(true) {
278                    RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
279                    RustfmtStatus::Failed => result = Err(()),
280                }
281            }
282        }
283
284        // Await remaining children.
285        for mut child in children {
286            match child(true) {
287                RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
288                RustfmtStatus::Failed => result = Err(()),
289            }
290        }
291
292        result
293    });
294
295    let formatted_paths = Mutex::new(Vec::new());
296    let formatted_paths_ref = &formatted_paths;
297    walker.run(|| {
298        let tx = tx.clone();
299        Box::new(move |entry| {
300            let cwd = std::env::current_dir();
301            let entry = t!(entry);
302            if entry.file_type().is_some_and(|t| t.is_file()) {
303                formatted_paths_ref.lock().unwrap().push({
304                    // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter
305                    // relative path.
306                    let mut path = entry.clone().into_path();
307                    if let Ok(cwd) = cwd {
308                        if let Ok(path2) = path.strip_prefix(cwd) {
309                            path = path2.to_path_buf();
310                        }
311                    }
312                    path.display().to_string()
313                });
314                t!(tx.send(entry.into_path()));
315            }
316            ignore::WalkState::Continue
317        })
318    });
319    let mut paths = formatted_paths.into_inner().unwrap();
320    paths.sort();
321    print_paths(build, if check { "checked" } else { "formatted" }, adjective, &paths);
322
323    drop(tx);
324
325    let result = thread.join().unwrap();
326
327    if result.is_err() {
328        crate::exit!(1);
329    }
330
331    if !check {
332        update_rustfmt_version(build);
333    }
334}
bootstrap/core/build_steps/format.rs

bootstrap/core/build_steps/
format.rs
