bootstrap/core/build_steps/
format.rs

Help
1//! Runs rustfmt on the repository.
2
3use std::collections::VecDeque;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::sync::Mutex;
7use std::sync::mpsc::SyncSender;
8
9use build_helper::ci::CiEnv;
10use build_helper::git::get_git_modified_files;
11use ignore::WalkBuilder;
12
13use crate::core::builder::Builder;
14use crate::utils::build_stamp::BuildStamp;
15use crate::utils::exec::command;
16use crate::utils::helpers::{self, t};
17
18#[must_use]
19enum RustfmtStatus {
20    InProgress,
21    Ok,
22    Failed,
23}
24
25fn rustfmt(
26    src: &Path,
27    rustfmt: &Path,
28    paths: &[PathBuf],
29    check: bool,
30) -> impl FnMut(bool) -> RustfmtStatus {
31    let mut cmd = Command::new(rustfmt);
32    // Avoid the submodule config paths from coming into play. We only allow a single global config
33    // for the workspace for now.
34    cmd.arg("--config-path").arg(src.canonicalize().unwrap());
35    cmd.arg("--edition").arg("2021");
36    cmd.arg("--unstable-features");
37    cmd.arg("--skip-children");
38    if check {
39        cmd.arg("--check");
40    }
41    cmd.args(paths);
42    let mut cmd = cmd.spawn().expect("running rustfmt");
43    // Poor man's async: return a closure that might wait for rustfmt's completion (depending on
44    // the value of the `block` argument).
45    move |block: bool| -> RustfmtStatus {
46        let status = if !block {
47            match cmd.try_wait() {
48                Ok(Some(status)) => Ok(status),
49                Ok(None) => return RustfmtStatus::InProgress,
50                Err(err) => Err(err),
51            }
52        } else {
53            cmd.wait()
54        };
55        if status.unwrap().success() { RustfmtStatus::Ok } else { RustfmtStatus::Failed }
56    }
57}
58
59fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, BuildStamp)> {
60    let stamp_file = BuildStamp::new(&build.out).with_prefix("rustfmt");
61
62    let mut cmd = command(build.initial_rustfmt()?);
63    cmd.arg("--version");
64
65    let output = cmd.allow_failure().run_capture(build);
66    if output.is_failure() {
67        return None;
68    }
69    Some((output.stdout(), stamp_file))
70}
71
72/// Return whether the format cache can be reused.
73fn verify_rustfmt_version(build: &Builder<'_>) -> bool {
74    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
75        return false;
76    };
77    stamp_file.add_stamp(version).is_up_to_date()
78}
79
80/// Updates the last rustfmt version used.
81fn update_rustfmt_version(build: &Builder<'_>) {
82    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
83        return;
84    };
85    t!(std::fs::write(stamp_file.path(), version))
86}
87
88/// Returns the Rust files modified between the `merge-base` of HEAD and
89/// rust-lang/master and what is now on the disk. Does not include removed files.
90///
91/// Returns `None` if all files should be formatted.
92fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, String> {
93    if !verify_rustfmt_version(build) {
94        return Ok(None);
95    }
96
97    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"])
98}
99
100#[derive(serde_derive::Deserialize)]
101struct RustfmtConfig {
102    ignore: Vec<String>,
103}
104
105// Prints output describing a collection of paths, with lines such as "formatted modified file
106// foo/bar/baz" or "skipped 20 untracked files".
107fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) {
108    let len = paths.len();
109    let adjective =
110        if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() };
111    if len <= 10 {
112        for path in paths {
113            println!("fmt: {verb} {adjective}file {path}");
114        }
115    } else {
116        println!("fmt: {verb} {len} {adjective}files");
117    }
118    if len > 1000 && !CiEnv::is_ci() {
119        println!("hint: if this number seems too high, try running `git fetch origin master`");
120    }
121}
122
123pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) {
124    if !paths.is_empty() {
125        eprintln!(
126            "fmt error: path arguments are no longer accepted; use `--all` to format everything"
127        );
128        crate::exit!(1);
129    };
130    if build.config.dry_run() {
131        return;
132    }
133
134    // By default, we only check modified files locally to speed up runtime. Exceptions are if
135    // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in
136    // `get_modified_rs_files` letting regressions slip through; we also care about CI time less
137    // since this is still very fast compared to building the compiler.
138    let all = all || CiEnv::is_ci();
139
140    let mut builder = ignore::types::TypesBuilder::new();
141    builder.add_defaults();
142    builder.select("rust");
143    let matcher = builder.build().unwrap();
144    let rustfmt_config = build.src.join("rustfmt.toml");
145    if !rustfmt_config.exists() {
146        eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist.");
147        eprintln!("fmt error: This may happen in distributed tarballs.");
148        return;
149    }
150    let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
151    let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
152    let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src);
153    for ignore in rustfmt_config.ignore {
154        if ignore.starts_with('!') {
155            // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`,
156            // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an
157            // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse
158            // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine
159            // explicit whitelisted entries and traversal of unmentioned files, but for now just
160            // forbid such entries.
161            eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry");
162            crate::exit!(1);
163        } else {
164            override_builder.add(&format!("!{ignore}")).expect(&ignore);
165        }
166    }
167    let git_available =
168        helpers::git(None).allow_failure().arg("--version").run_capture(build).is_success();
169
170    let mut adjective = None;
171    if git_available {
172        let in_working_tree = helpers::git(Some(&build.src))
173            .allow_failure()
174            .arg("rev-parse")
175            .arg("--is-inside-work-tree")
176            .run_capture(build)
177            .is_success();
178        if in_working_tree {
179            let untracked_paths_output = helpers::git(Some(&build.src))
180                .arg("status")
181                .arg("--porcelain")
182                .arg("-z")
183                .arg("--untracked-files=normal")
184                .run_capture_stdout(build)
185                .stdout();
186            let untracked_paths: Vec<_> = untracked_paths_output
187                .split_terminator('\0')
188                .filter_map(
189                    |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match
190                )
191                .map(|x| x.to_string())
192                .collect();
193            print_paths("skipped", Some("untracked"), &untracked_paths);
194
195            for untracked_path in untracked_paths {
196                // The leading `/` makes it an exact match against the
197                // repository root, rather than a glob. Without that, if you
198                // have `foo.rs` in the repository root it will also match
199                // against anything like `compiler/rustc_foo/src/foo.rs`,
200                // preventing the latter from being formatted.
201                override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path);
202            }
203            if !all {
204                adjective = Some("modified");
205                match get_modified_rs_files(build) {
206                    Ok(Some(files)) => {
207                        if files.is_empty() {
208                            println!("fmt info: No modified files detected for formatting.");
209                            return;
210                        }
211
212                        for file in files {
213                            override_builder.add(&format!("/{file}")).expect(&file);
214                        }
215                    }
216                    Ok(None) => {}
217                    Err(err) => {
218                        eprintln!("fmt warning: Something went wrong running git commands:");
219                        eprintln!("fmt warning: {err}");
220                        eprintln!("fmt warning: Falling back to formatting all files.");
221                    }
222                }
223            }
224        } else {
225            eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks");
226        }
227    } else {
228        eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks");
229    }
230
231    let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword
232
233    let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| {
234        eprintln!("fmt error: `x fmt` is not supported on this channel");
235        crate::exit!(1);
236    });
237    assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
238    let src = build.src.clone();
239    let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
240    let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel();
241
242    // There is a lot of blocking involved in spawning a child process and reading files to format.
243    // Spawn more processes than available concurrency to keep the CPU busy.
244    let max_processes = build.jobs() as usize * 2;
245
246    // Spawn child processes on a separate thread so we can batch entries we have received from
247    // ignore.
248    let thread = std::thread::spawn(move || {
249        let mut result = Ok(());
250
251        let mut children = VecDeque::new();
252        while let Ok(path) = rx.recv() {
253            // Try getting more paths from the channel to amortize the overhead of spawning
254            // processes.
255            let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect();
256
257            let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
258            children.push_back(child);
259
260            // Poll completion before waiting.
261            for i in (0..children.len()).rev() {
262                match children[i](false) {
263                    RustfmtStatus::InProgress => {}
264                    RustfmtStatus::Failed => {
265                        result = Err(());
266                        children.swap_remove_back(i);
267                        break;
268                    }
269                    RustfmtStatus::Ok => {
270                        children.swap_remove_back(i);
271                        break;
272                    }
273                }
274            }
275
276            if children.len() >= max_processes {
277                // Await oldest child.
278                match children.pop_front().unwrap()(true) {
279                    RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
280                    RustfmtStatus::Failed => result = Err(()),
281                }
282            }
283        }
284
285        // Await remaining children.
286        for mut child in children {
287            match child(true) {
288                RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
289                RustfmtStatus::Failed => result = Err(()),
290            }
291        }
292
293        result
294    });
295
296    let formatted_paths = Mutex::new(Vec::new());
297    let formatted_paths_ref = &formatted_paths;
298    walker.run(|| {
299        let tx = tx.clone();
300        Box::new(move |entry| {
301            let cwd = std::env::current_dir();
302            let entry = t!(entry);
303            if entry.file_type().is_some_and(|t| t.is_file()) {
304                formatted_paths_ref.lock().unwrap().push({
305                    // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter
306                    // relative path.
307                    let mut path = entry.clone().into_path();
308                    if let Ok(cwd) = cwd {
309                        if let Ok(path2) = path.strip_prefix(cwd) {
310                            path = path2.to_path_buf();
311                        }
312                    }
313                    path.display().to_string()
314                });
315                t!(tx.send(entry.into_path()));
316            }
317            ignore::WalkState::Continue
318        })
319    });
320    let mut paths = formatted_paths.into_inner().unwrap();
321    paths.sort();
322    print_paths(if check { "checked" } else { "formatted" }, adjective, &paths);
323
324    drop(tx);
325
326    let result = thread.join().unwrap();
327
328    if result.is_err() {
329        crate::exit!(1);
330    }
331
332    if !check {
333        update_rustfmt_version(build);
334    }
335}
bootstrap/core/build_steps/format.rs

bootstrap/core/build_steps/
format.rs
