cargo/util/
rustc.rs

Help
1use std::collections::hash_map::HashMap;
2use std::env;
3use std::hash::{Hash, Hasher};
4use std::path::{Path, PathBuf};
5use std::sync::Mutex;
6
7use anyhow::Context as _;
8use cargo_util::{paths, ProcessBuilder, ProcessError};
9use filetime::FileTime;
10use serde::{Deserialize, Serialize};
11use tracing::{debug, info, warn};
12
13use crate::core::compiler::apply_env_config;
14use crate::util::interning::InternedString;
15use crate::util::{CargoResult, GlobalContext, StableHasher};
16
17/// Information on the `rustc` executable
18#[derive(Debug)]
19pub struct Rustc {
20    /// The location of the exe
21    pub path: PathBuf,
22    /// An optional program that will be passed the path of the rust exe as its first argument, and
23    /// rustc args following this.
24    pub wrapper: Option<PathBuf>,
25    /// An optional wrapper to be used in addition to `rustc.wrapper` for workspace crates
26    pub workspace_wrapper: Option<PathBuf>,
27    /// Verbose version information (the output of `rustc -vV`)
28    pub verbose_version: String,
29    /// The rustc version (`1.23.4-beta.2`), this comes from `verbose_version`.
30    pub version: semver::Version,
31    /// The host triple (arch-platform-OS), this comes from `verbose_version`.
32    pub host: InternedString,
33    /// The rustc full commit hash, this comes from `verbose_version`.
34    pub commit_hash: Option<String>,
35    cache: Mutex<Cache>,
36}
37
38impl Rustc {
39    /// Runs the compiler at `path` to learn various pieces of information about
40    /// it, with an optional wrapper.
41    ///
42    /// If successful this function returns a description of the compiler along
43    /// with a list of its capabilities.
44    #[tracing::instrument(skip(gctx))]
45    pub fn new(
46        path: PathBuf,
47        wrapper: Option<PathBuf>,
48        workspace_wrapper: Option<PathBuf>,
49        rustup_rustc: &Path,
50        cache_location: Option<PathBuf>,
51        gctx: &GlobalContext,
52    ) -> CargoResult<Rustc> {
53        let mut cache = Cache::load(
54            wrapper.as_deref(),
55            workspace_wrapper.as_deref(),
56            &path,
57            rustup_rustc,
58            cache_location,
59            gctx,
60        );
61
62        let mut cmd = ProcessBuilder::new(&path)
63            .wrapped(workspace_wrapper.as_ref())
64            .wrapped(wrapper.as_deref());
65        apply_env_config(gctx, &mut cmd)?;
66        cmd.arg("-vV");
67        let verbose_version = cache.cached_output(&cmd, 0)?.0;
68
69        let extract = |field: &str| -> CargoResult<&str> {
70            verbose_version
71                .lines()
72                .find_map(|l| l.strip_prefix(field))
73                .ok_or_else(|| {
74                    anyhow::format_err!(
75                        "`rustc -vV` didn't have a line for `{}`, got:\n{}",
76                        field.trim(),
77                        verbose_version
78                    )
79                })
80        };
81
82        let host = InternedString::new(extract("host: ")?);
83        let version = semver::Version::parse(extract("release: ")?).with_context(|| {
84            format!(
85                "rustc version does not appear to be a valid semver version, from:\n{}",
86                verbose_version
87            )
88        })?;
89        let commit_hash = extract("commit-hash: ").ok().map(|hash| {
90            // Possible commit-hash values from rustc are SHA hex string and "unknown". See:
91            // * https://github.com/rust-lang/rust/blob/531cb83fc/src/bootstrap/src/utils/channel.rs#L73
92            // * https://github.com/rust-lang/rust/blob/531cb83fc/compiler/rustc_driver_impl/src/lib.rs#L911-L913
93            #[cfg(debug_assertions)]
94            if hash != "unknown" {
95                debug_assert!(
96                    hash.chars().all(|ch| ch.is_ascii_hexdigit()),
97                    "commit hash must be a hex string, got: {hash:?}"
98                );
99                debug_assert!(
100                    hash.len() == 40 || hash.len() == 64,
101                    "hex string must be generated from sha1 or sha256 (i.e., it must be 40 or 64 characters long)\ngot: {hash:?}"
102                );
103            }
104            hash.to_string()
105        });
106
107        Ok(Rustc {
108            path,
109            wrapper,
110            workspace_wrapper,
111            verbose_version,
112            version,
113            host,
114            commit_hash,
115            cache: Mutex::new(cache),
116        })
117    }
118
119    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.
120    pub fn process(&self) -> ProcessBuilder {
121        let mut cmd = ProcessBuilder::new(self.path.as_path()).wrapped(self.wrapper.as_ref());
122        cmd.retry_with_argfile(true);
123        cmd
124    }
125
126    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.
127    pub fn workspace_process(&self) -> ProcessBuilder {
128        let mut cmd = ProcessBuilder::new(self.path.as_path())
129            .wrapped(self.workspace_wrapper.as_ref())
130            .wrapped(self.wrapper.as_ref());
131        cmd.retry_with_argfile(true);
132        cmd
133    }
134
135    pub fn process_no_wrapper(&self) -> ProcessBuilder {
136        let mut cmd = ProcessBuilder::new(&self.path);
137        cmd.retry_with_argfile(true);
138        cmd
139    }
140
141    /// Gets the output for the given command.
142    ///
143    /// This will return the cached value if available, otherwise it will run
144    /// the command and cache the output.
145    ///
146    /// `extra_fingerprint` is extra data to include in the cache fingerprint.
147    /// Use this if there is other information about the environment that may
148    /// affect the output that is not part of `cmd`.
149    ///
150    /// Returns a tuple of strings `(stdout, stderr)`.
151    pub fn cached_output(
152        &self,
153        cmd: &ProcessBuilder,
154        extra_fingerprint: u64,
155    ) -> CargoResult<(String, String)> {
156        self.cache
157            .lock()
158            .unwrap()
159            .cached_output(cmd, extra_fingerprint)
160    }
161}
162
163/// It is a well known fact that `rustc` is not the fastest compiler in the
164/// world.  What is less known is that even `rustc --version --verbose` takes
165/// about a hundred milliseconds! Because we need compiler version info even
166/// for no-op builds, we cache it here, based on compiler's mtime and rustup's
167/// current toolchain.
168///
169/// <https://github.com/rust-lang/cargo/issues/5315>
170/// <https://github.com/rust-lang/rust/issues/49761>
171#[derive(Debug)]
172struct Cache {
173    cache_location: Option<PathBuf>,
174    dirty: bool,
175    data: CacheData,
176}
177
178#[derive(Serialize, Deserialize, Debug, Default)]
179struct CacheData {
180    rustc_fingerprint: u64,
181    outputs: HashMap<u64, Output>,
182    successes: HashMap<u64, bool>,
183}
184
185#[derive(Serialize, Deserialize, Debug)]
186struct Output {
187    success: bool,
188    status: String,
189    code: Option<i32>,
190    stdout: String,
191    stderr: String,
192}
193
194impl Cache {
195    fn load(
196        wrapper: Option<&Path>,
197        workspace_wrapper: Option<&Path>,
198        rustc: &Path,
199        rustup_rustc: &Path,
200        cache_location: Option<PathBuf>,
201        gctx: &GlobalContext,
202    ) -> Cache {
203        match (
204            cache_location,
205            rustc_fingerprint(wrapper, workspace_wrapper, rustc, rustup_rustc, gctx),
206        ) {
207            (Some(cache_location), Ok(rustc_fingerprint)) => {
208                let empty = CacheData {
209                    rustc_fingerprint,
210                    outputs: HashMap::new(),
211                    successes: HashMap::new(),
212                };
213                let mut dirty = true;
214                let data = match read(&cache_location) {
215                    Ok(data) => {
216                        if data.rustc_fingerprint == rustc_fingerprint {
217                            debug!("reusing existing rustc info cache");
218                            dirty = false;
219                            data
220                        } else {
221                            debug!("different compiler, creating new rustc info cache");
222                            empty
223                        }
224                    }
225                    Err(e) => {
226                        debug!("failed to read rustc info cache: {}", e);
227                        empty
228                    }
229                };
230                return Cache {
231                    cache_location: Some(cache_location),
232                    dirty,
233                    data,
234                };
235
236                fn read(path: &Path) -> CargoResult<CacheData> {
237                    let json = paths::read(path)?;
238                    Ok(serde_json::from_str(&json)?)
239                }
240            }
241            (_, fingerprint) => {
242                if let Err(e) = fingerprint {
243                    warn!("failed to calculate rustc fingerprint: {}", e);
244                }
245                debug!("rustc info cache disabled");
246                Cache {
247                    cache_location: None,
248                    dirty: false,
249                    data: CacheData::default(),
250                }
251            }
252        }
253    }
254
255    fn cached_output(
256        &mut self,
257        cmd: &ProcessBuilder,
258        extra_fingerprint: u64,
259    ) -> CargoResult<(String, String)> {
260        let key = process_fingerprint(cmd, extra_fingerprint);
261        if self.data.outputs.contains_key(&key) {
262            debug!("rustc info cache hit");
263        } else {
264            debug!("rustc info cache miss");
265            debug!("running {}", cmd);
266            let output = cmd.output()?;
267            let stdout = String::from_utf8(output.stdout)
268                .map_err(|e| anyhow::anyhow!("{}: {:?}", e, e.as_bytes()))
269                .with_context(|| format!("`{}` didn't return utf8 output", cmd))?;
270            let stderr = String::from_utf8(output.stderr)
271                .map_err(|e| anyhow::anyhow!("{}: {:?}", e, e.as_bytes()))
272                .with_context(|| format!("`{}` didn't return utf8 output", cmd))?;
273            self.data.outputs.insert(
274                key,
275                Output {
276                    success: output.status.success(),
277                    status: if output.status.success() {
278                        String::new()
279                    } else {
280                        cargo_util::exit_status_to_string(output.status)
281                    },
282                    code: output.status.code(),
283                    stdout,
284                    stderr,
285                },
286            );
287            self.dirty = true;
288        }
289        let output = &self.data.outputs[&key];
290        if output.success {
291            Ok((output.stdout.clone(), output.stderr.clone()))
292        } else {
293            Err(ProcessError::new_raw(
294                &format!("process didn't exit successfully: {}", cmd),
295                output.code,
296                &output.status,
297                Some(output.stdout.as_ref()),
298                Some(output.stderr.as_ref()),
299            )
300            .into())
301        }
302    }
303}
304
305impl Drop for Cache {
306    fn drop(&mut self) {
307        if !self.dirty {
308            return;
309        }
310        if let Some(ref path) = self.cache_location {
311            let json = serde_json::to_string(&self.data).unwrap();
312            match paths::write(path, json.as_bytes()) {
313                Ok(()) => info!("updated rustc info cache"),
314                Err(e) => warn!("failed to update rustc info cache: {}", e),
315            }
316        }
317    }
318}
319
320fn rustc_fingerprint(
321    wrapper: Option<&Path>,
322    workspace_wrapper: Option<&Path>,
323    rustc: &Path,
324    rustup_rustc: &Path,
325    gctx: &GlobalContext,
326) -> CargoResult<u64> {
327    let mut hasher = StableHasher::new();
328
329    let hash_exe = |hasher: &mut _, path| -> CargoResult<()> {
330        let path = paths::resolve_executable(path)?;
331        path.hash(hasher);
332
333        let meta = paths::metadata(&path)?;
334        meta.len().hash(hasher);
335
336        // Often created and modified are the same, but not all filesystems support the former,
337        // and distro reproducible builds may clamp the latter, so we try to use both.
338        FileTime::from_creation_time(&meta).hash(hasher);
339        FileTime::from_last_modification_time(&meta).hash(hasher);
340        Ok(())
341    };
342
343    hash_exe(&mut hasher, rustc)?;
344    if let Some(wrapper) = wrapper {
345        hash_exe(&mut hasher, wrapper)?;
346    }
347    if let Some(workspace_wrapper) = workspace_wrapper {
348        hash_exe(&mut hasher, workspace_wrapper)?;
349    }
350
351    // Rustup can change the effective compiler without touching
352    // the `rustc` binary, so we try to account for this here.
353    // If we see rustup's env vars, we mix them into the fingerprint,
354    // but we also mix in the mtime of the actual compiler (and not
355    // the rustup shim at `~/.cargo/bin/rustup`), because `RUSTUP_TOOLCHAIN`
356    // could be just `stable-x86_64-unknown-linux-gnu`, i.e, it could
357    // not mention the version of Rust at all, which changes after
358    // `rustup update`.
359    //
360    // If we don't see rustup env vars, but it looks like the compiler
361    // is managed by rustup, we conservatively bail out.
362    let maybe_rustup = rustup_rustc == rustc;
363    match (
364        maybe_rustup,
365        gctx.get_env("RUSTUP_HOME"),
366        gctx.get_env("RUSTUP_TOOLCHAIN"),
367    ) {
368        (_, Ok(rustup_home), Ok(rustup_toolchain)) => {
369            debug!("adding rustup info to rustc fingerprint");
370            rustup_toolchain.hash(&mut hasher);
371            rustup_home.hash(&mut hasher);
372            let real_rustc = Path::new(&rustup_home)
373                .join("toolchains")
374                .join(rustup_toolchain)
375                .join("bin")
376                .join("rustc")
377                .with_extension(env::consts::EXE_EXTENSION);
378            paths::mtime(&real_rustc)?.hash(&mut hasher);
379        }
380        (true, _, _) => anyhow::bail!("probably rustup rustc, but without rustup's env vars"),
381        _ => (),
382    }
383
384    Ok(Hasher::finish(&hasher))
385}
386
387fn process_fingerprint(cmd: &ProcessBuilder, extra_fingerprint: u64) -> u64 {
388    let mut hasher = StableHasher::new();
389    extra_fingerprint.hash(&mut hasher);
390    cmd.get_args().for_each(|arg| arg.hash(&mut hasher));
391    let mut env = cmd.get_envs().iter().collect::<Vec<_>>();
392    env.sort_unstable();
393    env.hash(&mut hasher);
394    Hasher::finish(&hasher)
395}
cargo/util/rustc.rs

cargo/util/
rustc.rs
