bootstrap/core/
download.rs

1use std::env;
2use std::ffi::OsString;
3use std::fs::{self, File};
4use std::io::{BufRead, BufReader, BufWriter, ErrorKind, Write};
5use std::path::{Path, PathBuf};
6use std::sync::OnceLock;
7
8use xz2::bufread::XzDecoder;
9
10use crate::core::config::{BUILDER_CONFIG_FILENAME, TargetSelection};
11use crate::utils::build_stamp::BuildStamp;
12use crate::utils::exec::{ExecutionContext, command};
13use crate::utils::helpers::{exe, hex_encode, move_file};
14use crate::{Config, t};
15
16static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new();
17
18fn extract_curl_version(out: String) -> semver::Version {
19    // The output should look like this: "curl <major>.<minor>.<patch> ..."
20    out.lines()
21        .next()
22        .and_then(|line| line.split(" ").nth(1))
23        .and_then(|version| semver::Version::parse(version).ok())
24        .unwrap_or(semver::Version::new(1, 0, 0))
25}
26
27/// Generic helpers that are useful anywhere in bootstrap.
28impl Config {
29    pub fn is_verbose(&self) -> bool {
30        self.exec_ctx.is_verbose()
31    }
32
33    pub(crate) fn create<P: AsRef<Path>>(&self, path: P, s: &str) {
34        if self.dry_run() {
35            return;
36        }
37        t!(fs::write(path, s));
38    }
39
40    pub(crate) fn remove(&self, f: &Path) {
41        remove(&self.exec_ctx, f);
42    }
43
44    /// Create a temporary directory in `out` and return its path.
45    ///
46    /// NOTE: this temporary directory is shared between all steps;
47    /// if you need an empty directory, create a new subdirectory inside it.
48    pub(crate) fn tempdir(&self) -> PathBuf {
49        let tmp = self.out.join("tmp");
50        t!(fs::create_dir_all(&tmp));
51        tmp
52    }
53
54    /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true
55    /// on NixOS
56    fn should_fix_bins_and_dylibs(&self) -> bool {
57        should_fix_bins_and_dylibs(self.patch_binaries_for_nix, &self.exec_ctx)
58    }
59
60    /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
61    /// or the RPATH section, to fix the dynamic library search path
62    ///
63    /// This is only required on NixOS and uses the PatchELF utility to
64    /// change the interpreter/RPATH of ELF executables.
65    ///
66    /// Please see <https://nixos.org/patchelf.html> for more information
67    fn fix_bin_or_dylib(&self, fname: &Path) {
68        fix_bin_or_dylib(&self.out, fname, &self.exec_ctx);
69    }
70
71    fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
72        let dwn_ctx: DownloadContext<'_> = self.into();
73        download_file(dwn_ctx, url, dest_path, help_on_error);
74    }
75
76    fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
77        unpack(&self.exec_ctx, tarball, dst, pattern);
78    }
79
80    /// Returns whether the SHA256 checksum of `path` matches `expected`.
81    #[cfg(test)]
82    pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
83        verify(&self.exec_ctx, path, expected)
84    }
85}
86
87fn recorded_entries(dst: &Path, pattern: &str) -> Option<BufWriter<File>> {
88    let name = if pattern == "rustc-dev" {
89        ".rustc-dev-contents"
90    } else if pattern.starts_with("rust-std") {
91        ".rust-std-contents"
92    } else {
93        return None;
94    };
95    Some(BufWriter::new(t!(File::create(dst.join(name)))))
96}
97
98#[derive(Clone)]
99enum DownloadSource {
100    CI,
101    Dist,
102}
103
104/// Functions that are only ever called once, but named for clarity and to avoid thousand-line functions.
105impl Config {
106    pub(crate) fn download_clippy(&self) -> PathBuf {
107        self.verbose(|| println!("downloading stage0 clippy artifacts"));
108
109        let date = &self.stage0_metadata.compiler.date;
110        let version = &self.stage0_metadata.compiler.version;
111        let host = self.host_target;
112
113        let clippy_stamp =
114            BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").add_stamp(date);
115        let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host));
116        if cargo_clippy.exists() && clippy_stamp.is_up_to_date() {
117            return cargo_clippy;
118        }
119
120        let filename = format!("clippy-{version}-{host}.tar.xz");
121        self.download_component(DownloadSource::Dist, filename, "clippy-preview", date, "stage0");
122        if self.should_fix_bins_and_dylibs() {
123            self.fix_bin_or_dylib(&cargo_clippy);
124            self.fix_bin_or_dylib(&cargo_clippy.with_file_name(exe("clippy-driver", host)));
125        }
126
127        t!(clippy_stamp.write());
128        cargo_clippy
129    }
130
131    pub(crate) fn ci_rust_std_contents(&self) -> Vec<String> {
132        self.ci_component_contents(".rust-std-contents")
133    }
134
135    pub(crate) fn ci_rustc_dev_contents(&self) -> Vec<String> {
136        self.ci_component_contents(".rustc-dev-contents")
137    }
138
139    fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
140        assert!(self.download_rustc());
141        if self.dry_run() {
142            return vec![];
143        }
144
145        let ci_rustc_dir = self.ci_rustc_dir();
146        let stamp_file = ci_rustc_dir.join(stamp_file);
147        let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
148        t!(BufReader::new(contents_file).lines().collect())
149    }
150
151    pub(crate) fn download_ci_rustc(&self, commit: &str) {
152        self.verbose(|| println!("using downloaded stage2 artifacts from CI (commit {commit})"));
153
154        let version = self.artifact_version_part(commit);
155        // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the
156        // `rustc_private` crates for tools.
157        let extra_components = ["rustc-dev"];
158
159        self.download_toolchain(
160            &version,
161            "ci-rustc",
162            &format!("{commit}-{}", self.llvm_assertions),
163            &extra_components,
164            Self::download_ci_component,
165        );
166    }
167
168    fn download_toolchain(
169        &self,
170        version: &str,
171        sysroot: &str,
172        stamp_key: &str,
173        extra_components: &[&str],
174        download_component: fn(&Config, String, &str, &str),
175    ) {
176        let host = self.host_target.triple;
177        let bin_root = self.out.join(host).join(sysroot);
178        let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key);
179
180        if !bin_root.join("bin").join(exe("rustc", self.host_target)).exists()
181            || !rustc_stamp.is_up_to_date()
182        {
183            if bin_root.exists() {
184                t!(fs::remove_dir_all(&bin_root));
185            }
186            let filename = format!("rust-std-{version}-{host}.tar.xz");
187            let pattern = format!("rust-std-{host}");
188            download_component(self, filename, &pattern, stamp_key);
189            let filename = format!("rustc-{version}-{host}.tar.xz");
190            download_component(self, filename, "rustc", stamp_key);
191
192            for component in extra_components {
193                let filename = format!("{component}-{version}-{host}.tar.xz");
194                download_component(self, filename, component, stamp_key);
195            }
196
197            if self.should_fix_bins_and_dylibs() {
198                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
199                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
200                self.fix_bin_or_dylib(
201                    &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
202                );
203                let lib_dir = bin_root.join("lib");
204                for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
205                    let lib = t!(lib);
206                    if path_is_dylib(&lib.path()) {
207                        self.fix_bin_or_dylib(&lib.path());
208                    }
209                }
210            }
211
212            t!(rustc_stamp.write());
213        }
214    }
215
216    /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
217    // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
218    fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) {
219        Self::download_component(
220            self,
221            DownloadSource::CI,
222            filename,
223            prefix,
224            commit_with_assertions,
225            "ci-rustc",
226        )
227    }
228
229    fn download_component(
230        &self,
231        mode: DownloadSource,
232        filename: String,
233        prefix: &str,
234        key: &str,
235        destination: &str,
236    ) {
237        let dwn_ctx: DownloadContext<'_> = self.into();
238        download_component(dwn_ctx, mode, filename, prefix, key, destination);
239    }
240
241    #[cfg(test)]
242    pub(crate) fn maybe_download_ci_llvm(&self) {}
243
244    #[cfg(not(test))]
245    pub(crate) fn maybe_download_ci_llvm(&self) {
246        use build_helper::exit;
247        use build_helper::git::PathFreshness;
248
249        use crate::core::build_steps::llvm::detect_llvm_freshness;
250        use crate::core::config::toml::llvm::check_incompatible_options_for_ci_llvm;
251
252        if !self.llvm_from_ci {
253            return;
254        }
255
256        let llvm_root = self.ci_llvm_root();
257        let llvm_freshness =
258            detect_llvm_freshness(self, self.rust_info.is_managed_git_subrepository());
259        self.verbose(|| {
260            eprintln!("LLVM freshness: {llvm_freshness:?}");
261        });
262        let llvm_sha = match llvm_freshness {
263            PathFreshness::LastModifiedUpstream { upstream } => upstream,
264            PathFreshness::HasLocalModifications { upstream } => upstream,
265            PathFreshness::MissingUpstream => {
266                eprintln!("error: could not find commit hash for downloading LLVM");
267                eprintln!("HELP: maybe your repository history is too shallow?");
268                eprintln!("HELP: consider disabling `download-ci-llvm`");
269                eprintln!("HELP: or fetch enough history to include one upstream commit");
270                crate::exit!(1);
271            }
272        };
273        let stamp_key = format!("{}{}", llvm_sha, self.llvm_assertions);
274        let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").add_stamp(stamp_key);
275        if !llvm_stamp.is_up_to_date() && !self.dry_run() {
276            self.download_ci_llvm(&llvm_sha);
277
278            if self.should_fix_bins_and_dylibs() {
279                for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
280                    self.fix_bin_or_dylib(&t!(entry).path());
281                }
282            }
283
284            // Update the timestamp of llvm-config to force rustc_llvm to be
285            // rebuilt. This is a hacky workaround for a deficiency in Cargo where
286            // the rerun-if-changed directive doesn't handle changes very well.
287            // https://github.com/rust-lang/cargo/issues/10791
288            // Cargo only compares the timestamp of the file relative to the last
289            // time `rustc_llvm` build script ran. However, the timestamps of the
290            // files in the tarball are in the past, so it doesn't trigger a
291            // rebuild.
292            let now = std::time::SystemTime::now();
293            let file_times = fs::FileTimes::new().set_accessed(now).set_modified(now);
294
295            let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.host_target));
296            t!(crate::utils::helpers::set_file_times(llvm_config, file_times));
297
298            if self.should_fix_bins_and_dylibs() {
299                let llvm_lib = llvm_root.join("lib");
300                for entry in t!(fs::read_dir(llvm_lib)) {
301                    let lib = t!(entry).path();
302                    if path_is_dylib(&lib) {
303                        self.fix_bin_or_dylib(&lib);
304                    }
305                }
306            }
307
308            t!(llvm_stamp.write());
309        }
310
311        if let Some(config_path) = &self.config {
312            let current_config_toml = Self::get_toml(config_path).unwrap();
313
314            match self.get_builder_toml("ci-llvm") {
315                Ok(ci_config_toml) => {
316                    t!(check_incompatible_options_for_ci_llvm(current_config_toml, ci_config_toml));
317                }
318                Err(e) if e.to_string().contains("unknown field") => {
319                    println!(
320                        "WARNING: CI LLVM has some fields that are no longer supported in bootstrap; download-ci-llvm will be disabled."
321                    );
322                    println!("HELP: Consider rebasing to a newer commit if available.");
323                }
324                Err(e) => {
325                    eprintln!("ERROR: Failed to parse CI LLVM bootstrap.toml: {e}");
326                    exit!(2);
327                }
328            };
329        };
330    }
331
332    #[cfg(not(test))]
333    fn download_ci_llvm(&self, llvm_sha: &str) {
334        let llvm_assertions = self.llvm_assertions;
335
336        let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}");
337        let cache_dst =
338            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
339
340        let rustc_cache = cache_dst.join(cache_prefix);
341        if !rustc_cache.exists() {
342            t!(fs::create_dir_all(&rustc_cache));
343        }
344        let base = if llvm_assertions {
345            &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
346        } else {
347            &self.stage0_metadata.config.artifacts_server
348        };
349        let version = self.artifact_version_part(llvm_sha);
350        let filename = format!("rust-dev-{}-{}.tar.xz", version, self.host_target.triple);
351        let tarball = rustc_cache.join(&filename);
352        if !tarball.exists() {
353            let help_on_error = "ERROR: failed to download llvm from ci
354
355    HELP: There could be two reasons behind this:
356        1) The host triple is not supported for `download-ci-llvm`.
357        2) Old builds get deleted after a certain time.
358    HELP: In either case, disable `download-ci-llvm` in your bootstrap.toml:
359
360    [llvm]
361    download-ci-llvm = false
362    ";
363            self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
364        }
365        let llvm_root = self.ci_llvm_root();
366        self.unpack(&tarball, &llvm_root, "rust-dev");
367    }
368
369    pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) {
370        let cache_prefix = format!("gcc-{gcc_sha}");
371        let cache_dst =
372            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
373
374        let gcc_cache = cache_dst.join(cache_prefix);
375        if !gcc_cache.exists() {
376            t!(fs::create_dir_all(&gcc_cache));
377        }
378        let base = &self.stage0_metadata.config.artifacts_server;
379        let version = self.artifact_version_part(gcc_sha);
380        let filename = format!("gcc-{version}-{}.tar.xz", self.host_target.triple);
381        let tarball = gcc_cache.join(&filename);
382        if !tarball.exists() {
383            let help_on_error = "ERROR: failed to download gcc from ci
384
385    HELP: There could be two reasons behind this:
386        1) The host triple is not supported for `download-ci-gcc`.
387        2) Old builds get deleted after a certain time.
388    HELP: In either case, disable `download-ci-gcc` in your bootstrap.toml:
389
390    [gcc]
391    download-ci-gcc = false
392    ";
393            self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
394        }
395        self.unpack(&tarball, root_dir, "gcc");
396    }
397}
398
399/// Only should be used for pre config initialization downloads.
400pub(crate) struct DownloadContext<'a> {
401    host_target: TargetSelection,
402    out: &'a Path,
403    patch_binaries_for_nix: Option<bool>,
404    exec_ctx: &'a ExecutionContext,
405    stage0_metadata: &'a build_helper::stage0_parser::Stage0,
406    llvm_assertions: bool,
407    bootstrap_cache_path: &'a Option<PathBuf>,
408    is_running_on_ci: bool,
409}
410
411impl<'a> AsRef<DownloadContext<'a>> for DownloadContext<'a> {
412    fn as_ref(&self) -> &DownloadContext<'a> {
413        self
414    }
415}
416
417impl<'a> From<&'a Config> for DownloadContext<'a> {
418    fn from(value: &'a Config) -> Self {
419        DownloadContext {
420            host_target: value.host_target,
421            out: &value.out,
422            patch_binaries_for_nix: value.patch_binaries_for_nix,
423            exec_ctx: &value.exec_ctx,
424            stage0_metadata: &value.stage0_metadata,
425            llvm_assertions: value.llvm_assertions,
426            bootstrap_cache_path: &value.bootstrap_cache_path,
427            is_running_on_ci: value.is_running_on_ci,
428        }
429    }
430}
431
432fn path_is_dylib(path: &Path) -> bool {
433    // The .so is not necessarily the extension, it might be libLLVM.so.18.1
434    path.to_str().is_some_and(|path| path.contains(".so"))
435}
436
437/// Checks whether the CI rustc is available for the given target triple.
438pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
439    // All tier 1 targets and tier 2 targets with host tools.
440    const SUPPORTED_PLATFORMS: &[&str] = &[
441        "aarch64-apple-darwin",
442        "aarch64-pc-windows-msvc",
443        "aarch64-unknown-linux-gnu",
444        "aarch64-unknown-linux-musl",
445        "arm-unknown-linux-gnueabi",
446        "arm-unknown-linux-gnueabihf",
447        "armv7-unknown-linux-gnueabihf",
448        "i686-pc-windows-gnu",
449        "i686-pc-windows-msvc",
450        "i686-unknown-linux-gnu",
451        "loongarch64-unknown-linux-gnu",
452        "powerpc-unknown-linux-gnu",
453        "powerpc64-unknown-linux-gnu",
454        "powerpc64le-unknown-linux-gnu",
455        "powerpc64le-unknown-linux-musl",
456        "riscv64gc-unknown-linux-gnu",
457        "s390x-unknown-linux-gnu",
458        "x86_64-apple-darwin",
459        "x86_64-pc-windows-gnu",
460        "x86_64-pc-windows-msvc",
461        "x86_64-unknown-freebsd",
462        "x86_64-unknown-illumos",
463        "x86_64-unknown-linux-gnu",
464        "x86_64-unknown-linux-musl",
465        "x86_64-unknown-netbsd",
466    ];
467
468    const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
469        &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
470
471    if llvm_assertions {
472        SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
473    } else {
474        SUPPORTED_PLATFORMS.contains(&target_triple)
475    }
476}
477
478#[cfg(test)]
479pub(crate) fn maybe_download_rustfmt<'a>(
480    dwn_ctx: impl AsRef<DownloadContext<'a>>,
481) -> Option<PathBuf> {
482    Some(PathBuf::new())
483}
484
485/// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't
486/// reuse target directories or artifacts
487#[cfg(not(test))]
488pub(crate) fn maybe_download_rustfmt<'a>(
489    dwn_ctx: impl AsRef<DownloadContext<'a>>,
490) -> Option<PathBuf> {
491    use build_helper::stage0_parser::VersionMetadata;
492
493    let dwn_ctx = dwn_ctx.as_ref();
494
495    if dwn_ctx.exec_ctx.dry_run() {
496        return Some(PathBuf::new());
497    }
498
499    let VersionMetadata { date, version } = dwn_ctx.stage0_metadata.rustfmt.as_ref()?;
500    let channel = format!("{version}-{date}");
501
502    let host = dwn_ctx.host_target;
503    let bin_root = dwn_ctx.out.join(host).join("rustfmt");
504    let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host));
505    let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel);
506    if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() {
507        return Some(rustfmt_path);
508    }
509
510    download_component(
511        dwn_ctx,
512        DownloadSource::Dist,
513        format!("rustfmt-{version}-{build}.tar.xz", build = host.triple),
514        "rustfmt-preview",
515        date,
516        "rustfmt",
517    );
518
519    download_component(
520        dwn_ctx,
521        DownloadSource::Dist,
522        format!("rustc-{version}-{build}.tar.xz", build = host.triple),
523        "rustc",
524        date,
525        "rustfmt",
526    );
527
528    if should_fix_bins_and_dylibs(dwn_ctx.patch_binaries_for_nix, dwn_ctx.exec_ctx) {
529        fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustfmt"), dwn_ctx.exec_ctx);
530        fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("cargo-fmt"), dwn_ctx.exec_ctx);
531        let lib_dir = bin_root.join("lib");
532        for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
533            let lib = t!(lib);
534            if path_is_dylib(&lib.path()) {
535                fix_bin_or_dylib(dwn_ctx.out, &lib.path(), dwn_ctx.exec_ctx);
536            }
537        }
538    }
539
540    t!(rustfmt_stamp.write());
541    Some(rustfmt_path)
542}
543
544#[cfg(test)]
545pub(crate) fn download_beta_toolchain<'a>(dwn_ctx: impl AsRef<DownloadContext<'a>>) {}
546
547#[cfg(not(test))]
548pub(crate) fn download_beta_toolchain<'a>(dwn_ctx: impl AsRef<DownloadContext<'a>>) {
549    let dwn_ctx = dwn_ctx.as_ref();
550    dwn_ctx.exec_ctx.verbose(|| {
551        println!("downloading stage0 beta artifacts");
552    });
553
554    let date = dwn_ctx.stage0_metadata.compiler.date.clone();
555    let version = dwn_ctx.stage0_metadata.compiler.version.clone();
556    let extra_components = ["cargo"];
557    let sysroot = "stage0";
558    download_toolchain(
559        dwn_ctx,
560        &version,
561        sysroot,
562        &date,
563        &extra_components,
564        "stage0",
565        DownloadSource::Dist,
566    );
567}
568
569fn download_toolchain<'a>(
570    dwn_ctx: impl AsRef<DownloadContext<'a>>,
571    version: &str,
572    sysroot: &str,
573    stamp_key: &str,
574    extra_components: &[&str],
575    destination: &str,
576    mode: DownloadSource,
577) {
578    let dwn_ctx = dwn_ctx.as_ref();
579    let host = dwn_ctx.host_target.triple;
580    let bin_root = dwn_ctx.out.join(host).join(sysroot);
581    let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key);
582
583    if !bin_root.join("bin").join(exe("rustc", dwn_ctx.host_target)).exists()
584        || !rustc_stamp.is_up_to_date()
585    {
586        if bin_root.exists() {
587            t!(fs::remove_dir_all(&bin_root));
588        }
589        let filename = format!("rust-std-{version}-{host}.tar.xz");
590        let pattern = format!("rust-std-{host}");
591        download_component(dwn_ctx, mode.clone(), filename, &pattern, stamp_key, destination);
592        let filename = format!("rustc-{version}-{host}.tar.xz");
593        download_component(dwn_ctx, mode.clone(), filename, "rustc", stamp_key, destination);
594
595        for component in extra_components {
596            let filename = format!("{component}-{version}-{host}.tar.xz");
597            download_component(dwn_ctx, mode.clone(), filename, component, stamp_key, destination);
598        }
599
600        if should_fix_bins_and_dylibs(dwn_ctx.patch_binaries_for_nix, dwn_ctx.exec_ctx) {
601            fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustc"), dwn_ctx.exec_ctx);
602            fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustdoc"), dwn_ctx.exec_ctx);
603            fix_bin_or_dylib(
604                dwn_ctx.out,
605                &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
606                dwn_ctx.exec_ctx,
607            );
608            let lib_dir = bin_root.join("lib");
609            for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
610                let lib = t!(lib);
611                if path_is_dylib(&lib.path()) {
612                    fix_bin_or_dylib(dwn_ctx.out, &lib.path(), dwn_ctx.exec_ctx);
613                }
614            }
615        }
616
617        t!(rustc_stamp.write());
618    }
619}
620
621pub(crate) fn remove(exec_ctx: &ExecutionContext, f: &Path) {
622    if exec_ctx.dry_run() {
623        return;
624    }
625    fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {f:?}"));
626}
627
628fn fix_bin_or_dylib(out: &Path, fname: &Path, exec_ctx: &ExecutionContext) {
629    assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true));
630    println!("attempting to patch {}", fname.display());
631
632    // Only build `.nix-deps` once.
633    static NIX_DEPS_DIR: OnceLock<PathBuf> = OnceLock::new();
634    let mut nix_build_succeeded = true;
635    let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
636        // Run `nix-build` to "build" each dependency (which will likely reuse
637        // the existing `/nix/store` copy, or at most download a pre-built copy).
638        //
639        // Importantly, we create a gc-root called `.nix-deps` in the `build/`
640        // directory, but still reference the actual `/nix/store` path in the rpath
641        // as it makes it significantly more robust against changes to the location of
642        // the `.nix-deps` location.
643        //
644        // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
645        // zlib: Needed as a system dependency of `libLLVM-*.so`.
646        // patchelf: Needed for patching ELF binaries (see doc comment above).
647        let nix_deps_dir = out.join(".nix-deps");
648        const NIX_EXPR: &str = "
649        with (import <nixpkgs> {});
650        symlinkJoin {
651            name = \"rust-stage0-dependencies\";
652            paths = [
653                zlib
654                patchelf
655                stdenv.cc.bintools
656            ];
657        }
658        ";
659        nix_build_succeeded = command("nix-build")
660            .allow_failure()
661            .args([Path::new("-E"), Path::new(NIX_EXPR), Path::new("-o"), &nix_deps_dir])
662            .run_capture_stdout(exec_ctx)
663            .is_success();
664        nix_deps_dir
665    });
666    if !nix_build_succeeded {
667        return;
668    }
669
670    let mut patchelf = command(nix_deps_dir.join("bin/patchelf"));
671    patchelf.args(&[
672        OsString::from("--add-rpath"),
673        OsString::from(t!(fs::canonicalize(nix_deps_dir)).join("lib")),
674    ]);
675    if !path_is_dylib(fname) {
676        // Finally, set the correct .interp for binaries
677        let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
678        let dynamic_linker = t!(fs::read_to_string(dynamic_linker_path));
679        patchelf.args(["--set-interpreter", dynamic_linker.trim_end()]);
680    }
681    patchelf.arg(fname);
682    let _ = patchelf.allow_failure().run_capture_stdout(exec_ctx);
683}
684
685fn should_fix_bins_and_dylibs(
686    patch_binaries_for_nix: Option<bool>,
687    exec_ctx: &ExecutionContext,
688) -> bool {
689    let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| {
690        let uname = command("uname").allow_failure().arg("-s").run_capture_stdout(exec_ctx);
691        if uname.is_failure() {
692            return false;
693        }
694        let output = uname.stdout();
695        if !output.starts_with("Linux") {
696            return false;
697        }
698        // If the user has asked binaries to be patched for Nix, then
699        // don't check for NixOS or `/lib`.
700        // NOTE: this intentionally comes after the Linux check:
701        // - patchelf only works with ELF files, so no need to run it on Mac or Windows
702        // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
703        if let Some(explicit_value) = patch_binaries_for_nix {
704            return explicit_value;
705        }
706
707        // Use `/etc/os-release` instead of `/etc/NIXOS`.
708        // The latter one does not exist on NixOS when using tmpfs as root.
709        let is_nixos = match File::open("/etc/os-release") {
710            Err(e) if e.kind() == ErrorKind::NotFound => false,
711            Err(e) => panic!("failed to access /etc/os-release: {e}"),
712            Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
713                let l = l.expect("reading /etc/os-release");
714                matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
715            }),
716        };
717        if !is_nixos {
718            let in_nix_shell = env::var("IN_NIX_SHELL");
719            if let Ok(in_nix_shell) = in_nix_shell {
720                eprintln!(
721                    "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \
722                     you may need to set `patch-binaries-for-nix=true` in bootstrap.toml"
723                );
724            }
725        }
726        is_nixos
727    });
728    if val {
729        eprintln!("INFO: You seem to be using Nix.");
730    }
731    val
732}
733
734fn download_component<'a>(
735    dwn_ctx: impl AsRef<DownloadContext<'a>>,
736    mode: DownloadSource,
737    filename: String,
738    prefix: &str,
739    key: &str,
740    destination: &str,
741) {
742    let dwn_ctx = dwn_ctx.as_ref();
743
744    if dwn_ctx.exec_ctx.dry_run() {
745        return;
746    }
747
748    let cache_dst =
749        dwn_ctx.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| dwn_ctx.out.join("cache"));
750
751    let cache_dir = cache_dst.join(key);
752    if !cache_dir.exists() {
753        t!(fs::create_dir_all(&cache_dir));
754    }
755
756    let bin_root = dwn_ctx.out.join(dwn_ctx.host_target).join(destination);
757    let tarball = cache_dir.join(&filename);
758    let (base_url, url, should_verify) = match mode {
759        DownloadSource::CI => {
760            let dist_server = if dwn_ctx.llvm_assertions {
761                dwn_ctx.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone()
762            } else {
763                dwn_ctx.stage0_metadata.config.artifacts_server.clone()
764            };
765            let url = format!(
766                "{}/{filename}",
767                key.strip_suffix(&format!("-{}", dwn_ctx.llvm_assertions)).unwrap()
768            );
769            (dist_server, url, false)
770        }
771        DownloadSource::Dist => {
772            let dist_server = env::var("RUSTUP_DIST_SERVER")
773                .unwrap_or(dwn_ctx.stage0_metadata.config.dist_server.to_string());
774            // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0
775            (dist_server, format!("dist/{key}/{filename}"), true)
776        }
777    };
778
779    // For the stage0 compiler, put special effort into ensuring the checksums are valid.
780    let checksum = if should_verify {
781        let error = format!(
782            "src/stage0 doesn't contain a checksum for {url}. \
783            Pre-built artifacts might not be available for this \
784            target at this time, see https://doc.rust-lang.org/nightly\
785            /rustc/platform-support.html for more information."
786        );
787        let sha256 = dwn_ctx.stage0_metadata.checksums_sha256.get(&url).expect(&error);
788        if tarball.exists() {
789            if verify(dwn_ctx.exec_ctx, &tarball, sha256) {
790                unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
791                return;
792            } else {
793                dwn_ctx.exec_ctx.verbose(|| {
794                    println!(
795                        "ignoring cached file {} due to failed verification",
796                        tarball.display()
797                    )
798                });
799                remove(dwn_ctx.exec_ctx, &tarball);
800            }
801        }
802        Some(sha256)
803    } else if tarball.exists() {
804        unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
805        return;
806    } else {
807        None
808    };
809
810    let mut help_on_error = "";
811    if destination == "ci-rustc" {
812        help_on_error = "ERROR: failed to download pre-built rustc from CI
813
814NOTE: old builds get deleted after a certain time
815HELP: if trying to compile an old commit of rustc, disable `download-rustc` in bootstrap.toml:
816
817[rust]
818download-rustc = false
819";
820    }
821    download_file(dwn_ctx, &format!("{base_url}/{url}"), &tarball, help_on_error);
822    if let Some(sha256) = checksum
823        && !verify(dwn_ctx.exec_ctx, &tarball, sha256)
824    {
825        panic!("failed to verify {}", tarball.display());
826    }
827
828    unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
829}
830
831pub(crate) fn verify(exec_ctx: &ExecutionContext, path: &Path, expected: &str) -> bool {
832    use sha2::Digest;
833
834    exec_ctx.verbose(|| {
835        println!("verifying {}", path.display());
836    });
837
838    if exec_ctx.dry_run() {
839        return false;
840    }
841
842    let mut hasher = sha2::Sha256::new();
843
844    let file = t!(File::open(path));
845    let mut reader = BufReader::new(file);
846
847    loop {
848        let buffer = t!(reader.fill_buf());
849        let l = buffer.len();
850        // break if EOF
851        if l == 0 {
852            break;
853        }
854        hasher.update(buffer);
855        reader.consume(l);
856    }
857
858    let checksum = hex_encode(hasher.finalize().as_slice());
859    let verified = checksum == expected;
860
861    if !verified {
862        println!(
863            "invalid checksum: \n\
864            found:    {checksum}\n\
865            expected: {expected}",
866        );
867    }
868
869    verified
870}
871
872fn unpack(exec_ctx: &ExecutionContext, tarball: &Path, dst: &Path, pattern: &str) {
873    eprintln!("extracting {} to {}", tarball.display(), dst.display());
874    if !dst.exists() {
875        t!(fs::create_dir_all(dst));
876    }
877
878    // `tarball` ends with `.tar.xz`; strip that suffix
879    // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
880    let uncompressed_filename =
881        Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
882    let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
883
884    // decompress the file
885    let data = t!(File::open(tarball), format!("file {} not found", tarball.display()));
886    let decompressor = XzDecoder::new(BufReader::new(data));
887
888    let mut tar = tar::Archive::new(decompressor);
889
890    let is_ci_rustc = dst.ends_with("ci-rustc");
891    let is_ci_llvm = dst.ends_with("ci-llvm");
892
893    // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding
894    // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow.
895    // Cache the entries when we extract it so we only have to read it once.
896    let mut recorded_entries = if is_ci_rustc { recorded_entries(dst, pattern) } else { None };
897
898    for member in t!(tar.entries()) {
899        let mut member = t!(member);
900        let original_path = t!(member.path()).into_owned();
901        // skip the top-level directory
902        if original_path == directory_prefix {
903            continue;
904        }
905        let mut short_path = t!(original_path.strip_prefix(directory_prefix));
906        let is_builder_config = short_path.to_str() == Some(BUILDER_CONFIG_FILENAME);
907
908        if !(short_path.starts_with(pattern) || ((is_ci_rustc || is_ci_llvm) && is_builder_config))
909        {
910            continue;
911        }
912        short_path = short_path.strip_prefix(pattern).unwrap_or(short_path);
913        let dst_path = dst.join(short_path);
914
915        exec_ctx.verbose(|| {
916            println!("extracting {} to {}", original_path.display(), dst.display());
917        });
918
919        if !t!(member.unpack_in(dst)) {
920            panic!("path traversal attack ??");
921        }
922        if let Some(record) = &mut recorded_entries {
923            t!(writeln!(record, "{}", short_path.to_str().unwrap()));
924        }
925        let src_path = dst.join(original_path);
926        if src_path.is_dir() && dst_path.exists() {
927            continue;
928        }
929        t!(move_file(src_path, dst_path));
930    }
931    let dst_dir = dst.join(directory_prefix);
932    if dst_dir.exists() {
933        t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display()));
934    }
935}
936
937fn download_file<'a>(
938    dwn_ctx: impl AsRef<DownloadContext<'a>>,
939    url: &str,
940    dest_path: &Path,
941    help_on_error: &str,
942) {
943    let dwn_ctx = dwn_ctx.as_ref();
944
945    dwn_ctx.exec_ctx.verbose(|| {
946        println!("download {url}");
947    });
948    // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
949    let tempfile = tempdir(dwn_ctx.out).join(dest_path.file_name().unwrap());
950    // While bootstrap itself only supports http and https downloads, downstream forks might
951    // need to download components from other protocols. The match allows them adding more
952    // protocols without worrying about merge conflicts if we change the HTTP implementation.
953    match url.split_once("://").map(|(proto, _)| proto) {
954        Some("http") | Some("https") => download_http_with_retries(
955            dwn_ctx.host_target,
956            dwn_ctx.is_running_on_ci,
957            dwn_ctx.exec_ctx,
958            &tempfile,
959            url,
960            help_on_error,
961        ),
962        Some(other) => panic!("unsupported protocol {other} in {url}"),
963        None => panic!("no protocol in {url}"),
964    }
965    t!(move_file(&tempfile, dest_path), format!("failed to rename {tempfile:?} to {dest_path:?}"));
966}
967
968/// Create a temporary directory in `out` and return its path.
969///
970/// NOTE: this temporary directory is shared between all steps;
971/// if you need an empty directory, create a new subdirectory inside it.
972pub(crate) fn tempdir(out: &Path) -> PathBuf {
973    let tmp = out.join("tmp");
974    t!(fs::create_dir_all(&tmp));
975    tmp
976}
977
978fn download_http_with_retries(
979    host_target: TargetSelection,
980    is_running_on_ci: bool,
981    exec_ctx: &ExecutionContext,
982    tempfile: &Path,
983    url: &str,
984    help_on_error: &str,
985) {
986    println!("downloading {url}");
987    // Try curl. If that fails and we are on windows, fallback to PowerShell.
988    // options should be kept in sync with
989    // src/bootstrap/src/core/download.rs
990    // for consistency
991    let mut curl = command("curl").allow_failure();
992    curl.args([
993        // follow redirect
994        "--location",
995        // timeout if speed is < 10 bytes/sec for > 30 seconds
996        "--speed-time",
997        "30",
998        "--speed-limit",
999        "10",
1000        // timeout if cannot connect within 30 seconds
1001        "--connect-timeout",
1002        "30",
1003        // output file
1004        "--output",
1005        tempfile.to_str().unwrap(),
1006        // if there is an error, don't restart the download,
1007        // instead continue where it left off.
1008        "--continue-at",
1009        "-",
1010        // retry up to 3 times.  note that this means a maximum of 4
1011        // attempts will be made, since the first attempt isn't a *re*try.
1012        "--retry",
1013        "3",
1014        // show errors, even if --silent is specified
1015        "--show-error",
1016        // set timestamp of downloaded file to that of the server
1017        "--remote-time",
1018        // fail on non-ok http status
1019        "--fail",
1020    ]);
1021    // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful.
1022    if is_running_on_ci {
1023        curl.arg("--silent");
1024    } else {
1025        curl.arg("--progress-bar");
1026    }
1027    // --retry-all-errors was added in 7.71.0, don't use it if curl is old.
1028    if curl_version(exec_ctx) >= semver::Version::new(7, 71, 0) {
1029        curl.arg("--retry-all-errors");
1030    }
1031    curl.arg(url);
1032    if !curl.run(exec_ctx) {
1033        if host_target.contains("windows-msvc") {
1034            eprintln!("Fallback to PowerShell");
1035            for _ in 0..3 {
1036                let powershell = command("PowerShell.exe").allow_failure().args([
1037                    "/nologo",
1038                    "-Command",
1039                    "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
1040                    &format!(
1041                        "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
1042                        url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
1043                    ),
1044                ]).run_capture_stdout(exec_ctx);
1045
1046                if powershell.is_failure() {
1047                    return;
1048                }
1049
1050                eprintln!("\nspurious failure, trying again");
1051            }
1052        }
1053        if !help_on_error.is_empty() {
1054            eprintln!("{help_on_error}");
1055        }
1056        crate::exit!(1);
1057    }
1058}
1059
1060fn curl_version(exec_ctx: &ExecutionContext) -> semver::Version {
1061    let mut curl = command("curl");
1062    curl.arg("-V");
1063    let curl = curl.run_capture_stdout(exec_ctx);
1064    if curl.is_failure() {
1065        return semver::Version::new(1, 0, 0);
1066    }
1067    let output = curl.stdout();
1068    extract_curl_version(output)
1069}